From 604cff4887fb165168510efbec04e60a41416fbc Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 17 Jul 2024 12:15:57 +0800 Subject: [PATCH 001/131] feat: add rule tag to rule_engine log --- apps/emqx_rule_engine/include/rule_engine.hrl | 1 + .../src/emqx_rule_actions.erl | 12 +++- .../src/emqx_rule_api_schema.erl | 13 +++-- .../emqx_rule_engine/src/emqx_rule_engine.erl | 6 +- .../src/emqx_rule_engine_api.erl | 55 ++++++++++++------- .../src/emqx_rule_sqltester.erl | 14 +++-- 6 files changed, 70 insertions(+), 31 deletions(-) diff --git a/apps/emqx_rule_engine/include/rule_engine.hrl b/apps/emqx_rule_engine/include/rule_engine.hrl index 7d0000a1c..cf0aae4d8 100644 --- a/apps/emqx_rule_engine/include/rule_engine.hrl +++ b/apps/emqx_rule_engine/include/rule_engine.hrl @@ -128,3 +128,4 @@ -define(KEY_PATH, [rule_engine, rules]). -define(RULE_PATH(RULE), [rule_engine, rules, RULE]). +-define(TAG, "RULE"). diff --git a/apps/emqx_rule_engine/src/emqx_rule_actions.erl b/apps/emqx_rule_engine/src/emqx_rule_actions.erl index dcd9024ef..950bdcf4a 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_actions.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_actions.erl @@ -134,7 +134,15 @@ republish( }, _Args ) -> - ?SLOG(error, #{msg => "recursive_republish_detected", topic => Topic}); + ?SLOG( + error, + #{ + msg => "recursive_republish_detected", + topic => Topic, + rule_id => RuleId + }, + #{tag => ?TAG} + ); republish( Selected, #{metadata := #{rule_id := RuleId}} = Env, @@ -311,6 +319,8 @@ render_pub_props(UserPropertiesTemplate, Selected, Env) -> rule_id => emqx_utils_maps:deep_get([metadata, rule_id], ENV, undefined), reason => REASON, property => K + }#{ + tag => ?TAG } ) ). diff --git a/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl b/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl index 2450253c1..ad1e29f17 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl @@ -21,6 +21,7 @@ -include_lib("typerefl/include/types.hrl"). -include_lib("hocon/include/hoconsc.hrl"). -include_lib("emqx/include/logger.hrl"). +-include("rule_engine.hrl"). -export([check_params/2]). @@ -36,10 +37,14 @@ check_params(Params, Tag) -> #{Tag := Checked} -> {ok, Checked} catch throw:Reason -> - ?SLOG(error, #{ - msg => "check_rule_params_failed", - reason => Reason - }), + ?SLOG( + info, + #{ + msg => "check_rule_params_failed", + reason => Reason + }, + #{tag => ?TAG} + ), {error, Reason} end. diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.erl b/apps/emqx_rule_engine/src/emqx_rule_engine.erl index 2adf511a2..246b9c06a 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.erl @@ -437,15 +437,15 @@ handle_call({delete_rule, Rule}, _From, State) -> ok = do_delete_rule(Rule), {reply, ok, State}; handle_call(Req, _From, State) -> - ?SLOG(error, #{msg => "unexpected_call", request => Req}), + ?SLOG(error, #{msg => "unexpected_call", request => Req}, #{tag => ?TAG}), {reply, ignored, State}. handle_cast(Msg, State) -> - ?SLOG(error, #{msg => "unexpected_cast", request => Msg}), + ?SLOG(error, #{msg => "unexpected_cast", request => Msg}, #{tag => ?TAG}), {noreply, State}. handle_info(Info, State) -> - ?SLOG(error, #{msg => "unexpected_info", request => Info}), + ?SLOG(error, #{msg => "unexpected_info", request => Info}, #{tag => ?TAG}), {noreply, State}. terminate(_Reason, _State) -> diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl index b1308f008..d2ac0389d 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl @@ -388,11 +388,15 @@ param_path_id() -> {ok, #{post_config_update := #{emqx_rule_engine := Rule}}} -> {201, format_rule_info_resp(Rule)}; {error, Reason} -> - ?SLOG(error, #{ - msg => "create_rule_failed", - id => Id, - reason => Reason - }), + ?SLOG( + info, + #{ + msg => "create_rule_failed", + rule_id => Id, + reason => Reason + }, + #{tag => ?TAG} + ), {400, #{code => 'BAD_REQUEST', message => ?ERR_BADARGS(Reason)}} end end @@ -448,11 +452,15 @@ param_path_id() -> {ok, #{post_config_update := #{emqx_rule_engine := Rule}}} -> {200, format_rule_info_resp(Rule)}; {error, Reason} -> - ?SLOG(error, #{ - msg => "update_rule_failed", - id => Id, - reason => Reason - }), + ?SLOG( + info, + #{ + msg => "update_rule_failed", + rule_id => Id, + reason => Reason + }, + #{tag => ?TAG} + ), {400, #{code => 'BAD_REQUEST', message => ?ERR_BADARGS(Reason)}} end; '/rules/:id'(delete, #{bindings := #{id := Id}}) -> @@ -463,11 +471,15 @@ param_path_id() -> {ok, _} -> {204}; {error, Reason} -> - ?SLOG(error, #{ - msg => "delete_rule_failed", - id => Id, - reason => Reason - }), + ?SLOG( + error, + #{ + msg => "delete_rule_failed", + rule_id => Id, + reason => Reason + }, + #{tag => ?TAG} + ), {500, #{code => 'INTERNAL_ERROR', message => ?ERR_BADARGS(Reason)}} end; not_found -> @@ -587,10 +599,15 @@ get_rule_metrics(Id) -> NodeMetrics = [format_metrics(Node, Metrics) || {Node, {ok, Metrics}} <- NodeResults], NodeErrors = [Result || Result = {_Node, {NOk, _}} <- NodeResults, NOk =/= ok], NodeErrors == [] orelse - ?SLOG(warning, #{ - msg => "rpc_get_rule_metrics_errors", - errors => NodeErrors - }), + ?SLOG( + warning, + #{ + msg => "rpc_get_rule_metrics_errors", + rule_id => Id, + errors => NodeErrors + }, + #{tag => ?TAG} + ), NodeMetrics. format_metrics(Node, #{ diff --git a/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl b/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl index f766cd273..8b83a7217 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl @@ -15,6 +15,7 @@ -module(emqx_rule_sqltester). -include_lib("emqx/include/logger.hrl"). +-include("rule_engine.hrl"). -export([ test/1, @@ -127,10 +128,15 @@ test(#{sql := Sql, context := Context}) -> end end; {error, Reason} -> - ?SLOG(debug, #{ - msg => "rulesql_parse_error", - detail => Reason - }), + ?SLOG( + debug, + #{ + msg => "rulesql_parse_error", + sql => Sql, + reason => Reason + }, + #{tag => ?TAG} + ), {error, Reason} end. From 52031441cf08d1ad712bb1b77405de849e0a8489 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 17 Jul 2024 12:19:50 +0800 Subject: [PATCH 002/131] chore: add authz tag to match_rule_error log --- .../src/emqx_authz/emqx_authz_rule.erl | 2 +- .../src/emqx_authz/emqx_authz_utils.erl | 56 +++++++++++++-- .../src/emqx_auth_mysql.app.src | 2 +- apps/emqx_auth_mysql/src/emqx_authz_mysql.erl | 20 ++---- .../src/emqx_auth_postgresql.app.src | 2 +- .../src/emqx_authz_postgresql.erl | 21 ++---- .../src/emqx_auth_redis.app.src | 2 +- apps/emqx_auth_redis/src/emqx_authz_redis.erl | 68 ++++++++----------- 8 files changed, 90 insertions(+), 83 deletions(-) diff --git a/apps/emqx_auth/src/emqx_authz/emqx_authz_rule.erl b/apps/emqx_auth/src/emqx_authz/emqx_authz_rule.erl index 8d7e12fcd..442852329 100644 --- a/apps/emqx_auth/src/emqx_authz/emqx_authz_rule.erl +++ b/apps/emqx_auth/src/emqx_authz/emqx_authz_rule.erl @@ -163,7 +163,7 @@ qos_from_opts(Opts) -> ) end catch - {bad_qos, QoS} -> + throw:{bad_qos, QoS} -> throw(#{ reason => invalid_authorization_qos, qos => QoS diff --git a/apps/emqx_auth/src/emqx_authz/emqx_authz_utils.erl b/apps/emqx_auth/src/emqx_authz/emqx_authz_utils.erl index 5c7b0965c..e4343b6fa 100644 --- a/apps/emqx_auth/src/emqx_authz/emqx_authz_utils.erl +++ b/apps/emqx_auth/src/emqx_authz/emqx_authz_utils.erl @@ -16,6 +16,8 @@ -module(emqx_authz_utils). +-feature(maybe_expr, enable). + -include_lib("emqx/include/emqx_placeholder.hrl"). -include_lib("emqx_authz.hrl"). -include_lib("snabbkaffe/include/trace.hrl"). @@ -37,7 +39,7 @@ render_sql_params/2, client_vars/1, vars_for_rule_query/2, - parse_rule_from_row/2 + do_authorize/6 ]). -export([ @@ -221,14 +223,18 @@ content_type(Headers) when is_list(Headers) -> -define(RAW_RULE_KEYS, [<<"permission">>, <<"action">>, <<"topic">>, <<"qos">>, <<"retain">>]). -parse_rule_from_row(ColumnNames, Row) -> - RuleRaw = maps:with(?RAW_RULE_KEYS, maps:from_list(lists:zip(ColumnNames, to_list(Row)))), - case emqx_authz_rule_raw:parse_rule(RuleRaw) of +-spec parse_rule_from_row([binary()], [binary()] | map()) -> + {ok, emqx_authz_rule:rule()} | {error, term()}. +parse_rule_from_row(_ColumnNames, RuleMap = #{}) -> + case emqx_authz_rule_raw:parse_rule(RuleMap) of {ok, {Permission, Action, Topics}} -> - emqx_authz_rule:compile({Permission, all, Action, Topics}); + {ok, emqx_authz_rule:compile({Permission, all, Action, Topics})}; {error, Reason} -> - error(Reason) - end. + {error, Reason} + end; +parse_rule_from_row(ColumnNames, Row) -> + RuleMap = maps:with(?RAW_RULE_KEYS, maps:from_list(lists:zip(ColumnNames, to_list(Row)))), + parse_rule_from_row(ColumnNames, RuleMap). vars_for_rule_query(Client, ?authz_action(PubSub, Qos) = Action) -> Client#{ @@ -281,3 +287,39 @@ to_list(Tuple) when is_tuple(Tuple) -> tuple_to_list(Tuple); to_list(List) when is_list(List) -> List. + +do_authorize(Type, Client, Action, Topic, ColumnNames, Row) -> + try + maybe + {ok, Rule} ?= parse_rule_from_row(ColumnNames, Row), + {matched, Permission} ?= emqx_authz_rule:match(Client, Action, Topic, Rule), + {matched, Permission} + else + nomatch -> + nomatch; + {error, Reason0} -> + log_match_rule_error(Type, Row, Reason0), + nomatch + end + catch + throw:Reason1 -> + log_match_rule_error(Type, Row, Reason1), + nomatch + end. + +log_match_rule_error(Type, Row, Reason0) -> + Msg0 = #{ + msg => "match_rule_error", + rule => Row, + type => Type + }, + Msg1 = + case is_map(Reason0) of + true -> maps:merge(Msg0, Reason0); + false -> Msg0#{reason => Reason0} + end, + ?SLOG( + error, + Msg1, + #{tag => "AUTHZ"} + ). diff --git a/apps/emqx_auth_mysql/src/emqx_auth_mysql.app.src b/apps/emqx_auth_mysql/src/emqx_auth_mysql.app.src index 29bf97ac8..abd9a7e27 100644 --- a/apps/emqx_auth_mysql/src/emqx_auth_mysql.app.src +++ b/apps/emqx_auth_mysql/src/emqx_auth_mysql.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_auth_mysql, [ {description, "EMQX MySQL Authentication and Authorization"}, - {vsn, "0.2.0"}, + {vsn, "0.2.1"}, {registered, []}, {mod, {emqx_auth_mysql_app, []}}, {applications, [ diff --git a/apps/emqx_auth_mysql/src/emqx_authz_mysql.erl b/apps/emqx_auth_mysql/src/emqx_authz_mysql.erl index 59ed878ab..0e2b77005 100644 --- a/apps/emqx_auth_mysql/src/emqx_authz_mysql.erl +++ b/apps/emqx_auth_mysql/src/emqx_authz_mysql.erl @@ -101,19 +101,9 @@ authorize( do_authorize(_Client, _Action, _Topic, _ColumnNames, []) -> nomatch; do_authorize(Client, Action, Topic, ColumnNames, [Row | Tail]) -> - try - emqx_authz_rule:match( - Client, Action, Topic, emqx_authz_utils:parse_rule_from_row(ColumnNames, Row) - ) - of - {matched, Permission} -> {matched, Permission}; - nomatch -> do_authorize(Client, Action, Topic, ColumnNames, Tail) - catch - error:Reason -> - ?SLOG(error, #{ - msg => "match_rule_error", - reason => Reason, - rule => Row - }), - do_authorize(Client, Action, Topic, ColumnNames, Tail) + case emqx_authz_utils:do_authorize(mysql, Client, Action, Topic, ColumnNames, Row) of + nomatch -> + do_authorize(Client, Action, Topic, ColumnNames, Tail); + {matched, Permission} -> + {matched, Permission} end. diff --git a/apps/emqx_auth_postgresql/src/emqx_auth_postgresql.app.src b/apps/emqx_auth_postgresql/src/emqx_auth_postgresql.app.src index 3978f7dbc..1eabc93f0 100644 --- a/apps/emqx_auth_postgresql/src/emqx_auth_postgresql.app.src +++ b/apps/emqx_auth_postgresql/src/emqx_auth_postgresql.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_auth_postgresql, [ {description, "EMQX PostgreSQL Authentication and Authorization"}, - {vsn, "0.2.0"}, + {vsn, "0.2.1"}, {registered, []}, {mod, {emqx_auth_postgresql_app, []}}, {applications, [ diff --git a/apps/emqx_auth_postgresql/src/emqx_authz_postgresql.erl b/apps/emqx_auth_postgresql/src/emqx_authz_postgresql.erl index a77f0a424..d1a0b32ea 100644 --- a/apps/emqx_auth_postgresql/src/emqx_authz_postgresql.erl +++ b/apps/emqx_auth_postgresql/src/emqx_authz_postgresql.erl @@ -107,22 +107,11 @@ authorize( do_authorize(_Client, _Action, _Topic, _ColumnNames, []) -> nomatch; do_authorize(Client, Action, Topic, ColumnNames, [Row | Tail]) -> - try - emqx_authz_rule:match( - Client, Action, Topic, emqx_authz_utils:parse_rule_from_row(ColumnNames, Row) - ) - of - {matched, Permission} -> {matched, Permission}; - nomatch -> do_authorize(Client, Action, Topic, ColumnNames, Tail) - catch - error:Reason:Stack -> - ?SLOG(error, #{ - msg => "match_rule_error", - reason => Reason, - rule => Row, - stack => Stack - }), - do_authorize(Client, Action, Topic, ColumnNames, Tail) + case emqx_authz_utils:do_authorize(postgresql, Client, Action, Topic, ColumnNames, Row) of + nomatch -> + do_authorize(Client, Action, Topic, ColumnNames, Tail); + {matched, Permission} -> + {matched, Permission} end. column_names(Columns) -> diff --git a/apps/emqx_auth_redis/src/emqx_auth_redis.app.src b/apps/emqx_auth_redis/src/emqx_auth_redis.app.src index 74168495b..9b43eca2c 100644 --- a/apps/emqx_auth_redis/src/emqx_auth_redis.app.src +++ b/apps/emqx_auth_redis/src/emqx_auth_redis.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_auth_redis, [ {description, "EMQX Redis Authentication and Authorization"}, - {vsn, "0.2.0"}, + {vsn, "0.2.1"}, {registered, []}, {mod, {emqx_auth_redis_app, []}}, {applications, [ diff --git a/apps/emqx_auth_redis/src/emqx_authz_redis.erl b/apps/emqx_auth_redis/src/emqx_authz_redis.erl index a7f88f7c6..8ce975033 100644 --- a/apps/emqx_auth_redis/src/emqx_authz_redis.erl +++ b/apps/emqx_auth_redis/src/emqx_authz_redis.erl @@ -92,44 +92,30 @@ authorize( do_authorize(_Client, _Action, _Topic, []) -> nomatch; do_authorize(Client, Action, Topic, [TopicFilterRaw, RuleEncoded | Tail]) -> - try - emqx_authz_rule:match( - Client, - Action, - Topic, - compile_rule(RuleEncoded, TopicFilterRaw) - ) - of - {matched, Permission} -> {matched, Permission}; - nomatch -> do_authorize(Client, Action, Topic, Tail) - catch - error:Reason:Stack -> - ?SLOG(error, #{ - msg => "match_rule_error", - reason => Reason, - rule_encoded => RuleEncoded, - topic_filter_raw => TopicFilterRaw, - stacktrace => Stack + case parse_rule(RuleEncoded) of + {ok, RuleMap0} -> + RuleMap = + maps:merge( + #{ + <<"permission">> => <<"allow">>, + <<"topic">> => TopicFilterRaw + }, + RuleMap0 + ), + case emqx_authz_utils:do_authorize(redis, Client, Action, Topic, undefined, RuleMap) of + nomatch -> + do_authorize(Client, Action, Topic, Tail); + {matched, Permission} -> + {matched, Permission} + end; + {error, Reason} -> + ?SLOG(error, Reason#{ + msg => "parse_rule_error", + rule => RuleEncoded }), do_authorize(Client, Action, Topic, Tail) end. -compile_rule(RuleBin, TopicFilterRaw) -> - RuleRaw = - maps:merge( - #{ - <<"permission">> => <<"allow">>, - <<"topic">> => TopicFilterRaw - }, - parse_rule(RuleBin) - ), - case emqx_authz_rule_raw:parse_rule(RuleRaw) of - {ok, {Permission, Action, Topics}} -> - emqx_authz_rule:compile({Permission, all, Action, Topics}); - {error, Reason} -> - error(Reason) - end. - parse_cmd(Query) -> case emqx_redis_command:split(Query) of {ok, Cmd} -> @@ -154,17 +140,17 @@ validate_cmd(Cmd) -> end. parse_rule(<<"publish">>) -> - #{<<"action">> => <<"publish">>}; + {ok, #{<<"action">> => <<"publish">>}}; parse_rule(<<"subscribe">>) -> - #{<<"action">> => <<"subscribe">>}; + {ok, #{<<"action">> => <<"subscribe">>}}; parse_rule(<<"all">>) -> - #{<<"action">> => <<"all">>}; + {ok, #{<<"action">> => <<"all">>}}; parse_rule(Bin) when is_binary(Bin) -> case emqx_utils_json:safe_decode(Bin, [return_maps]) of {ok, Map} when is_map(Map) -> - maps:with([<<"qos">>, <<"action">>, <<"retain">>], Map); + {ok, maps:with([<<"qos">>, <<"action">>, <<"retain">>], Map)}; {ok, _} -> - error({invalid_topic_rule, Bin, notamap}); - {error, Error} -> - error({invalid_topic_rule, Bin, Error}) + {error, #{reason => invalid_topic_rule_not_map, value => Bin}}; + {error, _Error} -> + {error, #{reason => invalid_topic_rule_not_json, value => Bin}} end. From 65544f34ec0a2dad8e868c5b9822445692b3ff8d Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 19 Jul 2024 17:25:18 -0300 Subject: [PATCH 003/131] chore: bump hocon -> 0.43.2 --- apps/emqx/rebar.config | 2 +- mix.exs | 2 +- rebar.config | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index 7a6ec9810..8a3e35537 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -31,7 +31,7 @@ {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.11.3"}}}, {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.19.5"}}}, {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "3.3.1"}}}, - {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.43.1"}}}, + {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.43.2"}}}, {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.3"}}}, {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}, {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}}, diff --git a/mix.exs b/mix.exs index dc2953683..a14beba7f 100644 --- a/mix.exs +++ b/mix.exs @@ -184,7 +184,7 @@ defmodule EMQXUmbrella.MixProject do def common_dep(:ekka), do: {:ekka, github: "emqx/ekka", tag: "0.19.5", override: true} def common_dep(:esockd), do: {:esockd, github: "emqx/esockd", tag: "5.11.3", override: true} def common_dep(:gproc), do: {:gproc, github: "emqx/gproc", tag: "0.9.0.1", override: true} - def common_dep(:hocon), do: {:hocon, github: "emqx/hocon", tag: "0.43.1", override: true} + def common_dep(:hocon), do: {:hocon, github: "emqx/hocon", tag: "0.43.2", override: true} def common_dep(:lc), do: {:lc, github: "emqx/lc", tag: "0.3.2", override: true} # in conflict by ehttpc and emqtt def common_dep(:gun), do: {:gun, github: "emqx/gun", tag: "1.3.11", override: true} diff --git a/rebar.config b/rebar.config index c7d9b019b..6cd70dd88 100644 --- a/rebar.config +++ b/rebar.config @@ -98,7 +98,7 @@ {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.5"}}}, {getopt, "1.0.2"}, {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.10"}}}, - {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.43.1"}}}, + {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.43.2"}}}, {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.3"}}}, {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.2.1"}}}, {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.2"}}}, From 8ae54ac325820484281b8e4671ac04e0606a6750 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 22 Jul 2024 11:31:10 -0300 Subject: [PATCH 004/131] fix(connector resource): use configuration `resource_opts` for health check interval when starting up Fixes https://emqx.atlassian.net/browse/EMQX-12738 --- apps/emqx_connector/src/emqx_connector_resource.erl | 6 ++++-- changes/ce/fix-13503.en.md | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 changes/ce/fix-13503.en.md diff --git a/apps/emqx_connector/src/emqx_connector_resource.erl b/apps/emqx_connector/src/emqx_connector_resource.erl index be8d3a32d..c5b0abf7b 100644 --- a/apps/emqx_connector/src/emqx_connector_resource.erl +++ b/apps/emqx_connector/src/emqx_connector_resource.erl @@ -351,8 +351,10 @@ safe_atom(Bin) when is_binary(Bin) -> binary_to_existing_atom(Bin, utf8); safe_atom(Atom) when is_atom(Atom) -> Atom. parse_opts(Conf, Opts0) -> - Opts1 = override_start_after_created(Conf, Opts0), - set_no_buffer_workers(Opts1). + Opts1 = emqx_resource:fetch_creation_opts(Conf), + Opts2 = maps:merge(Opts1, Opts0), + Opts = override_start_after_created(Conf, Opts2), + set_no_buffer_workers(Opts). override_start_after_created(Config, Opts) -> Enabled = maps:get(enable, Config, true), diff --git a/changes/ce/fix-13503.en.md b/changes/ce/fix-13503.en.md new file mode 100644 index 000000000..a4f0eb811 --- /dev/null +++ b/changes/ce/fix-13503.en.md @@ -0,0 +1 @@ +Fixed an issue where a connector wouldn't respect the configured health check interval when first starting up, and would need an update/restart for the correct value to take effect. From 69f5b6fa6c50560afd2391daebebe39dd29d43c0 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 19 Jul 2024 17:34:17 -0300 Subject: [PATCH 005/131] chore: hide `enable` fields from docgen Fixes https://emqx.atlassian.net/browse/EMQX-12730 --- apps/emqx/src/emqx_schema.erl | 13 ++++- .../src/emqx_authn/emqx_authn_schema.erl | 1 + .../src/emqx_authz/emqx_authz_schema.erl | 7 ++- .../src/schema/emqx_bridge_schema.erl | 1 + .../src/schema/emqx_bridge_v2_schema.erl | 17 ++++-- .../src/emqx_bridge_azure_event_hub.app.src | 2 +- .../src/emqx_bridge_azure_event_hub.erl | 11 +--- .../src/emqx_bridge_confluent.app.src | 2 +- .../src/emqx_bridge_confluent_producer.erl | 11 +--- .../src/emqx_bridge_http_schema.erl | 52 ++++++++----------- .../src/emqx_bridge_kafka.erl | 10 +--- .../src/emqx_bridge_syskeeper.app.src | 2 +- .../src/emqx_bridge_syskeeper.erl | 34 ++++-------- .../src/emqx_cluster_link_schema.erl | 7 ++- apps/emqx_conf/src/emqx_conf_schema.erl | 4 +- .../src/schema/emqx_connector_schema.erl | 7 ++- .../src/emqx_dashboard_swagger.erl | 2 +- .../src/emqx_dashboard_sso_schema.erl | 1 + .../src/emqx_enterprise_schema.erl | 1 - apps/emqx_exhook/src/emqx_exhook.app.src | 2 +- apps/emqx_exhook/src/emqx_exhook_schema.erl | 1 + apps/emqx_ft/src/emqx_ft_schema.erl | 2 + apps/emqx_gateway/src/emqx_gateway_schema.erl | 2 + apps/emqx_modules/src/emqx_modules_schema.erl | 7 ++- .../src/emqx_opentelemetry.app.src | 2 +- .../src/emqx_otel_schema.erl | 5 +- apps/emqx_plugins/src/emqx_plugins_schema.erl | 2 + .../src/emqx_prometheus_schema.erl | 2 + apps/emqx_psk/src/emqx_psk.app.src | 2 +- apps/emqx_psk/src/emqx_psk_schema.erl | 1 + .../src/emqx_retainer_schema.erl | 3 +- .../emqx_slow_subs/src/emqx_slow_subs.app.src | 2 +- .../src/emqx_slow_subs_schema.erl | 5 +- 33 files changed, 119 insertions(+), 104 deletions(-) diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index db1d5350f..eb4c7cf95 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -351,6 +351,7 @@ fields("authz_cache") -> #{ default => true, required => true, + importance => ?IMPORTANCE_NO_DOC, desc => ?DESC(fields_cache_enable) } )}, @@ -387,6 +388,7 @@ fields("flapping_detect") -> boolean(), #{ default => false, + importance => ?IMPORTANCE_NO_DOC, desc => ?DESC(flapping_detect_enable) } )}, @@ -423,6 +425,7 @@ fields("force_shutdown") -> boolean(), #{ default => true, + importance => ?IMPORTANCE_NO_DOC, desc => ?DESC(force_shutdown_enable) } )}, @@ -452,6 +455,7 @@ fields("overload_protection") -> boolean(), #{ desc => ?DESC(overload_protection_enable), + importance => ?IMPORTANCE_NO_DOC, default => false } )}, @@ -512,7 +516,11 @@ fields("force_gc") -> {"enable", sc( boolean(), - #{default => true, desc => ?DESC(force_gc_enable)} + #{ + default => true, + importance => ?IMPORTANCE_NO_DOC, + desc => ?DESC(force_gc_enable) + } )}, {"count", sc( @@ -1665,6 +1673,7 @@ fields("durable_sessions") -> sc( boolean(), #{ desc => ?DESC(durable_sessions_enable), + importance => ?IMPORTANCE_NO_DOC, default => false } )}, @@ -1888,6 +1897,7 @@ base_listener(Bind) -> #{ default => true, aliases => [enabled], + importance => ?IMPORTANCE_NO_DOC, desc => ?DESC(fields_listener_enabled) } )}, @@ -2416,6 +2426,7 @@ client_ssl_opts_schema(Defaults) -> boolean(), #{ default => false, + importance => ?IMPORTANCE_NO_DOC, desc => ?DESC(client_ssl_opts_schema_enable) } )}, diff --git a/apps/emqx_auth/src/emqx_authn/emqx_authn_schema.erl b/apps/emqx_auth/src/emqx_authn/emqx_authn_schema.erl index 57f524a0c..75612550b 100644 --- a/apps/emqx_auth/src/emqx_authn/emqx_authn_schema.erl +++ b/apps/emqx_auth/src/emqx_authn/emqx_authn_schema.erl @@ -203,6 +203,7 @@ common_fields() -> enable(type) -> boolean(); enable(default) -> true; +enable(importance) -> ?IMPORTANCE_NO_DOC; enable(desc) -> ?DESC(?FUNCTION_NAME); enable(_) -> undefined. diff --git a/apps/emqx_auth/src/emqx_authz/emqx_authz_schema.erl b/apps/emqx_auth/src/emqx_authz/emqx_authz_schema.erl index 24deb0161..4095767b3 100644 --- a/apps/emqx_auth/src/emqx_authz/emqx_authz_schema.erl +++ b/apps/emqx_auth/src/emqx_authz/emqx_authz_schema.erl @@ -170,7 +170,12 @@ api_authz_refs() -> authz_common_fields(Type) -> [ {type, ?HOCON(Type, #{required => true, desc => ?DESC(type)})}, - {enable, ?HOCON(boolean(), #{default => true, desc => ?DESC(enable)})} + {enable, + ?HOCON(boolean(), #{ + default => true, + importance => ?IMPORTANCE_NO_DOC, + desc => ?DESC(enable) + })} ]. source_types() -> diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl b/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl index aea9f8a86..1c9d861a1 100644 --- a/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl +++ b/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl @@ -123,6 +123,7 @@ common_bridge_fields() -> boolean(), #{ desc => ?DESC("desc_enable"), + importance => ?IMPORTANCE_NO_DOC, default => true } )}, diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl b/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl index 6dbad456b..e5a3465ff 100644 --- a/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl +++ b/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl @@ -65,6 +65,7 @@ -export([ make_producer_action_schema/1, make_producer_action_schema/2, make_consumer_action_schema/1, make_consumer_action_schema/2, + common_fields/0, top_level_common_action_keys/0, top_level_common_source_keys/0, project_to_actions_resource_opts/1, @@ -507,16 +508,26 @@ make_consumer_action_schema(ParametersRef, Opts) -> })} ]. -common_schema(ParametersRef, _Opts) -> +common_fields() -> [ - {enable, mk(boolean(), #{desc => ?DESC("config_enable"), default => true})}, + {enable, + mk(boolean(), #{ + desc => ?DESC("config_enable"), + importance => ?IMPORTANCE_NO_DOC, + default => true + })}, {connector, mk(binary(), #{ desc => ?DESC(emqx_connector_schema, "connector_field"), required => true })}, {tags, emqx_schema:tags_schema()}, - {description, emqx_schema:description_schema()}, + {description, emqx_schema:description_schema()} + ]. + +common_schema(ParametersRef, _Opts) -> + [ {parameters, ParametersRef} + | common_fields() ]. project_to_actions_resource_opts(OldResourceOpts) -> diff --git a/apps/emqx_bridge_azure_event_hub/src/emqx_bridge_azure_event_hub.app.src b/apps/emqx_bridge_azure_event_hub/src/emqx_bridge_azure_event_hub.app.src index 7e70fffff..69348b60a 100644 --- a/apps/emqx_bridge_azure_event_hub/src/emqx_bridge_azure_event_hub.app.src +++ b/apps/emqx_bridge_azure_event_hub/src/emqx_bridge_azure_event_hub.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_azure_event_hub, [ {description, "EMQX Enterprise Azure Event Hub Bridge"}, - {vsn, "0.1.7"}, + {vsn, "0.1.8"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_azure_event_hub/src/emqx_bridge_azure_event_hub.erl b/apps/emqx_bridge_azure_event_hub/src/emqx_bridge_azure_event_hub.erl index 213c2331c..197bdd0a7 100644 --- a/apps/emqx_bridge_azure_event_hub/src/emqx_bridge_azure_event_hub.erl +++ b/apps/emqx_bridge_azure_event_hub/src/emqx_bridge_azure_event_hub.erl @@ -129,16 +129,7 @@ fields(actions) -> override( emqx_bridge_kafka:producer_opts(action), bridge_v2_overrides() - ) ++ - [ - {enable, mk(boolean(), #{desc => ?DESC("config_enable"), default => true})}, - {connector, - mk(binary(), #{ - desc => ?DESC(emqx_connector_schema, "connector_field"), required => true - })}, - {tags, emqx_schema:tags_schema()}, - {description, emqx_schema:description_schema()} - ], + ) ++ emqx_bridge_v2_schema:common_fields(), override_documentations(Fields); fields(Method) -> Fields = emqx_bridge_kafka:fields(Method), diff --git a/apps/emqx_bridge_confluent/src/emqx_bridge_confluent.app.src b/apps/emqx_bridge_confluent/src/emqx_bridge_confluent.app.src index 46d6617c3..de3074ae6 100644 --- a/apps/emqx_bridge_confluent/src/emqx_bridge_confluent.app.src +++ b/apps/emqx_bridge_confluent/src/emqx_bridge_confluent.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_confluent, [ {description, "EMQX Enterprise Confluent Connector and Action"}, - {vsn, "0.1.2"}, + {vsn, "0.1.3"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_confluent/src/emqx_bridge_confluent_producer.erl b/apps/emqx_bridge_confluent/src/emqx_bridge_confluent_producer.erl index bf01626e3..93d644ef7 100644 --- a/apps/emqx_bridge_confluent/src/emqx_bridge_confluent_producer.erl +++ b/apps/emqx_bridge_confluent/src/emqx_bridge_confluent_producer.erl @@ -116,16 +116,7 @@ fields(actions) -> override( emqx_bridge_kafka:producer_opts(action), bridge_v2_overrides() - ) ++ - [ - {enable, mk(boolean(), #{desc => ?DESC("config_enable"), default => true})}, - {connector, - mk(binary(), #{ - desc => ?DESC(emqx_connector_schema, "connector_field"), required => true - })}, - {tags, emqx_schema:tags_schema()}, - {description, emqx_schema:description_schema()} - ], + ) ++ emqx_bridge_v2_schema:common_fields(), override_documentations(Fields); fields(Method) -> Fields = emqx_bridge_kafka:fields(Method), diff --git a/apps/emqx_bridge_http/src/emqx_bridge_http_schema.erl b/apps/emqx_bridge_http/src/emqx_bridge_http_schema.erl index cadbcf0d2..7868d6694 100644 --- a/apps/emqx_bridge_http/src/emqx_bridge_http_schema.erl +++ b/apps/emqx_bridge_http/src/emqx_bridge_http_schema.erl @@ -72,35 +72,29 @@ fields(action) -> } )}; fields("http_action") -> - [ - {enable, mk(boolean(), #{desc => ?DESC("config_enable_bridge"), default => true})}, - {connector, - mk(binary(), #{ - desc => ?DESC(emqx_connector_schema, "connector_field"), required => true - })}, - {tags, emqx_schema:tags_schema()}, - {description, emqx_schema:description_schema()}, - %% Note: there's an implicit convention in `emqx_bridge' that, - %% for egress bridges with this config, the published messages - %% will be forwarded to such bridges. - {local_topic, - mk( - binary(), - #{ - required => false, - desc => ?DESC("config_local_topic"), - importance => ?IMPORTANCE_HIDDEN - } - )}, - %% Since e5.3.2, we split the http bridge to two parts: a) connector. b) actions. - %% some fields are moved to connector, some fields are moved to actions and composed into the - %% `parameters` field. - {parameters, - mk(ref("parameters_opts"), #{ - required => true, - desc => ?DESC("config_parameters_opts") - })} - ] ++ + emqx_bridge_v2_schema:common_fields() ++ + [ + %% Note: there's an implicit convention in `emqx_bridge' that, + %% for egress bridges with this config, the published messages + %% will be forwarded to such bridges. + {local_topic, + mk( + binary(), + #{ + required => false, + desc => ?DESC("config_local_topic"), + importance => ?IMPORTANCE_HIDDEN + } + )}, + %% Since e5.3.2, we split the http bridge to two parts: a) connector. b) actions. + %% some fields are moved to connector, some fields are moved to actions and composed into the + %% `parameters` field. + {parameters, + mk(ref("parameters_opts"), #{ + required => true, + desc => ?DESC("config_parameters_opts") + })} + ] ++ emqx_connector_schema:resource_opts_ref( ?MODULE, action_resource_opts, fun legacy_action_resource_opts_converter/2 ); diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl index 9d15a26ee..212d288ba 100644 --- a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl +++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl @@ -297,15 +297,7 @@ fields("config_consumer") -> fields(kafka_producer) -> connector_config_fields() ++ producer_opts(v1); fields(kafka_producer_action) -> - [ - {enable, mk(boolean(), #{desc => ?DESC("config_enable"), default => true})}, - {connector, - mk(binary(), #{ - desc => ?DESC(emqx_connector_schema, "connector_field"), required => true - })}, - {tags, emqx_schema:tags_schema()}, - {description, emqx_schema:description_schema()} - ] ++ producer_opts(action); + emqx_bridge_v2_schema:common_fields() ++ producer_opts(action); fields(kafka_consumer) -> connector_config_fields() ++ fields(consumer_opts); fields(ssl_client_opts) -> diff --git a/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper.app.src b/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper.app.src index 5ae95ca67..cd1d51b01 100644 --- a/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper.app.src +++ b/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_syskeeper, [ {description, "EMQX Enterprise Data bridge for Syskeeper"}, - {vsn, "0.1.3"}, + {vsn, "0.1.4"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper.erl b/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper.erl index 547562f26..16f28e40d 100644 --- a/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper.erl +++ b/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper.erl @@ -84,30 +84,16 @@ fields(action) -> } )}; fields(config) -> - [ - {enable, mk(boolean(), #{desc => ?DESC("config_enable"), default => true})}, - {tags, emqx_schema:tags_schema()}, - {description, emqx_schema:description_schema()}, - {connector, - mk(binary(), #{ - desc => ?DESC(emqx_connector_schema, "connector_field"), required => true - })}, - {parameters, - mk( - ref(?MODULE, "parameters"), - #{required => true, desc => ?DESC("parameters")} - )}, - {local_topic, mk(binary(), #{required => false, desc => ?DESC(mqtt_topic)})}, - {resource_opts, - mk( - ref(?MODULE, "creation_opts"), - #{ - required => false, - default => #{}, - desc => ?DESC(emqx_resource_schema, <<"resource_opts">>) - } - )} - ]; + emqx_bridge_v2_schema:make_producer_action_schema( + mk( + ref(?MODULE, "parameters"), + #{ + required => true, + desc => ?DESC("parameters") + } + ), + #{resource_opts_ref => ref(?MODULE, "creation_opts")} + ); fields("parameters") -> [ {target_topic, diff --git a/apps/emqx_cluster_link/src/emqx_cluster_link_schema.erl b/apps/emqx_cluster_link/src/emqx_cluster_link_schema.erl index f46249a4f..74460b210 100644 --- a/apps/emqx_cluster_link/src/emqx_cluster_link_schema.erl +++ b/apps/emqx_cluster_link/src/emqx_cluster_link_schema.erl @@ -39,7 +39,12 @@ links_schema(Meta) -> fields("link") -> [ - {enable, ?HOCON(boolean(), #{default => true, desc => ?DESC(enable)})}, + {enable, + ?HOCON(boolean(), #{ + default => true, + importance => ?IMPORTANCE_NO_DOC, + desc => ?DESC(enable) + })}, {name, ?HOCON(binary(), #{required => true, desc => ?DESC(link_name)})}, {server, emqx_schema:servers_sc(#{required => true, desc => ?DESC(server)}, ?MQTT_HOST_OPTS)}, diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index df906911e..a83349ff0 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -997,6 +997,7 @@ fields("log_overload_kill") -> boolean(), #{ default => true, + importance => ?IMPORTANCE_NO_DOC, desc => ?DESC("log_overload_kill_enable") } )}, @@ -1032,6 +1033,7 @@ fields("log_burst_limit") -> boolean(), #{ default => true, + importance => ?IMPORTANCE_NO_DOC, desc => ?DESC("log_burst_limit_enable") } )}, @@ -1285,7 +1287,7 @@ log_handler_common_confs(Handler, Default) -> #{ default => Enable, desc => ?DESC("common_handler_enable"), - importance => ?IMPORTANCE_MEDIUM + importance => ?IMPORTANCE_NO_DOC } )}, {"formatter", diff --git a/apps/emqx_connector/src/schema/emqx_connector_schema.erl b/apps/emqx_connector/src/schema/emqx_connector_schema.erl index 060f3ba83..872bbd2c7 100644 --- a/apps/emqx_connector/src/schema/emqx_connector_schema.erl +++ b/apps/emqx_connector/src/schema/emqx_connector_schema.erl @@ -489,7 +489,12 @@ api_fields("put_connector", _Type, Fields) -> common_fields() -> [ - {enable, mk(boolean(), #{desc => ?DESC("config_enable"), default => true})}, + {enable, + mk(boolean(), #{ + desc => ?DESC("config_enable"), + importance => ?IMPORTANCE_NO_DOC, + default => true + })}, {tags, emqx_schema:tags_schema()}, {description, emqx_schema:description_schema()} ]. diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl index e8cea1db5..c73a06a73 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl @@ -1008,7 +1008,7 @@ parse_object_loop([{Name, Hocon} | Rest], Module, Options, Props, Required, Refs %% return true if the field has 'importance' set to 'hidden' is_hidden(Hocon) -> - hocon_schema:is_hidden(Hocon, #{include_importance_up_from => ?IMPORTANCE_LOW}). + hocon_schema:is_hidden(Hocon, #{include_importance_up_from => ?IMPORTANCE_NO_DOC}). is_required(Hocon) -> hocon_schema:field_schema(Hocon, required) =:= true. diff --git a/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_schema.erl b/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_schema.erl index d6184b42a..314839ee4 100644 --- a/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_schema.erl +++ b/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_schema.erl @@ -47,6 +47,7 @@ common_backend_schema(Backend) -> mk( boolean(), #{ desc => ?DESC(backend_enable), + importance => ?IMPORTANCE_NO_DOC, required => false, default => false } diff --git a/apps/emqx_enterprise/src/emqx_enterprise_schema.erl b/apps/emqx_enterprise/src/emqx_enterprise_schema.erl index f593dc877..a157b6553 100644 --- a/apps/emqx_enterprise/src/emqx_enterprise_schema.erl +++ b/apps/emqx_enterprise/src/emqx_enterprise_schema.erl @@ -53,7 +53,6 @@ fields("log_audit_handler") -> importance => ?IMPORTANCE_HIDDEN } )}, - {"path", hoconsc:mk( string(), diff --git a/apps/emqx_exhook/src/emqx_exhook.app.src b/apps/emqx_exhook/src/emqx_exhook.app.src index a12ec7d3a..a0961f660 100644 --- a/apps/emqx_exhook/src/emqx_exhook.app.src +++ b/apps/emqx_exhook/src/emqx_exhook.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_exhook, [ {description, "EMQX Extension for Hook"}, - {vsn, "5.0.17"}, + {vsn, "5.0.18"}, {modules, []}, {registered, []}, {mod, {emqx_exhook_app, []}}, diff --git a/apps/emqx_exhook/src/emqx_exhook_schema.erl b/apps/emqx_exhook/src/emqx_exhook_schema.erl index 6f37ea385..895515cf9 100644 --- a/apps/emqx_exhook/src/emqx_exhook_schema.erl +++ b/apps/emqx_exhook/src/emqx_exhook_schema.erl @@ -54,6 +54,7 @@ fields(server) -> {enable, ?HOCON(boolean(), #{ default => true, + importance => ?IMPORTANCE_NO_DOC, desc => ?DESC(enable) })}, {url, diff --git a/apps/emqx_ft/src/emqx_ft_schema.erl b/apps/emqx_ft/src/emqx_ft_schema.erl index adf8b4241..770b0509c 100644 --- a/apps/emqx_ft/src/emqx_ft_schema.erl +++ b/apps/emqx_ft/src/emqx_ft_schema.erl @@ -66,6 +66,7 @@ fields(file_transfer) -> boolean(), #{ desc => ?DESC("enable"), + importance => ?IMPORTANCE_NO_DOC, required => false, default => false } @@ -242,6 +243,7 @@ common_backend_fields() -> mk( boolean(), #{ desc => ?DESC("backend_enable"), + importance => ?IMPORTANCE_NO_DOC, required => false, default => true } diff --git a/apps/emqx_gateway/src/emqx_gateway_schema.erl b/apps/emqx_gateway/src/emqx_gateway_schema.erl index 11488d1a3..c59736a16 100644 --- a/apps/emqx_gateway/src/emqx_gateway_schema.erl +++ b/apps/emqx_gateway/src/emqx_gateway_schema.erl @@ -240,6 +240,7 @@ gateway_common_options() -> boolean(), #{ default => true, + importance => ?IMPORTANCE_NO_DOC, desc => ?DESC(gateway_common_enable) } )}, @@ -413,6 +414,7 @@ common_listener_opts() -> boolean(), #{ default => true, + importance => ?IMPORTANCE_NO_DOC, desc => ?DESC(gateway_common_listener_enable) } )}, diff --git a/apps/emqx_modules/src/emqx_modules_schema.erl b/apps/emqx_modules/src/emqx_modules_schema.erl index 48c5a4ab5..8df7d91dc 100644 --- a/apps/emqx_modules/src/emqx_modules_schema.erl +++ b/apps/emqx_modules/src/emqx_modules_schema.erl @@ -79,7 +79,12 @@ rewrite_validator(Rules) -> fields("delayed") -> [ - {enable, ?HOCON(boolean(), #{default => true, desc => ?DESC(enable)})}, + {enable, + ?HOCON(boolean(), #{ + default => true, + importance => ?IMPORTANCE_NO_DOC, + desc => ?DESC(enable) + })}, {max_delayed_messages, ?HOCON(integer(), #{desc => ?DESC(max_delayed_messages), default => 0})} ]; diff --git a/apps/emqx_opentelemetry/src/emqx_opentelemetry.app.src b/apps/emqx_opentelemetry/src/emqx_opentelemetry.app.src index 6a84ae043..8ed649cda 100644 --- a/apps/emqx_opentelemetry/src/emqx_opentelemetry.app.src +++ b/apps/emqx_opentelemetry/src/emqx_opentelemetry.app.src @@ -1,6 +1,6 @@ {application, emqx_opentelemetry, [ {description, "OpenTelemetry for EMQX Broker"}, - {vsn, "0.2.6"}, + {vsn, "0.2.7"}, {registered, []}, {mod, {emqx_otel_app, []}}, {applications, [ diff --git a/apps/emqx_opentelemetry/src/emqx_otel_schema.erl b/apps/emqx_opentelemetry/src/emqx_otel_schema.erl index e89591fa9..d4c6dca0a 100644 --- a/apps/emqx_opentelemetry/src/emqx_otel_schema.erl +++ b/apps/emqx_opentelemetry/src/emqx_otel_schema.erl @@ -72,6 +72,7 @@ fields("otel_metrics") -> boolean(), #{ default => false, + importance => ?IMPORTANCE_NO_DOC, required => true, desc => ?DESC(enable) } @@ -104,7 +105,7 @@ fields("otel_logs") -> #{ default => false, desc => ?DESC(enable), - importance => ?IMPORTANCE_HIGH + importance => ?IMPORTANCE_NO_DOC } )}, {max_queue_size, @@ -143,7 +144,7 @@ fields("otel_traces") -> #{ default => false, desc => ?DESC(enable), - importance => ?IMPORTANCE_HIGH + importance => ?IMPORTANCE_NO_DOC } )}, {max_queue_size, diff --git a/apps/emqx_plugins/src/emqx_plugins_schema.erl b/apps/emqx_plugins/src/emqx_plugins_schema.erl index 8932e3ab6..d750a029c 100644 --- a/apps/emqx_plugins/src/emqx_plugins_schema.erl +++ b/apps/emqx_plugins/src/emqx_plugins_schema.erl @@ -56,6 +56,8 @@ state_fields() -> ?HOCON( boolean(), #{ + default => true, + importance => ?IMPORTANCE_NO_DOC, desc => ?DESC(enable), required => true } diff --git a/apps/emqx_prometheus/src/emqx_prometheus_schema.erl b/apps/emqx_prometheus/src/emqx_prometheus_schema.erl index 84925c7b6..09f19bdb1 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus_schema.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus_schema.erl @@ -78,6 +78,7 @@ fields(push_gateway) -> #{ default => false, required => true, + importance => ?IMPORTANCE_NO_DOC, desc => ?DESC(push_gateway_enable) } )}, @@ -229,6 +230,7 @@ fields(legacy_deprecated_setting) -> #{ default => false, required => true, + importance => ?IMPORTANCE_NO_DOC, desc => ?DESC(legacy_enable) } )}, diff --git a/apps/emqx_psk/src/emqx_psk.app.src b/apps/emqx_psk/src/emqx_psk.app.src index 14c6ba0cc..88dd0ccef 100644 --- a/apps/emqx_psk/src/emqx_psk.app.src +++ b/apps/emqx_psk/src/emqx_psk.app.src @@ -2,7 +2,7 @@ {application, emqx_psk, [ {description, "EMQX PSK"}, % strict semver, bump manually! - {vsn, "5.0.6"}, + {vsn, "5.0.7"}, {modules, []}, {registered, [emqx_psk_sup]}, {applications, [kernel, stdlib]}, diff --git a/apps/emqx_psk/src/emqx_psk_schema.erl b/apps/emqx_psk/src/emqx_psk_schema.erl index 652f61b08..286877d99 100644 --- a/apps/emqx_psk/src/emqx_psk_schema.erl +++ b/apps/emqx_psk/src/emqx_psk_schema.erl @@ -42,6 +42,7 @@ fields() -> [ {enable, ?HOCON(boolean(), #{ + importance => ?IMPORTANCE_NO_DOC, default => false, require => true, desc => ?DESC(enable) diff --git a/apps/emqx_retainer/src/emqx_retainer_schema.erl b/apps/emqx_retainer/src/emqx_retainer_schema.erl index b89ee7db6..26b65faa8 100644 --- a/apps/emqx_retainer/src/emqx_retainer_schema.erl +++ b/apps/emqx_retainer/src/emqx_retainer_schema.erl @@ -43,7 +43,7 @@ roots() -> fields("retainer") -> [ - {enable, sc(boolean(), enable, true)}, + {enable, sc(boolean(), enable, true, ?IMPORTANCE_NO_DOC)}, {msg_expiry_interval, sc( %% not used in a `receive ... after' block, just timestamp comparison @@ -126,6 +126,7 @@ fields(mnesia_config) -> {enable, ?HOCON(boolean(), #{ desc => ?DESC(mnesia_enable), + importance => ?IMPORTANCE_NO_DOC, required => false, default => true })} diff --git a/apps/emqx_slow_subs/src/emqx_slow_subs.app.src b/apps/emqx_slow_subs/src/emqx_slow_subs.app.src index 6a24bc90b..a1485337e 100644 --- a/apps/emqx_slow_subs/src/emqx_slow_subs.app.src +++ b/apps/emqx_slow_subs/src/emqx_slow_subs.app.src @@ -1,7 +1,7 @@ {application, emqx_slow_subs, [ {description, "EMQX Slow Subscribers Statistics"}, % strict semver, bump manually! - {vsn, "1.0.7"}, + {vsn, "1.0.8"}, {modules, []}, {registered, [emqx_slow_subs_sup]}, {applications, [kernel, stdlib, emqx]}, diff --git a/apps/emqx_slow_subs/src/emqx_slow_subs_schema.erl b/apps/emqx_slow_subs/src/emqx_slow_subs_schema.erl index 37ca9327f..febb0af0a 100644 --- a/apps/emqx_slow_subs/src/emqx_slow_subs_schema.erl +++ b/apps/emqx_slow_subs/src/emqx_slow_subs_schema.erl @@ -27,7 +27,7 @@ roots() -> fields("slow_subs") -> [ - {enable, sc(boolean(), false, enable)}, + {enable, sc(boolean(), false, enable, ?IMPORTANCE_NO_DOC)}, {threshold, sc( %% not used in a `receive ... after' block, just timestamp comparison @@ -66,3 +66,6 @@ desc(_) -> %%-------------------------------------------------------------------- sc(Type, Default, Desc) -> ?HOCON(Type, #{default => Default, desc => ?DESC(Desc)}). + +sc(Type, Default, Desc, Importance) -> + ?HOCON(Type, #{default => Default, desc => ?DESC(Desc), importance => Importance}). From d7112921a6a177d9f7056d74e26f2b1ed76bd84f Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 19 Jul 2024 12:26:23 -0300 Subject: [PATCH 006/131] docs: remove `enable` from config examples Fixes https://emqx.atlassian.net/browse/EMQX-12730 --- .../file_transfer-with-local-exporter.conf.example | 3 --- .../ee-examples/file_transfer-with-s3-exporter.conf.example | 5 ----- rel/config/examples/delayed.conf.example | 2 -- rel/config/examples/exhook.conf.example | 3 --- rel/config/examples/flapping_detect.conf.example | 3 --- rel/config/examples/force_gc.conf.example | 3 --- rel/config/examples/force_shutdown.conf.example | 3 --- rel/config/examples/gateway.exproto.conf.example | 1 - rel/config/examples/listeners.ssl.conf.example | 1 - rel/config/examples/listeners.ws.conf.example | 1 - rel/config/examples/listeners.wss.conf.example | 1 - rel/config/examples/log.console.conf.example | 3 --- rel/config/examples/log.file.conf.example | 3 --- rel/config/examples/plugins.conf.example | 4 +--- rel/config/examples/prometheus-pushgateway.conf.example | 3 --- rel/config/examples/prometheus.conf.example | 1 - rel/config/examples/psk_authentication.conf.example | 3 --- rel/config/examples/retainer.conf.example | 3 --- 18 files changed, 1 insertion(+), 45 deletions(-) diff --git a/rel/config/ee-examples/file_transfer-with-local-exporter.conf.example b/rel/config/ee-examples/file_transfer-with-local-exporter.conf.example index dcb1e88d1..4eeef33c7 100644 --- a/rel/config/ee-examples/file_transfer-with-local-exporter.conf.example +++ b/rel/config/ee-examples/file_transfer-with-local-exporter.conf.example @@ -3,9 +3,6 @@ ## NOTE: This configuration is only applicable in EMQX Enterprise edition 5.1 or later. file_transfer { - ## Enable the File Transfer feature - enable = true - ## Storage backend settings storage { ## Local file system backend setting diff --git a/rel/config/ee-examples/file_transfer-with-s3-exporter.conf.example b/rel/config/ee-examples/file_transfer-with-s3-exporter.conf.example index 94061223e..7a738c097 100644 --- a/rel/config/ee-examples/file_transfer-with-s3-exporter.conf.example +++ b/rel/config/ee-examples/file_transfer-with-s3-exporter.conf.example @@ -4,9 +4,6 @@ ## Note: This configuration is only applicable for EMQX Enterprise edition 5.1 or later. file_transfer { - ## Enable the File Transfer feature - enable = true - ## Storage backend settings storage { ## Local file system backend setting @@ -51,8 +48,6 @@ file_transfer { ## Enable the HTTPS transport_options { - ssl.enable = true - ## Timeout for connection attempts connect_timeout = 15s } diff --git a/rel/config/examples/delayed.conf.example b/rel/config/examples/delayed.conf.example index 7b0d243c2..b5b2c2c08 100644 --- a/rel/config/examples/delayed.conf.example +++ b/rel/config/examples/delayed.conf.example @@ -7,8 +7,6 @@ ## you should copy and paste the below data into the emqx.conf for working delayed { - enable = true ## false for disabled - ## Maximum number of delayed messages ## Default: 0 (0 is no limit) max_delayed_messages = 0 diff --git a/rel/config/examples/exhook.conf.example b/rel/config/examples/exhook.conf.example index cf9fffc71..45a186586 100644 --- a/rel/config/examples/exhook.conf.example +++ b/rel/config/examples/exhook.conf.example @@ -7,9 +7,6 @@ exhook.servers = [ ## Name of the exhook server name = "server_1" - ## Feature switch - enable = false - ## URL of gRPC server url = "http://127.0.0.1:9090" diff --git a/rel/config/examples/flapping_detect.conf.example b/rel/config/examples/flapping_detect.conf.example index fe4522b1e..6878d1f41 100644 --- a/rel/config/examples/flapping_detect.conf.example +++ b/rel/config/examples/flapping_detect.conf.example @@ -3,9 +3,6 @@ ## Ban the client when the times of connections exceed the limit in the configured time window flapping_detect { - ## use 'true' to enable this feature - enable = false - ## Time window for flapping detection window_time = 1m diff --git a/rel/config/examples/force_gc.conf.example b/rel/config/examples/force_gc.conf.example index ee07ee4d2..08543b533 100644 --- a/rel/config/examples/force_gc.conf.example +++ b/rel/config/examples/force_gc.conf.example @@ -1,9 +1,6 @@ ## Force Elrang VM garbage collection force_gc { - ## set 'false' to disable this feature - enable = true - ## GC the process after this many received messages count = 16000 diff --git a/rel/config/examples/force_shutdown.conf.example b/rel/config/examples/force_shutdown.conf.example index d0a466aa0..1dd6747a8 100644 --- a/rel/config/examples/force_shutdown.conf.example +++ b/rel/config/examples/force_shutdown.conf.example @@ -3,9 +3,6 @@ ## Forced shutdown MQTT clients for overload protection force_shutdown { - ## set 'false' to disable force shutdown feature - enable = true - ## Maximum mailbox size for each Erlang process ## Note: Do not modify this unless you know what this is for max_mailbox_size = 1000 diff --git a/rel/config/examples/gateway.exproto.conf.example b/rel/config/examples/gateway.exproto.conf.example index fcedb944b..303bd2872 100644 --- a/rel/config/examples/gateway.exproto.conf.example +++ b/rel/config/examples/gateway.exproto.conf.example @@ -16,7 +16,6 @@ gateway.exproto { ## Configurations for request to ConnectionHandler service handler { address = "http://127.0.0.1:9001" - ssl_options {enable = false} } listeners.tcp.default { diff --git a/rel/config/examples/listeners.ssl.conf.example b/rel/config/examples/listeners.ssl.conf.example index 4a74f1a27..ffc79db54 100644 --- a/rel/config/examples/listeners.ssl.conf.example +++ b/rel/config/examples/listeners.ssl.conf.example @@ -3,7 +3,6 @@ listeners.ssl.my_ssl_listener_name { ## Port or Address to listen on, 0 means disable bind = 8883 ## or with an IP e.g. "127.0.0.1:8883" - enabled = true acceptors = 16 enable_authn = true max_connections = infinity diff --git a/rel/config/examples/listeners.ws.conf.example b/rel/config/examples/listeners.ws.conf.example index f6a6adae8..f8c07e84a 100644 --- a/rel/config/examples/listeners.ws.conf.example +++ b/rel/config/examples/listeners.ws.conf.example @@ -3,7 +3,6 @@ listeners.ws.my_ws_listener_name { ## Port or Address to listen on, 0 means disable bind = "0.0.0.0:8083" # or just a port number, e.g. 8083 - enabled = true enable_authn = true max_connections = infinity proxy_protocol = false diff --git a/rel/config/examples/listeners.wss.conf.example b/rel/config/examples/listeners.wss.conf.example index cbc632e30..d0a03b777 100644 --- a/rel/config/examples/listeners.wss.conf.example +++ b/rel/config/examples/listeners.wss.conf.example @@ -3,7 +3,6 @@ listeners.wss.my_wss_listener_name = { ## Port or Address to listen on, 0 means disable bind = 8084 ## or with an IP, e.g. "127.0.0.1:8084" - enabled = true enable_authn = true max_connections = infinity proxy_protocol = false diff --git a/rel/config/examples/log.console.conf.example b/rel/config/examples/log.console.conf.example index 9bb01b0d9..3b8a0436e 100644 --- a/rel/config/examples/log.console.conf.example +++ b/rel/config/examples/log.console.conf.example @@ -1,9 +1,6 @@ ## Log to console log.console { - ## set true to enable this - enable = false - ## Log level ## Type: debug | info | notice | warning | error | critical | alert | emergency level = warning diff --git a/rel/config/examples/log.file.conf.example b/rel/config/examples/log.file.conf.example index e6f408f61..f6dce6eaf 100644 --- a/rel/config/examples/log.file.conf.example +++ b/rel/config/examples/log.file.conf.example @@ -1,9 +1,6 @@ ## Log to file log.file { - ## Enable file log handler - enable = true - ## Log level ## Type: debug | info | notice | warning | error | critical | alert | emergency level = warning diff --git a/rel/config/examples/plugins.conf.example b/rel/config/examples/plugins.conf.example index 6fcc09cbb..9388f4f24 100644 --- a/rel/config/examples/plugins.conf.example +++ b/rel/config/examples/plugins.conf.example @@ -9,10 +9,8 @@ plugins { ## Format: {name}-{version} ## Note: name and version should be what it is in the plugin application name_vsn = "my_acl-0.1.0", - - enable = true ## enable this plugin }, - {name_vsn = "my_rule-0.1.1", enable = false} + {name_vsn = "my_rule-0.1.1"} ] ## The installation directory for the external plugins diff --git a/rel/config/examples/prometheus-pushgateway.conf.example b/rel/config/examples/prometheus-pushgateway.conf.example index 70b74794a..f463056e3 100644 --- a/rel/config/examples/prometheus-pushgateway.conf.example +++ b/rel/config/examples/prometheus-pushgateway.conf.example @@ -5,9 +5,6 @@ ## If you want to use push-gateway prometheus { - ## Set to true to make EMQX send metrics to push-gateway - enable = false - ## URL of push-gateway server push_gateway_server = "http://127.0.0.1:9091" diff --git a/rel/config/examples/prometheus.conf.example b/rel/config/examples/prometheus.conf.example index 049b11ee3..e31d0fe1d 100644 --- a/rel/config/examples/prometheus.conf.example +++ b/rel/config/examples/prometheus.conf.example @@ -7,7 +7,6 @@ prometheus { enable_basic_auth = false push_gateway { - enable = false url = "http://127.0.0.1:9091" headers {Authorization = "Basic YWRtaW46Y2JraG55eWd5QDE="} interval = 15s diff --git a/rel/config/examples/psk_authentication.conf.example b/rel/config/examples/psk_authentication.conf.example index 6c3482638..cea6f21c2 100644 --- a/rel/config/examples/psk_authentication.conf.example +++ b/rel/config/examples/psk_authentication.conf.example @@ -1,9 +1,6 @@ ## Pre-Shared Keys authentication psk_authentication { - ## Set to false to disable - enable = true - ## If init_file is specified, EMQX will import PSKs from the file into the built-in database at startup for use by the runtime init_file = "psk" diff --git a/rel/config/examples/retainer.conf.example b/rel/config/examples/retainer.conf.example index d78119ec2..b698020d4 100644 --- a/rel/config/examples/retainer.conf.example +++ b/rel/config/examples/retainer.conf.example @@ -5,9 +5,6 @@ ##-------------------------------------------------------------------- retainer { - ## set to false to disable retainer - enable = true - ## Message retention time, default is 0 means the message will never expire msg_expiry_interval = 5s From 0b1f0db73c1ca6b065875de70c15493e78c5586e Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 15 Jul 2024 17:54:21 -0300 Subject: [PATCH 007/131] chore(cluster link): refactor HTTP API for CRUD operations Fixes https://emqx.atlassian.net/browse/EMQX-12627 --- .../src/emqx_cluster_link_api.erl | 168 +++++++++++++++--- .../src/emqx_cluster_link_config.erl | 102 ++++++++++- .../src/emqx_cluster_link_schema.erl | 5 +- .../test/emqx_cluster_link_api_SUITE.erl | 144 ++++++++++----- .../test/emqx_mgmt_api_test_util.erl | 27 +++ apps/emqx_utils/include/emqx_utils_api.hrl | 2 + 6 files changed, 380 insertions(+), 68 deletions(-) diff --git a/apps/emqx_cluster_link/src/emqx_cluster_link_api.erl b/apps/emqx_cluster_link/src/emqx_cluster_link_api.erl index 33634607e..324d6dd68 100644 --- a/apps/emqx_cluster_link/src/emqx_cluster_link_api.erl +++ b/apps/emqx_cluster_link/src/emqx_cluster_link_api.erl @@ -7,6 +7,7 @@ -include_lib("hocon/include/hoconsc.hrl"). -include_lib("emqx/include/http_api.hrl"). +-include_lib("emqx_utils/include/emqx_utils_api.hrl"). -export([ api_spec/0, @@ -14,7 +15,10 @@ schema/1 ]). --export([config/2]). +-export([ + '/cluster/links'/2, + '/cluster/links/:name'/2 +]). -define(CONF_PATH, [cluster, links]). -define(TAGS, [<<"Cluster">>]). @@ -24,12 +28,13 @@ api_spec() -> paths() -> [ - "/cluster/links" + "/cluster/links", + "/cluster/links/:name" ]. schema("/cluster/links") -> #{ - 'operationId' => config, + 'operationId' => '/cluster/links', get => #{ description => "Get cluster links configuration", @@ -37,14 +42,63 @@ schema("/cluster/links") -> responses => #{200 => links_config_schema()} }, - put => + post => #{ - description => "Update cluster links configuration", + description => "Create a cluster link configuration", tags => ?TAGS, - 'requestBody' => links_config_schema(), + 'requestBody' => link_config_schema(), responses => #{ - 200 => links_config_schema(), + 200 => link_config_schema(), + 400 => + emqx_dashboard_swagger:error_codes( + [?BAD_REQUEST, ?ALREADY_EXISTS], + <<"Update Config Failed">> + ) + } + } + }; +schema("/cluster/links/:name") -> + #{ + 'operationId' => '/cluster/links/:name', + get => + #{ + description => "Get a cluster link configuration", + tags => ?TAGS, + parameters => [param_path_name()], + responses => + #{ + 200 => link_config_schema(), + 404 => emqx_dashboard_swagger:error_codes( + [?NOT_FOUND], <<"Cluster link not found">> + ) + } + }, + delete => + #{ + description => "Delete a cluster link configuration", + tags => ?TAGS, + parameters => [param_path_name()], + responses => + #{ + 204 => <<"Link deleted">>, + 404 => emqx_dashboard_swagger:error_codes( + [?NOT_FOUND], <<"Cluster link not found">> + ) + } + }, + put => + #{ + description => "Update a cluster link configuration", + tags => ?TAGS, + parameters => [param_path_name()], + 'requestBody' => update_link_config_schema(), + responses => + #{ + 200 => link_config_schema(), + 404 => emqx_dashboard_swagger:error_codes( + [?NOT_FOUND], <<"Cluster link not found">> + ), 400 => emqx_dashboard_swagger:error_codes( [?BAD_REQUEST], <<"Update Config Failed">> @@ -57,28 +111,66 @@ schema("/cluster/links") -> %% API Handler funcs %%-------------------------------------------------------------------- -config(get, _Params) -> - {200, get_raw()}; -config(put, #{body := Body}) -> - case emqx_cluster_link_config:update(Body) of - {ok, NewConfig} -> - {200, NewConfig}; - {error, Reason} -> - Message = list_to_binary(io_lib:format("Update config failed ~p", [Reason])), - {400, ?BAD_REQUEST, Message} - end. +'/cluster/links'(get, _Params) -> + ?OK(get_raw()); +'/cluster/links'(post, #{body := Body = #{<<"name">> := Name}}) -> + with_link( + Name, + return(?BAD_REQUEST('ALREADY_EXISTS', <<"Cluster link already exists">>)), + fun() -> + case emqx_cluster_link_config:create(Body) of + {ok, Res} -> + ?CREATED(Res); + {error, Reason} -> + Message = list_to_binary(io_lib:format("Create link failed ~p", [Reason])), + ?BAD_REQUEST(Message) + end + end + ). + +'/cluster/links/:name'(get, #{bindings := #{name := Name}}) -> + with_link(Name, fun(Link) -> ?OK(Link) end, not_found()); +'/cluster/links/:name'(put, #{bindings := #{name := Name}, body := Params0}) -> + with_link( + Name, + fun(Link) -> + Params = Params0#{<<"name">> => Name}, + case emqx_cluster_link_config:update_one_link(Params) of + {ok, Res} -> + ?OK(Res); + {error, Reason} -> + Message = list_to_binary(io_lib:format("Update link failed ~p", [Reason])), + ?BAD_REQUEST(Message) + end + end, + not_found() + ); +'/cluster/links/:name'(delete, #{bindings := #{name := Name}}) -> + with_link( + Name, + fun() -> + case emqx_cluster_link_config:delete(Name) of + ok -> + ?NO_CONTENT; + {error, Reason} -> + Message = list_to_binary(io_lib:format("Delete link failed ~p", [Reason])), + ?BAD_REQUEST(Message) + end + end, + not_found() + ). %%-------------------------------------------------------------------- %% Internal funcs %%-------------------------------------------------------------------- get_raw() -> - #{<<"links">> := Conf} = + #{<<"cluster">> := #{<<"links">> := Links}} = emqx_config:fill_defaults( - #{<<"links">> => emqx_conf:get_raw(?CONF_PATH)}, + #{<<"cluster">> => #{<<"links">> => emqx_conf:get_raw(?CONF_PATH)}}, #{obfuscate_sensitive_values => true} ), - Conf. + Links. links_config_schema() -> emqx_cluster_link_schema:links_schema( @@ -87,6 +179,24 @@ links_config_schema() -> } ). +link_config_schema() -> + emqx_cluster_link_schema:link_schema(). + +param_path_name() -> + {name, + hoconsc:mk( + binary(), + #{ + in => path, + required => true, + example => <<"my_link">>, + desc => ?DESC("param_path_name") + } + )}. + +update_link_config_schema() -> + proplists:delete(name, emqx_cluster_link_schema:fields("link")). + links_config_example() -> [ #{ @@ -114,3 +224,21 @@ links_config_example() -> <<"name">> => <<"emqxcl_c">> } ]. + +with_link(Name, FoundFn, NotFoundFn) -> + case emqx_cluster_link_config:link_raw(Name) of + undefined -> + NotFoundFn(); + Link = #{} -> + {arity, Arity} = erlang:fun_info(FoundFn, arity), + case Arity of + 1 -> FoundFn(Link); + 0 -> FoundFn() + end + end. + +return(Response) -> + fun() -> Response end. + +not_found() -> + return(?NOT_FOUND(<<"Cluster link not found">>)). diff --git a/apps/emqx_cluster_link/src/emqx_cluster_link_config.erl b/apps/emqx_cluster_link/src/emqx_cluster_link_config.erl index f27c7702e..36655460b 100644 --- a/apps/emqx_cluster_link/src/emqx_cluster_link_config.erl +++ b/apps/emqx_cluster_link/src/emqx_cluster_link_config.erl @@ -4,6 +4,8 @@ -module(emqx_cluster_link_config). +-feature(maybe_expr, enable). + -behaviour(emqx_config_handler). -include_lib("emqx/include/logger.hrl"). @@ -28,11 +30,15 @@ -export([ %% General + create/1, + delete/1, + update_one_link/1, update/1, cluster/0, enabled_links/0, links/0, link/1, + link_raw/1, topic_filters/1, %% Connections emqtt_options/1, @@ -55,6 +61,52 @@ %% +create(LinkConfig) -> + #{<<"name">> := Name} = LinkConfig, + case + emqx_conf:update( + ?LINKS_PATH, + {create, LinkConfig}, + #{rawconf_with_defaults => true, override_to => cluster} + ) + of + {ok, #{raw_config := NewConfigRows}} -> + NewLinkConfig = find_link(Name, NewConfigRows), + {ok, NewLinkConfig}; + {error, Reason} -> + {error, Reason} + end. + +delete(Name) -> + case + emqx_conf:update( + ?LINKS_PATH, + {delete, Name}, + #{rawconf_with_defaults => true, override_to => cluster} + ) + of + {ok, _} -> + ok; + {error, Reason} -> + {error, Reason} + end. + +update_one_link(LinkConfig) -> + #{<<"name">> := Name} = LinkConfig, + case + emqx_conf:update( + ?LINKS_PATH, + {update, LinkConfig}, + #{rawconf_with_defaults => true, override_to => cluster} + ) + of + {ok, #{raw_config := NewConfigRows}} -> + NewLinkConfig = find_link(Name, NewConfigRows), + {ok, NewLinkConfig}; + {error, Reason} -> + {error, Reason} + end. + update(Config) -> case emqx_conf:update( @@ -75,11 +127,20 @@ cluster() -> links() -> emqx:get_config(?LINKS_PATH, []). +links_raw() -> + emqx:get_raw_config(?LINKS_PATH, []). + enabled_links() -> [L || L = #{enable := true} <- links()]. link(Name) -> - case lists:dropwhile(fun(L) -> Name =/= upstream_name(L) end, links()) of + find_link(Name, links()). + +link_raw(Name) -> + find_link(Name, links_raw()). + +find_link(Name, Links) -> + case lists:dropwhile(fun(L) -> Name =/= upstream_name(L) end, Links) of [LinkConf | _] -> LinkConf; [] -> undefined end. @@ -133,6 +194,37 @@ remove_handler() -> pre_config_update(?LINKS_PATH, RawConf, RawConf) -> {ok, RawConf}; +pre_config_update(?LINKS_PATH, {create, LinkRawConf}, OldRawConf) -> + #{<<"name">> := Name} = LinkRawConf, + maybe + undefined ?= find_link(Name, OldRawConf), + NewRawConf0 = OldRawConf ++ [LinkRawConf], + NewRawConf = convert_certs(maybe_increment_ps_actor_incr(NewRawConf0, OldRawConf)), + {ok, NewRawConf} + else + _ -> + {error, already_exists} + end; +pre_config_update(?LINKS_PATH, {update, LinkRawConf}, OldRawConf) -> + #{<<"name">> := Name} = LinkRawConf, + maybe + {ok, {_Found, Front, Rear}} = safe_take(Name, OldRawConf), + NewRawConf0 = Front ++ [LinkRawConf] ++ Rear, + NewRawConf = convert_certs(maybe_increment_ps_actor_incr(NewRawConf0, OldRawConf)), + {ok, NewRawConf} + else + not_found -> + {error, not_found} + end; +pre_config_update(?LINKS_PATH, {delete, Name}, OldRawConf) -> + maybe + {ok, {_Found, Front, Rear}} = safe_take(Name, OldRawConf), + NewRawConf = Front ++ Rear, + {ok, NewRawConf} + else + _ -> + {error, not_found} + end; pre_config_update(?LINKS_PATH, NewRawConf, OldRawConf) -> {ok, convert_certs(maybe_increment_ps_actor_incr(NewRawConf, OldRawConf))}. @@ -320,3 +412,11 @@ do_convert_certs(LinkName, SSLOpts) -> ), throw({bad_ssl_config, Reason}) end. + +safe_take(Name, Transformations) -> + case lists:splitwith(fun(#{<<"name">> := N}) -> N =/= Name end, Transformations) of + {_Front, []} -> + not_found; + {Front, [Found | Rear]} -> + {ok, {Found, Front, Rear}} + end. diff --git a/apps/emqx_cluster_link/src/emqx_cluster_link_schema.erl b/apps/emqx_cluster_link/src/emqx_cluster_link_schema.erl index f46249a4f..a6073d677 100644 --- a/apps/emqx_cluster_link/src/emqx_cluster_link_schema.erl +++ b/apps/emqx_cluster_link/src/emqx_cluster_link_schema.erl @@ -12,7 +12,7 @@ -export([injected_fields/0]). %% Used in emqx_cluster_link_api --export([links_schema/1]). +-export([links_schema/1, link_schema/0]). -export([ roots/0, @@ -37,6 +37,9 @@ links_schema(Meta) -> default => [], validator => fun links_validator/1, desc => ?DESC("links") }). +link_schema() -> + hoconsc:ref(?MODULE, "link"). + fields("link") -> [ {enable, ?HOCON(boolean(), #{default => true, desc => ?DESC(enable)})}, diff --git a/apps/emqx_cluster_link/test/emqx_cluster_link_api_SUITE.erl b/apps/emqx_cluster_link/test/emqx_cluster_link_api_SUITE.erl index c5ec8da6c..5bb1c377a 100644 --- a/apps/emqx_cluster_link/test/emqx_cluster_link_api_SUITE.erl +++ b/apps/emqx_cluster_link/test/emqx_cluster_link_api_SUITE.erl @@ -37,6 +37,10 @@ "-----END CERTIFICATE-----" >>). +%%------------------------------------------------------------------------------ +%% CT boilerplate +%%------------------------------------------------------------------------------ + all() -> emqx_common_test_helpers:all(?MODULE). @@ -47,7 +51,7 @@ init_per_suite(Config) -> [ emqx_conf, emqx_management, - {emqx_dashboard, "dashboard.listeners.http { enable = true, bind = 18083 }"}, + emqx_mgmt_api_test_util:emqx_dashboard(), emqx_cluster_link ], #{work_dir => emqx_cth_suite:work_dir(Config)} @@ -61,8 +65,7 @@ end_per_suite(Config) -> ok. auth_header() -> - {ok, API} = emqx_common_test_http:create_default_app(), - emqx_common_test_http:auth_header(API). + emqx_mgmt_api_test_util:auth_header_(). init_per_testcase(_TC, Config) -> {ok, _} = emqx_cluster_link_config:update([]), @@ -71,62 +74,111 @@ init_per_testcase(_TC, Config) -> end_per_testcase(_TC, _Config) -> ok. -t_put_get_valid(Config) -> - Auth = ?config(auth, Config), - Path = ?API_PATH, - {ok, Resp} = emqx_mgmt_api_test_util:request_api(get, Path, Auth), - ?assertMatch([], emqx_utils_json:decode(Resp)), +%%------------------------------------------------------------------------------ +%% Helper fns +%%------------------------------------------------------------------------------ - Link1 = #{ +api_root() -> + <<"cluster/links">>. + +list() -> + Path = emqx_mgmt_api_test_util:api_path([api_root()]), + emqx_mgmt_api_test_util:simple_request(get, Path, _Params = ""). + +get_link(Name) -> + Path = emqx_mgmt_api_test_util:api_path([api_root(), Name]), + emqx_mgmt_api_test_util:simple_request(get, Path, _Params = ""). + +delete_link(Name) -> + Path = emqx_mgmt_api_test_util:api_path([api_root(), Name]), + emqx_mgmt_api_test_util:simple_request(delete, Path, _Params = ""). + +update_link(Name, Params) -> + Path = emqx_mgmt_api_test_util:api_path([api_root(), Name]), + emqx_mgmt_api_test_util:simple_request(put, Path, Params). + +create_link(Name, Params0) -> + Params = Params0#{<<"name">> => Name}, + Path = emqx_mgmt_api_test_util:api_path([api_root()]), + emqx_mgmt_api_test_util:simple_request(post, Path, Params). + +link_params() -> + link_params(_Overrides = #{}). + +link_params(Overrides) -> + Default = #{ + <<"clientid">> => <<"linkclientid">>, + <<"username">> => <<"myusername">>, <<"pool_size">> => 1, <<"server">> => <<"emqxcl_2.nohost:31883">>, - <<"topics">> => [<<"t/test-topic">>, <<"t/test/#">>], - <<"name">> => <<"emqcl_1">> + <<"topics">> => [<<"t/test-topic">>, <<"t/test/#">>] }, - Link2 = #{ - <<"pool_size">> => 1, - <<"server">> => <<"emqxcl_2.nohost:41883">>, - <<"topics">> => [<<"t/test-topic">>, <<"t/test/#">>], - <<"name">> => <<"emqcl_2">> - }, - ?assertMatch({ok, _}, emqx_mgmt_api_test_util:request_api(put, Path, "", Auth, [Link1, Link2])), + emqx_utils_maps:deep_merge(Default, Overrides). - {ok, Resp1} = emqx_mgmt_api_test_util:request_api(get, Path, Auth), - ?assertMatch([Link1, Link2], emqx_utils_json:decode(Resp1)), +%%------------------------------------------------------------------------------ +%% Test cases +%%------------------------------------------------------------------------------ + +t_put_get_valid(_Config) -> + ?assertMatch({200, []}, list()), + + Name1 = <<"emqcl_1">>, + Link1 = link_params(#{ + <<"server">> => <<"emqxcl_2.nohost:31883">>, + <<"name">> => Name1 + }), + Name2 = <<"emqcl_2">>, + Link2 = link_params(#{ + <<"server">> => <<"emqxcl_2.nohost:41883">>, + <<"name">> => Name2 + }), + ?assertMatch({201, _}, create_link(Name1, Link1)), + ?assertMatch({201, _}, create_link(Name2, Link2)), + ?assertMatch({200, [_, _]}, list()), DisabledLink1 = Link1#{<<"enable">> => false}, - ?assertMatch( - {ok, _}, emqx_mgmt_api_test_util:request_api(put, Path, "", Auth, [DisabledLink1, Link2]) - ), - - {ok, Resp2} = emqx_mgmt_api_test_util:request_api(get, Path, Auth), - ?assertMatch([DisabledLink1, Link2], emqx_utils_json:decode(Resp2)), + ?assertMatch({200, _}, update_link(Name1, maps:remove(<<"name">>, DisabledLink1))), + ?assertMatch({200, #{<<"enable">> := false}}, get_link(Name1)), + ?assertMatch({200, #{<<"enable">> := true}}, get_link(Name2)), SSL = #{<<"enable">> => true, <<"cacertfile">> => ?CACERT}, SSLLink1 = Link1#{<<"ssl">> => SSL}, + ?assertMatch({200, _}, update_link(Name1, maps:remove(<<"name">>, SSLLink1))), ?assertMatch( - {ok, _}, emqx_mgmt_api_test_util:request_api(put, Path, "", Auth, [Link2, SSLLink1]) + {200, #{<<"ssl">> := #{<<"enable">> := true, <<"cacertfile">> := _Path}}}, + get_link(Name1) ), - {ok, Resp3} = emqx_mgmt_api_test_util:request_api(get, Path, Auth), + ok. +t_put_invalid(_Config) -> + Name = <<"l1">>, + {201, _} = create_link(Name, link_params()), ?assertMatch( - [Link2, #{<<"ssl">> := #{<<"enable">> := true, <<"cacertfile">> := _Path}}], - emqx_utils_json:decode(Resp3) + {400, _}, + update_link(Name, maps:remove(<<"server">>, link_params())) ). -t_put_invalid(Config) -> - Auth = ?config(auth, Config), - Path = ?API_PATH, - Link = #{ - <<"pool_size">> => 1, - <<"server">> => <<"emqxcl_2.nohost:31883">>, - <<"topics">> => [<<"t/test-topic">>, <<"t/test/#">>], - <<"name">> => <<"emqcl_1">> - }, - ?assertMatch( - {error, {_, 400, _}}, emqx_mgmt_api_test_util:request_api(put, Path, "", Auth, [Link, Link]) - ), - ?assertMatch( - {error, {_, 400, _}}, - emqx_mgmt_api_test_util:request_api(put, Path, "", Auth, [maps:remove(<<"name">>, Link)]) - ). +t_crud(_Config) -> + %% No links initially. + ?assertMatch({200, []}, list()), + NameA = <<"a">>, + ?assertMatch({404, _}, get_link(NameA)), + ?assertMatch({404, _}, delete_link(NameA)), + ?assertMatch({404, _}, update_link(NameA, link_params())), + + Params1 = link_params(), + ?assertMatch({201, #{<<"name">> := NameA}}, create_link(NameA, Params1)), + ?assertMatch({400, #{<<"code">> := <<"ALREADY_EXISTS">>}}, create_link(NameA, Params1)), + ?assertMatch({200, [#{<<"name">> := NameA}]}, list()), + ?assertMatch({200, #{<<"name">> := NameA}}, get_link(NameA)), + + Params2 = Params1#{<<"pool_size">> := 2}, + ?assertMatch({200, #{<<"name">> := NameA}}, update_link(NameA, Params2)), + + ?assertMatch({204, _}, delete_link(NameA)), + ?assertMatch({404, _}, delete_link(NameA)), + ?assertMatch({404, _}, get_link(NameA)), + ?assertMatch({404, _}, update_link(NameA, Params1)), + ?assertMatch({200, []}, list()), + + ok. diff --git a/apps/emqx_management/test/emqx_mgmt_api_test_util.erl b/apps/emqx_management/test/emqx_mgmt_api_test_util.erl index 106a65a9c..4b1d40651 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_test_util.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_test_util.erl @@ -293,3 +293,30 @@ format_multipart_formdata(Data, Params, Name, FileNames, MimeType, Boundary) -> FileNames ), erlang:iolist_to_binary([WithPaths, StartBoundary, <<"--">>, LineSeparator]). + +maybe_json_decode(X) -> + case emqx_utils_json:safe_decode(X, [return_maps]) of + {ok, Decoded} -> Decoded; + {error, _} -> X + end. + +simple_request(Method, Path, Params) -> + AuthHeader = auth_header_(), + Opts = #{return_all => true}, + case request_api(Method, Path, "", AuthHeader, Params, Opts) of + {ok, {{_, Status, _}, _Headers, Body0}} -> + Body = maybe_json_decode(Body0), + {Status, Body}; + {error, {{_, Status, _}, _Headers, Body0}} -> + Body = + case emqx_utils_json:safe_decode(Body0, [return_maps]) of + {ok, Decoded0 = #{<<"message">> := Msg0}} -> + Msg = maybe_json_decode(Msg0), + Decoded0#{<<"message">> := Msg}; + {ok, Decoded0} -> + Decoded0; + {error, _} -> + Body0 + end, + {Status, Body} + end. diff --git a/apps/emqx_utils/include/emqx_utils_api.hrl b/apps/emqx_utils/include/emqx_utils_api.hrl index ba2941a4f..0876b9829 100644 --- a/apps/emqx_utils/include/emqx_utils_api.hrl +++ b/apps/emqx_utils/include/emqx_utils_api.hrl @@ -21,6 +21,8 @@ -define(OK(CONTENT), {200, CONTENT}). +-define(CREATED(CONTENT), {201, CONTENT}). + -define(NO_CONTENT, 204). -define(BAD_REQUEST(CODE, REASON), {400, ?ERROR_MSG(CODE, REASON)}). From ba3cbe02e33c9e5bc74c0954967074c1ffc39901 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 16 Jul 2024 16:47:25 -0300 Subject: [PATCH 008/131] feat(cluster link api): add status to responses Fixes https://emqx.atlassian.net/browse/EMQX-12627 --- apps/emqx/priv/bpapi.versions | 1 + .../src/emqx_cluster_link_api.erl | 246 +++++++++++++++--- .../src/emqx_cluster_link_mqtt.erl | 74 +++++- .../src/proto/emqx_cluster_link_proto_v1.erl | 31 +++ .../test/emqx_cluster_link_api_SUITE.erl | 212 ++++++++++++++- 5 files changed, 524 insertions(+), 40 deletions(-) create mode 100644 apps/emqx_cluster_link/src/proto/emqx_cluster_link_proto_v1.erl diff --git a/apps/emqx/priv/bpapi.versions b/apps/emqx/priv/bpapi.versions index 6e3cc046f..a5cc6b335 100644 --- a/apps/emqx/priv/bpapi.versions +++ b/apps/emqx/priv/bpapi.versions @@ -10,6 +10,7 @@ {emqx_bridge,5}. {emqx_bridge,6}. {emqx_broker,1}. +{emqx_cluster_link,1}. {emqx_cm,1}. {emqx_cm,2}. {emqx_cm,3}. diff --git a/apps/emqx_cluster_link/src/emqx_cluster_link_api.erl b/apps/emqx_cluster_link/src/emqx_cluster_link_api.erl index 324d6dd68..a184ab36d 100644 --- a/apps/emqx_cluster_link/src/emqx_cluster_link_api.erl +++ b/apps/emqx_cluster_link/src/emqx_cluster_link_api.erl @@ -8,10 +8,14 @@ -include_lib("hocon/include/hoconsc.hrl"). -include_lib("emqx/include/http_api.hrl"). -include_lib("emqx_utils/include/emqx_utils_api.hrl"). +-include_lib("emqx_resource/include/emqx_resource.hrl"). +-include_lib("emqx/include/logger.hrl"). -export([ api_spec/0, paths/0, + namespace/0, + fields/1, schema/1 ]). @@ -23,6 +27,10 @@ -define(CONF_PATH, [cluster, links]). -define(TAGS, [<<"Cluster">>]). +-type cluster_name() :: binary(). + +namespace() -> "cluster_link". + api_spec() -> emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}). @@ -40,7 +48,7 @@ schema("/cluster/links") -> description => "Get cluster links configuration", tags => ?TAGS, responses => - #{200 => links_config_schema()} + #{200 => links_config_schema_response()} }, post => #{ @@ -49,7 +57,7 @@ schema("/cluster/links") -> 'requestBody' => link_config_schema(), responses => #{ - 200 => link_config_schema(), + 200 => link_config_schema_response(), 400 => emqx_dashboard_swagger:error_codes( [?BAD_REQUEST, ?ALREADY_EXISTS], @@ -68,7 +76,7 @@ schema("/cluster/links/:name") -> parameters => [param_path_name()], responses => #{ - 200 => link_config_schema(), + 200 => link_config_schema_response(), 404 => emqx_dashboard_swagger:error_codes( [?NOT_FOUND], <<"Cluster link not found">> ) @@ -95,7 +103,7 @@ schema("/cluster/links/:name") -> 'requestBody' => update_link_config_schema(), responses => #{ - 200 => link_config_schema(), + 200 => link_config_schema_response(), 404 => emqx_dashboard_swagger:error_codes( [?NOT_FOUND], <<"Cluster link not found">> ), @@ -107,44 +115,30 @@ schema("/cluster/links/:name") -> } }. +fields(link_config_response) -> + [ + {node, hoconsc:mk(binary(), #{desc => ?DESC("node")})}, + {status, hoconsc:mk(status(), #{desc => ?DESC("status")})} + | emqx_cluster_link_schema:fields("link") + ]. + %%-------------------------------------------------------------------- %% API Handler funcs %%-------------------------------------------------------------------- '/cluster/links'(get, _Params) -> - ?OK(get_raw()); + handle_list(); '/cluster/links'(post, #{body := Body = #{<<"name">> := Name}}) -> with_link( Name, return(?BAD_REQUEST('ALREADY_EXISTS', <<"Cluster link already exists">>)), - fun() -> - case emqx_cluster_link_config:create(Body) of - {ok, Res} -> - ?CREATED(Res); - {error, Reason} -> - Message = list_to_binary(io_lib:format("Create link failed ~p", [Reason])), - ?BAD_REQUEST(Message) - end - end + fun() -> handle_create(Name, Body) end ). '/cluster/links/:name'(get, #{bindings := #{name := Name}}) -> - with_link(Name, fun(Link) -> ?OK(Link) end, not_found()); + with_link(Name, fun(Link) -> handle_lookup(Name, Link) end, not_found()); '/cluster/links/:name'(put, #{bindings := #{name := Name}, body := Params0}) -> - with_link( - Name, - fun(Link) -> - Params = Params0#{<<"name">> => Name}, - case emqx_cluster_link_config:update_one_link(Params) of - {ok, Res} -> - ?OK(Res); - {error, Reason} -> - Message = list_to_binary(io_lib:format("Update link failed ~p", [Reason])), - ?BAD_REQUEST(Message) - end - end, - not_found() - ); + with_link(Name, fun() -> handle_update(Name, Params0) end, not_found()); '/cluster/links/:name'(delete, #{bindings := #{name := Name}}) -> with_link( Name, @@ -164,6 +158,48 @@ schema("/cluster/links/:name") -> %% Internal funcs %%-------------------------------------------------------------------- +handle_list() -> + Links = get_raw(), + NodeResults = get_all_link_status_cluster(), + NameToStatus = collect_all_status(NodeResults), + EmptyStatus = #{status => inconsistent, node_status => []}, + Response = + lists:map( + fun(#{<<"name">> := Name} = Link) -> + Status = maps:get(Name, NameToStatus, EmptyStatus), + maps:merge(Link, Status) + end, + Links + ), + ?OK(Response). + +handle_create(Name, Params) -> + case emqx_cluster_link_config:create(Params) of + {ok, Link} -> + ?CREATED(add_status(Name, Link)); + {error, Reason} -> + Message = list_to_binary(io_lib:format("Create link failed ~p", [Reason])), + ?BAD_REQUEST(Message) + end. + +handle_lookup(Name, Link) -> + ?OK(add_status(Name, Link)). + +add_status(Name, Link) -> + NodeResults = get_link_status_cluster(Name), + Status = collect_single_status(NodeResults), + maps:merge(Link, Status). + +handle_update(Name, Params0) -> + Params = Params0#{<<"name">> => Name}, + case emqx_cluster_link_config:update_one_link(Params) of + {ok, Link} -> + ?OK(add_status(Name, Link)); + {error, Reason} -> + Message = list_to_binary(io_lib:format("Update link failed ~p", [Reason])), + ?BAD_REQUEST(Message) + end. + get_raw() -> #{<<"cluster">> := #{<<"links">> := Links}} = emqx_config:fill_defaults( @@ -172,15 +208,130 @@ get_raw() -> ), Links. -links_config_schema() -> - emqx_cluster_link_schema:links_schema( +get_all_link_status_cluster() -> + case emqx_cluster_link_mqtt:get_all_resources_cluster() of + {error, BadResults} -> + ?SLOG(warning, #{ + msg => "cluster_link_api_all_status_bad_erpc_results", + results => BadResults + }), + []; + {ok, NodeResults} -> + NodeResults + end. + +get_link_status_cluster(Name) -> + case emqx_cluster_link_mqtt:get_resource_cluster(Name) of + {error, BadResults} -> + ?SLOG(warning, #{ + msg => "cluster_link_api_lookup_status_bad_erpc_results", + results => BadResults + }), + []; + {ok, NodeResults} -> + NodeResults + end. + +-spec collect_all_status([{node(), #{cluster_name() => _}}]) -> + #{ + cluster_name() => #{ + node := node(), + status := emqx_resource:resource_status() | inconsistent + } + }. +collect_all_status(NodeResults) -> + Reindexed = lists:foldl( + fun({Node, AllLinkData}, Acc) -> + maps:fold( + fun(Name, Data, AccIn) -> + collect_all_status1(Node, Name, Data, AccIn) + end, + Acc, + AllLinkData + ) + end, + #{}, + NodeResults + ), + maps:fold( + fun(Name, NodeToData, Acc) -> + OnlyStatus = [S || #{status := S} <- maps:values(NodeToData)], + SummaryStatus = + case lists:usort(OnlyStatus) of + [SameStatus] -> SameStatus; + _ -> inconsistent + end, + NodeStatus = lists:map( + fun({Node, #{status := S}}) -> + #{node => Node, status => S} + end, + maps:to_list(NodeToData) + ), + Acc#{ + Name => #{ + status => SummaryStatus, + node_status => NodeStatus + } + } + end, + #{}, + Reindexed + ). + +collect_all_status1(Node, Name, Data, Acc) -> + maps:update_with( + Name, + fun(Old) -> Old#{Node => Data} end, + #{Node => Data}, + Acc + ). + +collect_single_status(NodeResults) -> + NodeStatus = + lists:map( + fun + ({Node, {ok, #{status := S}}}) -> + #{node => Node, status => S}; + ({Node, {error, _}}) -> + #{node => Node, status => ?status_disconnected}; + ({Node, _}) -> + #{node => Node, status => inconsistent} + end, + NodeResults + ), + OnlyStatus = [S || #{status := S} <- NodeStatus], + SummaryStatus = + case lists:usort(OnlyStatus) of + [SameStatus] -> SameStatus; + _ -> inconsistent + end, + #{ + status => SummaryStatus, + node_status => NodeStatus + }. + +links_config_schema_response() -> + hoconsc:mk(hoconsc:array(hoconsc:ref(?MODULE, link_config_response)), #{ + examples => #{<<"example">> => links_config_response_example()} + }). + +link_config_schema() -> + hoconsc:mk(emqx_cluster_link_schema:link_schema(), #{ + examples => #{<<"example">> => hd(links_config_example())} + }). + +link_config_schema_response() -> + hoconsc:mk( + hoconsc:ref(?MODULE, link_config_response), #{ - examples => #{<<"example">> => links_config_example()} + examples => #{ + <<"example">> => hd(links_config_response_example()) + } } ). -link_config_schema() -> - emqx_cluster_link_schema:link_schema(). +status() -> + hoconsc:enum([?status_connected, ?status_disconnected, ?status_connecting, inconsistent]). param_path_name() -> {name, @@ -197,6 +348,22 @@ param_path_name() -> update_link_config_schema() -> proplists:delete(name, emqx_cluster_link_schema:fields("link")). +links_config_response_example() -> + lists:map( + fun(LinkEx) -> + LinkEx#{ + <<"status">> => <<"connected">>, + <<"node_status">> => [ + #{ + <<"node">> => <<"emqx1@emqx.net">>, + <<"status">> => <<"connected">> + } + ] + } + end, + links_config_example() + ). + links_config_example() -> [ #{ @@ -229,7 +396,8 @@ with_link(Name, FoundFn, NotFoundFn) -> case emqx_cluster_link_config:link_raw(Name) of undefined -> NotFoundFn(); - Link = #{} -> + Link0 = #{} -> + Link = fill_defaults_single(Link0), {arity, Arity} = erlang:fun_info(FoundFn, arity), case Arity of 1 -> FoundFn(Link); @@ -237,6 +405,14 @@ with_link(Name, FoundFn, NotFoundFn) -> end end. +fill_defaults_single(Link0) -> + #{<<"cluster">> := #{<<"links">> := [Link]}} = + emqx_config:fill_defaults( + #{<<"cluster">> => #{<<"links">> => [Link0]}}, + #{obfuscate_sensitive_values => true} + ), + Link. + return(Response) -> fun() -> Response end. diff --git a/apps/emqx_cluster_link/src/emqx_cluster_link_mqtt.erl b/apps/emqx_cluster_link/src/emqx_cluster_link_mqtt.erl index 5185803b6..3b37a304e 100644 --- a/apps/emqx_cluster_link/src/emqx_cluster_link_mqtt.erl +++ b/apps/emqx_cluster_link/src/emqx_cluster_link_mqtt.erl @@ -46,6 +46,16 @@ forward/2 ]). +-export([ + get_all_resources_cluster/0, + get_resource_cluster/1 +]). +%% BpAPI / RPC Targets +-export([ + get_resource_local_v1/1, + get_all_resources_local_v1/0 +]). + -define(MSG_CLIENTID_SUFFIX, ":msg:"). -define(MQTT_HOST_OPTS, #{default_port => 1883}). @@ -80,6 +90,8 @@ -define(PUB_TIMEOUT, 10_000). +-type cluster_name() :: binary(). + -spec ensure_msg_fwd_resource(map()) -> {ok, emqx_resource:resource_data() | already_started} | {error, Reason :: term()}. ensure_msg_fwd_resource(#{name := Name, resource_opts := ResOpts} = ClusterConf) -> @@ -89,10 +101,57 @@ ensure_msg_fwd_resource(#{name := Name, resource_opts := ResOpts} = ClusterConf) }, emqx_resource:create_local(?MSG_RES_ID(Name), ?RES_GROUP, ?MODULE, ClusterConf, ResOpts1). --spec remove_msg_fwd_resource(binary() | map()) -> ok | {error, Reason :: term()}. +-spec remove_msg_fwd_resource(cluster_name()) -> ok | {error, Reason :: term()}. remove_msg_fwd_resource(ClusterName) -> emqx_resource:remove_local(?MSG_RES_ID(ClusterName)). +-spec get_all_resources_cluster() -> + {ok, [{node(), #{cluster_name() => emqx_resource:resource_data()}}]} + | {error, [term()]}. +get_all_resources_cluster() -> + Nodes = emqx:running_nodes(), + Results = emqx_cluster_link_proto_v1:get_all_resources(Nodes), + sequence_multicall_results(Nodes, Results). + +-spec get_resource_cluster(cluster_name()) -> + {ok, [{node(), {ok, emqx_resource:resource_data()} | {error, not_found}}]} + | {error, [term()]}. +get_resource_cluster(ClusterName) -> + Nodes = emqx:running_nodes(), + Results = emqx_cluster_link_proto_v1:get_resource(Nodes, ClusterName), + sequence_multicall_results(Nodes, Results). + +%% RPC Target in `emqx_cluster_link_proto_v1'. +-spec get_resource_local_v1(cluster_name()) -> + {ok, emqx_resource:resource_data()} | {error, not_found}. +get_resource_local_v1(ClusterName) -> + case emqx_resource:get_instance(?MSG_RES_ID(ClusterName)) of + {ok, _ResourceGroup, ResourceData} -> + {ok, ResourceData}; + {error, not_found} -> + {error, not_found} + end. + +%% RPC Target in `emqx_cluster_link_proto_v1'. +-spec get_all_resources_local_v1() -> #{cluster_name() => emqx_resource:resource_data()}. +get_all_resources_local_v1() -> + lists:foldl( + fun + (?MSG_RES_ID(Name) = Id, Acc) -> + case emqx_resource:get_instance(Id) of + {ok, ?RES_GROUP, ResourceData} -> + Acc#{Name => ResourceData}; + _ -> + Acc + end; + (_Id, Acc) -> + %% Doesn't follow the naming pattern; manually crafted? + Acc + end, + #{}, + emqx_resource:list_group_instances(?RES_GROUP) + ). + %%-------------------------------------------------------------------- %% emqx_resource callbacks (message forwarding) %%-------------------------------------------------------------------- @@ -419,3 +478,16 @@ emqtt_client_opts(ClientIdSuffix, ClusterConf) -> #{clientid := BaseClientId} = Opts = emqx_cluster_link_config:mk_emqtt_options(ClusterConf), ClientId = emqx_bridge_mqtt_lib:clientid_base([BaseClientId, ClientIdSuffix]), Opts#{clientid => ClientId}. + +-spec sequence_multicall_results([node()], emqx_rpc:erpc_multicall(term())) -> + {ok, [{node(), term()}]} | {error, [term()]}. +sequence_multicall_results(Nodes, Results) -> + case lists:partition(fun is_ok/1, lists:zip(Nodes, Results)) of + {OkResults, []} -> + {ok, [{Node, Res} || {Node, {ok, Res}} <- OkResults]}; + {_OkResults, BadResults} -> + {error, BadResults} + end. + +is_ok({_Node, {ok, _}}) -> true; +is_ok(_) -> false. diff --git a/apps/emqx_cluster_link/src/proto/emqx_cluster_link_proto_v1.erl b/apps/emqx_cluster_link/src/proto/emqx_cluster_link_proto_v1.erl new file mode 100644 index 000000000..725bb8afc --- /dev/null +++ b/apps/emqx_cluster_link/src/proto/emqx_cluster_link_proto_v1.erl @@ -0,0 +1,31 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_cluster_link_proto_v1). + +-behaviour(emqx_bpapi). + +-export([ + introduced_in/0, + + get_resource/2, + get_all_resources/1 +]). + +-include_lib("emqx/include/bpapi.hrl"). + +-define(TIMEOUT, 15000). + +introduced_in() -> + "5.7.2". + +-spec get_resource([node()], binary()) -> + emqx_rpc:erpc_multicall({ok, emqx_resource:resource_data()} | {error, not_found}). +get_resource(Nodes, ClusterName) -> + erpc:multicall(Nodes, emqx_cluster_link_mqtt, get_resource_local_v1, [ClusterName], ?TIMEOUT). + +-spec get_all_resources([node()]) -> + emqx_rpc:erpc_multicall(#{binary() => emqx_resource:resource_data()}). +get_all_resources(Nodes) -> + erpc:multicall(Nodes, emqx_cluster_link_mqtt, get_all_resources_local_v1, [], ?TIMEOUT). diff --git a/apps/emqx_cluster_link/test/emqx_cluster_link_api_SUITE.erl b/apps/emqx_cluster_link/test/emqx_cluster_link_api_SUITE.erl index 5bb1c377a..5c136925d 100644 --- a/apps/emqx_cluster_link/test/emqx_cluster_link_api_SUITE.erl +++ b/apps/emqx_cluster_link/test/emqx_cluster_link_api_SUITE.erl @@ -37,6 +37,8 @@ "-----END CERTIFICATE-----" >>). +-define(ON(NODE, BODY), erpc:call(NODE, fun() -> BODY end)). + %%------------------------------------------------------------------------------ %% CT boilerplate %%------------------------------------------------------------------------------ @@ -67,10 +69,36 @@ end_per_suite(Config) -> auth_header() -> emqx_mgmt_api_test_util:auth_header_(). +init_per_testcase(t_status = TestCase, Config) -> + ok = emqx_cth_suite:stop_apps([emqx_dashboard]), + SourceClusterSpec = emqx_cluster_link_SUITE:mk_source_cluster(TestCase, Config), + TargetClusterSpec = emqx_cluster_link_SUITE:mk_target_cluster(TestCase, Config), + SourceNodes = [SN1 | _] = emqx_cth_cluster:start(SourceClusterSpec), + TargetNodes = emqx_cth_cluster:start(TargetClusterSpec), + emqx_cluster_link_SUITE:start_cluster_link(SourceNodes ++ TargetNodes, Config), + erpc:call(SN1, emqx_cth_suite, start_apps, [ + [emqx_management, emqx_mgmt_api_test_util:emqx_dashboard()], + #{work_dir => emqx_cth_suite:work_dir(TestCase, Config)} + ]), + [ + {source_nodes, SourceNodes}, + {target_nodes, TargetNodes} + | Config + ]; init_per_testcase(_TC, Config) -> {ok, _} = emqx_cluster_link_config:update([]), Config. +end_per_testcase(t_status, Config) -> + SourceNodes = ?config(source_nodes, Config), + TargetNodes = ?config(target_nodes, Config), + ok = emqx_cth_cluster:stop(SourceNodes), + ok = emqx_cth_cluster:stop(TargetNodes), + _ = emqx_cth_suite:start_apps( + [emqx_mgmt_api_test_util:emqx_dashboard()], + #{work_dir => emqx_cth_suite:work_dir(Config)} + ), + ok; end_per_testcase(_TC, _Config) -> ok. @@ -158,6 +186,8 @@ t_put_invalid(_Config) -> update_link(Name, maps:remove(<<"server">>, link_params())) ). +%% Tests a sequence of CRUD operations and their expected responses, for common use cases +%% and configuration states. t_crud(_Config) -> %% No links initially. ?assertMatch({200, []}, list()), @@ -167,13 +197,43 @@ t_crud(_Config) -> ?assertMatch({404, _}, update_link(NameA, link_params())), Params1 = link_params(), - ?assertMatch({201, #{<<"name">> := NameA}}, create_link(NameA, Params1)), + ?assertMatch( + {201, #{ + <<"name">> := NameA, + <<"status">> := _, + <<"node_status">> := [#{<<"node">> := _, <<"status">> := _} | _] + }}, + create_link(NameA, Params1) + ), ?assertMatch({400, #{<<"code">> := <<"ALREADY_EXISTS">>}}, create_link(NameA, Params1)), - ?assertMatch({200, [#{<<"name">> := NameA}]}, list()), - ?assertMatch({200, #{<<"name">> := NameA}}, get_link(NameA)), + ?assertMatch( + {200, [ + #{ + <<"name">> := NameA, + <<"status">> := _, + <<"node_status">> := [#{<<"node">> := _, <<"status">> := _} | _] + } + ]}, + list() + ), + ?assertMatch( + {200, #{ + <<"name">> := NameA, + <<"status">> := _, + <<"node_status">> := [#{<<"node">> := _, <<"status">> := _} | _] + }}, + get_link(NameA) + ), Params2 = Params1#{<<"pool_size">> := 2}, - ?assertMatch({200, #{<<"name">> := NameA}}, update_link(NameA, Params2)), + ?assertMatch( + {200, #{ + <<"name">> := NameA, + <<"status">> := _, + <<"node_status">> := [#{<<"node">> := _, <<"status">> := _} | _] + }}, + update_link(NameA, Params2) + ), ?assertMatch({204, _}, delete_link(NameA)), ?assertMatch({404, _}, delete_link(NameA)), @@ -182,3 +242,147 @@ t_crud(_Config) -> ?assertMatch({200, []}, list()), ok. + +%% Verifies the behavior of reported status under different conditions when listing all +%% links and when fetching a specific link. +t_status(Config) -> + [SN1 | _] = ?config(source_nodes, Config), + Name = <<"cl.target">>, + ?assertMatch( + {200, [ + #{ + <<"status">> := <<"connected">>, + <<"node_status">> := [ + #{ + <<"node">> := _, + <<"status">> := <<"connected">> + }, + #{ + <<"node">> := _, + <<"status">> := <<"connected">> + } + ] + } + ]}, + list() + ), + ?assertMatch( + {200, #{ + <<"status">> := <<"connected">>, + <<"node_status">> := [ + #{ + <<"node">> := _, + <<"status">> := <<"connected">> + }, + #{ + <<"node">> := _, + <<"status">> := <<"connected">> + } + ] + }}, + get_link(Name) + ), + + %% If one of the nodes reports a different status, the cluster is inconsistent. + ProtoMod = emqx_cluster_link_proto_v1, + ?ON(SN1, begin + ok = meck:new(ProtoMod, [no_link, passthrough, no_history]), + meck:expect(ProtoMod, get_all_resources, fun(Nodes) -> + [Res1, {ok, Res2A} | Rest] = meck:passthrough([Nodes]), + %% Res2A :: #{cluster_name() => emqx_resource:resource_data()} + Res2B = maps:map(fun(_, Data) -> Data#{status := disconnected} end, Res2A), + [Res1, {ok, Res2B} | Rest] + end), + meck:expect(ProtoMod, get_resource, fun(Nodes, LinkName) -> + [Res1, {ok, {ok, Res2A}} | Rest] = meck:passthrough([Nodes, LinkName]), + Res2B = Res2A#{status := disconnected}, + [Res1, {ok, {ok, Res2B}} | Rest] + end) + end), + ?assertMatch( + {200, [ + #{ + <<"status">> := <<"inconsistent">>, + <<"node_status">> := [ + #{ + <<"node">> := _, + <<"status">> := <<"connected">> + }, + #{ + <<"node">> := _, + <<"status">> := <<"disconnected">> + } + ] + } + ]}, + list() + ), + ?assertMatch( + {200, #{ + <<"status">> := <<"inconsistent">>, + <<"node_status">> := [ + #{ + <<"node">> := _, + <<"status">> := <<"connected">> + }, + #{ + <<"node">> := _, + <<"status">> := <<"disconnected">> + } + ] + }}, + get_link(Name) + ), + + %% Simulating erpc failures + ?ON(SN1, begin + meck:expect(ProtoMod, get_all_resources, fun(Nodes) -> + [Res1, _ | Rest] = meck:passthrough([Nodes]), + [Res1, {error, {erpc, noconnection}} | Rest] + end), + meck:expect(ProtoMod, get_resource, fun(Nodes, LinkName) -> + [Res1, _ | Rest] = meck:passthrough([Nodes, LinkName]), + [Res1, {error, {erpc, noconnection}} | Rest] + end) + end), + ?assertMatch( + {200, [ + #{ + <<"status">> := <<"inconsistent">>, + <<"node_status">> := [] + } + ]}, + list() + ), + ?assertMatch( + {200, #{ + <<"status">> := <<"inconsistent">>, + <<"node_status">> := [] + }}, + get_link(Name) + ), + %% Simulate another inconsistency + ?ON(SN1, begin + meck:expect(ProtoMod, get_resource, fun(Nodes, LinkName) -> + [Res1, _ | Rest] = meck:passthrough([Nodes, LinkName]), + [Res1, {ok, {error, not_found}} | Rest] + end) + end), + ?assertMatch( + {200, #{ + <<"status">> := <<"inconsistent">>, + <<"node_status">> := [ + #{ + <<"node">> := _, + <<"status">> := <<"connected">> + }, + #{ + <<"node">> := _, + <<"status">> := <<"disconnected">> + } + ] + }}, + get_link(Name) + ), + + ok. From 07cb147d383bc12e3eb3319cfd33b6de8749e03a Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 17 Jul 2024 18:00:28 -0300 Subject: [PATCH 009/131] fix(cluster link schema): username is not required --- apps/emqx_cluster_link/src/emqx_cluster_link_schema.erl | 4 ++-- apps/emqx_cluster_link/test/emqx_cluster_link_api_SUITE.erl | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/apps/emqx_cluster_link/src/emqx_cluster_link_schema.erl b/apps/emqx_cluster_link/src/emqx_cluster_link_schema.erl index a6073d677..a369429d5 100644 --- a/apps/emqx_cluster_link/src/emqx_cluster_link_schema.erl +++ b/apps/emqx_cluster_link/src/emqx_cluster_link_schema.erl @@ -47,8 +47,8 @@ fields("link") -> {server, emqx_schema:servers_sc(#{required => true, desc => ?DESC(server)}, ?MQTT_HOST_OPTS)}, {clientid, ?HOCON(binary(), #{desc => ?DESC(clientid)})}, - {username, ?HOCON(binary(), #{desc => ?DESC(username)})}, - {password, emqx_schema_secret:mk(#{desc => ?DESC(password)})}, + {username, ?HOCON(binary(), #{required => false, desc => ?DESC(username)})}, + {password, emqx_schema_secret:mk(#{required => false, desc => ?DESC(password)})}, {ssl, #{ type => ?R_REF(emqx_schema, "ssl_client_opts"), default => #{<<"enable">> => false}, diff --git a/apps/emqx_cluster_link/test/emqx_cluster_link_api_SUITE.erl b/apps/emqx_cluster_link/test/emqx_cluster_link_api_SUITE.erl index 5c136925d..535e8521a 100644 --- a/apps/emqx_cluster_link/test/emqx_cluster_link_api_SUITE.erl +++ b/apps/emqx_cluster_link/test/emqx_cluster_link_api_SUITE.erl @@ -136,7 +136,6 @@ link_params() -> link_params(Overrides) -> Default = #{ <<"clientid">> => <<"linkclientid">>, - <<"username">> => <<"myusername">>, <<"pool_size">> => 1, <<"server">> => <<"emqxcl_2.nohost:31883">>, <<"topics">> => [<<"t/test-topic">>, <<"t/test/#">>] From 6a5849488c767a567ba39a6ec785b728948c8164 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 17 Jul 2024 18:01:52 -0300 Subject: [PATCH 010/131] feat(cluster link): add metrics Fixes https://emqx.atlassian.net/browse/EMQX-12627 --- .../include/emqx_cluster_link.hrl | 4 + .../src/emqx_cluster_link.erl | 57 ++++ .../src/emqx_cluster_link_api.erl | 106 ++++++- .../src/emqx_cluster_link_app.erl | 12 +- .../src/emqx_cluster_link_config.erl | 6 +- .../src/emqx_cluster_link_extrouter.erl | 24 +- .../src/emqx_cluster_link_sup.erl | 5 +- .../test/emqx_cluster_link_api_SUITE.erl | 265 +++++++++++++++++- 8 files changed, 450 insertions(+), 29 deletions(-) diff --git a/apps/emqx_cluster_link/include/emqx_cluster_link.hrl b/apps/emqx_cluster_link/include/emqx_cluster_link.hrl index 08dc7f4ad..32c675d8d 100644 --- a/apps/emqx_cluster_link/include/emqx_cluster_link.hrl +++ b/apps/emqx_cluster_link/include/emqx_cluster_link.hrl @@ -17,3 +17,7 @@ %% Fairly compact text encoding. -define(SHARED_ROUTE_ID(Topic, Group), <<"$s/", Group/binary, "/", Topic/binary>>). -define(PERSISTENT_ROUTE_ID(Topic, ID), <<"$p/", ID/binary, "/", Topic/binary>>). + +-define(METRIC_NAME, cluster_link). + +-define(route_metric, 'routes'). diff --git a/apps/emqx_cluster_link/src/emqx_cluster_link.erl b/apps/emqx_cluster_link/src/emqx_cluster_link.erl index 76228c052..e3bc04a29 100644 --- a/apps/emqx_cluster_link/src/emqx_cluster_link.erl +++ b/apps/emqx_cluster_link/src/emqx_cluster_link.erl @@ -26,11 +26,29 @@ on_message_publish/1 ]). +%% metrics API +-export([ + maybe_create_metrics/1, + drop_metrics/1, + + get_metrics/1, + routes_inc/2 +]). + -include("emqx_cluster_link.hrl"). -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/emqx_hooks.hrl"). -include_lib("emqx/include/logger.hrl"). +%%-------------------------------------------------------------------- +%% Type definitions +%%-------------------------------------------------------------------- + +-define(METRICS, [ + ?route_metric +]). +-define(RATE_METRICS, []). + %%-------------------------------------------------------------------- %% emqx_external_broker API %%-------------------------------------------------------------------- @@ -132,6 +150,32 @@ put_hook() -> delete_hook() -> emqx_hooks:del('message.publish', {?MODULE, on_message_publish, []}). +%%-------------------------------------------------------------------- +%% metrics API +%%-------------------------------------------------------------------- + +get_metrics(ClusterName) -> + Nodes = emqx:running_nodes(), + Timeout = 15_000, + Results = emqx_metrics_proto_v2:get_metrics(Nodes, ?METRIC_NAME, ClusterName, Timeout), + sequence_multicall_results(Nodes, Results). + +maybe_create_metrics(ClusterName) -> + case emqx_metrics_worker:has_metrics(?METRIC_NAME, ClusterName) of + true -> + ok = emqx_metrics_worker:reset_metrics(?METRIC_NAME, ClusterName); + false -> + ok = emqx_metrics_worker:create_metrics( + ?METRIC_NAME, ClusterName, ?METRICS, ?RATE_METRICS + ) + end. + +drop_metrics(ClusterName) -> + ok = emqx_metrics_worker:clear_metrics(?METRIC_NAME, ClusterName). + +routes_inc(ClusterName, Val) -> + catch emqx_metrics_worker:inc(?METRIC_NAME, ClusterName, ?route_metric, Val). + %%-------------------------------------------------------------------- %% Internal functions %%-------------------------------------------------------------------- @@ -253,3 +297,16 @@ maybe_filter_incomming_msg(#message{topic = T} = Msg, ClusterName) -> true -> with_sender_name(Msg, ClusterName); false -> [] end. + +-spec sequence_multicall_results([node()], emqx_rpc:erpc_multicall(term())) -> + {ok, [{node(), term()}]} | {error, [term()]}. +sequence_multicall_results(Nodes, Results) -> + case lists:partition(fun is_ok/1, lists:zip(Nodes, Results)) of + {OkResults, []} -> + {ok, [{Node, Res} || {Node, {ok, Res}} <- OkResults]}; + {_OkResults, BadResults} -> + {error, BadResults} + end. + +is_ok({_Node, {ok, _}}) -> true; +is_ok(_) -> false. diff --git a/apps/emqx_cluster_link/src/emqx_cluster_link_api.erl b/apps/emqx_cluster_link/src/emqx_cluster_link_api.erl index a184ab36d..77a77610b 100644 --- a/apps/emqx_cluster_link/src/emqx_cluster_link_api.erl +++ b/apps/emqx_cluster_link/src/emqx_cluster_link_api.erl @@ -10,6 +10,7 @@ -include_lib("emqx_utils/include/emqx_utils_api.hrl"). -include_lib("emqx_resource/include/emqx_resource.hrl"). -include_lib("emqx/include/logger.hrl"). +-include("emqx_cluster_link.hrl"). -export([ api_spec/0, @@ -21,7 +22,8 @@ -export([ '/cluster/links'/2, - '/cluster/links/:name'/2 + '/cluster/links/:name'/2, + '/cluster/links/:name/metrics'/2 ]). -define(CONF_PATH, [cluster, links]). @@ -37,7 +39,8 @@ api_spec() -> paths() -> [ "/cluster/links", - "/cluster/links/:name" + "/cluster/links/:name", + "/cluster/links/:name/metrics" ]. schema("/cluster/links") -> @@ -113,6 +116,23 @@ schema("/cluster/links/:name") -> ) } } + }; +schema("/cluster/links/:name/metrics") -> + #{ + 'operationId' => '/cluster/links/:name/metrics', + get => + #{ + description => "Get a cluster link metrics", + tags => ?TAGS, + parameters => [param_path_name()], + responses => + #{ + 200 => link_metrics_schema_response(), + 404 => emqx_dashboard_swagger:error_codes( + [?NOT_FOUND], <<"Cluster link not found">> + ) + } + } }. fields(link_config_response) -> @@ -120,6 +140,24 @@ fields(link_config_response) -> {node, hoconsc:mk(binary(), #{desc => ?DESC("node")})}, {status, hoconsc:mk(status(), #{desc => ?DESC("status")})} | emqx_cluster_link_schema:fields("link") + ]; +fields(metrics) -> + [ + {metrics, hoconsc:mk(map(), #{desc => ?DESC("metrics")})} + ]; +fields(link_metrics_response) -> + [ + {node_metrics, + hoconsc:mk( + hoconsc:array(hoconsc:ref(?MODULE, node_metrics)), + #{desc => ?DESC("node_metrics")} + )} + | fields(metrics) + ]; +fields(node_metrics) -> + [ + {node, hoconsc:mk(atom(), #{desc => ?DESC("node")})} + | fields(metrics) ]. %%-------------------------------------------------------------------- @@ -154,6 +192,9 @@ fields(link_config_response) -> not_found() ). +'/cluster/links/:name/metrics'(get, #{bindings := #{name := Name}}) -> + with_link(Name, fun() -> handle_metrics(Name) end, not_found()). + %%-------------------------------------------------------------------- %% Internal funcs %%-------------------------------------------------------------------- @@ -185,6 +226,46 @@ handle_create(Name, Params) -> handle_lookup(Name, Link) -> ?OK(add_status(Name, Link)). +handle_metrics(Name) -> + case emqx_cluster_link:get_metrics(Name) of + {error, BadResults} -> + ?SLOG(warning, #{ + msg => "cluster_link_api_metrics_bad_erpc_results", + results => BadResults + }), + ?OK(#{metrics => #{}, node_metrics => []}); + {ok, NodeResults} -> + NodeMetrics = + lists:map( + fun({Node, Metrics}) -> + format_metrics(Node, Metrics) + end, + NodeResults + ), + AggregatedMetrics = aggregate_metrics(NodeMetrics), + Response = #{metrics => AggregatedMetrics, node_metrics => NodeMetrics}, + ?OK(Response) + end. + +aggregate_metrics(NodeMetrics) -> + ErrorLogger = fun(_) -> ok end, + lists:foldl( + fun(#{metrics := Metrics}, Acc) -> + emqx_utils_maps:best_effort_recursive_sum(Metrics, Acc, ErrorLogger) + end, + #{}, + NodeMetrics + ). + +format_metrics(Node, Metrics) -> + Routes = emqx_utils_maps:deep_get([counters, ?route_metric], Metrics, 0), + #{ + node => Node, + metrics => #{ + ?route_metric => Routes + } + }. + add_status(Name, Link) -> NodeResults = get_link_status_cluster(Name), Status = collect_single_status(NodeResults), @@ -330,6 +411,16 @@ link_config_schema_response() -> } ). +link_metrics_schema_response() -> + hoconsc:mk( + hoconsc:ref(?MODULE, link_metrics_response), + #{ + examples => #{ + <<"example">> => link_metrics_response_example() + } + } + ). + status() -> hoconsc:enum([?status_connected, ?status_disconnected, ?status_connecting, inconsistent]). @@ -392,6 +483,17 @@ links_config_example() -> } ]. +link_metrics_response_example() -> + #{ + <<"metrics">> => #{<<"routes">> => 10240}, + <<"node_metrics">> => [ + #{ + <<"node">> => <<"emqx1@emqx.net">>, + <<"metrics">> => #{<<"routes">> => 10240} + } + ] + }. + with_link(Name, FoundFn, NotFoundFn) -> case emqx_cluster_link_config:link_raw(Name) of undefined -> diff --git a/apps/emqx_cluster_link/src/emqx_cluster_link_app.erl b/apps/emqx_cluster_link/src/emqx_cluster_link_app.erl index 41f1a0a77..f9625fae4 100644 --- a/apps/emqx_cluster_link/src/emqx_cluster_link_app.erl +++ b/apps/emqx_cluster_link/src/emqx_cluster_link_app.erl @@ -22,7 +22,9 @@ start(_StartType, _StartArgs) -> _ -> ok end, - emqx_cluster_link_sup:start_link(LinksConf). + {ok, Sup} = emqx_cluster_link_sup:start_link(LinksConf), + ok = create_metrics(LinksConf), + {ok, Sup}. prep_stop(State) -> emqx_cluster_link_config:remove_handler(), @@ -53,3 +55,11 @@ remove_msg_fwd_resources(LinksConf) -> end, LinksConf ). + +create_metrics(LinksConf) -> + lists:foreach( + fun(#{name := ClusterName}) -> + ok = emqx_cluster_link:maybe_create_metrics(ClusterName) + end, + LinksConf + ). diff --git a/apps/emqx_cluster_link/src/emqx_cluster_link_config.erl b/apps/emqx_cluster_link/src/emqx_cluster_link_config.erl index 36655460b..0455ab21c 100644 --- a/apps/emqx_cluster_link/src/emqx_cluster_link_config.erl +++ b/apps/emqx_cluster_link/src/emqx_cluster_link_config.erl @@ -277,9 +277,10 @@ all_ok(Results) -> add_links(LinksConf) -> [add_link(Link) || Link <- LinksConf]. -add_link(#{enable := true} = LinkConf) -> +add_link(#{name := ClusterName, enable := true} = LinkConf) -> {ok, _Pid} = emqx_cluster_link_sup:ensure_actor(LinkConf), {ok, _} = emqx_cluster_link_mqtt:ensure_msg_fwd_resource(LinkConf), + ok = emqx_cluster_link:maybe_create_metrics(ClusterName), ok; add_link(_DisabledLinkConf) -> ok. @@ -289,7 +290,8 @@ remove_links(LinksConf) -> remove_link(Name) -> _ = emqx_cluster_link_mqtt:remove_msg_fwd_resource(Name), - ensure_actor_stopped(Name). + _ = ensure_actor_stopped(Name), + emqx_cluster_link:drop_metrics(Name). update_links(LinksConf) -> [update_link(Link) || Link <- LinksConf]. diff --git a/apps/emqx_cluster_link/src/emqx_cluster_link_extrouter.erl b/apps/emqx_cluster_link/src/emqx_cluster_link_extrouter.erl index 79d96e207..c45b12ae0 100644 --- a/apps/emqx_cluster_link/src/emqx_cluster_link_extrouter.erl +++ b/apps/emqx_cluster_link/src/emqx_cluster_link_extrouter.erl @@ -256,31 +256,35 @@ actor_apply_operation( apply_actor_operation(ActorID, Incarnation, Entry, OpName, Lane) -> _ = assert_current_incarnation(ActorID, Incarnation), - apply_operation(Entry, OpName, Lane). + apply_operation(ActorID, Entry, OpName, Lane). -apply_operation(Entry, OpName, Lane) -> +apply_operation(ActorID, Entry, OpName, Lane) -> %% NOTE %% This is safe sequence of operations only on core nodes. On replicants, %% `mria:dirty_update_counter/3` will be replicated asynchronously, which %% means this read can be stale. case mnesia:dirty_read(?EXTROUTE_TAB, Entry) of [#extroute{mcounter = MCounter}] -> - apply_operation(Entry, MCounter, OpName, Lane); + apply_operation(ActorID, Entry, MCounter, OpName, Lane); [] -> - apply_operation(Entry, 0, OpName, Lane) + apply_operation(ActorID, Entry, 0, OpName, Lane) end. -apply_operation(Entry, MCounter, OpName, Lane) -> +apply_operation(ActorID, Entry, MCounter, OpName, Lane) -> %% NOTE %% We are relying on the fact that changes to each individual lane of this %% multi-counter are synchronized. Without this, such counter updates would %% be unsafe. Instead, we would have to use another, more complex approach, %% that runs `ets:lookup/2` + `ets:select_replace/2` in a loop until the %% counter is updated accordingly. + ?ACTOR_ID(ClusterName, _Actor) = ActorID, Marker = 1 bsl Lane, case MCounter band Marker of 0 when OpName =:= add -> - mria:dirty_update_counter(?EXTROUTE_TAB, Entry, Marker); + Res = mria:dirty_update_counter(?EXTROUTE_TAB, Entry, Marker), + _ = emqx_cluster_link:routes_inc(ClusterName, 1), + ?tp("cluster_link_extrouter_route_added", #{}), + Res; Marker when OpName =:= add -> %% Already added. MCounter; @@ -289,6 +293,8 @@ apply_operation(Entry, MCounter, OpName, Lane) -> 0 -> Record = #extroute{entry = Entry, mcounter = 0}, ok = mria:dirty_delete_object(?EXTROUTE_TAB, Record), + _ = emqx_cluster_link:routes_inc(ClusterName, -1), + ?tp("cluster_link_extrouter_route_deleted", #{}), 0; C -> C @@ -362,16 +368,16 @@ clean_incarnation(Rec = #actor{id = {Cluster, Actor}}) -> mnesia_clean_incarnation(#actor{id = Actor, incarnation = Incarnation, lane = Lane}) -> case mnesia:read(?EXTROUTE_ACTOR_TAB, Actor, write) of [#actor{incarnation = Incarnation}] -> - _ = clean_lane(Lane), + _ = clean_lane(Actor, Lane), mnesia:delete(?EXTROUTE_ACTOR_TAB, Actor, write); _Renewed -> stale end. -clean_lane(Lane) -> +clean_lane(ActorID, Lane) -> ets:foldl( fun(#extroute{entry = Entry, mcounter = MCounter}, _) -> - apply_operation(Entry, MCounter, delete, Lane) + apply_operation(ActorID, Entry, MCounter, delete, Lane) end, 0, ?EXTROUTE_TAB diff --git a/apps/emqx_cluster_link/src/emqx_cluster_link_sup.erl b/apps/emqx_cluster_link/src/emqx_cluster_link_sup.erl index 2025510fc..81b5afb4c 100644 --- a/apps/emqx_cluster_link/src/emqx_cluster_link_sup.erl +++ b/apps/emqx_cluster_link/src/emqx_cluster_link_sup.erl @@ -6,6 +6,8 @@ -behaviour(supervisor). +-include("emqx_cluster_link.hrl"). + -export([start_link/1]). -export([ @@ -27,12 +29,13 @@ init(LinksConf) -> intensity => 10, period => 5 }, + Metrics = emqx_metrics_worker:child_spec(metrics, ?METRIC_NAME), ExtrouterGC = extrouter_gc_spec(), RouteActors = [ sup_spec(Name, ?ACTOR_MODULE, [LinkConf]) || #{name := Name} = LinkConf <- LinksConf ], - {ok, {SupFlags, [ExtrouterGC | RouteActors]}}. + {ok, {SupFlags, [Metrics, ExtrouterGC | RouteActors]}}. extrouter_gc_spec() -> %% NOTE: This one is currently global, not per-link. diff --git a/apps/emqx_cluster_link/test/emqx_cluster_link_api_SUITE.erl b/apps/emqx_cluster_link/test/emqx_cluster_link_api_SUITE.erl index 535e8521a..f4a15a62a 100644 --- a/apps/emqx_cluster_link/test/emqx_cluster_link_api_SUITE.erl +++ b/apps/emqx_cluster_link/test/emqx_cluster_link_api_SUITE.erl @@ -11,6 +11,8 @@ -include_lib("common_test/include/ct.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). +-import(emqx_common_test_helpers, [on_exit/1]). + -define(API_PATH, emqx_mgmt_api_test_util:api_path(["cluster", "links"])). -define(CONF_PATH, [cluster, links]). @@ -44,7 +46,21 @@ %%------------------------------------------------------------------------------ all() -> - emqx_common_test_helpers:all(?MODULE). + AllTCs = emqx_common_test_helpers:all(?MODULE), + OtherTCs = AllTCs -- cluster_test_cases(), + [ + {group, cluster} + | OtherTCs + ]. + +groups() -> + [{cluster, cluster_test_cases()}]. + +cluster_test_cases() -> + [ + t_status, + t_metrics + ]. init_per_suite(Config) -> %% This is called by emqx_machine in EMQX release @@ -66,30 +82,35 @@ end_per_suite(Config) -> emqx_config:delete_override_conf_files(), ok. -auth_header() -> - emqx_mgmt_api_test_util:auth_header_(). - -init_per_testcase(t_status = TestCase, Config) -> +init_per_group(cluster = Group, Config) -> ok = emqx_cth_suite:stop_apps([emqx_dashboard]), - SourceClusterSpec = emqx_cluster_link_SUITE:mk_source_cluster(TestCase, Config), - TargetClusterSpec = emqx_cluster_link_SUITE:mk_target_cluster(TestCase, Config), + SourceClusterSpec = emqx_cluster_link_SUITE:mk_source_cluster(Group, Config), + TargetClusterSpec = emqx_cluster_link_SUITE:mk_target_cluster(Group, Config), SourceNodes = [SN1 | _] = emqx_cth_cluster:start(SourceClusterSpec), - TargetNodes = emqx_cth_cluster:start(TargetClusterSpec), + TargetNodes = [TN1 | _] = emqx_cth_cluster:start(TargetClusterSpec), emqx_cluster_link_SUITE:start_cluster_link(SourceNodes ++ TargetNodes, Config), erpc:call(SN1, emqx_cth_suite, start_apps, [ [emqx_management, emqx_mgmt_api_test_util:emqx_dashboard()], - #{work_dir => emqx_cth_suite:work_dir(TestCase, Config)} + #{work_dir => emqx_cth_suite:work_dir(Group, Config)} + ]), + erpc:call(TN1, emqx_cth_suite, start_apps, [ + [ + emqx_management, + emqx_mgmt_api_test_util:emqx_dashboard( + "dashboard.listeners.http { enable = true, bind = 28083 }" + ) + ], + #{work_dir => emqx_cth_suite:work_dir(Group, Config)} ]), [ {source_nodes, SourceNodes}, {target_nodes, TargetNodes} | Config ]; -init_per_testcase(_TC, Config) -> - {ok, _} = emqx_cluster_link_config:update([]), +init_per_group(_Group, Config) -> Config. -end_per_testcase(t_status, Config) -> +end_per_group(cluster, Config) -> SourceNodes = ?config(source_nodes, Config), TargetNodes = ?config(target_nodes, Config), ok = emqx_cth_cluster:stop(SourceNodes), @@ -99,7 +120,20 @@ end_per_testcase(t_status, Config) -> #{work_dir => emqx_cth_suite:work_dir(Config)} ), ok; +end_per_group(_Group, _Config) -> + ok. + +auth_header() -> + emqx_mgmt_api_test_util:auth_header_(). + +init_per_testcase(_TC, Config) -> + {ok, _} = emqx_cluster_link_config:update([]), + snabbkaffe:start_trace(), + Config. + end_per_testcase(_TC, _Config) -> + snabbkaffe:stop(), + emqx_common_test_helpers:call_janitor(), ok. %%------------------------------------------------------------------------------ @@ -114,7 +148,11 @@ list() -> emqx_mgmt_api_test_util:simple_request(get, Path, _Params = ""). get_link(Name) -> - Path = emqx_mgmt_api_test_util:api_path([api_root(), Name]), + get_link(source, Name). + +get_link(SourceOrTargetCluster, Name) -> + Host = host(SourceOrTargetCluster), + Path = emqx_mgmt_api_test_util:api_path(Host, [api_root(), Name]), emqx_mgmt_api_test_util:simple_request(get, Path, _Params = ""). delete_link(Name) -> @@ -122,7 +160,11 @@ delete_link(Name) -> emqx_mgmt_api_test_util:simple_request(delete, Path, _Params = ""). update_link(Name, Params) -> - Path = emqx_mgmt_api_test_util:api_path([api_root(), Name]), + update_link(source, Name, Params). + +update_link(SourceOrTargetCluster, Name, Params) -> + Host = host(SourceOrTargetCluster), + Path = emqx_mgmt_api_test_util:api_path(Host, [api_root(), Name]), emqx_mgmt_api_test_util:simple_request(put, Path, Params). create_link(Name, Params0) -> @@ -130,6 +172,17 @@ create_link(Name, Params0) -> Path = emqx_mgmt_api_test_util:api_path([api_root()]), emqx_mgmt_api_test_util:simple_request(post, Path, Params). +get_metrics(Name) -> + get_metrics(source, Name). + +get_metrics(SourceOrTargetCluster, Name) -> + Host = host(SourceOrTargetCluster), + Path = emqx_mgmt_api_test_util:api_path(Host, [api_root(), Name, "metrics"]), + emqx_mgmt_api_test_util:simple_request(get, Path, _Params = []). + +host(source) -> "http://127.0.0.1:18083"; +host(target) -> "http://127.0.0.1:28083". + link_params() -> link_params(_Overrides = #{}). @@ -194,6 +247,7 @@ t_crud(_Config) -> ?assertMatch({404, _}, get_link(NameA)), ?assertMatch({404, _}, delete_link(NameA)), ?assertMatch({404, _}, update_link(NameA, link_params())), + ?assertMatch({404, _}, get_metrics(NameA)), Params1 = link_params(), ?assertMatch( @@ -223,6 +277,7 @@ t_crud(_Config) -> }}, get_link(NameA) ), + ?assertMatch({200, _}, get_metrics(NameA)), Params2 = Params1#{<<"pool_size">> := 2}, ?assertMatch( @@ -238,6 +293,7 @@ t_crud(_Config) -> ?assertMatch({404, _}, delete_link(NameA)), ?assertMatch({404, _}, get_link(NameA)), ?assertMatch({404, _}, update_link(NameA, Params1)), + ?assertMatch({404, _}, get_metrics(NameA)), ?assertMatch({200, []}, list()), ok. @@ -298,6 +354,7 @@ t_status(Config) -> [Res1, {ok, {ok, Res2B}} | Rest] end) end), + on_exit(fun() -> catch ?ON(SN1, meck:unload()) end), ?assertMatch( {200, [ #{ @@ -385,3 +442,183 @@ t_status(Config) -> ), ok. + +t_metrics(Config) -> + ct:timetrap({seconds, 10}), + [SN1, SN2] = ?config(source_nodes, Config), + [TN1, TN2] = ?config(target_nodes, Config), + %% N.B. Link names on each cluster, so they are switched. + SourceName = <<"cl.target">>, + TargetName = <<"cl.source">>, + + ?assertMatch( + {200, #{ + <<"metrics">> := #{<<"routes">> := 0}, + <<"node_metrics">> := [ + #{ + <<"node">> := _, + <<"metrics">> := #{<<"routes">> := 0} + }, + #{ + <<"node">> := _, + <<"metrics">> := #{<<"routes">> := 0} + } + ] + }}, + get_metrics(source, SourceName) + ), + ?assertMatch( + {200, #{ + <<"metrics">> := #{<<"routes">> := 0}, + <<"node_metrics">> := [ + #{ + <<"node">> := _, + <<"metrics">> := #{<<"routes">> := 0} + }, + #{ + <<"node">> := _, + <<"metrics">> := #{<<"routes">> := 0} + } + ] + }}, + get_metrics(target, TargetName) + ), + + SourceC1 = emqx_cluster_link_SUITE:start_client(<<"sc1">>, SN1), + SourceC2 = emqx_cluster_link_SUITE:start_client(<<"sc2">>, SN2), + {ok, _, _} = emqtt:subscribe(SourceC1, <<"t/sc1">>), + {ok, _, _} = emqtt:subscribe(SourceC2, <<"t/sc2">>), + + %% Still no routes, as routes in the source cluster are replicated to the target + %% cluster. + ?assertMatch( + {200, #{ + <<"metrics">> := #{<<"routes">> := 0}, + <<"node_metrics">> := [ + #{ + <<"node">> := _, + <<"metrics">> := #{<<"routes">> := 0} + }, + #{ + <<"node">> := _, + <<"metrics">> := #{<<"routes">> := 0} + } + ] + }}, + get_metrics(source, SourceName) + ), + ?assertMatch( + {200, #{ + <<"metrics">> := #{<<"routes">> := 0}, + <<"node_metrics">> := [ + #{ + <<"node">> := _, + <<"metrics">> := #{<<"routes">> := 0} + }, + #{ + <<"node">> := _, + <<"metrics">> := #{<<"routes">> := 0} + } + ] + }}, + get_metrics(target, TargetName) + ), + + TargetC1 = emqx_cluster_link_SUITE:start_client(<<"tc1">>, TN1), + TargetC2 = emqx_cluster_link_SUITE:start_client(<<"tc2">>, TN2), + {ok, _, _} = emqtt:subscribe(TargetC1, <<"t/tc1">>), + {ok, _, _} = emqtt:subscribe(TargetC2, <<"t/tc2">>), + {_, {ok, _}} = + ?wait_async_action( + begin + {ok, _, _} = emqtt:subscribe(TargetC1, <<"t/tc1">>), + {ok, _, _} = emqtt:subscribe(TargetC2, <<"t/tc2">>) + end, + #{?snk_kind := clink_route_sync_complete} + ), + + %% Routes = 2 in source cluster, because the target cluster has some topic filters + %% configured and subscribers to them, which were replicated to the source cluster. + ?assertMatch( + {200, #{ + <<"metrics">> := #{<<"routes">> := 2}, + <<"node_metrics">> := _ + }}, + get_metrics(source, SourceName) + ), + ?assertMatch( + {200, #{ + <<"metrics">> := #{<<"routes">> := 0}, + <<"node_metrics">> := _ + }}, + get_metrics(target, TargetName) + ), + + %% Unsubscribe and remove route. + ct:pal("unsubscribing"), + {_, {ok, _}} = + ?wait_async_action( + begin + {ok, _, _} = emqtt:unsubscribe(TargetC1, <<"t/tc1">>) + end, + #{?snk_kind := clink_route_sync_complete} + ), + + ?assertMatch( + {200, #{ + <<"metrics">> := #{<<"routes">> := 1}, + <<"node_metrics">> := _ + }}, + get_metrics(source, SourceName) + ), + + %% Disabling the link should remove the routes. + ct:pal("disabling"), + {200, TargetLink0} = get_link(target, TargetName), + TargetLink1 = maps:without([<<"status">>, <<"node_status">>], TargetLink0), + TargetLink2 = TargetLink1#{<<"enable">> := false}, + {_, {ok, _}} = + ?wait_async_action( + begin + {200, _} = update_link(target, TargetName, TargetLink2), + %% Note that only when the GC runs and collects the stopped actor it'll actually + %% remove the routes + NowMS = erlang:system_time(millisecond), + TTL = emqx_cluster_link_config:actor_ttl(), + ct:pal("gc"), + %% 2 Actors: one for normal routes, one for PS routes + 1 = ?ON(SN1, emqx_cluster_link_extrouter:actor_gc(#{timestamp => NowMS + TTL * 3})), + 1 = ?ON(SN1, emqx_cluster_link_extrouter:actor_gc(#{timestamp => NowMS + TTL * 3})), + ct:pal("gc done"), + ok + end, + #{?snk_kind := "cluster_link_extrouter_route_deleted"} + ), + + ?assertMatch( + {200, #{ + <<"metrics">> := #{<<"routes">> := 0}, + <<"node_metrics">> := _ + }}, + get_metrics(source, SourceName) + ), + + %% Enabling again + TargetLink3 = TargetLink2#{<<"enable">> := true}, + {_, {ok, _}} = + ?wait_async_action( + begin + {200, _} = update_link(target, TargetName, TargetLink3) + end, + #{?snk_kind := "cluster_link_extrouter_route_added"} + ), + + ?assertMatch( + {200, #{ + <<"metrics">> := #{<<"routes">> := 1}, + <<"node_metrics">> := _ + }}, + get_metrics(source, SourceName) + ), + + ok. From d9832252d8f1cafe30796fe555a4da088421359a Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 22 Jul 2024 09:55:16 -0300 Subject: [PATCH 011/131] refactor: add namespace to avoid clashes with operations or other resources --- .../src/emqx_cluster_link_api.erl | 24 +++++++++---------- .../test/emqx_cluster_link_api_SUITE.erl | 8 +++---- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/apps/emqx_cluster_link/src/emqx_cluster_link_api.erl b/apps/emqx_cluster_link/src/emqx_cluster_link_api.erl index 77a77610b..7e70a9ccc 100644 --- a/apps/emqx_cluster_link/src/emqx_cluster_link_api.erl +++ b/apps/emqx_cluster_link/src/emqx_cluster_link_api.erl @@ -22,8 +22,8 @@ -export([ '/cluster/links'/2, - '/cluster/links/:name'/2, - '/cluster/links/:name/metrics'/2 + '/cluster/links/link/:name'/2, + '/cluster/links/link/:name/metrics'/2 ]). -define(CONF_PATH, [cluster, links]). @@ -39,8 +39,8 @@ api_spec() -> paths() -> [ "/cluster/links", - "/cluster/links/:name", - "/cluster/links/:name/metrics" + "/cluster/links/link/:name", + "/cluster/links/link/:name/metrics" ]. schema("/cluster/links") -> @@ -69,9 +69,9 @@ schema("/cluster/links") -> } } }; -schema("/cluster/links/:name") -> +schema("/cluster/links/link/:name") -> #{ - 'operationId' => '/cluster/links/:name', + 'operationId' => '/cluster/links/link/:name', get => #{ description => "Get a cluster link configuration", @@ -117,9 +117,9 @@ schema("/cluster/links/:name") -> } } }; -schema("/cluster/links/:name/metrics") -> +schema("/cluster/links/link/:name/metrics") -> #{ - 'operationId' => '/cluster/links/:name/metrics', + 'operationId' => '/cluster/links/link/:name/metrics', get => #{ description => "Get a cluster link metrics", @@ -173,11 +173,11 @@ fields(node_metrics) -> fun() -> handle_create(Name, Body) end ). -'/cluster/links/:name'(get, #{bindings := #{name := Name}}) -> +'/cluster/links/link/:name'(get, #{bindings := #{name := Name}}) -> with_link(Name, fun(Link) -> handle_lookup(Name, Link) end, not_found()); -'/cluster/links/:name'(put, #{bindings := #{name := Name}, body := Params0}) -> +'/cluster/links/link/:name'(put, #{bindings := #{name := Name}, body := Params0}) -> with_link(Name, fun() -> handle_update(Name, Params0) end, not_found()); -'/cluster/links/:name'(delete, #{bindings := #{name := Name}}) -> +'/cluster/links/link/:name'(delete, #{bindings := #{name := Name}}) -> with_link( Name, fun() -> @@ -192,7 +192,7 @@ fields(node_metrics) -> not_found() ). -'/cluster/links/:name/metrics'(get, #{bindings := #{name := Name}}) -> +'/cluster/links/link/:name/metrics'(get, #{bindings := #{name := Name}}) -> with_link(Name, fun() -> handle_metrics(Name) end, not_found()). %%-------------------------------------------------------------------- diff --git a/apps/emqx_cluster_link/test/emqx_cluster_link_api_SUITE.erl b/apps/emqx_cluster_link/test/emqx_cluster_link_api_SUITE.erl index f4a15a62a..27cbe7532 100644 --- a/apps/emqx_cluster_link/test/emqx_cluster_link_api_SUITE.erl +++ b/apps/emqx_cluster_link/test/emqx_cluster_link_api_SUITE.erl @@ -152,11 +152,11 @@ get_link(Name) -> get_link(SourceOrTargetCluster, Name) -> Host = host(SourceOrTargetCluster), - Path = emqx_mgmt_api_test_util:api_path(Host, [api_root(), Name]), + Path = emqx_mgmt_api_test_util:api_path(Host, [api_root(), "link", Name]), emqx_mgmt_api_test_util:simple_request(get, Path, _Params = ""). delete_link(Name) -> - Path = emqx_mgmt_api_test_util:api_path([api_root(), Name]), + Path = emqx_mgmt_api_test_util:api_path([api_root(), "link", Name]), emqx_mgmt_api_test_util:simple_request(delete, Path, _Params = ""). update_link(Name, Params) -> @@ -164,7 +164,7 @@ update_link(Name, Params) -> update_link(SourceOrTargetCluster, Name, Params) -> Host = host(SourceOrTargetCluster), - Path = emqx_mgmt_api_test_util:api_path(Host, [api_root(), Name]), + Path = emqx_mgmt_api_test_util:api_path(Host, [api_root(), "link", Name]), emqx_mgmt_api_test_util:simple_request(put, Path, Params). create_link(Name, Params0) -> @@ -177,7 +177,7 @@ get_metrics(Name) -> get_metrics(SourceOrTargetCluster, Name) -> Host = host(SourceOrTargetCluster), - Path = emqx_mgmt_api_test_util:api_path(Host, [api_root(), Name, "metrics"]), + Path = emqx_mgmt_api_test_util:api_path(Host, [api_root(), "link", Name, "metrics"]), emqx_mgmt_api_test_util:simple_request(get, Path, _Params = []). host(source) -> "http://127.0.0.1:18083"; From cba3f532f894fdf3e1a4631e67e36cf60cb2151e Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Thu, 11 Jul 2024 14:16:18 +0800 Subject: [PATCH 012/131] feat: don't record dry_run log --- .../src/emqx_bridge_kafka_impl_consumer.erl | 12 +- .../src/emqx_bridge_kafka_impl_producer.erl | 9 +- .../src/emqx_bridge_pulsar.app.src | 2 +- .../src/emqx_bridge_pulsar_connector.erl | 10 +- apps/emqx_resource/src/emqx_resource.erl | 9 ++ .../src/emqx_resource_manager.erl | 103 +++++++++++------- 6 files changed, 76 insertions(+), 69 deletions(-) diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_consumer.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_consumer.erl index c4f66dfff..35ffbc90b 100644 --- a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_consumer.erl +++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_consumer.erl @@ -628,16 +628,6 @@ consumer_group_id(BridgeName0) -> BridgeName = to_bin(BridgeName0), <<"emqx-kafka-consumer-", BridgeName/binary>>. --spec is_dry_run(connector_resource_id()) -> boolean(). -is_dry_run(ConnectorResId) -> - TestIdStart = string:find(ConnectorResId, ?TEST_ID_PREFIX), - case TestIdStart of - nomatch -> - false; - _ -> - string:equal(TestIdStart, ConnectorResId) - end. - -spec check_client_connectivity(pid()) -> ?status_connected | ?status_disconnected @@ -673,7 +663,7 @@ maybe_clean_error(Reason) -> -spec make_client_id(connector_resource_id(), binary(), atom() | binary()) -> atom(). make_client_id(ConnectorResId, BridgeType, BridgeName) -> - case is_dry_run(ConnectorResId) of + case emqx_resource:is_dry_run(ConnectorResId) of false -> ClientID0 = emqx_bridge_kafka_impl:make_client_id(BridgeType, BridgeName), binary_to_atom(ClientID0); diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl index b819925ac..8b6326545 100644 --- a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl +++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl @@ -137,14 +137,7 @@ create_producers_for_bridge_v2( KafkaHeadersValEncodeMode = maps:get(kafka_header_value_encode_mode, KafkaConfig, none), MaxPartitions = maps:get(partitions_limit, KafkaConfig, all_partitions), #{name := BridgeName} = emqx_bridge_v2:parse_id(BridgeV2Id), - TestIdStart = string:find(BridgeV2Id, ?TEST_ID_PREFIX), - IsDryRun = - case TestIdStart of - nomatch -> - false; - _ -> - string:equal(TestIdStart, InstId) - end, + IsDryRun = emqx_resource:is_dry_run(BridgeV2Id), ok = check_topic_and_leader_connections(ClientId, KafkaTopic, MaxPartitions), WolffProducerConfig = producers_config( BridgeType, BridgeName, KafkaConfig, IsDryRun, BridgeV2Id diff --git a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.app.src b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.app.src index a8eeba483..dcb86a3ca 100644 --- a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.app.src +++ b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_pulsar, [ {description, "EMQX Pulsar Bridge"}, - {vsn, "0.2.2"}, + {vsn, "0.2.3"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_connector.erl b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_connector.erl index 9d269493d..4157deec2 100644 --- a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_connector.erl +++ b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_connector.erl @@ -255,7 +255,7 @@ format_servers(Servers0) -> -spec make_client_id(resource_id()) -> pulsar_client_id(). make_client_id(InstanceId) -> - case is_dry_run(InstanceId) of + case emqx_resource:is_dry_run(InstanceId) of true -> pulsar_producer_probe; false -> @@ -269,14 +269,6 @@ make_client_id(InstanceId) -> binary_to_atom(ClientIdBin) end. --spec is_dry_run(resource_id()) -> boolean(). -is_dry_run(InstanceId) -> - TestIdStart = string:find(InstanceId, ?TEST_ID_PREFIX), - case TestIdStart of - nomatch -> false; - _ -> string:equal(TestIdStart, InstanceId) - end. - conn_opts(#{authentication := none}) -> #{}; conn_opts(#{authentication := #{username := Username, password := Password}}) -> diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl index 15c0a0293..b6f01fde5 100644 --- a/apps/emqx_resource/src/emqx_resource.erl +++ b/apps/emqx_resource/src/emqx_resource.erl @@ -140,6 +140,8 @@ validate_name/1 ]). +-export([is_dry_run/1]). + -export_type([ query_mode/0, resource_id/0, @@ -769,6 +771,13 @@ validate_name(Name) -> _ = validate_name(Name, #{atom_name => false}), ok. +-spec is_dry_run(resource_id()) -> boolean(). +is_dry_run(ResId) -> + case string:find(ResId, ?TEST_ID_PREFIX) of + nomatch -> false; + TestIdStart -> string:equal(TestIdStart, ResId) + end. + validate_name(<<>>, _Opts) -> invalid_data("Name cannot be empty string"); validate_name(Name, _Opts) when size(Name) >= 255 -> diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl index 50a25620c..95b1271f4 100644 --- a/apps/emqx_resource/src/emqx_resource_manager.erl +++ b/apps/emqx_resource/src/emqx_resource_manager.erl @@ -752,7 +752,8 @@ handle_remove_event(From, ClearMetrics, Data) -> start_resource(Data, From) -> %% in case the emqx_resource:call_start/2 hangs, the lookup/1 can read status from the cache - case emqx_resource:call_start(Data#data.id, Data#data.mod, Data#data.config) of + ResId = Data#data.id, + case emqx_resource:call_start(ResId, Data#data.mod, Data#data.config) of {ok, ResourceState} -> UpdatedData1 = Data#data{status = ?status_connecting, state = ResourceState}, %% Perform an initial health_check immediately before transitioning into a connected state @@ -760,12 +761,13 @@ start_resource(Data, From) -> Actions = maybe_reply([{state_timeout, 0, health_check}], From, ok), {next_state, ?state_connecting, update_state(UpdatedData2, Data), Actions}; {error, Reason} = Err -> - ?SLOG(warning, #{ + IsDryRun = emqx_resource:is_dry_run(ResId), + ?SLOG(log_level(IsDryRun), #{ msg => "start_resource_failed", - id => Data#data.id, + id => ResId, reason => Reason }), - _ = maybe_alarm(?status_disconnected, Data#data.id, Err, Data#data.error), + _ = maybe_alarm(?status_disconnected, IsDryRun, ResId, Err, Data#data.error), %% Add channels and raise alarms NewData1 = channels_health_check(?status_disconnected, add_channels(Data)), %% Keep track of the error reason why the connection did not work @@ -796,9 +798,10 @@ add_channels(Data) -> add_channels_in_list([], Data) -> Data; add_channels_in_list([{ChannelID, ChannelConfig} | Rest], Data) -> + Id = Data#data.id, case emqx_resource:call_add_channel( - Data#data.id, Data#data.mod, Data#data.state, ChannelID, ChannelConfig + Id, Data#data.mod, Data#data.state, ChannelID, ChannelConfig ) of {ok, NewState} -> @@ -816,9 +819,10 @@ add_channels_in_list([{ChannelID, ChannelConfig} | Rest], Data) -> }, add_channels_in_list(Rest, NewData); {error, Reason} = Error -> - ?SLOG(warning, #{ - msg => add_channel_failed, - id => Data#data.id, + IsDryRun = emqx_resource:is_dry_run(Id), + ?SLOG(log_level(IsDryRun), #{ + msg => "add_channel_failed", + id => Id, channel_id => ChannelID, reason => Reason }), @@ -832,7 +836,7 @@ add_channels_in_list([{ChannelID, ChannelConfig} | Rest], Data) -> added_channels = NewAddedChannelsMap }, %% Raise an alarm since the channel could not be added - _ = maybe_alarm(?status_disconnected, ChannelID, Error, no_prev_error), + _ = maybe_alarm(?status_disconnected, IsDryRun, ChannelID, Error, no_prev_error), add_channels_in_list(Rest, NewData) end. @@ -856,7 +860,8 @@ stop_resource(#data{id = ResId} = Data) -> false -> ok end, - _ = maybe_clear_alarm(ResId), + IsDryRun = emqx_resource:is_dry_run(ResId), + _ = maybe_clear_alarm(IsDryRun, ResId), ok = emqx_metrics_worker:reset_metrics(?RES_METRICS, ResId), NewData#data{status = ?rm_status_stopped}. @@ -868,15 +873,17 @@ remove_channels_in_list([], Data, _KeepInChannelMap) -> Data; remove_channels_in_list([ChannelID | Rest], Data, KeepInChannelMap) -> AddedChannelsMap = Data#data.added_channels, + Id = Data#data.id, + IsDryRun = emqx_resource:is_dry_run(Id), NewAddedChannelsMap = case KeepInChannelMap of true -> AddedChannelsMap; false -> - _ = maybe_clear_alarm(ChannelID), + _ = maybe_clear_alarm(IsDryRun, ChannelID), maps:remove(ChannelID, AddedChannelsMap) end, - case safe_call_remove_channel(Data#data.id, Data#data.mod, Data#data.state, ChannelID) of + case safe_call_remove_channel(Id, Data#data.mod, Data#data.state, ChannelID) of {ok, NewState} -> NewData = Data#data{ state = NewState, @@ -884,9 +891,9 @@ remove_channels_in_list([ChannelID | Rest], Data, KeepInChannelMap) -> }, remove_channels_in_list(Rest, NewData, KeepInChannelMap); {error, Reason} -> - ?SLOG(warning, #{ - msg => remove_channel_failed, - id => Data#data.id, + ?SLOG(log_level(IsDryRun), #{ + msg => "remove_channel_failed", + id => Id, channel_id => ChannelID, reason => Reason }), @@ -968,8 +975,8 @@ handle_not_connected_add_channel(From, ChannelId, ChannelConfig, State, Data) -> handle_remove_channel(From, ChannelId, Data) -> Channels = Data#data.added_channels, - %% Deactivate alarm - _ = maybe_clear_alarm(ChannelId), + IsDryRun = emqx_resource:is_dry_run(Data#data.id), + _ = maybe_clear_alarm(IsDryRun, ChannelId), case channel_status_is_channel_added( maps:get(ChannelId, Channels, channel_status_not_added(undefined)) @@ -990,13 +997,13 @@ handle_remove_channel(From, ChannelId, Data) -> end. handle_remove_channel_exists(From, ChannelId, Data) -> + #data{id = Id, added_channels = AddedChannelsMap} = Data, case emqx_resource:call_remove_channel( - Data#data.id, Data#data.mod, Data#data.state, ChannelId + Id, Data#data.mod, Data#data.state, ChannelId ) of {ok, NewState} -> - AddedChannelsMap = Data#data.added_channels, NewAddedChannelsMap = maps:remove(ChannelId, AddedChannelsMap), UpdatedData = Data#data{ state = NewState, @@ -1004,10 +1011,10 @@ handle_remove_channel_exists(From, ChannelId, Data) -> }, {keep_state, update_state(UpdatedData, Data), [{reply, From, ok}]}; {error, Reason} = Error -> - %% Log the error as a warning - ?SLOG(warning, #{ - msg => remove_channel_failed, - id => Data#data.id, + IsDryRun = emqx_resource:is_dry_run(Id), + ?SLOG(log_level(IsDryRun), #{ + msg => "remove_channel_failed", + id => Id, channel_id => ChannelId, reason => Reason }), @@ -1021,7 +1028,8 @@ handle_not_connected_and_not_connecting_remove_channel(From, ChannelId, Data) -> Channels = Data#data.added_channels, NewChannels = maps:remove(ChannelId, Channels), NewData = Data#data{added_channels = NewChannels}, - _ = maybe_clear_alarm(ChannelId), + IsDryRun = emqx_resource:is_dry_run(Data#data.id), + _ = maybe_clear_alarm(IsDryRun, ChannelId), {keep_state, update_state(NewData, Data), [{reply, From, ok}]}. handle_manual_resource_health_check(From, Data0 = #data{hc_workers = #{resource := HCWorkers}}) when @@ -1090,7 +1098,8 @@ continue_with_health_check(#data{} = Data0, CurrentState, HCRes) -> error = PrevError } = Data0, {NewStatus, NewState, Err} = parse_health_check_result(HCRes, Data0), - _ = maybe_alarm(NewStatus, ResId, Err, PrevError), + IsDryRun = emqx_resource:is_dry_run(ResId), + _ = maybe_alarm(NewStatus, IsDryRun, ResId, Err, PrevError), ok = maybe_resume_resource_workers(ResId, NewStatus), Data1 = Data0#data{ state = NewState, status = NewStatus, error = Err @@ -1114,7 +1123,8 @@ continue_resource_health_check_connected(NewStatus, Data0) -> Actions = Replies ++ resource_health_check_actions(Data), {keep_state, Data, Actions}; _ -> - ?SLOG(warning, #{ + IsDryRun = emqx_resource:is_dry_run(Data0#data.id), + ?SLOG(log_level(IsDryRun), #{ msg => "health_check_failed", id => Data0#data.id, status => NewStatus @@ -1214,7 +1224,7 @@ channels_health_check(?status_connected = _ConnectorStatus, Data0) -> channels_health_check(?status_connecting = _ConnectorStatus, Data0) -> %% Whenever the resource is connecting: %% 1. Change the status of all added channels to connecting - %% 2. Raise alarms (TODO: if it is a probe we should not raise alarms) + %% 2. Raise alarms Channels = Data0#data.added_channels, ChannelsToChangeStatusFor = [ {ChannelId, Config} @@ -1240,9 +1250,10 @@ channels_health_check(?status_connecting = _ConnectorStatus, Data0) -> || {ChannelId, NewStatus} <- maps:to_list(NewChannels) ], %% Raise alarms for all channels + IsDryRun = emqx_resource:is_dry_run(Data0#data.id), lists:foreach( fun({ChannelId, Status, PrevStatus}) -> - maybe_alarm(?status_connecting, ChannelId, Status, PrevStatus) + maybe_alarm(?status_connecting, IsDryRun, ChannelId, Status, PrevStatus) end, ChannelsWithNewAndPrevErrorStatuses ), @@ -1275,9 +1286,10 @@ channels_health_check(ConnectorStatus, Data0) -> || {ChannelId, #{config := Config} = OldStatus} <- maps:to_list(Data1#data.added_channels) ], %% Raise alarms + IsDryRun = emqx_resource:is_dry_run(Data1#data.id), _ = lists:foreach( fun({ChannelId, OldStatus, NewStatus}) -> - _ = maybe_alarm(NewStatus, ChannelId, NewStatus, OldStatus) + _ = maybe_alarm(NewStatus, IsDryRun, ChannelId, NewStatus, OldStatus) end, ChannelsWithNewAndOldStatuses ), @@ -1386,13 +1398,14 @@ continue_channel_health_check_connected_no_update_during_check(ChannelId, OldSta NewStatus = maps:get(ChannelId, Data1#data.added_channels), ChannelsToRemove = [ChannelId || not channel_status_is_channel_added(NewStatus)], Data = remove_channels_in_list(ChannelsToRemove, Data1, true), + IsDryRun = emqx_resource:is_dry_run(Data1#data.id), %% Raise/clear alarms case NewStatus of #{status := ?status_connected} -> - _ = maybe_clear_alarm(ChannelId), + _ = maybe_clear_alarm(IsDryRun, ChannelId), ok; _ -> - _ = maybe_alarm(NewStatus, ChannelId, NewStatus, OldStatus), + _ = maybe_alarm(NewStatus, IsDryRun, ChannelId, NewStatus, OldStatus), ok end, Data. @@ -1556,15 +1569,21 @@ remove_runtime_data(#data{} = Data0) -> health_check_interval(Opts) -> maps:get(health_check_interval, Opts, ?HEALTHCHECK_INTERVAL). --spec maybe_alarm(resource_status(), resource_id(), _Error :: term(), _PrevError :: term()) -> ok. -maybe_alarm(?status_connected, _ResId, _Error, _PrevError) -> +-spec maybe_alarm( + resource_status(), + boolean(), + resource_id(), + _Error :: term(), + _PrevError :: term() +) -> ok. +maybe_alarm(?status_connected, _IsDryRun, _ResId, _Error, _PrevError) -> ok; -maybe_alarm(_Status, <>, _Error, _PrevError) -> +maybe_alarm(_Status, true, _ResId, _Error, _PrevError) -> ok; %% Assume that alarm is already active -maybe_alarm(_Status, _ResId, Error, Error) -> +maybe_alarm(_Status, _IsDryRun, _ResId, Error, Error) -> ok; -maybe_alarm(_Status, ResId, Error, _PrevError) -> +maybe_alarm(_Status, false, ResId, Error, _PrevError) -> HrError = case Error of {error, undefined} -> @@ -1596,10 +1615,10 @@ maybe_resume_resource_workers(ResId, ?status_connected) -> maybe_resume_resource_workers(_, _) -> ok. --spec maybe_clear_alarm(resource_id()) -> ok | {error, not_found}. -maybe_clear_alarm(<>) -> +-spec maybe_clear_alarm(boolean(), resource_id()) -> ok | {error, not_found}. +maybe_clear_alarm(true, _ResId) -> ok; -maybe_clear_alarm(ResId) -> +maybe_clear_alarm(false, ResId) -> emqx_alarm:safe_deactivate(ResId). parse_health_check_result(Status, Data) when ?IS_STATUS(Status) -> @@ -1767,10 +1786,14 @@ add_or_update_channel_status(Data, ChannelId, ChannelConfig, State) -> ChannelStatus = channel_status({error, resource_not_operational}, ChannelConfig), NewChannels = maps:put(ChannelId, ChannelStatus, Channels), ResStatus = state_to_status(State), - maybe_alarm(ResStatus, ChannelId, ChannelStatus, no_prev), + IsDryRun = emqx_resource:is_dry_run(ChannelId), + maybe_alarm(ResStatus, IsDryRun, ChannelId, ChannelStatus, no_prev), Data#data{added_channels = NewChannels}. state_to_status(?state_stopped) -> ?rm_status_stopped; state_to_status(?state_connected) -> ?status_connected; state_to_status(?state_connecting) -> ?status_connecting; state_to_status(?state_disconnected) -> ?status_disconnected. + +log_level(true) -> info; +log_level(false) -> warning. From 0a04b1ad6e73dc0ae29e1b6c15ad3ff23edd0b2e Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Thu, 11 Jul 2024 17:09:10 +0800 Subject: [PATCH 013/131] feat: add group/type to resource slog --- apps/emqx_auth/include/emqx_authn.hrl | 2 +- apps/emqx_auth/include/emqx_authz.hrl | 2 +- .../src/emqx_authn/emqx_authn_utils.erl | 11 +- .../src/emqx_authz/emqx_authz_utils.erl | 14 ++- apps/emqx_auth_http/src/emqx_authn_http.erl | 5 +- apps/emqx_auth_http/src/emqx_authz_http.erl | 6 +- apps/emqx_auth_jwt/src/emqx_authn_jwt.erl | 1 + .../emqx_auth_ldap/src/emqx_auth_ldap.app.src | 2 +- apps/emqx_auth_ldap/src/emqx_authn_ldap.erl | 4 +- apps/emqx_auth_ldap/src/emqx_authz_ldap.erl | 4 +- .../test/emqx_authn_ldap_SUITE.erl | 1 + .../test/emqx_authn_ldap_bind_SUITE.erl | 1 + .../test/emqx_authz_ldap_SUITE.erl | 1 + .../src/emqx_auth_mongodb.app.src | 2 +- .../src/emqx_authn_mongodb.erl | 3 +- .../src/emqx_authz_mongodb.erl | 4 +- apps/emqx_auth_mysql/src/emqx_authn_mysql.erl | 4 +- apps/emqx_auth_mysql/src/emqx_authz_mysql.erl | 4 +- .../test/emqx_authn_mysql_SUITE.erl | 1 + .../test/emqx_authz_mysql_SUITE.erl | 1 + .../src/emqx_authn_postgresql.erl | 3 +- .../src/emqx_authz_postgresql.erl | 2 + .../test/emqx_authn_postgresql_SUITE.erl | 5 +- .../test/emqx_authz_postgresql_SUITE.erl | 1 + apps/emqx_auth_redis/src/emqx_authn_redis.erl | 3 +- apps/emqx_auth_redis/src/emqx_authz_redis.erl | 4 +- .../test/emqx_authn_redis_SUITE.erl | 1 + .../test/emqx_authz_redis_SUITE.erl | 1 + apps/emqx_bridge/src/emqx_bridge_resource.erl | 6 +- .../emqx_bridge_cassandra_connector_SUITE.erl | 1 + ...emqx_bridge_clickhouse_connector_SUITE.erl | 2 + ...emqx_bridge_greptimedb_connector_SUITE.erl | 1 + .../emqx_bridge_influxdb_connector_SUITE.erl | 2 + .../src/emqx_bridge_pulsar_connector.erl | 2 +- .../emqx_bridge_rabbitmq_connector_SUITE.erl | 1 + .../emqx_connector/include/emqx_connector.hrl | 2 +- .../src/emqx_connector_resource.erl | 7 +- .../src/emqx_dashboard_sso_manager.erl | 5 +- .../test/emqx_dashboard_sso_ldap_SUITE.erl | 2 +- apps/emqx_ldap/test/emqx_ldap_SUITE.erl | 2 + apps/emqx_mongodb/test/emqx_mongodb_SUITE.erl | 1 + apps/emqx_mysql/test/emqx_mysql_SUITE.erl | 1 + .../test/emqx_postgresql_SUITE.erl | 1 + apps/emqx_redis/test/emqx_redis_SUITE.erl | 1 + apps/emqx_resource/include/emqx_resource.hrl | 1 + apps/emqx_resource/src/emqx_resource.erl | 70 ++++++----- .../src/emqx_resource_manager.erl | 118 ++++++++++++------ .../src/emqx_resource_manager_sup.erl | 15 ++- .../test/emqx_resource_SUITE.erl | 20 ++- 49 files changed, 232 insertions(+), 122 deletions(-) diff --git a/apps/emqx_auth/include/emqx_authn.hrl b/apps/emqx_auth/include/emqx_authn.hrl index 1f2c6b8b9..8ffab1203 100644 --- a/apps/emqx_auth/include/emqx_authn.hrl +++ b/apps/emqx_auth/include/emqx_authn.hrl @@ -30,6 +30,6 @@ -type authenticator_id() :: binary(). --define(AUTHN_RESOURCE_GROUP, <<"emqx_authn">>). +-define(AUTHN_RESOURCE_GROUP, <<"authn">>). -endif. diff --git a/apps/emqx_auth/include/emqx_authz.hrl b/apps/emqx_auth/include/emqx_authz.hrl index 6dc80cb3f..73ff44c25 100644 --- a/apps/emqx_auth/include/emqx_authz.hrl +++ b/apps/emqx_auth/include/emqx_authz.hrl @@ -158,7 +158,7 @@ count => 1 }). --define(AUTHZ_RESOURCE_GROUP, <<"emqx_authz">>). +-define(AUTHZ_RESOURCE_GROUP, <<"authz">>). -define(AUTHZ_FEATURES, [rich_actions]). diff --git a/apps/emqx_auth/src/emqx_authn/emqx_authn_utils.erl b/apps/emqx_auth/src/emqx_authn/emqx_authn_utils.erl index a08ac260c..e81145f2c 100644 --- a/apps/emqx_auth/src/emqx_authn/emqx_authn_utils.erl +++ b/apps/emqx_auth/src/emqx_authn/emqx_authn_utils.erl @@ -21,8 +21,8 @@ -include_lib("snabbkaffe/include/trace.hrl"). -export([ - create_resource/3, - update_resource/3, + create_resource/4, + update_resource/4, check_password_from_selected_map/3, parse_deep/1, parse_str/1, @@ -66,8 +66,9 @@ %% APIs %%-------------------------------------------------------------------- -create_resource(ResourceId, Module, Config) -> +create_resource(Type, ResourceId, Module, Config) -> Result = emqx_resource:create_local( + Type, ResourceId, ?AUTHN_RESOURCE_GROUP, Module, @@ -76,9 +77,9 @@ create_resource(ResourceId, Module, Config) -> ), start_resource_if_enabled(Result, ResourceId, Config). -update_resource(Module, Config, ResourceId) -> +update_resource(Type, Module, Config, ResourceId) -> Result = emqx_resource:recreate_local( - ResourceId, Module, Config, ?DEFAULT_RESOURCE_OPTS + Type, ResourceId, Module, Config, ?DEFAULT_RESOURCE_OPTS ), start_resource_if_enabled(Result, ResourceId, Config). diff --git a/apps/emqx_auth/src/emqx_authz/emqx_authz_utils.erl b/apps/emqx_auth/src/emqx_authz/emqx_authz_utils.erl index e4343b6fa..533f982c1 100644 --- a/apps/emqx_auth/src/emqx_authz/emqx_authz_utils.erl +++ b/apps/emqx_auth/src/emqx_authz/emqx_authz_utils.erl @@ -25,9 +25,9 @@ -export([ cleanup_resources/0, make_resource_id/1, - create_resource/2, create_resource/3, - update_resource/2, + create_resource/4, + update_resource/3, remove_resource/1, update_config/2, parse_deep/2, @@ -57,12 +57,13 @@ %% APIs %%-------------------------------------------------------------------- -create_resource(Module, Config) -> +create_resource(Type, Module, Config) -> ResourceId = make_resource_id(Module), - create_resource(ResourceId, Module, Config). + create_resource(Type, ResourceId, Module, Config). -create_resource(ResourceId, Module, Config) -> +create_resource(Type, ResourceId, Module, Config) -> Result = emqx_resource:create_local( + Type, ResourceId, ?AUTHZ_RESOURCE_GROUP, Module, @@ -71,10 +72,11 @@ create_resource(ResourceId, Module, Config) -> ), start_resource_if_enabled(Result, ResourceId, Config). -update_resource(Module, #{annotations := #{id := ResourceId}} = Config) -> +update_resource(Type, Module, #{annotations := #{id := ResourceId}} = Config) -> Result = case emqx_resource:recreate_local( + Type, ResourceId, Module, Config, diff --git a/apps/emqx_auth_http/src/emqx_authn_http.erl b/apps/emqx_auth_http/src/emqx_authn_http.erl index d9c5c5ed5..ed151428f 100644 --- a/apps/emqx_auth_http/src/emqx_authn_http.erl +++ b/apps/emqx_auth_http/src/emqx_authn_http.erl @@ -40,6 +40,7 @@ create(Config0) -> ResourceId = emqx_authn_utils:make_resource_id(?MODULE), % {Config, State} = parse_config(Config0), {ok, _Data} = emqx_authn_utils:create_resource( + http, ResourceId, emqx_bridge_http_connector, Config @@ -50,7 +51,9 @@ create(Config0) -> update(Config0, #{resource_id := ResourceId} = _State) -> with_validated_config(Config0, fun(Config, NState) -> % {Config, NState} = parse_config(Config0), - case emqx_authn_utils:update_resource(emqx_bridge_http_connector, Config, ResourceId) of + case + emqx_authn_utils:update_resource(http, emqx_bridge_http_connector, Config, ResourceId) + of {error, Reason} -> error({load_config_error, Reason}); {ok, _} -> diff --git a/apps/emqx_auth_http/src/emqx_authz_http.erl b/apps/emqx_auth_http/src/emqx_authz_http.erl index 6b0152b7d..2a1a2638b 100644 --- a/apps/emqx_auth_http/src/emqx_authz_http.erl +++ b/apps/emqx_auth_http/src/emqx_authz_http.erl @@ -66,12 +66,14 @@ description() -> create(Config) -> NConfig = parse_config(Config), ResourceId = emqx_authn_utils:make_resource_id(?MODULE), - {ok, _Data} = emqx_authz_utils:create_resource(ResourceId, emqx_bridge_http_connector, NConfig), + {ok, _Data} = emqx_authz_utils:create_resource( + http, ResourceId, emqx_bridge_http_connector, NConfig + ), NConfig#{annotations => #{id => ResourceId}}. update(Config) -> NConfig = parse_config(Config), - case emqx_authz_utils:update_resource(emqx_bridge_http_connector, NConfig) of + case emqx_authz_utils:update_resource(http, emqx_bridge_http_connector, NConfig) of {error, Reason} -> error({load_config_error, Reason}); {ok, Id} -> NConfig#{annotations => #{id => Id}} end. diff --git a/apps/emqx_auth_jwt/src/emqx_authn_jwt.erl b/apps/emqx_auth_jwt/src/emqx_authn_jwt.erl index 18c954ab5..e01121a01 100644 --- a/apps/emqx_auth_jwt/src/emqx_authn_jwt.erl +++ b/apps/emqx_auth_jwt/src/emqx_authn_jwt.erl @@ -183,6 +183,7 @@ do_create( ) -> ResourceId = emqx_authn_utils:make_resource_id(?MODULE), {ok, _Data} = emqx_resource:create_local( + jwt, ResourceId, ?AUTHN_RESOURCE_GROUP, emqx_authn_jwks_connector, diff --git a/apps/emqx_auth_ldap/src/emqx_auth_ldap.app.src b/apps/emqx_auth_ldap/src/emqx_auth_ldap.app.src index d84d6ff81..a58117356 100644 --- a/apps/emqx_auth_ldap/src/emqx_auth_ldap.app.src +++ b/apps/emqx_auth_ldap/src/emqx_auth_ldap.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_auth_ldap, [ {description, "EMQX LDAP Authentication and Authorization"}, - {vsn, "0.1.2"}, + {vsn, "0.1.3"}, {registered, []}, {mod, {emqx_auth_ldap_app, []}}, {applications, [ diff --git a/apps/emqx_auth_ldap/src/emqx_authn_ldap.erl b/apps/emqx_auth_ldap/src/emqx_authn_ldap.erl index a18236d15..8a9b5650a 100644 --- a/apps/emqx_auth_ldap/src/emqx_authn_ldap.erl +++ b/apps/emqx_auth_ldap/src/emqx_authn_ldap.erl @@ -40,12 +40,12 @@ create(_AuthenticatorID, Config) -> do_create(Module, Config) -> ResourceId = emqx_authn_utils:make_resource_id(Module), State = parse_config(Config), - {ok, _Data} = emqx_authn_utils:create_resource(ResourceId, emqx_ldap, Config), + {ok, _Data} = emqx_authn_utils:create_resource(ldap, ResourceId, emqx_ldap, Config), {ok, State#{resource_id => ResourceId}}. update(Config, #{resource_id := ResourceId} = _State) -> NState = parse_config(Config), - case emqx_authn_utils:update_resource(emqx_ldap, Config, ResourceId) of + case emqx_authn_utils:update_resource(ldap, emqx_ldap, Config, ResourceId) of {error, Reason} -> error({load_config_error, Reason}); {ok, _} -> diff --git a/apps/emqx_auth_ldap/src/emqx_authz_ldap.erl b/apps/emqx_auth_ldap/src/emqx_authz_ldap.erl index 24bd8c008..e70b840a3 100644 --- a/apps/emqx_auth_ldap/src/emqx_authz_ldap.erl +++ b/apps/emqx_auth_ldap/src/emqx_authz_ldap.erl @@ -56,12 +56,12 @@ description() -> create(Source) -> ResourceId = emqx_authz_utils:make_resource_id(?MODULE), - {ok, _Data} = emqx_authz_utils:create_resource(ResourceId, emqx_ldap, Source), + {ok, _Data} = emqx_authz_utils:create_resource(ldap, ResourceId, emqx_ldap, Source), Annotations = new_annotations(#{id => ResourceId}, Source), Source#{annotations => Annotations}. update(Source) -> - case emqx_authz_utils:update_resource(emqx_ldap, Source) of + case emqx_authz_utils:update_resource(ldap, emqx_ldap, Source) of {error, Reason} -> error({load_config_error, Reason}); {ok, Id} -> diff --git a/apps/emqx_auth_ldap/test/emqx_authn_ldap_SUITE.erl b/apps/emqx_auth_ldap/test/emqx_authn_ldap_SUITE.erl index ac941f268..f6ee582be 100644 --- a/apps/emqx_auth_ldap/test/emqx_authn_ldap_SUITE.erl +++ b/apps/emqx_auth_ldap/test/emqx_authn_ldap_SUITE.erl @@ -47,6 +47,7 @@ init_per_suite(Config) -> work_dir => ?config(priv_dir, Config) }), {ok, _} = emqx_resource:create_local( + ldap, ?LDAP_RESOURCE, ?AUTHN_RESOURCE_GROUP, emqx_ldap, diff --git a/apps/emqx_auth_ldap/test/emqx_authn_ldap_bind_SUITE.erl b/apps/emqx_auth_ldap/test/emqx_authn_ldap_bind_SUITE.erl index d8e9c0d8e..c39225e74 100644 --- a/apps/emqx_auth_ldap/test/emqx_authn_ldap_bind_SUITE.erl +++ b/apps/emqx_auth_ldap/test/emqx_authn_ldap_bind_SUITE.erl @@ -47,6 +47,7 @@ init_per_suite(Config) -> work_dir => ?config(priv_dir, Config) }), {ok, _} = emqx_resource:create_local( + ldap, ?LDAP_RESOURCE, ?AUTHN_RESOURCE_GROUP, emqx_ldap, diff --git a/apps/emqx_auth_ldap/test/emqx_authz_ldap_SUITE.erl b/apps/emqx_auth_ldap/test/emqx_authz_ldap_SUITE.erl index 09875a3fa..3f8c0ba63 100644 --- a/apps/emqx_auth_ldap/test/emqx_authz_ldap_SUITE.erl +++ b/apps/emqx_auth_ldap/test/emqx_authz_ldap_SUITE.erl @@ -178,6 +178,7 @@ stop_apps(Apps) -> create_ldap_resource() -> {ok, _} = emqx_resource:create_local( + ldap, ?LDAP_RESOURCE, ?AUTHZ_RESOURCE_GROUP, emqx_ldap, diff --git a/apps/emqx_auth_mongodb/src/emqx_auth_mongodb.app.src b/apps/emqx_auth_mongodb/src/emqx_auth_mongodb.app.src index df3cc1268..5ffc59787 100644 --- a/apps/emqx_auth_mongodb/src/emqx_auth_mongodb.app.src +++ b/apps/emqx_auth_mongodb/src/emqx_auth_mongodb.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_auth_mongodb, [ {description, "EMQX MongoDB Authentication and Authorization"}, - {vsn, "0.2.0"}, + {vsn, "0.2.1"}, {registered, []}, {mod, {emqx_auth_mongodb_app, []}}, {applications, [ diff --git a/apps/emqx_auth_mongodb/src/emqx_authn_mongodb.erl b/apps/emqx_auth_mongodb/src/emqx_authn_mongodb.erl index 75a474c0c..ffe78159d 100644 --- a/apps/emqx_auth_mongodb/src/emqx_authn_mongodb.erl +++ b/apps/emqx_auth_mongodb/src/emqx_authn_mongodb.erl @@ -37,6 +37,7 @@ create(Config0) -> ResourceId = emqx_authn_utils:make_resource_id(?MODULE), {Config, State} = parse_config(Config0), {ok, _Data} = emqx_authn_utils:create_resource( + mongodb, ResourceId, emqx_mongodb, Config @@ -45,7 +46,7 @@ create(Config0) -> update(Config0, #{resource_id := ResourceId} = _State) -> {Config, NState} = parse_config(Config0), - case emqx_authn_utils:update_resource(emqx_mongodb, Config, ResourceId) of + case emqx_authn_utils:update_resource(mongodb, emqx_mongodb, Config, ResourceId) of {error, Reason} -> error({load_config_error, Reason}); {ok, _} -> diff --git a/apps/emqx_auth_mongodb/src/emqx_authz_mongodb.erl b/apps/emqx_auth_mongodb/src/emqx_authz_mongodb.erl index 0bab6ef90..0d65afe0d 100644 --- a/apps/emqx_auth_mongodb/src/emqx_authz_mongodb.erl +++ b/apps/emqx_auth_mongodb/src/emqx_authz_mongodb.erl @@ -49,13 +49,13 @@ description() -> create(#{filter := Filter} = Source) -> ResourceId = emqx_authz_utils:make_resource_id(?MODULE), - {ok, _Data} = emqx_authz_utils:create_resource(ResourceId, emqx_mongodb, Source), + {ok, _Data} = emqx_authz_utils:create_resource(mongodb, ResourceId, emqx_mongodb, Source), FilterTemp = emqx_authz_utils:parse_deep(Filter, ?ALLOWED_VARS), Source#{annotations => #{id => ResourceId}, filter_template => FilterTemp}. update(#{filter := Filter} = Source) -> FilterTemp = emqx_authz_utils:parse_deep(Filter, ?ALLOWED_VARS), - case emqx_authz_utils:update_resource(emqx_mongodb, Source) of + case emqx_authz_utils:update_resource(mongodb, emqx_mongodb, Source) of {error, Reason} -> error({load_config_error, Reason}); {ok, Id} -> diff --git a/apps/emqx_auth_mysql/src/emqx_authn_mysql.erl b/apps/emqx_auth_mysql/src/emqx_authn_mysql.erl index f68c74a14..7d03ca856 100644 --- a/apps/emqx_auth_mysql/src/emqx_authn_mysql.erl +++ b/apps/emqx_auth_mysql/src/emqx_authn_mysql.erl @@ -39,12 +39,12 @@ create(_AuthenticatorID, Config) -> create(Config0) -> ResourceId = emqx_authn_utils:make_resource_id(?MODULE), {Config, State} = parse_config(Config0), - {ok, _Data} = emqx_authn_utils:create_resource(ResourceId, emqx_mysql, Config), + {ok, _Data} = emqx_authn_utils:create_resource(mysql, ResourceId, emqx_mysql, Config), {ok, State#{resource_id => ResourceId}}. update(Config0, #{resource_id := ResourceId} = _State) -> {Config, NState} = parse_config(Config0), - case emqx_authn_utils:update_resource(emqx_mysql, Config, ResourceId) of + case emqx_authn_utils:update_resource(mysql, emqx_mysql, Config, ResourceId) of {error, Reason} -> error({load_config_error, Reason}); {ok, _} -> diff --git a/apps/emqx_auth_mysql/src/emqx_authz_mysql.erl b/apps/emqx_auth_mysql/src/emqx_authz_mysql.erl index 0e2b77005..557eed14f 100644 --- a/apps/emqx_auth_mysql/src/emqx_authz_mysql.erl +++ b/apps/emqx_auth_mysql/src/emqx_authz_mysql.erl @@ -53,13 +53,13 @@ create(#{query := SQL} = Source0) -> {PrepareSQL, TmplToken} = emqx_authz_utils:parse_sql(SQL, '?', ?ALLOWED_VARS), ResourceId = emqx_authz_utils:make_resource_id(?MODULE), Source = Source0#{prepare_statement => #{?PREPARE_KEY => PrepareSQL}}, - {ok, _Data} = emqx_authz_utils:create_resource(ResourceId, emqx_mysql, Source), + {ok, _Data} = emqx_authz_utils:create_resource(mysql, ResourceId, emqx_mysql, Source), Source#{annotations => #{id => ResourceId, tmpl_token => TmplToken}}. update(#{query := SQL} = Source0) -> {PrepareSQL, TmplToken} = emqx_authz_utils:parse_sql(SQL, '?', ?ALLOWED_VARS), Source = Source0#{prepare_statement => #{?PREPARE_KEY => PrepareSQL}}, - case emqx_authz_utils:update_resource(emqx_mysql, Source) of + case emqx_authz_utils:update_resource(mysql, emqx_mysql, Source) of {error, Reason} -> error({load_config_error, Reason}); {ok, Id} -> diff --git a/apps/emqx_auth_mysql/test/emqx_authn_mysql_SUITE.erl b/apps/emqx_auth_mysql/test/emqx_authn_mysql_SUITE.erl index 8ab812fc0..78b8aa8b3 100644 --- a/apps/emqx_auth_mysql/test/emqx_authn_mysql_SUITE.erl +++ b/apps/emqx_auth_mysql/test/emqx_authn_mysql_SUITE.erl @@ -58,6 +58,7 @@ init_per_suite(Config) -> work_dir => ?config(priv_dir, Config) }), {ok, _} = emqx_resource:create_local( + mysql, ?MYSQL_RESOURCE, ?AUTHN_RESOURCE_GROUP, emqx_mysql, diff --git a/apps/emqx_auth_mysql/test/emqx_authz_mysql_SUITE.erl b/apps/emqx_auth_mysql/test/emqx_authz_mysql_SUITE.erl index ce30e203e..5fe020f1b 100644 --- a/apps/emqx_auth_mysql/test/emqx_authz_mysql_SUITE.erl +++ b/apps/emqx_auth_mysql/test/emqx_authz_mysql_SUITE.erl @@ -446,6 +446,7 @@ stop_apps(Apps) -> create_mysql_resource() -> {ok, _} = emqx_resource:create_local( + mysql, ?MYSQL_RESOURCE, ?AUTHZ_RESOURCE_GROUP, emqx_mysql, diff --git a/apps/emqx_auth_postgresql/src/emqx_authn_postgresql.erl b/apps/emqx_auth_postgresql/src/emqx_authn_postgresql.erl index 980c2ddd8..95e8ac7ce 100644 --- a/apps/emqx_auth_postgresql/src/emqx_authn_postgresql.erl +++ b/apps/emqx_auth_postgresql/src/emqx_authn_postgresql.erl @@ -45,6 +45,7 @@ create(Config0) -> ResourceId = emqx_authn_utils:make_resource_id(?MODULE), {Config, State} = parse_config(Config0, ResourceId), {ok, _Data} = emqx_authn_utils:create_resource( + postgresql, ResourceId, emqx_postgresql, Config @@ -53,7 +54,7 @@ create(Config0) -> update(Config0, #{resource_id := ResourceId} = _State) -> {Config, NState} = parse_config(Config0, ResourceId), - case emqx_authn_utils:update_resource(emqx_postgresql, Config, ResourceId) of + case emqx_authn_utils:update_resource(postgresql, emqx_postgresql, Config, ResourceId) of {error, Reason} -> error({load_config_error, Reason}); {ok, _} -> diff --git a/apps/emqx_auth_postgresql/src/emqx_authz_postgresql.erl b/apps/emqx_auth_postgresql/src/emqx_authz_postgresql.erl index d1a0b32ea..e0fc9e032 100644 --- a/apps/emqx_auth_postgresql/src/emqx_authz_postgresql.erl +++ b/apps/emqx_auth_postgresql/src/emqx_authz_postgresql.erl @@ -53,6 +53,7 @@ create(#{query := SQL0} = Source) -> {SQL, PlaceHolders} = emqx_authz_utils:parse_sql(SQL0, '$n', ?ALLOWED_VARS), ResourceID = emqx_authz_utils:make_resource_id(emqx_postgresql), {ok, _Data} = emqx_authz_utils:create_resource( + postgresql, ResourceID, emqx_postgresql, Source#{prepare_statement => #{ResourceID => SQL}} @@ -63,6 +64,7 @@ update(#{query := SQL0, annotations := #{id := ResourceID}} = Source) -> {SQL, PlaceHolders} = emqx_authz_utils:parse_sql(SQL0, '$n', ?ALLOWED_VARS), case emqx_authz_utils:update_resource( + postgresql, emqx_postgresql, Source#{prepare_statement => #{ResourceID => SQL}} ) diff --git a/apps/emqx_auth_postgresql/test/emqx_authn_postgresql_SUITE.erl b/apps/emqx_auth_postgresql/test/emqx_authn_postgresql_SUITE.erl index 50bff634d..e899edbd9 100644 --- a/apps/emqx_auth_postgresql/test/emqx_authn_postgresql_SUITE.erl +++ b/apps/emqx_auth_postgresql/test/emqx_authn_postgresql_SUITE.erl @@ -79,6 +79,7 @@ init_per_suite(Config) -> work_dir => ?config(priv_dir, Config) }), {ok, _} = emqx_resource:create_local( + postgresql, ?PGSQL_RESOURCE, ?AUTHN_RESOURCE_GROUP, emqx_postgresql, @@ -198,9 +199,9 @@ test_user_auth(#{ t_authenticate_disabled_prepared_statements(_Config) -> ResConfig = maps:merge(pgsql_config(), #{disable_prepared_statements => true}), - {ok, _} = emqx_resource:recreate_local(?PGSQL_RESOURCE, emqx_postgresql, ResConfig), + {ok, _} = emqx_resource:recreate_local(postgresql, ?PGSQL_RESOURCE, emqx_postgresql, ResConfig), on_exit(fun() -> - emqx_resource:recreate_local(?PGSQL_RESOURCE, emqx_postgresql, pgsql_config()) + emqx_resource:recreate_local(postgresql, ?PGSQL_RESOURCE, emqx_postgresql, pgsql_config()) end), ok = lists:foreach( fun(Sample0) -> diff --git a/apps/emqx_auth_postgresql/test/emqx_authz_postgresql_SUITE.erl b/apps/emqx_auth_postgresql/test/emqx_authz_postgresql_SUITE.erl index 78b1e17a8..9346bf863 100644 --- a/apps/emqx_auth_postgresql/test/emqx_authz_postgresql_SUITE.erl +++ b/apps/emqx_auth_postgresql/test/emqx_authz_postgresql_SUITE.erl @@ -437,6 +437,7 @@ pgsql_config() -> create_pgsql_resource() -> emqx_resource:create_local( + postgresql, ?PGSQL_RESOURCE, ?AUTHZ_RESOURCE_GROUP, emqx_postgresql, diff --git a/apps/emqx_auth_redis/src/emqx_authn_redis.erl b/apps/emqx_auth_redis/src/emqx_authn_redis.erl index 779c58e39..4eb5a36a3 100644 --- a/apps/emqx_auth_redis/src/emqx_authn_redis.erl +++ b/apps/emqx_auth_redis/src/emqx_authn_redis.erl @@ -42,6 +42,7 @@ create(Config0) -> Res; {Config, State} -> {ok, _Data} = emqx_authn_utils:create_resource( + redis, ResourceId, emqx_redis, Config @@ -51,7 +52,7 @@ create(Config0) -> update(Config0, #{resource_id := ResourceId} = _State) -> {Config, NState} = parse_config(Config0), - case emqx_authn_utils:update_resource(emqx_redis, Config, ResourceId) of + case emqx_authn_utils:update_resource(redis, emqx_redis, Config, ResourceId) of {error, Reason} -> error({load_config_error, Reason}); {ok, _} -> diff --git a/apps/emqx_auth_redis/src/emqx_authz_redis.erl b/apps/emqx_auth_redis/src/emqx_authz_redis.erl index 8ce975033..b83d1abad 100644 --- a/apps/emqx_auth_redis/src/emqx_authz_redis.erl +++ b/apps/emqx_auth_redis/src/emqx_authz_redis.erl @@ -50,12 +50,12 @@ description() -> create(#{cmd := CmdStr} = Source) -> CmdTemplate = parse_cmd(CmdStr), ResourceId = emqx_authz_utils:make_resource_id(?MODULE), - {ok, _Data} = emqx_authz_utils:create_resource(ResourceId, emqx_redis, Source), + {ok, _Data} = emqx_authz_utils:create_resource(redis, ResourceId, emqx_redis, Source), Source#{annotations => #{id => ResourceId}, cmd_template => CmdTemplate}. update(#{cmd := CmdStr} = Source) -> CmdTemplate = parse_cmd(CmdStr), - case emqx_authz_utils:update_resource(emqx_redis, Source) of + case emqx_authz_utils:update_resource(redis, emqx_redis, Source) of {error, Reason} -> error({load_config_error, Reason}); {ok, Id} -> diff --git a/apps/emqx_auth_redis/test/emqx_authn_redis_SUITE.erl b/apps/emqx_auth_redis/test/emqx_authn_redis_SUITE.erl index e8c8760de..1e9d825d2 100644 --- a/apps/emqx_auth_redis/test/emqx_authn_redis_SUITE.erl +++ b/apps/emqx_auth_redis/test/emqx_authn_redis_SUITE.erl @@ -63,6 +63,7 @@ init_per_suite(Config) -> work_dir => ?config(priv_dir, Config) }), {ok, _} = emqx_resource:create_local( + redis, ?REDIS_RESOURCE, ?AUTHN_RESOURCE_GROUP, emqx_redis, diff --git a/apps/emqx_auth_redis/test/emqx_authz_redis_SUITE.erl b/apps/emqx_auth_redis/test/emqx_authz_redis_SUITE.erl index 5818eea07..d0c695c73 100644 --- a/apps/emqx_auth_redis/test/emqx_authz_redis_SUITE.erl +++ b/apps/emqx_auth_redis/test/emqx_authz_redis_SUITE.erl @@ -384,6 +384,7 @@ stop_apps(Apps) -> create_redis_resource() -> {ok, _} = emqx_resource:create_local( + redis, ?REDIS_RESOURCE, ?AUTHZ_RESOURCE_GROUP, emqx_redis, diff --git a/apps/emqx_bridge/src/emqx_bridge_resource.erl b/apps/emqx_bridge/src/emqx_bridge_resource.erl index d2408ca73..7e9f5300a 100644 --- a/apps/emqx_bridge/src/emqx_bridge_resource.erl +++ b/apps/emqx_bridge/src/emqx_bridge_resource.erl @@ -195,8 +195,9 @@ create(Type, Name, Conf0, Opts) -> TypeBin = bin(Type), Conf = Conf0#{bridge_type => TypeBin, bridge_name => Name}, {ok, _Data} = emqx_resource:create_local( + Type, resource_id(Type, Name), - <<"emqx_bridge">>, + <<"bridge">>, bridge_to_resource_type(Type), parse_confs(TypeBin, Name, Conf), parse_opts(Conf, Opts) @@ -264,6 +265,7 @@ recreate(Type, Name, Conf0, Opts) -> TypeBin = bin(Type), Conf = Conf0#{bridge_type => TypeBin, bridge_name => Name}, emqx_resource:recreate_local( + Type, resource_id(Type, Name), bridge_to_resource_type(Type), parse_confs(TypeBin, Name, Conf), @@ -300,7 +302,7 @@ create_dry_run_bridge_v1(Type, Conf0) -> {error, Reason}; {ok, ConfNew} -> ParseConf = parse_confs(TypeBin, TmpName, ConfNew), - emqx_resource:create_dry_run_local(bridge_to_resource_type(Type), ParseConf) + emqx_resource:create_dry_run_local(Type, bridge_to_resource_type(Type), ParseConf) end catch %% validation errors diff --git a/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_connector_SUITE.erl b/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_connector_SUITE.erl index b784d36c0..02c102832 100644 --- a/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_connector_SUITE.erl +++ b/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_connector_SUITE.erl @@ -212,6 +212,7 @@ check_config(Config) -> create_local_resource(ResourceId, CheckedConfig) -> {ok, Bridge} = emqx_resource:create_local( + cassandra, ResourceId, ?CONNECTOR_RESOURCE_GROUP, ?CASSANDRA_RESOURCE_MOD, diff --git a/apps/emqx_bridge_clickhouse/test/emqx_bridge_clickhouse_connector_SUITE.erl b/apps/emqx_bridge_clickhouse/test/emqx_bridge_clickhouse_connector_SUITE.erl index 1c83961a5..0b0acd78a 100644 --- a/apps/emqx_bridge_clickhouse/test/emqx_bridge_clickhouse_connector_SUITE.erl +++ b/apps/emqx_bridge_clickhouse/test/emqx_bridge_clickhouse_connector_SUITE.erl @@ -109,6 +109,7 @@ t_start_passfile(Config) -> ?assertMatch( {ok, #{status := connected}}, emqx_resource:create_local( + clickhouse, ResourceID, ?CONNECTOR_RESOURCE_GROUP, ?CLICKHOUSE_RESOURCE_MOD, @@ -138,6 +139,7 @@ perform_lifecycle_check(ResourceID, InitialConfig) -> status := InitialStatus }} = emqx_resource:create_local( + clickhouse, ResourceID, ?CONNECTOR_RESOURCE_GROUP, ?CLICKHOUSE_RESOURCE_MOD, diff --git a/apps/emqx_bridge_greptimedb/test/emqx_bridge_greptimedb_connector_SUITE.erl b/apps/emqx_bridge_greptimedb/test/emqx_bridge_greptimedb_connector_SUITE.erl index be36cb167..68a32e9c2 100644 --- a/apps/emqx_bridge_greptimedb/test/emqx_bridge_greptimedb_connector_SUITE.erl +++ b/apps/emqx_bridge_greptimedb/test/emqx_bridge_greptimedb_connector_SUITE.erl @@ -83,6 +83,7 @@ perform_lifecycle_check(PoolName, InitialConfig) -> state := #{client := #{pool := ReturnedPoolName}} = State, status := InitialStatus }} = emqx_resource:create_local( + greptimedb, PoolName, ?CONNECTOR_RESOURCE_GROUP, ?GREPTIMEDB_RESOURCE_MOD, diff --git a/apps/emqx_bridge_influxdb/test/emqx_bridge_influxdb_connector_SUITE.erl b/apps/emqx_bridge_influxdb/test/emqx_bridge_influxdb_connector_SUITE.erl index 0ca693171..a7f78f253 100644 --- a/apps/emqx_bridge_influxdb/test/emqx_bridge_influxdb_connector_SUITE.erl +++ b/apps/emqx_bridge_influxdb/test/emqx_bridge_influxdb_connector_SUITE.erl @@ -86,6 +86,7 @@ perform_lifecycle_check(PoolName, InitialConfig) -> state := #{client := #{pool := ReturnedPoolName}} = State, status := InitialStatus }} = emqx_resource:create_local( + influxdb, PoolName, ?CONNECTOR_RESOURCE_GROUP, ?INFLUXDB_RESOURCE_MOD, @@ -197,6 +198,7 @@ perform_tls_opts_check(PoolName, InitialConfig, VerifyReturn) -> config := #{ssl := #{enable := SslEnabled}}, status := Status }} = emqx_resource:create_local( + influxdb, PoolName, ?CONNECTOR_RESOURCE_GROUP, ?INFLUXDB_RESOURCE_MOD, diff --git a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_connector.erl b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_connector.erl index 4157deec2..835536bda 100644 --- a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_connector.erl +++ b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_connector.erl @@ -289,7 +289,7 @@ replayq_dir(ClientId) -> filename:join([emqx:data_dir(), "pulsar", emqx_utils_conv:bin(ClientId)]). producer_name(InstanceId, ChannelId) -> - case is_dry_run(InstanceId) of + case emqx_resource:is_dry_run(InstanceId) of %% do not create more atom true -> pulsar_producer_probe_worker; diff --git a/apps/emqx_bridge_rabbitmq/test/emqx_bridge_rabbitmq_connector_SUITE.erl b/apps/emqx_bridge_rabbitmq/test/emqx_bridge_rabbitmq_connector_SUITE.erl index 77482ae0f..fe288a185 100644 --- a/apps/emqx_bridge_rabbitmq/test/emqx_bridge_rabbitmq_connector_SUITE.erl +++ b/apps/emqx_bridge_rabbitmq/test/emqx_bridge_rabbitmq_connector_SUITE.erl @@ -135,6 +135,7 @@ check_config(Config) -> create_local_resource(ResourceID, CheckedConfig) -> {ok, Bridge} = emqx_resource:create_local( + rabbitmq, ResourceID, ?CONNECTOR_RESOURCE_GROUP, emqx_bridge_rabbitmq_connector, diff --git a/apps/emqx_connector/include/emqx_connector.hrl b/apps/emqx_connector/include/emqx_connector.hrl index 4b29dd5ce..0004cd72c 100644 --- a/apps/emqx_connector/include/emqx_connector.hrl +++ b/apps/emqx_connector/include/emqx_connector.hrl @@ -37,4 +37,4 @@ "The " ++ TYPE ++ " default port " ++ DEFAULT_PORT ++ " is used if `[:Port]` is not specified." ). --define(CONNECTOR_RESOURCE_GROUP, <<"emqx_connector">>). +-define(CONNECTOR_RESOURCE_GROUP, <<"connector">>). diff --git a/apps/emqx_connector/src/emqx_connector_resource.erl b/apps/emqx_connector/src/emqx_connector_resource.erl index be8d3a32d..2a5b3bcfc 100644 --- a/apps/emqx_connector/src/emqx_connector_resource.erl +++ b/apps/emqx_connector/src/emqx_connector_resource.erl @@ -18,6 +18,7 @@ -include("../../emqx_bridge/include/emqx_bridge_resource.hrl"). -include_lib("emqx/include/logger.hrl"). -include_lib("emqx_resource/include/emqx_resource.hrl"). +-include("emqx_connector.hrl"). -export([ connector_to_resource_type/1, @@ -125,8 +126,9 @@ create(Type, Name, Conf0, Opts) -> ResourceId = resource_id(Type, Name), Conf = Conf0#{connector_type => TypeBin, connector_name => Name}, {ok, _Data} = emqx_resource:create_local( + Type, ResourceId, - <<"emqx_connector">>, + ?CONNECTOR_RESOURCE_GROUP, ?MODULE:connector_to_resource_type(Type), parse_confs(TypeBin, Name, Conf), parse_opts(Conf, Opts) @@ -198,6 +200,7 @@ recreate(Type, Name, Conf) -> recreate(Type, Name, Conf, Opts) -> TypeBin = bin(Type), emqx_resource:recreate_local( + Type, resource_id(Type, Name), ?MODULE:connector_to_resource_type(Type), parse_confs(TypeBin, Name, Conf), @@ -234,7 +237,7 @@ create_dry_run(Type, Conf0, Callback) -> {ok, ConfNew} -> ParseConf = parse_confs(bin(Type), TmpName, ConfNew), emqx_resource:create_dry_run_local( - TmpName, ?MODULE:connector_to_resource_type(Type), ParseConf, Callback + Type, TmpName, ?MODULE:connector_to_resource_type(Type), ParseConf, Callback ) end catch diff --git a/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_manager.erl b/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_manager.erl index 6834da9e9..6ac02efc6 100644 --- a/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_manager.erl +++ b/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_manager.erl @@ -45,7 +45,7 @@ -define(MOD_TAB, emqx_dashboard_sso). -define(MOD_KEY_PATH, [dashboard, sso]). -define(MOD_KEY_PATH(Sub), [dashboard, sso, Sub]). --define(RESOURCE_GROUP, <<"emqx_dashboard_sso">>). +-define(RESOURCE_GROUP, <<"dashboard_sso">>). -define(NO_ERROR, <<>>). -define(DEFAULT_RESOURCE_OPTS, #{ start_after_created => false @@ -132,6 +132,7 @@ make_resource_id(Backend) -> create_resource(ResourceId, Module, Config) -> Result = emqx_resource:create_local( + dashboard_sso, ResourceId, ?RESOURCE_GROUP, Module, @@ -142,7 +143,7 @@ create_resource(ResourceId, Module, Config) -> update_resource(ResourceId, Module, Config) -> Result = emqx_resource:recreate_local( - ResourceId, Module, Config, ?DEFAULT_RESOURCE_OPTS + dashboard_sso, ResourceId, Module, Config, ?DEFAULT_RESOURCE_OPTS ), start_resource_if_enabled(ResourceId, Result, Config). diff --git a/apps/emqx_dashboard_sso/test/emqx_dashboard_sso_ldap_SUITE.erl b/apps/emqx_dashboard_sso/test/emqx_dashboard_sso_ldap_SUITE.erl index 51524f0fd..40c5de9e5 100644 --- a/apps/emqx_dashboard_sso/test/emqx_dashboard_sso_ldap_SUITE.erl +++ b/apps/emqx_dashboard_sso/test/emqx_dashboard_sso_ldap_SUITE.erl @@ -24,7 +24,7 @@ -define(MOD_TAB, emqx_dashboard_sso). -define(MOD_KEY_PATH, [dashboard, sso, ldap]). --define(RESOURCE_GROUP, <<"emqx_dashboard_sso">>). +-define(RESOURCE_GROUP, <<"dashboard_sso">>). -import(emqx_mgmt_api_test_util, [request/2, request/3, uri/1, request_api/3]). diff --git a/apps/emqx_ldap/test/emqx_ldap_SUITE.erl b/apps/emqx_ldap/test/emqx_ldap_SUITE.erl index a15ff2775..413cbc3a5 100644 --- a/apps/emqx_ldap/test/emqx_ldap_SUITE.erl +++ b/apps/emqx_ldap/test/emqx_ldap_SUITE.erl @@ -96,6 +96,7 @@ perform_lifecycle_check(ResourceId, InitialConfig) -> state := #{pool_name := PoolName} = State, status := InitialStatus }} = emqx_resource:create_local( + ldap, ResourceId, ?CONNECTOR_RESOURCE_GROUP, ?LDAP_RESOURCE_MOD, @@ -171,6 +172,7 @@ t_get_status(Config) -> ?LDAP_RESOURCE_MOD, ldap_config(Config) ), {ok, _} = emqx_resource:create_local( + ldap, ResourceId, ?CONNECTOR_RESOURCE_GROUP, ?LDAP_RESOURCE_MOD, diff --git a/apps/emqx_mongodb/test/emqx_mongodb_SUITE.erl b/apps/emqx_mongodb/test/emqx_mongodb_SUITE.erl index 8af05e0d3..850683d99 100644 --- a/apps/emqx_mongodb/test/emqx_mongodb_SUITE.erl +++ b/apps/emqx_mongodb/test/emqx_mongodb_SUITE.erl @@ -143,6 +143,7 @@ check_config(Config) -> create_local_resource(ResourceId, CheckedConfig) -> {ok, Bridge} = emqx_resource:create_local( + mongodb, ResourceId, ?CONNECTOR_RESOURCE_GROUP, ?MONGO_RESOURCE_MOD, diff --git a/apps/emqx_mysql/test/emqx_mysql_SUITE.erl b/apps/emqx_mysql/test/emqx_mysql_SUITE.erl index 03e6c6797..be69140fc 100644 --- a/apps/emqx_mysql/test/emqx_mysql_SUITE.erl +++ b/apps/emqx_mysql/test/emqx_mysql_SUITE.erl @@ -67,6 +67,7 @@ perform_lifecycle_check(ResourceId, InitialConfig) -> state := #{pool_name := PoolName} = State, status := InitialStatus }} = emqx_resource:create_local( + mysql, ResourceId, ?CONNECTOR_RESOURCE_GROUP, ?MYSQL_RESOURCE_MOD, diff --git a/apps/emqx_postgresql/test/emqx_postgresql_SUITE.erl b/apps/emqx_postgresql/test/emqx_postgresql_SUITE.erl index d771d80d8..06210be86 100644 --- a/apps/emqx_postgresql/test/emqx_postgresql_SUITE.erl +++ b/apps/emqx_postgresql/test/emqx_postgresql_SUITE.erl @@ -75,6 +75,7 @@ perform_lifecycle_check(ResourceId, InitialConfig) -> status := InitialStatus }} = emqx_resource:create_local( + postgresql, ResourceId, ?CONNECTOR_RESOURCE_GROUP, ?PGSQL_RESOURCE_MOD, diff --git a/apps/emqx_redis/test/emqx_redis_SUITE.erl b/apps/emqx_redis/test/emqx_redis_SUITE.erl index a9064f184..06ac82143 100644 --- a/apps/emqx_redis/test/emqx_redis_SUITE.erl +++ b/apps/emqx_redis/test/emqx_redis_SUITE.erl @@ -115,6 +115,7 @@ perform_lifecycle_check(ResourceId, InitialConfig, RedisCommand) -> state := #{pool_name := PoolName} = State, status := InitialStatus }} = emqx_resource:create_local( + redis, ResourceId, ?CONNECTOR_RESOURCE_GROUP, ?REDIS_RESOURCE_MOD, diff --git a/apps/emqx_resource/include/emqx_resource.hrl b/apps/emqx_resource/include/emqx_resource.hrl index 587786cb2..aa86fe239 100644 --- a/apps/emqx_resource/include/emqx_resource.hrl +++ b/apps/emqx_resource/include/emqx_resource.hrl @@ -23,6 +23,7 @@ %% remind us of that. -define(rm_status_stopped, stopped). +-type type() :: atom() | binary(). -type resource_type() :: module(). -type resource_id() :: binary(). -type channel_id() :: binary(). diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl index b6f01fde5..71ccaa696 100644 --- a/apps/emqx_resource/src/emqx_resource.erl +++ b/apps/emqx_resource/src/emqx_resource.erl @@ -28,9 +28,9 @@ -export([ check_config/2, - check_and_create_local/4, check_and_create_local/5, - check_and_recreate_local/4 + check_and_create_local/6, + check_and_recreate_local/5 ]). %% Sync resource instances and files @@ -39,13 +39,13 @@ -export([ %% store the config and start the instance - create_local/4, create_local/5, - create_dry_run_local/2, + create_local/6, create_dry_run_local/3, create_dry_run_local/4, - recreate_local/3, + create_dry_run_local/5, recreate_local/4, + recreate_local/5, %% remove the config and stop the instance remove_local/1, reset_metrics/1, @@ -282,12 +282,13 @@ is_resource_mod(Module) -> %% APIs for resource instances %% ================================================================================= --spec create_local(resource_id(), resource_group(), resource_type(), resource_config()) -> +-spec create_local(type(), resource_id(), resource_group(), resource_type(), resource_config()) -> {ok, resource_data() | 'already_created'} | {error, Reason :: term()}. -create_local(ResId, Group, ResourceType, Config) -> - create_local(ResId, Group, ResourceType, Config, #{}). +create_local(Type, ResId, Group, ResourceType, Config) -> + create_local(Type, ResId, Group, ResourceType, Config, #{}). -spec create_local( + type(), resource_id(), resource_group(), resource_type(), @@ -295,33 +296,39 @@ create_local(ResId, Group, ResourceType, Config) -> creation_opts() ) -> {ok, resource_data()}. -create_local(ResId, Group, ResourceType, Config, Opts) -> - emqx_resource_manager:ensure_resource(ResId, Group, ResourceType, Config, Opts). +create_local(Type, ResId, Group, ResourceType, Config, Opts) -> + emqx_resource_manager:ensure_resource(Type, ResId, Group, ResourceType, Config, Opts). --spec create_dry_run_local(resource_type(), resource_config()) -> +-spec create_dry_run_local(type(), resource_type(), resource_config()) -> ok | {error, Reason :: term()}. -create_dry_run_local(ResourceType, Config) -> - emqx_resource_manager:create_dry_run(ResourceType, Config). +create_dry_run_local(Type, ResourceType, Config) -> + emqx_resource_manager:create_dry_run(Type, ResourceType, Config). -create_dry_run_local(ResId, ResourceType, Config) -> - emqx_resource_manager:create_dry_run(ResId, ResourceType, Config). +create_dry_run_local(Type, ResId, ResourceType, Config) -> + emqx_resource_manager:create_dry_run(Type, ResId, ResourceType, Config). --spec create_dry_run_local(resource_id(), resource_type(), resource_config(), OnReadyCallback) -> +-spec create_dry_run_local( + type(), + resource_id(), + resource_type(), + resource_config(), + OnReadyCallback +) -> ok | {error, Reason :: term()} when OnReadyCallback :: fun((resource_id()) -> ok | {error, Reason :: term()}). -create_dry_run_local(ResId, ResourceType, Config, OnReadyCallback) -> - emqx_resource_manager:create_dry_run(ResId, ResourceType, Config, OnReadyCallback). +create_dry_run_local(Type, ResId, ResourceType, Config, OnReadyCallback) -> + emqx_resource_manager:create_dry_run(Type, ResId, ResourceType, Config, OnReadyCallback). --spec recreate_local(resource_id(), resource_type(), resource_config()) -> +-spec recreate_local(type(), resource_id(), resource_type(), resource_config()) -> {ok, resource_data()} | {error, Reason :: term()}. -recreate_local(ResId, ResourceType, Config) -> - recreate_local(ResId, ResourceType, Config, #{}). +recreate_local(Type, ResId, ResourceType, Config) -> + recreate_local(Type, ResId, ResourceType, Config, #{}). --spec recreate_local(resource_id(), resource_type(), resource_config(), creation_opts()) -> +-spec recreate_local(type(), resource_id(), resource_type(), resource_config(), creation_opts()) -> {ok, resource_data()} | {error, Reason :: term()}. -recreate_local(ResId, ResourceType, Config, Opts) -> - emqx_resource_manager:recreate(ResId, ResourceType, Config, Opts). +recreate_local(Type, ResId, ResourceType, Config, Opts) -> + emqx_resource_manager:recreate(Type, ResId, ResourceType, Config, Opts). -spec remove_local(resource_id()) -> ok. remove_local(ResId) -> @@ -607,41 +614,44 @@ check_config(ResourceType, Conf) -> emqx_hocon:check(ResourceType, Conf). -spec check_and_create_local( + type(), resource_id(), resource_group(), resource_type(), raw_resource_config() ) -> {ok, resource_data()} | {error, term()}. -check_and_create_local(ResId, Group, ResourceType, RawConfig) -> - check_and_create_local(ResId, Group, ResourceType, RawConfig, #{}). +check_and_create_local(Type, ResId, Group, ResourceType, RawConfig) -> + check_and_create_local(Type, ResId, Group, ResourceType, RawConfig, #{}). -spec check_and_create_local( + type(), resource_id(), resource_group(), resource_type(), raw_resource_config(), creation_opts() ) -> {ok, resource_data()} | {error, term()}. -check_and_create_local(ResId, Group, ResourceType, RawConfig, Opts) -> +check_and_create_local(Type, ResId, Group, ResourceType, RawConfig, Opts) -> check_and_do( ResourceType, RawConfig, - fun(ResConf) -> create_local(ResId, Group, ResourceType, ResConf, Opts) end + fun(ResConf) -> create_local(Type, ResId, Group, ResourceType, ResConf, Opts) end ). -spec check_and_recreate_local( + type(), resource_id(), resource_type(), raw_resource_config(), creation_opts() ) -> {ok, resource_data()} | {error, term()}. -check_and_recreate_local(ResId, ResourceType, RawConfig, Opts) -> +check_and_recreate_local(Type, ResId, ResourceType, RawConfig, Opts) -> check_and_do( ResourceType, RawConfig, - fun(ResConf) -> recreate_local(ResId, ResourceType, ResConf, Opts) end + fun(ResConf) -> recreate_local(Type, ResId, ResourceType, ResConf, Opts) end ). check_and_do(ResourceType, RawConfig, Do) when is_function(Do) -> diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl index 95b1271f4..1c0b74edd 100644 --- a/apps/emqx_resource/src/emqx_resource_manager.erl +++ b/apps/emqx_resource/src/emqx_resource_manager.erl @@ -25,12 +25,12 @@ % API -export([ - ensure_resource/5, - recreate/4, + ensure_resource/6, + recreate/5, remove/1, - create_dry_run/2, create_dry_run/3, create_dry_run/4, + create_dry_run/5, restart/2, start/2, stop/1, @@ -59,7 +59,7 @@ ]). % Server --export([start_link/5]). +-export([start_link/6]). % Behaviour -export([init/1, callback_mode/0, handle_event/4, terminate/3]). @@ -75,6 +75,7 @@ -record(data, { id, group, + type, mod, callback_mode, query_mode, @@ -161,43 +162,44 @@ %% Triggers the emqx_resource_manager_sup supervisor to actually create %% and link the process itself if not already started. -spec ensure_resource( + type(), resource_id(), resource_group(), resource_type(), resource_config(), creation_opts() ) -> {ok, resource_data()}. -ensure_resource(ResId, Group, ResourceType, Config, Opts) -> +ensure_resource(Type, ResId, Group, ResourceType, Config, Opts) -> case lookup(ResId) of {ok, _Group, Data} -> {ok, Data}; {error, not_found} -> - create_and_return_data(ResId, Group, ResourceType, Config, Opts) + create_and_return_data(Type, ResId, Group, ResourceType, Config, Opts) end. %% @doc Called from emqx_resource when recreating a resource which may or may not exist --spec recreate(resource_id(), resource_type(), resource_config(), creation_opts()) -> +-spec recreate(type(), resource_id(), resource_type(), resource_config(), creation_opts()) -> {ok, resource_data()} | {error, not_found} | {error, updating_to_incorrect_resource_type}. -recreate(ResId, ResourceType, NewConfig, Opts) -> +recreate(Type, ResId, ResourceType, NewConfig, Opts) -> case lookup(ResId) of {ok, Group, #{mod := ResourceType, status := _} = _Data} -> _ = remove(ResId, false), - create_and_return_data(ResId, Group, ResourceType, NewConfig, Opts); + create_and_return_data(Type, ResId, Group, ResourceType, NewConfig, Opts); {ok, _, #{mod := Mod}} when Mod =/= ResourceType -> {error, updating_to_incorrect_resource_type}; {error, not_found} -> {error, not_found} end. -create_and_return_data(ResId, Group, ResourceType, Config, Opts) -> - _ = create(ResId, Group, ResourceType, Config, Opts), +create_and_return_data(Type, ResId, Group, ResourceType, Config, Opts) -> + _ = create(Type, ResId, Group, ResourceType, Config, Opts), {ok, _Group, Data} = lookup(ResId), {ok, Data}. %% @doc Create a resource_manager and wait until it is running -create(ResId, Group, ResourceType, Config, Opts) -> +create(Type, ResId, Group, ResourceType, Config, Opts) -> % The state machine will make the actual call to the callback/resource module after init - ok = emqx_resource_manager_sup:ensure_child(ResId, Group, ResourceType, Config, Opts), + ok = emqx_resource_manager_sup:ensure_child(Type, ResId, Group, ResourceType, Config, Opts), % Create metrics for the resource ok = emqx_resource:create_metrics(ResId), QueryMode = emqx_resource:query_mode(ResourceType, Config, Opts), @@ -219,30 +221,32 @@ create(ResId, Group, ResourceType, Config, Opts) -> %% @doc Called from `emqx_resource` when doing a dry run for creating a resource instance. %% %% Triggers the `emqx_resource_manager_sup` supervisor to actually create -%% and link the process itself if not already started, and then immedately stops. --spec create_dry_run(resource_type(), resource_config()) -> +%% and link the process itself if not already started, and then immediately stops. +-spec create_dry_run(type(), resource_type(), resource_config()) -> ok | {error, Reason :: term()}. -create_dry_run(ResourceType, Config) -> +create_dry_run(Type, ResourceType, Config) -> ResId = make_test_id(), - create_dry_run(ResId, ResourceType, Config). + create_dry_run(Type, ResId, ResourceType, Config). -create_dry_run(ResId, ResourceType, Config) -> - create_dry_run(ResId, ResourceType, Config, fun do_nothing_on_ready/1). +create_dry_run(Type, ResId, ResourceType, Config) -> + create_dry_run(Type, ResId, ResourceType, Config, fun do_nothing_on_ready/1). do_nothing_on_ready(_ResId) -> ok. --spec create_dry_run(resource_id(), resource_type(), resource_config(), OnReadyCallback) -> +-spec create_dry_run(type(), resource_id(), resource_type(), resource_config(), OnReadyCallback) -> ok | {error, Reason :: term()} when OnReadyCallback :: fun((resource_id()) -> ok | {error, Reason :: term()}). -create_dry_run(ResId, ResourceType, Config, OnReadyCallback) -> +create_dry_run(Type, ResId, ResourceType, Config, OnReadyCallback) -> Opts = case is_map(Config) of true -> maps:get(resource_opts, Config, #{}); false -> #{} end, - ok = emqx_resource_manager_sup:ensure_child(ResId, <<"dry_run">>, ResourceType, Config, Opts), + ok = emqx_resource_manager_sup:ensure_child( + Type, ResId, <<"dry_run">>, ResourceType, Config, Opts + ), HealthCheckInterval = maps:get(health_check_interval, Opts, ?HEALTHCHECK_INTERVAL), Timeout = emqx_utils:clamp(HealthCheckInterval, 5_000, 60_000), case wait_for_ready(ResId, Timeout) of @@ -491,7 +495,7 @@ try_clean_allocated_resources(ResId) -> %% Server start/stop callbacks %% @doc Function called from the supervisor to actually start the server -start_link(ResId, Group, ResourceType, Config, Opts) -> +start_link(Type, ResId, Group, ResourceType, Config, Opts) -> QueryMode = emqx_resource:query_mode( ResourceType, Config, @@ -499,6 +503,7 @@ start_link(ResId, Group, ResourceType, Config, Opts) -> ), Data = #data{ id = ResId, + type = Type, group = Group, mod = ResourceType, callback_mode = emqx_resource:get_callback_mode(ResourceType), @@ -683,6 +688,9 @@ handle_event(EventType, EventData, State, Data) -> error, #{ msg => "ignore_all_other_events", + resource_id => Data#data.id, + group => Data#data.group, + type => Data#data.type, event_type => EventType, event_data => EventData, state => State, @@ -752,8 +760,8 @@ handle_remove_event(From, ClearMetrics, Data) -> start_resource(Data, From) -> %% in case the emqx_resource:call_start/2 hangs, the lookup/1 can read status from the cache - ResId = Data#data.id, - case emqx_resource:call_start(ResId, Data#data.mod, Data#data.config) of + #data{id = ResId, mod = Mod, config = Config, group = Group, type = Type} = Data, + case emqx_resource:call_start(ResId, Mod, Config) of {ok, ResourceState} -> UpdatedData1 = Data#data{status = ?status_connecting, state = ResourceState}, %% Perform an initial health_check immediately before transitioning into a connected state @@ -764,7 +772,9 @@ start_resource(Data, From) -> IsDryRun = emqx_resource:is_dry_run(ResId), ?SLOG(log_level(IsDryRun), #{ msg => "start_resource_failed", - id => ResId, + resource_id => ResId, + group => Group, + type => Type, reason => Reason }), _ = maybe_alarm(?status_disconnected, IsDryRun, ResId, Err, Data#data.error), @@ -798,14 +808,20 @@ add_channels(Data) -> add_channels_in_list([], Data) -> Data; add_channels_in_list([{ChannelID, ChannelConfig} | Rest], Data) -> - Id = Data#data.id, + #data{ + id = ResId, + mod = Mod, + state = State, + added_channels = AddedChannelsMap, + group = Group, + type = Type + } = Data, case emqx_resource:call_add_channel( - Id, Data#data.mod, Data#data.state, ChannelID, ChannelConfig + ResId, Mod, State, ChannelID, ChannelConfig ) of {ok, NewState} -> - AddedChannelsMap = Data#data.added_channels, %% Set the channel status to connecting to indicate that %% we have not yet performed the initial health_check NewAddedChannelsMap = maps:put( @@ -819,10 +835,12 @@ add_channels_in_list([{ChannelID, ChannelConfig} | Rest], Data) -> }, add_channels_in_list(Rest, NewData); {error, Reason} = Error -> - IsDryRun = emqx_resource:is_dry_run(Id), + IsDryRun = emqx_resource:is_dry_run(ResId), ?SLOG(log_level(IsDryRun), #{ msg => "add_channel_failed", - id => Id, + resource_id => ResId, + type => Type, + group => Group, channel_id => ChannelID, reason => Reason }), @@ -872,9 +890,15 @@ remove_channels(Data) -> remove_channels_in_list([], Data, _KeepInChannelMap) -> Data; remove_channels_in_list([ChannelID | Rest], Data, KeepInChannelMap) -> - AddedChannelsMap = Data#data.added_channels, - Id = Data#data.id, - IsDryRun = emqx_resource:is_dry_run(Id), + #data{ + id = ResId, + added_channels = AddedChannelsMap, + mod = Mod, + state = State, + group = Group, + type = Type + } = Data, + IsDryRun = emqx_resource:is_dry_run(ResId), NewAddedChannelsMap = case KeepInChannelMap of true -> @@ -883,7 +907,7 @@ remove_channels_in_list([ChannelID | Rest], Data, KeepInChannelMap) -> _ = maybe_clear_alarm(IsDryRun, ChannelID), maps:remove(ChannelID, AddedChannelsMap) end, - case safe_call_remove_channel(Id, Data#data.mod, Data#data.state, ChannelID) of + case safe_call_remove_channel(ResId, Mod, State, ChannelID) of {ok, NewState} -> NewData = Data#data{ state = NewState, @@ -893,7 +917,9 @@ remove_channels_in_list([ChannelID | Rest], Data, KeepInChannelMap) -> {error, Reason} -> ?SLOG(log_level(IsDryRun), #{ msg => "remove_channel_failed", - id => Id, + resource_id => ResId, + group => Group, + type => Type, channel_id => ChannelID, reason => Reason }), @@ -997,7 +1023,12 @@ handle_remove_channel(From, ChannelId, Data) -> end. handle_remove_channel_exists(From, ChannelId, Data) -> - #data{id = Id, added_channels = AddedChannelsMap} = Data, + #data{ + id = Id, + group = Group, + type = Type, + added_channels = AddedChannelsMap + } = Data, case emqx_resource:call_remove_channel( Id, Data#data.mod, Data#data.state, ChannelId @@ -1014,7 +1045,9 @@ handle_remove_channel_exists(From, ChannelId, Data) -> IsDryRun = emqx_resource:is_dry_run(Id), ?SLOG(log_level(IsDryRun), #{ msg => "remove_channel_failed", - id => Id, + resource_id => Id, + group => Group, + type => Type, channel_id => ChannelId, reason => Reason }), @@ -1123,10 +1156,13 @@ continue_resource_health_check_connected(NewStatus, Data0) -> Actions = Replies ++ resource_health_check_actions(Data), {keep_state, Data, Actions}; _ -> - IsDryRun = emqx_resource:is_dry_run(Data0#data.id), + #data{id = ResId, group = Group, type = Type} = Data0, + IsDryRun = emqx_resource:is_dry_run(ResId), ?SLOG(log_level(IsDryRun), #{ msg => "health_check_failed", - id => Data0#data.id, + resource_id => ResId, + group => Group, + type => Type, status => NewStatus }), %% Note: works because, coincidentally, channel/resource status is a @@ -1633,6 +1669,8 @@ parse_health_check_result({error, Error}, Data) -> #{ msg => "health_check_exception", resource_id => Data#data.id, + type => Data#data.type, + group => Data#data.group, reason => Error } ), diff --git a/apps/emqx_resource/src/emqx_resource_manager_sup.erl b/apps/emqx_resource/src/emqx_resource_manager_sup.erl index 7af6eca81..c14b08f94 100644 --- a/apps/emqx_resource/src/emqx_resource_manager_sup.erl +++ b/apps/emqx_resource/src/emqx_resource_manager_sup.erl @@ -19,14 +19,16 @@ -include("emqx_resource.hrl"). --export([ensure_child/5, delete_child/1]). +-export([ensure_child/6, delete_child/1]). -export([start_link/0]). -export([init/1]). -ensure_child(ResId, Group, ResourceType, Config, Opts) -> - case supervisor:start_child(?MODULE, child_spec(ResId, Group, ResourceType, Config, Opts)) of +ensure_child(Type, ResId, Group, ResourceType, Config, Opts) -> + case + supervisor:start_child(?MODULE, child_spec(Type, ResId, Group, ResourceType, Config, Opts)) + of {error, Reason} -> %% This should not happen in production but it can be a huge time sink in %% development environments if the error is just silently ignored. @@ -55,13 +57,14 @@ init([]) -> SupFlags = #{strategy => one_for_one, intensity => 10, period => 10}, {ok, {SupFlags, ChildSpecs}}. -child_spec(ResId, Group, ResourceType, Config, Opts) -> +child_spec(Type, ResId, Group, ResourceType, Config, Opts) -> #{ id => ResId, - start => {emqx_resource_manager, start_link, [ResId, Group, ResourceType, Config, Opts]}, + start => + {emqx_resource_manager, start_link, [Type, ResId, Group, ResourceType, Config, Opts]}, restart => transient, %% never force kill a resource manager. - %% becasue otherwise it may lead to release leak, + %% because otherwise it may lead to release leak, %% resource_manager's terminate callback calls resource on_stop shutdown => infinity, type => worker, diff --git a/apps/emqx_resource/test/emqx_resource_SUITE.erl b/apps/emqx_resource/test/emqx_resource_SUITE.erl index af9abe95b..981c88edd 100644 --- a/apps/emqx_resource/test/emqx_resource_SUITE.erl +++ b/apps/emqx_resource/test/emqx_resource_SUITE.erl @@ -23,6 +23,7 @@ -include_lib("snabbkaffe/include/snabbkaffe.hrl"). -define(TEST_RESOURCE, emqx_connector_demo). +-define(TYPE, test). -define(ID, <<"id">>). -define(ID1, <<"id1">>). -define(DEFAULT_RESOURCE_GROUP, <<"default">>). @@ -90,6 +91,7 @@ t_create_remove(_) -> ?assertMatch( {error, _}, emqx_resource:check_and_create_local( + ?TYPE, ?ID, ?DEFAULT_RESOURCE_GROUP, ?TEST_RESOURCE, @@ -110,6 +112,7 @@ t_create_remove(_) -> ?assertMatch( {ok, _}, emqx_resource:recreate_local( + ?TYPE, ?ID, ?TEST_RESOURCE, #{name => test_resource}, @@ -135,6 +138,7 @@ t_create_remove_local(_) -> ?assertMatch( {error, _}, emqx_resource:check_and_create_local( + ?TYPE, ?ID, ?DEFAULT_RESOURCE_GROUP, ?TEST_RESOURCE, @@ -153,6 +157,7 @@ t_create_remove_local(_) -> ), emqx_resource:recreate_local( + ?TYPE, ?ID, ?TEST_RESOURCE, #{name => test_resource}, @@ -166,6 +171,7 @@ t_create_remove_local(_) -> emqx_resource:set_resource_status_connecting(?ID), emqx_resource:recreate_local( + ?TYPE, ?ID, ?TEST_RESOURCE, #{name => test_resource}, @@ -937,6 +943,7 @@ t_stop_start(_) -> ?assertMatch( {error, _}, emqx_resource:check_and_create_local( + ?TYPE, ?ID, ?DEFAULT_RESOURCE_GROUP, ?TEST_RESOURCE, @@ -947,6 +954,7 @@ t_stop_start(_) -> ?assertMatch( {ok, _}, emqx_resource:check_and_create_local( + ?TYPE, ?ID, ?DEFAULT_RESOURCE_GROUP, ?TEST_RESOURCE, @@ -964,6 +972,7 @@ t_stop_start(_) -> ?assertMatch( {ok, _}, emqx_resource:check_and_recreate_local( + ?TYPE, ?ID, ?TEST_RESOURCE, #{<<"name">> => <<"test_resource">>}, @@ -1013,6 +1022,7 @@ t_stop_start_local(_) -> ?assertMatch( {error, _}, emqx_resource:check_and_create_local( + ?TYPE, ?ID, ?DEFAULT_RESOURCE_GROUP, ?TEST_RESOURCE, @@ -1023,6 +1033,7 @@ t_stop_start_local(_) -> ?assertMatch( {ok, _}, emqx_resource:check_and_create_local( + ?TYPE, ?ID, ?DEFAULT_RESOURCE_GROUP, ?TEST_RESOURCE, @@ -1033,6 +1044,7 @@ t_stop_start_local(_) -> ?assertMatch( {ok, _}, emqx_resource:check_and_recreate_local( + ?TYPE, ?ID, ?TEST_RESOURCE, #{<<"name">> => <<"test_resource">>}, @@ -1108,6 +1120,7 @@ create_dry_run_local_succ() -> ?assertEqual( ok, emqx_resource:create_dry_run_local( + test, ?TEST_RESOURCE, #{name => test_resource, register => true} ) @@ -1118,6 +1131,7 @@ t_create_dry_run_local_failed(_) -> ct:timetrap({seconds, 120}), ct:pal("creating with creation error"), Res1 = emqx_resource:create_dry_run_local( + test, ?TEST_RESOURCE, #{create_error => true} ), @@ -1125,6 +1139,7 @@ t_create_dry_run_local_failed(_) -> ct:pal("creating with health check error"), Res2 = emqx_resource:create_dry_run_local( + test, ?TEST_RESOURCE, #{name => test_resource, health_check_error => true} ), @@ -1132,6 +1147,7 @@ t_create_dry_run_local_failed(_) -> ct:pal("creating with stop error"), Res3 = emqx_resource:create_dry_run_local( + test, ?TEST_RESOURCE, #{name => test_resource, stop_error => true} ), @@ -3490,10 +3506,10 @@ gauge_metric_set_fns() -> ]. create(Id, Group, Type, Config) -> - emqx_resource:create_local(Id, Group, Type, Config). + emqx_resource:create_local(test, Id, Group, Type, Config). create(Id, Group, Type, Config, Opts) -> - emqx_resource:create_local(Id, Group, Type, Config, Opts). + emqx_resource:create_local(test, Id, Group, Type, Config, Opts). log_consistency_prop() -> {"check state and cache consistency", fun ?MODULE:log_consistency_prop/1}. From e148d903e8bcdc0534a44c6f8435d70a7117cc2d Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Tue, 16 Jul 2024 15:51:10 +0800 Subject: [PATCH 014/131] feat: log resource_id --- .../src/emqx_bridge_mqtt.app.src | 2 +- .../src/emqx_bridge_mqtt_connector.erl | 17 ++++++++++------- apps/emqx_resource/include/emqx_resource.hrl | 1 + apps/emqx_resource/src/emqx_resource.erl | 2 +- apps/emqx_resource/src/emqx_resource_pool.erl | 11 +++++++---- 5 files changed, 20 insertions(+), 13 deletions(-) diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src index 26b8967f0..d43ec5591 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_bridge_mqtt, [ {description, "EMQX MQTT Broker Bridge"}, - {vsn, "0.2.2"}, + {vsn, "0.2.3"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl index d507d11b8..9c2506bab 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl @@ -207,7 +207,7 @@ start_mqtt_clients(ResourceId, Conf) -> start_mqtt_clients(ResourceId, Conf, ClientOpts). start_mqtt_clients(ResourceId, StartConf, ClientOpts) -> - PoolName = <>, + PoolName = ResourceId, #{ pool_size := PoolSize } = StartConf, @@ -227,7 +227,7 @@ start_mqtt_clients(ResourceId, StartConf, ClientOpts) -> on_stop(ResourceId, State) -> ?SLOG(info, #{ msg => "stopping_mqtt_connector", - connector => ResourceId + resource_id => ResourceId }), %% on_stop can be called with State = undefined StateMap = @@ -271,7 +271,7 @@ on_query( on_query(ResourceId, {_ChannelId, Msg}, #{}) -> ?SLOG(error, #{ msg => "forwarding_unavailable", - connector => ResourceId, + resource_id => ResourceId, message => Msg, reason => "Egress is not configured" }). @@ -298,7 +298,7 @@ on_query_async( on_query_async(ResourceId, {_ChannelId, Msg}, _Callback, #{}) -> ?SLOG(error, #{ msg => "forwarding_unavailable", - connector => ResourceId, + resource_id => ResourceId, message => Msg, reason => "Egress is not configured" }). @@ -463,8 +463,10 @@ connect(Options) -> {ok, Pid} -> connect(Pid, Name); {error, Reason} = Error -> - ?SLOG(error, #{ + IsDryRun = emqx_resource:is_dry_run(Name), + ?SLOG(?LOG_LEVEL(IsDryRun), #{ msg => "client_start_failed", + resource_id => Name, config => emqx_utils:redact(ClientOpts), reason => Reason }), @@ -508,10 +510,11 @@ connect(Pid, Name) -> {ok, _Props} -> {ok, Pid}; {error, Reason} = Error -> - ?SLOG(warning, #{ + IsDryRun = emqx_resource:is_dry_run(Name), + ?SLOG(?LOG_LEVEL(IsDryRun), #{ msg => "ingress_client_connect_failed", reason => Reason, - name => Name + resource_id => Name }), _ = catch emqtt:stop(Pid), Error diff --git a/apps/emqx_resource/include/emqx_resource.hrl b/apps/emqx_resource/include/emqx_resource.hrl index aa86fe239..3d141bca1 100644 --- a/apps/emqx_resource/include/emqx_resource.hrl +++ b/apps/emqx_resource/include/emqx_resource.hrl @@ -159,5 +159,6 @@ %% See `hocon_tconf` -define(TEST_ID_PREFIX, "t_probe_"). -define(RES_METRICS, resource_metrics). +-define(LOG_LEVEL(_L_), case _L_ of true -> info; false -> warning end). -define(RESOURCE_ALLOCATION_TAB, emqx_resource_allocations). diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl index 71ccaa696..76493b99f 100644 --- a/apps/emqx_resource/src/emqx_resource.erl +++ b/apps/emqx_resource/src/emqx_resource.erl @@ -340,7 +340,7 @@ remove_local(ResId) -> Error -> %% Only log, the ResId worker is always removed in manager's remove action. ?SLOG(warning, #{ - msg => "remove_local_resource_failed", + msg => "remove_resource_failed", error => Error, resource_id => ResId }), diff --git a/apps/emqx_resource/src/emqx_resource_pool.erl b/apps/emqx_resource/src/emqx_resource_pool.erl index 47e7ed7ff..516c483e3 100644 --- a/apps/emqx_resource/src/emqx_resource_pool.erl +++ b/apps/emqx_resource/src/emqx_resource_pool.erl @@ -26,6 +26,7 @@ ]). -include_lib("emqx/include/logger.hrl"). +-include("emqx_resource.hrl"). -ifndef(TEST). -define(HEALTH_CHECK_TIMEOUT, 15000). @@ -44,9 +45,10 @@ start(Name, Mod, Options) -> start(Name, Mod, Options); {error, Reason} -> NReason = parse_reason(Reason), - ?SLOG(error, #{ + IsDryRun = emqx_resource:is_dry_run(Name), + ?SLOG(?LOG_LEVEL(IsDryRun), #{ msg => "start_ecpool_error", - pool_name => Name, + resource_id => Name, reason => NReason }), {error, {start_pool_failed, Name, NReason}} @@ -59,9 +61,10 @@ stop(Name) -> {error, not_found} -> ok; {error, Reason} -> - ?SLOG(error, #{ + IsDryRun = emqx_resource:is_dry_run(Name), + ?SLOG(?LOG_LEVEL(IsDryRun), #{ msg => "stop_ecpool_failed", - pool_name => Name, + resource_id => Name, reason => Reason }), error({stop_pool_failed, Name, Reason}) From f29988ed8ebbda70ab15d6c24afb7778c978a961 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 17 Jul 2024 09:14:58 +0800 Subject: [PATCH 015/131] chore: add tag to resouce log --- apps/emqx_resource/include/emqx_resource.hrl | 7 +- .../src/emqx_resource_manager.erl | 102 ++++++++++-------- 2 files changed, 64 insertions(+), 45 deletions(-) diff --git a/apps/emqx_resource/include/emqx_resource.hrl b/apps/emqx_resource/include/emqx_resource.hrl index 3d141bca1..f26f6699c 100644 --- a/apps/emqx_resource/include/emqx_resource.hrl +++ b/apps/emqx_resource/include/emqx_resource.hrl @@ -159,6 +159,11 @@ %% See `hocon_tconf` -define(TEST_ID_PREFIX, "t_probe_"). -define(RES_METRICS, resource_metrics). --define(LOG_LEVEL(_L_), case _L_ of true -> info; false -> warning end). +-define(LOG_LEVEL(_L_), + case _L_ of + true -> info; + false -> warning + end +). -define(RESOURCE_ALLOCATION_TAB, emqx_resource_allocations). diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl index 1c0b74edd..a742c4486 100644 --- a/apps/emqx_resource/src/emqx_resource_manager.erl +++ b/apps/emqx_resource/src/emqx_resource_manager.erl @@ -689,13 +689,12 @@ handle_event(EventType, EventData, State, Data) -> #{ msg => "ignore_all_other_events", resource_id => Data#data.id, - group => Data#data.group, - type => Data#data.type, event_type => EventType, event_data => EventData, state => State, data => emqx_utils:redact(Data) - } + }, + #{tag => tag(Data#data.group, Data#data.type)} ), keep_state_and_data. @@ -770,13 +769,15 @@ start_resource(Data, From) -> {next_state, ?state_connecting, update_state(UpdatedData2, Data), Actions}; {error, Reason} = Err -> IsDryRun = emqx_resource:is_dry_run(ResId), - ?SLOG(log_level(IsDryRun), #{ - msg => "start_resource_failed", - resource_id => ResId, - group => Group, - type => Type, - reason => Reason - }), + ?SLOG( + log_level(IsDryRun), + #{ + msg => "start_resource_failed", + resource_id => ResId, + reason => Reason + }, + #{tag => tag(Group, Type)} + ), _ = maybe_alarm(?status_disconnected, IsDryRun, ResId, Err, Data#data.error), %% Add channels and raise alarms NewData1 = channels_health_check(?status_disconnected, add_channels(Data)), @@ -836,14 +837,16 @@ add_channels_in_list([{ChannelID, ChannelConfig} | Rest], Data) -> add_channels_in_list(Rest, NewData); {error, Reason} = Error -> IsDryRun = emqx_resource:is_dry_run(ResId), - ?SLOG(log_level(IsDryRun), #{ - msg => "add_channel_failed", - resource_id => ResId, - type => Type, - group => Group, - channel_id => ChannelID, - reason => Reason - }), + ?SLOG( + log_level(IsDryRun), + #{ + msg => "add_channel_failed", + resource_id => ResId, + channel_id => ChannelID, + reason => Reason + }, + #{tag => tag(Group, Type)} + ), AddedChannelsMap = Data#data.added_channels, NewAddedChannelsMap = maps:put( ChannelID, @@ -915,14 +918,18 @@ remove_channels_in_list([ChannelID | Rest], Data, KeepInChannelMap) -> }, remove_channels_in_list(Rest, NewData, KeepInChannelMap); {error, Reason} -> - ?SLOG(log_level(IsDryRun), #{ - msg => "remove_channel_failed", - resource_id => ResId, - group => Group, - type => Type, - channel_id => ChannelID, - reason => Reason - }), + ?SLOG( + log_level(IsDryRun), + #{ + msg => "remove_channel_failed", + resource_id => ResId, + group => Group, + type => Type, + channel_id => ChannelID, + reason => Reason + }, + #{tag => tag(Group, Type)} + ), NewData = Data#data{ added_channels = NewAddedChannelsMap }, @@ -1043,14 +1050,16 @@ handle_remove_channel_exists(From, ChannelId, Data) -> {keep_state, update_state(UpdatedData, Data), [{reply, From, ok}]}; {error, Reason} = Error -> IsDryRun = emqx_resource:is_dry_run(Id), - ?SLOG(log_level(IsDryRun), #{ - msg => "remove_channel_failed", - resource_id => Id, - group => Group, - type => Type, - channel_id => ChannelId, - reason => Reason - }), + ?SLOG( + log_level(IsDryRun), + #{ + msg => "remove_channel_failed", + resource_id => Id, + channel_id => ChannelId, + reason => Reason + }, + #{tag => tag(Group, Type)} + ), {keep_state_and_data, [{reply, From, Error}]} end. @@ -1158,13 +1167,15 @@ continue_resource_health_check_connected(NewStatus, Data0) -> _ -> #data{id = ResId, group = Group, type = Type} = Data0, IsDryRun = emqx_resource:is_dry_run(ResId), - ?SLOG(log_level(IsDryRun), #{ - msg => "health_check_failed", - resource_id => ResId, - group => Group, - type => Type, - status => NewStatus - }), + ?SLOG( + log_level(IsDryRun), + #{ + msg => "health_check_failed", + resource_id => ResId, + status => NewStatus + }, + #{tag => tag(Group, Type)} + ), %% Note: works because, coincidentally, channel/resource status is a %% subset of resource manager state... But there should be a conversion %% between the two here, as resource manager also has `stopped', which is @@ -1669,10 +1680,9 @@ parse_health_check_result({error, Error}, Data) -> #{ msg => "health_check_exception", resource_id => Data#data.id, - type => Data#data.type, - group => Data#data.group, reason => Error - } + }, + #{tag => tag(Data#data.group, Data#data.type)} ), {?status_disconnected, Data#data.state, {error, Error}}. @@ -1835,3 +1845,7 @@ state_to_status(?state_disconnected) -> ?status_disconnected. log_level(true) -> info; log_level(false) -> warning. + +tag(Group, Type) -> + Str = emqx_utils_conv:str(Group) ++ "/" ++ emqx_utils_conv:str(Type), + string:uppercase(Str). From 2bb062d3a377767e407d21ecb4f4c9ffc06ef0bf Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 17 Jul 2024 12:45:37 +0800 Subject: [PATCH 016/131] chore: create_local/5 for emqx_resource_proto_v1 --- apps/emqx_auth_jwt/src/emqx_authn_jwt.erl | 3 +- .../test/emqx_authn_postgresql_SUITE.erl | 8 +++- apps/emqx_resource/src/emqx_resource.erl | 45 +++++++++++-------- apps/emqx_resource/src/emqx_resource_pool.erl | 32 ++++++++----- .../test/emqx_resource_SUITE.erl | 2 +- 5 files changed, 56 insertions(+), 34 deletions(-) diff --git a/apps/emqx_auth_jwt/src/emqx_authn_jwt.erl b/apps/emqx_auth_jwt/src/emqx_authn_jwt.erl index e01121a01..2f8ebec36 100644 --- a/apps/emqx_auth_jwt/src/emqx_authn_jwt.erl +++ b/apps/emqx_auth_jwt/src/emqx_authn_jwt.erl @@ -187,7 +187,8 @@ do_create( ResourceId, ?AUTHN_RESOURCE_GROUP, emqx_authn_jwks_connector, - connector_opts(Config) + connector_opts(Config), + #{} ), {ok, #{ jwk_resource => ResourceId, diff --git a/apps/emqx_auth_postgresql/test/emqx_authn_postgresql_SUITE.erl b/apps/emqx_auth_postgresql/test/emqx_authn_postgresql_SUITE.erl index e899edbd9..dde5a184a 100644 --- a/apps/emqx_auth_postgresql/test/emqx_authn_postgresql_SUITE.erl +++ b/apps/emqx_auth_postgresql/test/emqx_authn_postgresql_SUITE.erl @@ -199,9 +199,13 @@ test_user_auth(#{ t_authenticate_disabled_prepared_statements(_Config) -> ResConfig = maps:merge(pgsql_config(), #{disable_prepared_statements => true}), - {ok, _} = emqx_resource:recreate_local(postgresql, ?PGSQL_RESOURCE, emqx_postgresql, ResConfig), + {ok, _} = emqx_resource:recreate_local( + postgresql, ?PGSQL_RESOURCE, emqx_postgresql, ResConfig, #{} + ), on_exit(fun() -> - emqx_resource:recreate_local(postgresql, ?PGSQL_RESOURCE, emqx_postgresql, pgsql_config()) + emqx_resource:recreate_local( + postgresql, ?PGSQL_RESOURCE, emqx_postgresql, pgsql_config(), #{} + ) end), ok = lists:foreach( fun(Sample0) -> diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl index 76493b99f..e0a2e9343 100644 --- a/apps/emqx_resource/src/emqx_resource.erl +++ b/apps/emqx_resource/src/emqx_resource.erl @@ -39,12 +39,10 @@ -export([ %% store the config and start the instance - create_local/5, create_local/6, create_dry_run_local/3, create_dry_run_local/4, create_dry_run_local/5, - recreate_local/4, recreate_local/5, %% remove the config and stop the instance remove_local/1, @@ -142,6 +140,9 @@ -export([is_dry_run/1]). +%% For emqx_resource_proto_v1 rpc only +-export([create_local/5, recreate_local/4, create_dry_run_local/2]). + -export_type([ query_mode/0, resource_id/0, @@ -281,12 +282,6 @@ is_resource_mod(Module) -> %% ================================================================================= %% APIs for resource instances %% ================================================================================= - --spec create_local(type(), resource_id(), resource_group(), resource_type(), resource_config()) -> - {ok, resource_data() | 'already_created'} | {error, Reason :: term()}. -create_local(Type, ResId, Group, ResourceType, Config) -> - create_local(Type, ResId, Group, ResourceType, Config, #{}). - -spec create_local( type(), resource_id(), @@ -320,11 +315,6 @@ when create_dry_run_local(Type, ResId, ResourceType, Config, OnReadyCallback) -> emqx_resource_manager:create_dry_run(Type, ResId, ResourceType, Config, OnReadyCallback). --spec recreate_local(type(), resource_id(), resource_type(), resource_config()) -> - {ok, resource_data()} | {error, Reason :: term()}. -recreate_local(Type, ResId, ResourceType, Config) -> - recreate_local(Type, ResId, ResourceType, Config, #{}). - -spec recreate_local(type(), resource_id(), resource_type(), resource_config(), creation_opts()) -> {ok, resource_data()} | {error, Reason :: term()}. recreate_local(Type, ResId, ResourceType, Config, Opts) -> @@ -339,11 +329,15 @@ remove_local(ResId) -> ok; Error -> %% Only log, the ResId worker is always removed in manager's remove action. - ?SLOG(warning, #{ - msg => "remove_resource_failed", - error => Error, - resource_id => ResId - }), + ?SLOG( + warning, + #{ + msg => "remove_resource_failed", + error => Error, + resource_id => ResId + }, + #{tag => ?TAG} + ), ok end. @@ -815,3 +809,18 @@ validate_name(Name, Opts) -> -spec invalid_data(binary()) -> no_return(). invalid_data(Reason) -> throw(#{kind => validation_error, reason => Reason}). + +%% Those functions is only used in the emqx_resource_proto_v1 +%% for versions that are less than version 5.6.0. +%% begin +-spec create_local( + resource_id(), resource_group(), resource_type(), resource_config(), creation_opts() +) -> + {ok, resource_data() | 'already_created'} | {error, Reason :: term()}. +create_local(ResId, Group, ResourceType, Config, Opts) -> + create_local(deprecated, ResId, Group, ResourceType, Config, Opts). +create_dry_run_local(ResourceType, Config) -> + create_dry_run_local(deprecated, ResourceType, Config). +recreate_local(ResId, ResourceType, Config, Opts) -> + recreate_local(deprecated, ResId, ResourceType, Config, Opts). +%% end diff --git a/apps/emqx_resource/src/emqx_resource_pool.erl b/apps/emqx_resource/src/emqx_resource_pool.erl index 516c483e3..ba286e35c 100644 --- a/apps/emqx_resource/src/emqx_resource_pool.erl +++ b/apps/emqx_resource/src/emqx_resource_pool.erl @@ -38,7 +38,7 @@ start(Name, Mod, Options) -> case ecpool:start_sup_pool(Name, Mod, Options) of {ok, _} -> - ?SLOG(info, #{msg => "start_ecpool_ok", pool_name => Name}), + ?SLOG(info, #{msg => "start_ecpool_ok", pool_name => Name}, #{tag => ?TAG}), ok; {error, {already_started, _Pid}} -> stop(Name), @@ -46,27 +46,35 @@ start(Name, Mod, Options) -> {error, Reason} -> NReason = parse_reason(Reason), IsDryRun = emqx_resource:is_dry_run(Name), - ?SLOG(?LOG_LEVEL(IsDryRun), #{ - msg => "start_ecpool_error", - resource_id => Name, - reason => NReason - }), + ?SLOG( + ?LOG_LEVEL(IsDryRun), + #{ + msg => "start_ecpool_error", + resource_id => Name, + reason => NReason + }, + #{tag => ?TAG} + ), {error, {start_pool_failed, Name, NReason}} end. stop(Name) -> case ecpool:stop_sup_pool(Name) of ok -> - ?SLOG(info, #{msg => "stop_ecpool_ok", pool_name => Name}); + ?SLOG(info, #{msg => "stop_ecpool_ok", pool_name => Name}, #{tag => ?TAG}); {error, not_found} -> ok; {error, Reason} -> IsDryRun = emqx_resource:is_dry_run(Name), - ?SLOG(?LOG_LEVEL(IsDryRun), #{ - msg => "stop_ecpool_failed", - resource_id => Name, - reason => Reason - }), + ?SLOG( + ?LOG_LEVEL(IsDryRun), + #{ + msg => "stop_ecpool_failed", + resource_id => Name, + reason => Reason + }, + #{tag => ?TAG} + ), error({stop_pool_failed, Name, Reason}) end. diff --git a/apps/emqx_resource/test/emqx_resource_SUITE.erl b/apps/emqx_resource/test/emqx_resource_SUITE.erl index 981c88edd..8e16ec26a 100644 --- a/apps/emqx_resource/test/emqx_resource_SUITE.erl +++ b/apps/emqx_resource/test/emqx_resource_SUITE.erl @@ -3506,7 +3506,7 @@ gauge_metric_set_fns() -> ]. create(Id, Group, Type, Config) -> - emqx_resource:create_local(test, Id, Group, Type, Config). + emqx_resource:create_local(test, Id, Group, Type, Config, #{}). create(Id, Group, Type, Config, Opts) -> emqx_resource:create_local(test, Id, Group, Type, Config, Opts). From 2a58a36e372bbedef63b01dfd7e6c51e132fd47d Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 17 Jul 2024 16:05:16 +0800 Subject: [PATCH 017/131] chore: add resource tag for log --- apps/emqx_resource/include/emqx_resource.hrl | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/emqx_resource/include/emqx_resource.hrl b/apps/emqx_resource/include/emqx_resource.hrl index f26f6699c..0c0cc8f6b 100644 --- a/apps/emqx_resource/include/emqx_resource.hrl +++ b/apps/emqx_resource/include/emqx_resource.hrl @@ -165,5 +165,6 @@ false -> warning end ). +-define(TAG, "RESOURCE"). -define(RESOURCE_ALLOCATION_TAB, emqx_resource_allocations). From e74a921d33a0dd637821eeeb889cfa3a6d298ea8 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 17 Jul 2024 16:18:53 +0800 Subject: [PATCH 018/131] chore: compile error --- .../src/emqx_resource_metrics.erl | 46 +++++++++++-------- 1 file changed, 27 insertions(+), 19 deletions(-) diff --git a/apps/emqx_resource/src/emqx_resource_metrics.erl b/apps/emqx_resource/src/emqx_resource_metrics.erl index 97a09b074..98a74dfad 100644 --- a/apps/emqx_resource/src/emqx_resource_metrics.erl +++ b/apps/emqx_resource/src/emqx_resource_metrics.erl @@ -17,6 +17,7 @@ -module(emqx_resource_metrics). -include_lib("emqx/include/logger.hrl"). +-include("emqx_resource.hrl"). -export([ events/0, @@ -74,7 +75,6 @@ success_get/1 ]). --define(RES_METRICS, resource_metrics). -define(TELEMETRY_PREFIX, emqx, resource). -spec events() -> [telemetry:event_name()]. @@ -127,15 +127,19 @@ handle_telemetry_event( %% We catch errors to avoid detaching the telemetry handler function. %% When restarting a resource while it's under load, there might be transient %% failures while the metrics are not yet created. - ?SLOG(warning, #{ - msg => "handle_resource_metrics_failed", - hint => "transient failures may occur when restarting a resource", - kind => Kind, - reason => Reason, - stacktrace => Stacktrace, - resource_id => ID, - event => Event - }), + ?SLOG( + warning, + #{ + msg => "handle_resource_metrics_failed", + hint => "transient failures may occur when restarting a resource", + kind => Kind, + reason => Reason, + stacktrace => Stacktrace, + resource_id => ID, + event => Event + }, + #{tag => ?TAG} + ), ok end; handle_telemetry_event( @@ -151,15 +155,19 @@ handle_telemetry_event( %% We catch errors to avoid detaching the telemetry handler function. %% When restarting a resource while it's under load, there might be transient %% failures while the metrics are not yet created. - ?SLOG(warning, #{ - msg => "handle_resource_metrics_failed", - hint => "transient failures may occur when restarting a resource", - kind => Kind, - reason => Reason, - stacktrace => Stacktrace, - resource_id => ID, - event => Event - }), + ?SLOG( + warning, + #{ + msg => "handle_resource_metrics_failed", + hint => "transient failures may occur when restarting a resource", + kind => Kind, + reason => Reason, + stacktrace => Stacktrace, + resource_id => ID, + event => Event + }, + #{tag => ?TAG} + ), ok end; handle_telemetry_event(_EventName, _Measurements, _Metadata, _HandlerConfig) -> From 878b21869264965f461195b3389287ebd4405b06 Mon Sep 17 00:00:00 2001 From: firest Date: Mon, 22 Jul 2024 23:09:19 +0800 Subject: [PATCH 019/131] feat(authn): added a HTTP backend for the authentication mechanism scram --- .../emqx_authn/emqx_authn_schema_SUITE.erl | 8 - .../emqx_auth_http/include/emqx_auth_http.hrl | 5 + .../emqx_auth_http/src/emqx_auth_http_app.erl | 2 + apps/emqx_auth_http/src/emqx_authn_http.erl | 7 + .../src/emqx_authn_http_schema.erl | 8 +- .../src/emqx_authn_scram_http.erl | 179 +++++++ .../src/emqx_authn_scram_http_schema.erl | 81 ++++ .../test/emqx_authn_scram_http_SUITE.erl | 438 ++++++++++++++++++ .../emqx_authn_scram_http_test_server.erl | 115 +++++ .../src/emqx_authn_scram_mnesia.erl | 69 +-- .../src/emqx_authn_scram_mnesia_schema.erl | 7 +- .../emqx_conf/src/emqx_conf_schema_inject.erl | 5 +- .../src/emqx_gateway_api_authn.erl | 5 +- apps/emqx_utils/src/emqx_utils_scram.erl | 81 ++++ 14 files changed, 929 insertions(+), 81 deletions(-) create mode 100644 apps/emqx_auth_http/src/emqx_authn_scram_http.erl create mode 100644 apps/emqx_auth_http/src/emqx_authn_scram_http_schema.erl create mode 100644 apps/emqx_auth_http/test/emqx_authn_scram_http_SUITE.erl create mode 100644 apps/emqx_auth_http/test/emqx_authn_scram_http_test_server.erl create mode 100644 apps/emqx_utils/src/emqx_utils_scram.erl diff --git a/apps/emqx_auth/test/emqx_authn/emqx_authn_schema_SUITE.erl b/apps/emqx_auth/test/emqx_authn/emqx_authn_schema_SUITE.erl index f2688fff9..46eb18b82 100644 --- a/apps/emqx_auth/test/emqx_authn/emqx_authn_schema_SUITE.erl +++ b/apps/emqx_auth/test/emqx_authn/emqx_authn_schema_SUITE.erl @@ -122,14 +122,6 @@ t_union_member_selector(_) -> }, check(BadMechanism) ), - BadCombination = Base#{<<"mechanism">> => <<"scram">>, <<"backend">> => <<"http">>}, - ?assertThrow( - #{ - reason := "unknown_mechanism", - expected := "password_based" - }, - check(BadCombination) - ), ok. t_http_auth_selector(_) -> diff --git a/apps/emqx_auth_http/include/emqx_auth_http.hrl b/apps/emqx_auth_http/include/emqx_auth_http.hrl index 9fc3b029e..c0bfa2177 100644 --- a/apps/emqx_auth_http/include/emqx_auth_http.hrl +++ b/apps/emqx_auth_http/include/emqx_auth_http.hrl @@ -22,8 +22,13 @@ -define(AUTHN_MECHANISM, password_based). -define(AUTHN_MECHANISM_BIN, <<"password_based">>). + +-define(AUTHN_MECHANISM_SCRAM, scram). +-define(AUTHN_MECHANISM_SCRAM_BIN, <<"scram">>). + -define(AUTHN_BACKEND, http). -define(AUTHN_BACKEND_BIN, <<"http">>). -define(AUTHN_TYPE, {?AUTHN_MECHANISM, ?AUTHN_BACKEND}). +-define(AUTHN_TYPE_SCRAM, {?AUTHN_MECHANISM_SCRAM, ?AUTHN_BACKEND}). -endif. diff --git a/apps/emqx_auth_http/src/emqx_auth_http_app.erl b/apps/emqx_auth_http/src/emqx_auth_http_app.erl index b97743b41..3d8ae0dad 100644 --- a/apps/emqx_auth_http/src/emqx_auth_http_app.erl +++ b/apps/emqx_auth_http/src/emqx_auth_http_app.erl @@ -25,10 +25,12 @@ start(_StartType, _StartArgs) -> ok = emqx_authz:register_source(?AUTHZ_TYPE, emqx_authz_http), ok = emqx_authn:register_provider(?AUTHN_TYPE, emqx_authn_http), + ok = emqx_authn:register_provider(?AUTHN_TYPE_SCRAM, emqx_authn_scram_http), {ok, Sup} = emqx_auth_http_sup:start_link(), {ok, Sup}. stop(_State) -> ok = emqx_authn:deregister_provider(?AUTHN_TYPE), + ok = emqx_authn:deregister_provider(?AUTHN_TYPE_SCRAM), ok = emqx_authz:unregister_source(?AUTHZ_TYPE), ok. diff --git a/apps/emqx_auth_http/src/emqx_authn_http.erl b/apps/emqx_auth_http/src/emqx_authn_http.erl index d9c5c5ed5..b294de24f 100644 --- a/apps/emqx_auth_http/src/emqx_authn_http.erl +++ b/apps/emqx_auth_http/src/emqx_authn_http.erl @@ -28,6 +28,13 @@ destroy/1 ]). +-export([ + with_validated_config/2, + generate_request/2, + request_for_log/2, + response_for_log/1 +]). + %%------------------------------------------------------------------------------ %% APIs %%------------------------------------------------------------------------------ diff --git a/apps/emqx_auth_http/src/emqx_authn_http_schema.erl b/apps/emqx_auth_http/src/emqx_authn_http_schema.erl index aff16b824..0167571c0 100644 --- a/apps/emqx_auth_http/src/emqx_authn_http_schema.erl +++ b/apps/emqx_auth_http/src/emqx_authn_http_schema.erl @@ -27,6 +27,8 @@ namespace/0 ]). +-export([url/1, headers/1, headers_no_content_type/1, request_timeout/1]). + -include("emqx_auth_http.hrl"). -include_lib("emqx_auth/include/emqx_authn.hrl"). -include_lib("hocon/include/hoconsc.hrl"). @@ -61,12 +63,6 @@ select_union_member( got => Else }) end; -select_union_member(#{<<"backend">> := ?AUTHN_BACKEND_BIN}) -> - throw(#{ - reason => "unknown_mechanism", - expected => "password_based", - got => undefined - }); select_union_member(_Value) -> undefined. diff --git a/apps/emqx_auth_http/src/emqx_authn_scram_http.erl b/apps/emqx_auth_http/src/emqx_authn_scram_http.erl new file mode 100644 index 000000000..0e6190b4b --- /dev/null +++ b/apps/emqx_auth_http/src/emqx_authn_scram_http.erl @@ -0,0 +1,179 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_authn_scram_http). + +-include_lib("emqx_auth/include/emqx_authn.hrl"). +-include_lib("emqx/include/logger.hrl"). + +-behaviour(emqx_authn_provider). + +-export([ + create/2, + update/2, + authenticate/2, + destroy/1 +]). + +-define(REQUIRED_USER_INFO_KEYS, [ + <<"stored_key">>, + <<"server_key">>, + <<"salt">> +]). + +-define(OPTIONAL_USER_INFO_KEYS, [ + <<"is_superuser">> +]). + +%%------------------------------------------------------------------------------ +%% APIs +%%------------------------------------------------------------------------------ + +create(_AuthenticatorID, Config) -> + create(Config). + +create(Config0) -> + emqx_authn_http:with_validated_config(Config0, fun(Config, State) -> + ResourceId = emqx_authn_utils:make_resource_id(?MODULE), + % {Config, State} = parse_config(Config0), + {ok, _Data} = emqx_authn_utils:create_resource( + ResourceId, + emqx_bridge_http_connector, + Config + ), + {ok, merge_scram_conf(Config, State#{resource_id => ResourceId})} + end). + +update(Config0, #{resource_id := ResourceId} = _State) -> + emqx_authn_http:with_validated_config(Config0, fun(Config, NState) -> + % {Config, NState} = parse_config(Config0), + case emqx_authn_utils:update_resource(emqx_bridge_http_connector, Config, ResourceId) of + {error, Reason} -> + error({load_config_error, Reason}); + {ok, _} -> + {ok, merge_scram_conf(Config, NState#{resource_id => ResourceId})} + end + end). + +authenticate( + #{ + auth_method := AuthMethod, + auth_data := AuthData, + auth_cache := AuthCache + } = Credential, + State +) -> + RetrieveFun = fun(Username) -> + retrieve(Username, Credential, State) + end, + OnErrFun = fun(Msg, Reason) -> + ?TRACE_AUTHN_PROVIDER(Msg, #{ + reason => Reason + }) + end, + emqx_utils_scram:authenticate(AuthMethod, AuthData, AuthCache, RetrieveFun, OnErrFun, State); +authenticate(_Credential, _State) -> + ignore. + +destroy(#{resource_id := ResourceId}) -> + _ = emqx_resource:remove_local(ResourceId), + ok. + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- + +retrieve( + Username, + Credential, + #{ + resource_id := ResourceId, + method := Method, + request_timeout := RequestTimeout + } = State +) -> + Request = emqx_authn_http:generate_request(Credential#{username := Username}, State), + Response = emqx_resource:simple_sync_query(ResourceId, {Method, Request, RequestTimeout}), + ?TRACE_AUTHN_PROVIDER("scram_http_response", #{ + request => emqx_authn_http:request_for_log(Credential, State), + response => emqx_authn_http:response_for_log(Response), + resource => ResourceId + }), + case Response of + {ok, 200, Headers, Body} -> + handle_response(Headers, Body); + {ok, _StatusCode, _Headers} -> + {error, bad_response}; + {ok, _StatusCode, _Headers, _Body} -> + {error, bad_response}; + {error, _Reason} = Error -> + Error + end. + +handle_response(Headers, Body) -> + ContentType = proplists:get_value(<<"content-type">>, Headers), + case safely_parse_body(ContentType, Body) of + {ok, NBody} -> + body_to_user_info(NBody); + {error, Reason} = Error -> + ?TRACE_AUTHN_PROVIDER( + error, + "parse_scram_http_response_failed", + #{content_type => ContentType, body => Body, reason => Reason} + ), + Error + end. + +body_to_user_info(Body) -> + Required0 = maps:with(?REQUIRED_USER_INFO_KEYS, Body), + case maps:size(Required0) =:= erlang:length(?REQUIRED_USER_INFO_KEYS) of + true -> + case safely_convert_hex(Required0) of + {ok, Required} -> + UserInfo0 = maps:merge(Required, maps:with(?OPTIONAL_USER_INFO_KEYS, Body)), + UserInfo1 = emqx_utils_maps:safe_atom_key_map(UserInfo0), + UserInfo = maps:merge(#{is_superuser => false}, UserInfo1), + {ok, UserInfo}; + Error -> + Error + end; + _ -> + ?TRACE_AUTHN_PROVIDER("bad_response_body", #{http_body => Body}), + {error, bad_response} + end. + +safely_parse_body(ContentType, Body) -> + try + parse_body(ContentType, Body) + catch + _Class:_Reason -> + {error, invalid_body} + end. + +safely_convert_hex(Required) -> + try + {ok, + maps:map( + fun(_Key, Hex) -> + binary:decode_hex(Hex) + end, + Required + )} + catch + _Class:Reason -> + {error, Reason} + end. + +parse_body(<<"application/json", _/binary>>, Body) -> + {ok, emqx_utils_json:decode(Body, [return_maps])}; +parse_body(<<"application/x-www-form-urlencoded", _/binary>>, Body) -> + Flags = ?REQUIRED_USER_INFO_KEYS ++ ?OPTIONAL_USER_INFO_KEYS, + RawMap = maps:from_list(cow_qs:parse_qs(Body)), + NBody = maps:with(Flags, RawMap), + {ok, NBody}; +parse_body(ContentType, _) -> + {error, {unsupported_content_type, ContentType}}. + +merge_scram_conf(Conf, State) -> + maps:merge(maps:with([algorithm, iteration_count], Conf), State). diff --git a/apps/emqx_auth_http/src/emqx_authn_scram_http_schema.erl b/apps/emqx_auth_http/src/emqx_authn_scram_http_schema.erl new file mode 100644 index 000000000..ca43fe3a6 --- /dev/null +++ b/apps/emqx_auth_http/src/emqx_authn_scram_http_schema.erl @@ -0,0 +1,81 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_authn_scram_http_schema). + +-behaviour(emqx_authn_schema). + +-export([ + fields/1, + validations/0, + desc/1, + refs/0, + select_union_member/1, + namespace/0 +]). + +-include("emqx_auth_http.hrl"). +-include_lib("emqx_auth/include/emqx_authn.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). + +namespace() -> "authn". + +refs() -> + [?R_REF(scram_http_get), ?R_REF(scram_http_post)]. + +select_union_member( + #{<<"mechanism">> := ?AUTHN_MECHANISM_SCRAM_BIN, <<"backend">> := ?AUTHN_BACKEND_BIN} = Value +) -> + case maps:get(<<"method">>, Value, undefined) of + <<"get">> -> + [?R_REF(scram_http_get)]; + <<"post">> -> + [?R_REF(scramm_http_post)]; + Else -> + throw(#{ + reason => "unknown_http_method", + expected => "get | post", + field_name => method, + got => Else + }) + end; +select_union_member(_Value) -> + undefined. + +fields(scram_http_get) -> + [ + {method, #{type => get, required => true, desc => ?DESC(emqx_authn_http_schema, method)}}, + {headers, fun emqx_authn_http_schema:headers_no_content_type/1} + ] ++ common_fields(); +fields(scram_http_post) -> + [ + {method, #{type => post, required => true, desc => ?DESC(emqx_authn_http_schema, method)}}, + {headers, fun emqx_authn_http_schema:headers/1} + ] ++ common_fields(). + +desc(scram_http_get) -> + ?DESC(emqx_authn_http_schema, get); +desc(scram_http_post) -> + ?DESC(emqx_authn_http_schema, post); +desc(_) -> + undefined. + +validations() -> + emqx_authn_http_schema:validations(). + +common_fields() -> + emqx_authn_schema:common_fields() ++ + [ + {mechanism, emqx_authn_schema:mechanism(?AUTHN_MECHANISM_SCRAM)}, + {backend, emqx_authn_schema:backend(?AUTHN_BACKEND)}, + {algorithm, fun emqx_authn_scram_mnesia_schema:algorithm/1}, + {iteration_count, fun emqx_authn_scram_mnesia_schema:iteration_count/1}, + {url, fun emqx_authn_http_schema:url/1}, + {body, + hoconsc:mk(typerefl:alias("map", map([{fuzzy, term(), binary()}])), #{ + required => false, desc => ?DESC(emqx_authn_http_schema, body) + })}, + {request_timeout, fun emqx_authn_http_schema:request_timeout/1} + ] ++ + proplists:delete(pool_type, emqx_bridge_http_connector:fields(config)). diff --git a/apps/emqx_auth_http/test/emqx_authn_scram_http_SUITE.erl b/apps/emqx_auth_http/test/emqx_authn_scram_http_SUITE.erl new file mode 100644 index 000000000..b00212cb1 --- /dev/null +++ b/apps/emqx_auth_http/test/emqx_authn_scram_http_SUITE.erl @@ -0,0 +1,438 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_authn_scram_http_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +-include_lib("emqx/include/emqx_mqtt.hrl"). +-include_lib("emqx_auth/include/emqx_authn.hrl"). + +-define(PATH, [authentication]). + +-define(HTTP_PORT, 34333). +-define(HTTP_PATH, "/user/[...]"). +-define(ALGORITHM, sha512). +-define(ALGORITHM_STR, <<"sha512">>). +-define(ITERATION_COUNT, 4096). + +-include_lib("emqx/include/emqx_placeholder.hrl"). + +all() -> + case emqx_release:edition() of + ce -> + []; + _ -> + emqx_common_test_helpers:all(?MODULE) + end. + +init_per_suite(Config) -> + Apps = emqx_cth_suite:start([cowboy, emqx, emqx_conf, emqx_auth, emqx_auth_http], #{ + work_dir => ?config(priv_dir, Config) + }), + + IdleTimeout = emqx_config:get([mqtt, idle_timeout]), + [{apps, Apps}, {idle_timeout, IdleTimeout} | Config]. + +end_per_suite(Config) -> + ok = emqx_config:put([mqtt, idle_timeout], ?config(idle_timeout, Config)), + emqx_authn_test_lib:delete_authenticators( + [authentication], + ?GLOBAL + ), + ok = emqx_cth_suite:stop(?config(apps, Config)), + ok. + +init_per_testcase(_Case, Config) -> + {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000), + emqx_authn_test_lib:delete_authenticators( + [authentication], + ?GLOBAL + ), + {ok, _} = emqx_authn_scram_http_test_server:start_link(?HTTP_PORT, ?HTTP_PATH), + Config. + +end_per_testcase(_Case, _Config) -> + ok = emqx_authn_scram_http_test_server:stop(). + +%%------------------------------------------------------------------------------ +%% Tests +%%------------------------------------------------------------------------------ + +t_create(_Config) -> + AuthConfig = raw_config(), + + {ok, _} = emqx:update_config( + ?PATH, + {create_authenticator, ?GLOBAL, AuthConfig} + ), + + {ok, [#{provider := emqx_authn_scram_http}]} = emqx_authn_chains:list_authenticators(?GLOBAL). + +t_create_invalid(_Config) -> + AuthConfig = raw_config(), + + InvalidConfigs = + [ + AuthConfig#{<<"headers">> => []}, + AuthConfig#{<<"method">> => <<"delete">>}, + AuthConfig#{<<"url">> => <<"localhost">>}, + AuthConfig#{<<"url">> => <<"http://foo.com/xxx#fragment">>}, + AuthConfig#{<<"url">> => <<"http://${foo}.com/xxx">>}, + AuthConfig#{<<"url">> => <<"//foo.com/xxx">>}, + AuthConfig#{<<"algorithm">> => <<"sha128">>} + ], + + lists:foreach( + fun(Config) -> + ct:pal("creating authenticator with invalid config: ~p", [Config]), + {error, _} = + try + emqx:update_config( + ?PATH, + {create_authenticator, ?GLOBAL, Config} + ) + catch + throw:Error -> + {error, Error} + end, + ?assertEqual( + {error, {not_found, {chain, ?GLOBAL}}}, + emqx_authn_chains:list_authenticators(?GLOBAL) + ) + end, + InvalidConfigs + ). + +t_authenticate(_Config) -> + Username = <<"u">>, + Password = <<"p">>, + + set_user_handler(Username, Password), + init_auth(), + + ok = emqx_config:put([mqtt, idle_timeout], 500), + + {ok, Pid} = emqx_authn_mqtt_test_client:start_link("127.0.0.1", 1883), + + ClientFirstMessage = esasl_scram:client_first_message(Username), + + ConnectPacket = ?CONNECT_PACKET( + #mqtt_packet_connect{ + proto_ver = ?MQTT_PROTO_V5, + properties = #{ + 'Authentication-Method' => <<"SCRAM-SHA-512">>, + 'Authentication-Data' => ClientFirstMessage + } + } + ), + + ok = emqx_authn_mqtt_test_client:send(Pid, ConnectPacket), + + %% Intentional sleep to trigger idle timeout for the connection not yet authenticated + ok = ct:sleep(1000), + + ?AUTH_PACKET( + ?RC_CONTINUE_AUTHENTICATION, + #{'Authentication-Data' := ServerFirstMessage} + ) = receive_packet(), + + {continue, ClientFinalMessage, ClientCache} = + esasl_scram:check_server_first_message( + ServerFirstMessage, + #{ + client_first_message => ClientFirstMessage, + password => Password, + algorithm => ?ALGORITHM + } + ), + + AuthContinuePacket = ?AUTH_PACKET( + ?RC_CONTINUE_AUTHENTICATION, + #{ + 'Authentication-Method' => <<"SCRAM-SHA-512">>, + 'Authentication-Data' => ClientFinalMessage + } + ), + + ok = emqx_authn_mqtt_test_client:send(Pid, AuthContinuePacket), + + ?CONNACK_PACKET( + ?RC_SUCCESS, + _, + #{'Authentication-Data' := ServerFinalMessage} + ) = receive_packet(), + + ok = esasl_scram:check_server_final_message( + ServerFinalMessage, ClientCache#{algorithm => ?ALGORITHM} + ). + +t_authenticate_bad_props(_Config) -> + Username = <<"u">>, + Password = <<"p">>, + + set_user_handler(Username, Password), + init_auth(), + + {ok, Pid} = emqx_authn_mqtt_test_client:start_link("127.0.0.1", 1883), + + ConnectPacket = ?CONNECT_PACKET( + #mqtt_packet_connect{ + proto_ver = ?MQTT_PROTO_V5, + properties = #{ + 'Authentication-Method' => <<"SCRAM-SHA-512">> + } + } + ), + + ok = emqx_authn_mqtt_test_client:send(Pid, ConnectPacket), + + ?CONNACK_PACKET(?RC_NOT_AUTHORIZED) = receive_packet(). + +t_authenticate_bad_username(_Config) -> + Username = <<"u">>, + Password = <<"p">>, + + set_user_handler(Username, Password), + init_auth(), + + {ok, Pid} = emqx_authn_mqtt_test_client:start_link("127.0.0.1", 1883), + + ClientFirstMessage = esasl_scram:client_first_message(<<"badusername">>), + + ConnectPacket = ?CONNECT_PACKET( + #mqtt_packet_connect{ + proto_ver = ?MQTT_PROTO_V5, + properties = #{ + 'Authentication-Method' => <<"SCRAM-SHA-512">>, + 'Authentication-Data' => ClientFirstMessage + } + } + ), + + ok = emqx_authn_mqtt_test_client:send(Pid, ConnectPacket), + + ?CONNACK_PACKET(?RC_NOT_AUTHORIZED) = receive_packet(). + +t_authenticate_bad_password(_Config) -> + Username = <<"u">>, + Password = <<"p">>, + + set_user_handler(Username, Password), + init_auth(), + + {ok, Pid} = emqx_authn_mqtt_test_client:start_link("127.0.0.1", 1883), + + ClientFirstMessage = esasl_scram:client_first_message(Username), + + ConnectPacket = ?CONNECT_PACKET( + #mqtt_packet_connect{ + proto_ver = ?MQTT_PROTO_V5, + properties = #{ + 'Authentication-Method' => <<"SCRAM-SHA-512">>, + 'Authentication-Data' => ClientFirstMessage + } + } + ), + + ok = emqx_authn_mqtt_test_client:send(Pid, ConnectPacket), + + ?AUTH_PACKET( + ?RC_CONTINUE_AUTHENTICATION, + #{'Authentication-Data' := ServerFirstMessage} + ) = receive_packet(), + + {continue, ClientFinalMessage, _ClientCache} = + esasl_scram:check_server_first_message( + ServerFirstMessage, + #{ + client_first_message => ClientFirstMessage, + password => <<"badpassword">>, + algorithm => ?ALGORITHM + } + ), + + AuthContinuePacket = ?AUTH_PACKET( + ?RC_CONTINUE_AUTHENTICATION, + #{ + 'Authentication-Method' => <<"SCRAM-SHA-512">>, + 'Authentication-Data' => ClientFinalMessage + } + ), + + ok = emqx_authn_mqtt_test_client:send(Pid, AuthContinuePacket), + + ?CONNACK_PACKET(?RC_NOT_AUTHORIZED) = receive_packet(). + +t_destroy(_Config) -> + Username = <<"u">>, + Password = <<"p">>, + + set_user_handler(Username, Password), + init_auth(), + + ok = emqx_config:put([mqtt, idle_timeout], 500), + + {ok, Pid} = emqx_authn_mqtt_test_client:start_link("127.0.0.1", 1883), + + ConnectPacket = ?CONNECT_PACKET( + #mqtt_packet_connect{ + proto_ver = ?MQTT_PROTO_V5, + properties = #{ + 'Authentication-Method' => <<"SCRAM-SHA-512">> + } + } + ), + + ok = emqx_authn_mqtt_test_client:send(Pid, ConnectPacket), + + ok = ct:sleep(1000), + + ?CONNACK_PACKET(?RC_NOT_AUTHORIZED) = receive_packet(), + + %% emqx_authn_mqtt_test_client:stop(Pid), + + emqx_authn_test_lib:delete_authenticators( + [authentication], + ?GLOBAL + ), + + {ok, Pid2} = emqx_authn_mqtt_test_client:start_link("127.0.0.1", 1883), + + ok = emqx_authn_mqtt_test_client:send(Pid2, ConnectPacket), + + ok = ct:sleep(1000), + + ?CONNACK_PACKET( + ?RC_SUCCESS, + _, + _ + ) = receive_packet(). + +t_is_superuser() -> + State = init_auth(), + ok = test_is_superuser(State, false), + ok = test_is_superuser(State, true), + ok = test_is_superuser(State, false). + +test_is_superuser(State, ExpectedIsSuperuser) -> + Username = <<"u">>, + Password = <<"p">>, + + set_user_handler(Username, Password, ExpectedIsSuperuser), + + ClientFirstMessage = esasl_scram:client_first_message(Username), + + {continue, ServerFirstMessage, ServerCache} = + emqx_authn_scram_http:authenticate( + #{ + auth_method => <<"SCRAM-SHA-512">>, + auth_data => ClientFirstMessage, + auth_cache => #{} + }, + State + ), + + {continue, ClientFinalMessage, ClientCache} = + esasl_scram:check_server_first_message( + ServerFirstMessage, + #{ + client_first_message => ClientFirstMessage, + password => Password, + algorithm => ?ALGORITHM + } + ), + + {ok, UserInfo1, ServerFinalMessage} = + emqx_authn_scram_http:authenticate( + #{ + auth_method => <<"SCRAM-SHA-512">>, + auth_data => ClientFinalMessage, + auth_cache => ServerCache + }, + State + ), + + ok = esasl_scram:check_server_final_message( + ServerFinalMessage, ClientCache#{algorithm => ?ALGORITHM} + ), + + ?assertMatch(#{is_superuser := ExpectedIsSuperuser}, UserInfo1). + +%%------------------------------------------------------------------------------ +%% Helpers +%%------------------------------------------------------------------------------ + +raw_config() -> + #{ + <<"mechanism">> => <<"scram">>, + <<"backend">> => <<"http">>, + <<"enable">> => <<"true">>, + <<"method">> => <<"get">>, + <<"url">> => <<"http://127.0.0.1:34333/user">>, + <<"body">> => #{<<"username">> => ?PH_USERNAME}, + <<"headers">> => #{<<"X-Test-Header">> => <<"Test Value">>}, + <<"algorithm">> => ?ALGORITHM_STR, + <<"iteration_count">> => ?ITERATION_COUNT + }. + +set_user_handler(Username, Password) -> + set_user_handler(Username, Password, false). +set_user_handler(Username, Password, IsSuperuser) -> + %% HTTP Server + Handler = fun(Req0, State) -> + #{ + username := Username + } = cowboy_req:match_qs([username], Req0), + + UserInfo = make_user_info(Password, ?ALGORITHM, ?ITERATION_COUNT, IsSuperuser), + Req = cowboy_req:reply( + 200, + #{<<"content-type">> => <<"application/json">>}, + emqx_utils_json:encode(UserInfo), + Req0 + ), + {ok, Req, State} + end, + ok = emqx_authn_scram_http_test_server:set_handler(Handler). + +init_auth() -> + init_auth(raw_config()). + +init_auth(Config) -> + {ok, _} = emqx:update_config( + ?PATH, + {create_authenticator, ?GLOBAL, Config} + ), + + {ok, [#{state := State}]} = emqx_authn_chains:list_authenticators(?GLOBAL), + State. + +make_user_info(Password, Algorithm, IterationCount, IsSuperuser) -> + {StoredKey, ServerKey, Salt} = esasl_scram:generate_authentication_info( + Password, + #{ + algorithm => Algorithm, + iteration_count => IterationCount + } + ), + #{ + stored_key => binary:encode_hex(StoredKey), + server_key => binary:encode_hex(ServerKey), + salt => binary:encode_hex(Salt), + is_superuser => IsSuperuser + }. + +receive_packet() -> + receive + {packet, Packet} -> + ct:pal("Delivered packet: ~p", [Packet]), + Packet + after 1000 -> + ct:fail("Deliver timeout") + end. diff --git a/apps/emqx_auth_http/test/emqx_authn_scram_http_test_server.erl b/apps/emqx_auth_http/test/emqx_authn_scram_http_test_server.erl new file mode 100644 index 000000000..5467df621 --- /dev/null +++ b/apps/emqx_auth_http/test/emqx_authn_scram_http_test_server.erl @@ -0,0 +1,115 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_authn_scram_http_test_server). + +-behaviour(supervisor). +-behaviour(cowboy_handler). + +% cowboy_server callbacks +-export([init/2]). + +% supervisor callbacks +-export([init/1]). + +% API +-export([ + start_link/2, + start_link/3, + stop/0, + set_handler/1 +]). + +%%------------------------------------------------------------------------------ +%% API +%%------------------------------------------------------------------------------ + +start_link(Port, Path) -> + start_link(Port, Path, false). + +start_link(Port, Path, SSLOpts) -> + supervisor:start_link({local, ?MODULE}, ?MODULE, [Port, Path, SSLOpts]). + +stop() -> + gen_server:stop(?MODULE). + +set_handler(F) when is_function(F, 2) -> + true = ets:insert(?MODULE, {handler, F}), + ok. + +%%------------------------------------------------------------------------------ +%% supervisor API +%%------------------------------------------------------------------------------ + +init([Port, Path, SSLOpts]) -> + Dispatch = cowboy_router:compile( + [ + {'_', [{Path, ?MODULE, []}]} + ] + ), + + ProtoOpts = #{env => #{dispatch => Dispatch}}, + + Tab = ets:new(?MODULE, [set, named_table, public]), + ets:insert(Tab, {handler, fun default_handler/2}), + + {Transport, TransOpts, CowboyModule} = transport_settings(Port, SSLOpts), + + ChildSpec = ranch:child_spec(?MODULE, Transport, TransOpts, CowboyModule, ProtoOpts), + + {ok, {#{}, [ChildSpec]}}. + +%%------------------------------------------------------------------------------ +%% cowboy_server API +%%------------------------------------------------------------------------------ + +init(Req, State) -> + [{handler, Handler}] = ets:lookup(?MODULE, handler), + Handler(Req, State). + +%%------------------------------------------------------------------------------ +%% Internal functions +%%------------------------------------------------------------------------------ + +transport_settings(Port, false) -> + TransOpts = #{ + socket_opts => [{port, Port}], + connection_type => supervisor + }, + {ranch_tcp, TransOpts, cowboy_clear}; +transport_settings(Port, SSLOpts) -> + TransOpts = #{ + socket_opts => [ + {port, Port}, + {next_protocols_advertised, [<<"h2">>, <<"http/1.1">>]}, + {alpn_preferred_protocols, [<<"h2">>, <<"http/1.1">>]} + | SSLOpts + ], + connection_type => supervisor + }, + {ranch_ssl, TransOpts, cowboy_tls}. + +default_handler(Req0, State) -> + Req = cowboy_req:reply( + 400, + #{<<"content-type">> => <<"text/plain">>}, + <<"">>, + Req0 + ), + {ok, Req, State}. + +make_user_info(Password, Algorithm, IterationCount) -> + {StoredKey, ServerKey, Salt} = esasl_scram:generate_authentication_info( + Password, + #{ + algorithm => Algorithm, + iteration_count => IterationCount + } + ), + #{ + stored_key => StoredKey, + server_key => ServerKey, + salt => Salt, + is_superuser => false + }. diff --git a/apps/emqx_auth_mnesia/src/emqx_authn_scram_mnesia.erl b/apps/emqx_auth_mnesia/src/emqx_authn_scram_mnesia.erl index 611469c5b..d59afea28 100644 --- a/apps/emqx_auth_mnesia/src/emqx_authn_scram_mnesia.erl +++ b/apps/emqx_auth_mnesia/src/emqx_authn_scram_mnesia.erl @@ -133,17 +133,15 @@ authenticate( }, State ) -> - case ensure_auth_method(AuthMethod, AuthData, State) of - true -> - case AuthCache of - #{next_step := client_final} -> - check_client_final_message(AuthData, AuthCache, State); - _ -> - check_client_first_message(AuthData, AuthCache, State) - end; - false -> - ignore - end; + RetrieveFun = fun(Username) -> + retrieve(Username, State) + end, + OnErrFun = fun(Msg, Reason) -> + ?TRACE_AUTHN_PROVIDER(Msg, #{ + reason => Reason + }) + end, + emqx_utils_scram:authenticate(AuthMethod, AuthData, AuthCache, RetrieveFun, OnErrFun, State); authenticate(_Credential, _State) -> ignore. @@ -257,55 +255,6 @@ run_fuzzy_filter( %% Internal functions %%------------------------------------------------------------------------------ -ensure_auth_method(_AuthMethod, undefined, _State) -> - false; -ensure_auth_method(<<"SCRAM-SHA-256">>, _AuthData, #{algorithm := sha256}) -> - true; -ensure_auth_method(<<"SCRAM-SHA-512">>, _AuthData, #{algorithm := sha512}) -> - true; -ensure_auth_method(_AuthMethod, _AuthData, _State) -> - false. - -check_client_first_message(Bin, _Cache, #{iteration_count := IterationCount} = State) -> - RetrieveFun = fun(Username) -> - retrieve(Username, State) - end, - case - esasl_scram:check_client_first_message( - Bin, - #{ - iteration_count => IterationCount, - retrieve => RetrieveFun - } - ) - of - {continue, ServerFirstMessage, Cache} -> - {continue, ServerFirstMessage, Cache}; - ignore -> - ignore; - {error, Reason} -> - ?TRACE_AUTHN_PROVIDER("check_client_first_message_error", #{ - reason => Reason - }), - {error, not_authorized} - end. - -check_client_final_message(Bin, #{is_superuser := IsSuperuser} = Cache, #{algorithm := Alg}) -> - case - esasl_scram:check_client_final_message( - Bin, - Cache#{algorithm => Alg} - ) - of - {ok, ServerFinalMessage} -> - {ok, #{is_superuser => IsSuperuser}, ServerFinalMessage}; - {error, Reason} -> - ?TRACE_AUTHN_PROVIDER("check_client_final_message_error", #{ - reason => Reason - }), - {error, not_authorized} - end. - user_info_record( #{ user_id := UserID, diff --git a/apps/emqx_auth_mnesia/src/emqx_authn_scram_mnesia_schema.erl b/apps/emqx_auth_mnesia/src/emqx_authn_scram_mnesia_schema.erl index 5d442cd57..dbad2118f 100644 --- a/apps/emqx_auth_mnesia/src/emqx_authn_scram_mnesia_schema.erl +++ b/apps/emqx_auth_mnesia/src/emqx_authn_scram_mnesia_schema.erl @@ -29,6 +29,8 @@ select_union_member/1 ]). +-export([algorithm/1, iteration_count/1]). + namespace() -> "authn". refs() -> @@ -38,11 +40,6 @@ select_union_member(#{ <<"mechanism">> := ?AUTHN_MECHANISM_SCRAM_BIN, <<"backend">> := ?AUTHN_BACKEND_BIN }) -> refs(); -select_union_member(#{<<"mechanism">> := ?AUTHN_MECHANISM_SCRAM_BIN}) -> - throw(#{ - reason => "unknown_backend", - expected => ?AUTHN_BACKEND - }); select_union_member(_) -> undefined. diff --git a/apps/emqx_conf/src/emqx_conf_schema_inject.erl b/apps/emqx_conf/src/emqx_conf_schema_inject.erl index baab7cfe8..5c155bbf5 100644 --- a/apps/emqx_conf/src/emqx_conf_schema_inject.erl +++ b/apps/emqx_conf/src/emqx_conf_schema_inject.erl @@ -49,7 +49,10 @@ authn_mods(ce) -> ]; authn_mods(ee) -> authn_mods(ce) ++ - [emqx_gcp_device_authn_schema]. + [ + emqx_gcp_device_authn_schema, + emqx_authn_scram_http_schema + ]. authz() -> [{emqx_authz_schema, authz_mods()}]. diff --git a/apps/emqx_gateway/src/emqx_gateway_api_authn.erl b/apps/emqx_gateway/src/emqx_gateway_api_authn.erl index 0ee16824d..0707c12aa 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api_authn.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api_authn.erl @@ -381,6 +381,9 @@ params_fuzzy_in_qs() -> schema_authn() -> emqx_dashboard_swagger:schema_with_examples( - emqx_authn_schema:authenticator_type_without([emqx_authn_scram_mnesia_schema]), + emqx_authn_schema:authenticator_type_without([ + emqx_authn_scram_mnesia_schema, + emqx_authn_scram_http_schema + ]), emqx_authn_api:authenticator_examples() ). diff --git a/apps/emqx_utils/src/emqx_utils_scram.erl b/apps/emqx_utils/src/emqx_utils_scram.erl new file mode 100644 index 000000000..9d0543703 --- /dev/null +++ b/apps/emqx_utils/src/emqx_utils_scram.erl @@ -0,0 +1,81 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021-2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_utils_scram). + +-export([authenticate/6]). + +%%------------------------------------------------------------------------------ +%% Authentication +%%------------------------------------------------------------------------------ +authenticate(AuthMethod, AuthData, AuthCache, RetrieveFun, OnErrFun, Conf) -> + case ensure_auth_method(AuthMethod, AuthData, Conf) of + true -> + case AuthCache of + #{next_step := client_final} -> + check_client_final_message(AuthData, AuthCache, Conf, OnErrFun); + _ -> + check_client_first_message(AuthData, AuthCache, Conf, RetrieveFun, OnErrFun) + end; + false -> + ignore + end. + +ensure_auth_method(_AuthMethod, undefined, _Conf) -> + false; +ensure_auth_method(<<"SCRAM-SHA-256">>, _AuthData, #{algorithm := sha256}) -> + true; +ensure_auth_method(<<"SCRAM-SHA-512">>, _AuthData, #{algorithm := sha512}) -> + true; +ensure_auth_method(_AuthMethod, _AuthData, _Conf) -> + false. + +check_client_first_message( + Bin, _Cache, #{iteration_count := IterationCount}, RetrieveFun, OnErrFun +) -> + case + esasl_scram:check_client_first_message( + Bin, + #{ + iteration_count => IterationCount, + retrieve => RetrieveFun + } + ) + of + {continue, ServerFirstMessage, Cache} -> + {continue, ServerFirstMessage, Cache}; + ignore -> + ignore; + {error, Reason} -> + OnErrFun("check_client_first_message_error", Reason), + {error, not_authorized} + end. + +check_client_final_message( + Bin, #{is_superuser := IsSuperuser} = Cache, #{algorithm := Alg}, OnErrFun +) -> + case + esasl_scram:check_client_final_message( + Bin, + Cache#{algorithm => Alg} + ) + of + {ok, ServerFinalMessage} -> + {ok, #{is_superuser => IsSuperuser}, ServerFinalMessage}; + {error, Reason} -> + OnErrFun("check_client_final_message_error", Reason), + {error, not_authorized} + end. From 7bf270a2423c1ff132ca17544244211cf6ebacc0 Mon Sep 17 00:00:00 2001 From: firest Date: Tue, 23 Jul 2024 07:20:51 +0800 Subject: [PATCH 020/131] chore: update changes --- changes/ee/feat-13504.en.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/ee/feat-13504.en.md diff --git a/changes/ee/feat-13504.en.md b/changes/ee/feat-13504.en.md new file mode 100644 index 000000000..c9b22f403 --- /dev/null +++ b/changes/ee/feat-13504.en.md @@ -0,0 +1 @@ +Added a HTTP backend for the authentication mechanism `scram`. From e7d07ea17c2e0d47b109af22bf0f85dbed05a307 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Tue, 23 Jul 2024 16:15:57 +0800 Subject: [PATCH 021/131] feat: add resource_type to emqx_resource behaviour --- .../src/emqx_authn/emqx_authn_utils.erl | 11 ++- .../src/emqx_authz/emqx_authz_utils.erl | 14 ++- apps/emqx_auth_http/src/emqx_authn_http.erl | 5 +- apps/emqx_auth_http/src/emqx_authz_http.erl | 6 +- .../src/emqx_authn_jwks_connector.erl | 3 + apps/emqx_auth_jwt/src/emqx_authn_jwt.erl | 1 - apps/emqx_auth_ldap/src/emqx_authn_ldap.erl | 4 +- apps/emqx_auth_ldap/src/emqx_authz_ldap.erl | 4 +- .../test/emqx_authn_ldap_SUITE.erl | 1 - .../test/emqx_authn_ldap_bind_SUITE.erl | 1 - .../test/emqx_authz_ldap_SUITE.erl | 1 - .../src/emqx_authn_mongodb.erl | 3 +- .../src/emqx_authz_mongodb.erl | 4 +- apps/emqx_auth_mysql/src/emqx_authn_mysql.erl | 4 +- apps/emqx_auth_mysql/src/emqx_authz_mysql.erl | 4 +- .../test/emqx_authn_mysql_SUITE.erl | 1 - .../test/emqx_authz_mysql_SUITE.erl | 1 - .../src/emqx_authn_postgresql.erl | 3 +- .../src/emqx_authz_postgresql.erl | 2 - .../test/emqx_authn_postgresql_SUITE.erl | 9 +- .../test/emqx_authz_postgresql_SUITE.erl | 1 - apps/emqx_auth_redis/src/emqx_authn_redis.erl | 3 +- apps/emqx_auth_redis/src/emqx_authz_redis.erl | 4 +- .../test/emqx_authn_redis_SUITE.erl | 1 - .../test/emqx_authz_redis_SUITE.erl | 1 - apps/emqx_bridge/src/emqx_bridge_resource.erl | 6 +- .../emqx_bridge/test/emqx_bridge_v2_SUITE.erl | 1 + .../test/emqx_bridge_v2_api_SUITE.erl | 1 + .../test/emqx_bridge_v2_dummy_connector.erl | 4 +- .../test/emqx_bridge_v2_test_connector.erl | 3 + .../src/emqx_bridge_cassandra.app.src | 2 +- .../src/emqx_bridge_cassandra_connector.erl | 2 + .../emqx_bridge_cassandra_connector_SUITE.erl | 1 - .../src/emqx_bridge_clickhouse.app.src | 2 +- .../src/emqx_bridge_clickhouse_connector.erl | 2 + ...emqx_bridge_clickhouse_connector_SUITE.erl | 2 - .../src/emqx_bridge_dynamo.app.src | 2 +- .../src/emqx_bridge_dynamo_connector.erl | 2 + .../emqx_bridge_es/src/emqx_bridge_es.app.src | 2 +- .../src/emqx_bridge_es_connector.erl | 3 + .../src/emqx_bridge_gcp_pubsub.app.src | 2 +- .../emqx_bridge_gcp_pubsub_impl_consumer.erl | 3 + .../emqx_bridge_gcp_pubsub_impl_producer.erl | 2 + .../src/emqx_bridge_greptimedb_connector.erl | 3 + ...emqx_bridge_greptimedb_connector_SUITE.erl | 1 - .../src/emqx_bridge_hstreamdb.app.src | 2 +- .../src/emqx_bridge_hstreamdb_connector.erl | 3 + .../src/emqx_bridge_http_connector.erl | 2 + .../src/emqx_bridge_influxdb.app.src | 2 +- .../src/emqx_bridge_influxdb_connector.erl | 3 + .../emqx_bridge_influxdb_connector_SUITE.erl | 2 - .../src/emqx_bridge_iotdb.app.src | 2 +- .../src/emqx_bridge_iotdb_connector.erl | 3 + .../src/emqx_bridge_kafka_impl_consumer.erl | 2 + .../src/emqx_bridge_kafka_impl_producer.erl | 3 + .../src/emqx_bridge_kinesis.app.src | 2 +- .../src/emqx_bridge_kinesis_impl_producer.erl | 2 + .../src/emqx_bridge_mongodb.app.src | 2 +- .../src/emqx_bridge_mongodb_connector.erl | 2 + .../src/emqx_bridge_mqtt_connector.erl | 3 + .../src/emqx_bridge_mysql.app.src | 2 +- .../src/emqx_bridge_mysql_connector.erl | 2 + .../src/emqx_bridge_opents.app.src | 2 +- .../src/emqx_bridge_opents_connector.erl | 3 + .../src/emqx_bridge_pulsar_connector.erl | 2 + .../src/emqx_bridge_rabbitmq.app.src | 2 +- .../src/emqx_bridge_rabbitmq_connector.erl | 2 + .../emqx_bridge_rabbitmq_connector_SUITE.erl | 1 - .../src/emqx_bridge_redis.app.src | 2 +- .../src/emqx_bridge_redis_connector.erl | 5 +- .../src/emqx_bridge_rocketmq.app.src | 2 +- .../src/emqx_bridge_rocketmq_connector.erl | 3 + .../src/emqx_bridge_s3_connector.erl | 3 + .../src/emqx_bridge_sqlserver.app.src | 2 +- .../src/emqx_bridge_sqlserver_connector.erl | 2 + .../src/emqx_bridge_syskeeper.app.src | 2 +- .../src/emqx_bridge_syskeeper_connector.erl | 2 + .../emqx_bridge_syskeeper_proxy_server.erl | 3 + .../src/emqx_bridge_tdengine.app.src | 2 +- .../src/emqx_bridge_tdengine_connector.erl | 2 + .../src/emqx_connector_resource.erl | 6 +- .../test/emqx_connector_SUITE.erl | 5 ++ .../test/emqx_connector_api_SUITE.erl | 1 + .../test/emqx_connector_dummy_impl.erl | 3 + .../src/emqx_dashboard_sso_manager.erl | 3 +- apps/emqx_ldap/src/emqx_ldap.app.src | 2 +- apps/emqx_ldap/src/emqx_ldap.erl | 3 + apps/emqx_ldap/test/emqx_ldap_SUITE.erl | 2 - apps/emqx_mongodb/src/emqx_mongodb.app.src | 2 +- apps/emqx_mongodb/src/emqx_mongodb.erl | 2 + apps/emqx_mongodb/test/emqx_mongodb_SUITE.erl | 1 - apps/emqx_mysql/src/emqx_mysql.app.src | 2 +- apps/emqx_mysql/src/emqx_mysql.erl | 3 + apps/emqx_mysql/test/emqx_mysql_SUITE.erl | 1 - apps/emqx_oracle/src/emqx_oracle.app.src | 2 +- apps/emqx_oracle/src/emqx_oracle.erl | 3 + .../src/emqx_postgresql.app.src | 2 +- apps/emqx_postgresql/src/emqx_postgresql.erl | 3 + .../test/emqx_postgresql_SUITE.erl | 1 - apps/emqx_redis/src/emqx_redis.erl | 3 + apps/emqx_redis/test/emqx_redis_SUITE.erl | 1 - apps/emqx_resource/include/emqx_resource.hrl | 4 +- apps/emqx_resource/src/emqx_resource.erl | 88 ++++++++----------- .../src/emqx_resource_manager.erl | 53 +++++------ .../src/emqx_resource_manager_sup.erl | 12 ++- .../src/proto/emqx_resource_proto_v1.erl | 6 +- .../test/emqx_connector_demo.erl | 3 + .../test/emqx_resource_SUITE.erl | 20 +---- .../test/emqx_rule_engine_test_connector.erl | 3 + 109 files changed, 248 insertions(+), 211 deletions(-) diff --git a/apps/emqx_auth/src/emqx_authn/emqx_authn_utils.erl b/apps/emqx_auth/src/emqx_authn/emqx_authn_utils.erl index e81145f2c..a08ac260c 100644 --- a/apps/emqx_auth/src/emqx_authn/emqx_authn_utils.erl +++ b/apps/emqx_auth/src/emqx_authn/emqx_authn_utils.erl @@ -21,8 +21,8 @@ -include_lib("snabbkaffe/include/trace.hrl"). -export([ - create_resource/4, - update_resource/4, + create_resource/3, + update_resource/3, check_password_from_selected_map/3, parse_deep/1, parse_str/1, @@ -66,9 +66,8 @@ %% APIs %%-------------------------------------------------------------------- -create_resource(Type, ResourceId, Module, Config) -> +create_resource(ResourceId, Module, Config) -> Result = emqx_resource:create_local( - Type, ResourceId, ?AUTHN_RESOURCE_GROUP, Module, @@ -77,9 +76,9 @@ create_resource(Type, ResourceId, Module, Config) -> ), start_resource_if_enabled(Result, ResourceId, Config). -update_resource(Type, Module, Config, ResourceId) -> +update_resource(Module, Config, ResourceId) -> Result = emqx_resource:recreate_local( - Type, ResourceId, Module, Config, ?DEFAULT_RESOURCE_OPTS + ResourceId, Module, Config, ?DEFAULT_RESOURCE_OPTS ), start_resource_if_enabled(Result, ResourceId, Config). diff --git a/apps/emqx_auth/src/emqx_authz/emqx_authz_utils.erl b/apps/emqx_auth/src/emqx_authz/emqx_authz_utils.erl index 533f982c1..e4343b6fa 100644 --- a/apps/emqx_auth/src/emqx_authz/emqx_authz_utils.erl +++ b/apps/emqx_auth/src/emqx_authz/emqx_authz_utils.erl @@ -25,9 +25,9 @@ -export([ cleanup_resources/0, make_resource_id/1, + create_resource/2, create_resource/3, - create_resource/4, - update_resource/3, + update_resource/2, remove_resource/1, update_config/2, parse_deep/2, @@ -57,13 +57,12 @@ %% APIs %%-------------------------------------------------------------------- -create_resource(Type, Module, Config) -> +create_resource(Module, Config) -> ResourceId = make_resource_id(Module), - create_resource(Type, ResourceId, Module, Config). + create_resource(ResourceId, Module, Config). -create_resource(Type, ResourceId, Module, Config) -> +create_resource(ResourceId, Module, Config) -> Result = emqx_resource:create_local( - Type, ResourceId, ?AUTHZ_RESOURCE_GROUP, Module, @@ -72,11 +71,10 @@ create_resource(Type, ResourceId, Module, Config) -> ), start_resource_if_enabled(Result, ResourceId, Config). -update_resource(Type, Module, #{annotations := #{id := ResourceId}} = Config) -> +update_resource(Module, #{annotations := #{id := ResourceId}} = Config) -> Result = case emqx_resource:recreate_local( - Type, ResourceId, Module, Config, diff --git a/apps/emqx_auth_http/src/emqx_authn_http.erl b/apps/emqx_auth_http/src/emqx_authn_http.erl index ed151428f..d9c5c5ed5 100644 --- a/apps/emqx_auth_http/src/emqx_authn_http.erl +++ b/apps/emqx_auth_http/src/emqx_authn_http.erl @@ -40,7 +40,6 @@ create(Config0) -> ResourceId = emqx_authn_utils:make_resource_id(?MODULE), % {Config, State} = parse_config(Config0), {ok, _Data} = emqx_authn_utils:create_resource( - http, ResourceId, emqx_bridge_http_connector, Config @@ -51,9 +50,7 @@ create(Config0) -> update(Config0, #{resource_id := ResourceId} = _State) -> with_validated_config(Config0, fun(Config, NState) -> % {Config, NState} = parse_config(Config0), - case - emqx_authn_utils:update_resource(http, emqx_bridge_http_connector, Config, ResourceId) - of + case emqx_authn_utils:update_resource(emqx_bridge_http_connector, Config, ResourceId) of {error, Reason} -> error({load_config_error, Reason}); {ok, _} -> diff --git a/apps/emqx_auth_http/src/emqx_authz_http.erl b/apps/emqx_auth_http/src/emqx_authz_http.erl index 2a1a2638b..a858ec7d7 100644 --- a/apps/emqx_auth_http/src/emqx_authz_http.erl +++ b/apps/emqx_auth_http/src/emqx_authz_http.erl @@ -67,13 +67,15 @@ create(Config) -> NConfig = parse_config(Config), ResourceId = emqx_authn_utils:make_resource_id(?MODULE), {ok, _Data} = emqx_authz_utils:create_resource( - http, ResourceId, emqx_bridge_http_connector, NConfig + ResourceId, + emqx_bridge_http_connector, + NConfig ), NConfig#{annotations => #{id => ResourceId}}. update(Config) -> NConfig = parse_config(Config), - case emqx_authz_utils:update_resource(http, emqx_bridge_http_connector, NConfig) of + case emqx_authz_utils:update_resource(emqx_bridge_http_connector, NConfig) of {error, Reason} -> error({load_config_error, Reason}); {ok, Id} -> NConfig#{annotations => #{id => Id}} end. diff --git a/apps/emqx_auth_jwt/src/emqx_authn_jwks_connector.erl b/apps/emqx_auth_jwt/src/emqx_authn_jwks_connector.erl index ffa8175b7..22aed8e57 100644 --- a/apps/emqx_auth_jwt/src/emqx_authn_jwks_connector.erl +++ b/apps/emqx_auth_jwt/src/emqx_authn_jwks_connector.erl @@ -22,6 +22,7 @@ %% callbacks of behaviour emqx_resource -export([ + resource_type/0, callback_mode/0, on_start/2, on_stop/2, @@ -32,6 +33,8 @@ -define(DEFAULT_POOL_SIZE, 8). +resource_type() -> jwks. + callback_mode() -> always_sync. on_start(InstId, Opts) -> diff --git a/apps/emqx_auth_jwt/src/emqx_authn_jwt.erl b/apps/emqx_auth_jwt/src/emqx_authn_jwt.erl index 2f8ebec36..1d0d58474 100644 --- a/apps/emqx_auth_jwt/src/emqx_authn_jwt.erl +++ b/apps/emqx_auth_jwt/src/emqx_authn_jwt.erl @@ -183,7 +183,6 @@ do_create( ) -> ResourceId = emqx_authn_utils:make_resource_id(?MODULE), {ok, _Data} = emqx_resource:create_local( - jwt, ResourceId, ?AUTHN_RESOURCE_GROUP, emqx_authn_jwks_connector, diff --git a/apps/emqx_auth_ldap/src/emqx_authn_ldap.erl b/apps/emqx_auth_ldap/src/emqx_authn_ldap.erl index 8a9b5650a..a18236d15 100644 --- a/apps/emqx_auth_ldap/src/emqx_authn_ldap.erl +++ b/apps/emqx_auth_ldap/src/emqx_authn_ldap.erl @@ -40,12 +40,12 @@ create(_AuthenticatorID, Config) -> do_create(Module, Config) -> ResourceId = emqx_authn_utils:make_resource_id(Module), State = parse_config(Config), - {ok, _Data} = emqx_authn_utils:create_resource(ldap, ResourceId, emqx_ldap, Config), + {ok, _Data} = emqx_authn_utils:create_resource(ResourceId, emqx_ldap, Config), {ok, State#{resource_id => ResourceId}}. update(Config, #{resource_id := ResourceId} = _State) -> NState = parse_config(Config), - case emqx_authn_utils:update_resource(ldap, emqx_ldap, Config, ResourceId) of + case emqx_authn_utils:update_resource(emqx_ldap, Config, ResourceId) of {error, Reason} -> error({load_config_error, Reason}); {ok, _} -> diff --git a/apps/emqx_auth_ldap/src/emqx_authz_ldap.erl b/apps/emqx_auth_ldap/src/emqx_authz_ldap.erl index e70b840a3..24bd8c008 100644 --- a/apps/emqx_auth_ldap/src/emqx_authz_ldap.erl +++ b/apps/emqx_auth_ldap/src/emqx_authz_ldap.erl @@ -56,12 +56,12 @@ description() -> create(Source) -> ResourceId = emqx_authz_utils:make_resource_id(?MODULE), - {ok, _Data} = emqx_authz_utils:create_resource(ldap, ResourceId, emqx_ldap, Source), + {ok, _Data} = emqx_authz_utils:create_resource(ResourceId, emqx_ldap, Source), Annotations = new_annotations(#{id => ResourceId}, Source), Source#{annotations => Annotations}. update(Source) -> - case emqx_authz_utils:update_resource(ldap, emqx_ldap, Source) of + case emqx_authz_utils:update_resource(emqx_ldap, Source) of {error, Reason} -> error({load_config_error, Reason}); {ok, Id} -> diff --git a/apps/emqx_auth_ldap/test/emqx_authn_ldap_SUITE.erl b/apps/emqx_auth_ldap/test/emqx_authn_ldap_SUITE.erl index f6ee582be..ac941f268 100644 --- a/apps/emqx_auth_ldap/test/emqx_authn_ldap_SUITE.erl +++ b/apps/emqx_auth_ldap/test/emqx_authn_ldap_SUITE.erl @@ -47,7 +47,6 @@ init_per_suite(Config) -> work_dir => ?config(priv_dir, Config) }), {ok, _} = emqx_resource:create_local( - ldap, ?LDAP_RESOURCE, ?AUTHN_RESOURCE_GROUP, emqx_ldap, diff --git a/apps/emqx_auth_ldap/test/emqx_authn_ldap_bind_SUITE.erl b/apps/emqx_auth_ldap/test/emqx_authn_ldap_bind_SUITE.erl index c39225e74..d8e9c0d8e 100644 --- a/apps/emqx_auth_ldap/test/emqx_authn_ldap_bind_SUITE.erl +++ b/apps/emqx_auth_ldap/test/emqx_authn_ldap_bind_SUITE.erl @@ -47,7 +47,6 @@ init_per_suite(Config) -> work_dir => ?config(priv_dir, Config) }), {ok, _} = emqx_resource:create_local( - ldap, ?LDAP_RESOURCE, ?AUTHN_RESOURCE_GROUP, emqx_ldap, diff --git a/apps/emqx_auth_ldap/test/emqx_authz_ldap_SUITE.erl b/apps/emqx_auth_ldap/test/emqx_authz_ldap_SUITE.erl index 3f8c0ba63..09875a3fa 100644 --- a/apps/emqx_auth_ldap/test/emqx_authz_ldap_SUITE.erl +++ b/apps/emqx_auth_ldap/test/emqx_authz_ldap_SUITE.erl @@ -178,7 +178,6 @@ stop_apps(Apps) -> create_ldap_resource() -> {ok, _} = emqx_resource:create_local( - ldap, ?LDAP_RESOURCE, ?AUTHZ_RESOURCE_GROUP, emqx_ldap, diff --git a/apps/emqx_auth_mongodb/src/emqx_authn_mongodb.erl b/apps/emqx_auth_mongodb/src/emqx_authn_mongodb.erl index ffe78159d..75a474c0c 100644 --- a/apps/emqx_auth_mongodb/src/emqx_authn_mongodb.erl +++ b/apps/emqx_auth_mongodb/src/emqx_authn_mongodb.erl @@ -37,7 +37,6 @@ create(Config0) -> ResourceId = emqx_authn_utils:make_resource_id(?MODULE), {Config, State} = parse_config(Config0), {ok, _Data} = emqx_authn_utils:create_resource( - mongodb, ResourceId, emqx_mongodb, Config @@ -46,7 +45,7 @@ create(Config0) -> update(Config0, #{resource_id := ResourceId} = _State) -> {Config, NState} = parse_config(Config0), - case emqx_authn_utils:update_resource(mongodb, emqx_mongodb, Config, ResourceId) of + case emqx_authn_utils:update_resource(emqx_mongodb, Config, ResourceId) of {error, Reason} -> error({load_config_error, Reason}); {ok, _} -> diff --git a/apps/emqx_auth_mongodb/src/emqx_authz_mongodb.erl b/apps/emqx_auth_mongodb/src/emqx_authz_mongodb.erl index 0d65afe0d..0bab6ef90 100644 --- a/apps/emqx_auth_mongodb/src/emqx_authz_mongodb.erl +++ b/apps/emqx_auth_mongodb/src/emqx_authz_mongodb.erl @@ -49,13 +49,13 @@ description() -> create(#{filter := Filter} = Source) -> ResourceId = emqx_authz_utils:make_resource_id(?MODULE), - {ok, _Data} = emqx_authz_utils:create_resource(mongodb, ResourceId, emqx_mongodb, Source), + {ok, _Data} = emqx_authz_utils:create_resource(ResourceId, emqx_mongodb, Source), FilterTemp = emqx_authz_utils:parse_deep(Filter, ?ALLOWED_VARS), Source#{annotations => #{id => ResourceId}, filter_template => FilterTemp}. update(#{filter := Filter} = Source) -> FilterTemp = emqx_authz_utils:parse_deep(Filter, ?ALLOWED_VARS), - case emqx_authz_utils:update_resource(mongodb, emqx_mongodb, Source) of + case emqx_authz_utils:update_resource(emqx_mongodb, Source) of {error, Reason} -> error({load_config_error, Reason}); {ok, Id} -> diff --git a/apps/emqx_auth_mysql/src/emqx_authn_mysql.erl b/apps/emqx_auth_mysql/src/emqx_authn_mysql.erl index 7d03ca856..f68c74a14 100644 --- a/apps/emqx_auth_mysql/src/emqx_authn_mysql.erl +++ b/apps/emqx_auth_mysql/src/emqx_authn_mysql.erl @@ -39,12 +39,12 @@ create(_AuthenticatorID, Config) -> create(Config0) -> ResourceId = emqx_authn_utils:make_resource_id(?MODULE), {Config, State} = parse_config(Config0), - {ok, _Data} = emqx_authn_utils:create_resource(mysql, ResourceId, emqx_mysql, Config), + {ok, _Data} = emqx_authn_utils:create_resource(ResourceId, emqx_mysql, Config), {ok, State#{resource_id => ResourceId}}. update(Config0, #{resource_id := ResourceId} = _State) -> {Config, NState} = parse_config(Config0), - case emqx_authn_utils:update_resource(mysql, emqx_mysql, Config, ResourceId) of + case emqx_authn_utils:update_resource(emqx_mysql, Config, ResourceId) of {error, Reason} -> error({load_config_error, Reason}); {ok, _} -> diff --git a/apps/emqx_auth_mysql/src/emqx_authz_mysql.erl b/apps/emqx_auth_mysql/src/emqx_authz_mysql.erl index 557eed14f..0e2b77005 100644 --- a/apps/emqx_auth_mysql/src/emqx_authz_mysql.erl +++ b/apps/emqx_auth_mysql/src/emqx_authz_mysql.erl @@ -53,13 +53,13 @@ create(#{query := SQL} = Source0) -> {PrepareSQL, TmplToken} = emqx_authz_utils:parse_sql(SQL, '?', ?ALLOWED_VARS), ResourceId = emqx_authz_utils:make_resource_id(?MODULE), Source = Source0#{prepare_statement => #{?PREPARE_KEY => PrepareSQL}}, - {ok, _Data} = emqx_authz_utils:create_resource(mysql, ResourceId, emqx_mysql, Source), + {ok, _Data} = emqx_authz_utils:create_resource(ResourceId, emqx_mysql, Source), Source#{annotations => #{id => ResourceId, tmpl_token => TmplToken}}. update(#{query := SQL} = Source0) -> {PrepareSQL, TmplToken} = emqx_authz_utils:parse_sql(SQL, '?', ?ALLOWED_VARS), Source = Source0#{prepare_statement => #{?PREPARE_KEY => PrepareSQL}}, - case emqx_authz_utils:update_resource(mysql, emqx_mysql, Source) of + case emqx_authz_utils:update_resource(emqx_mysql, Source) of {error, Reason} -> error({load_config_error, Reason}); {ok, Id} -> diff --git a/apps/emqx_auth_mysql/test/emqx_authn_mysql_SUITE.erl b/apps/emqx_auth_mysql/test/emqx_authn_mysql_SUITE.erl index 78b8aa8b3..8ab812fc0 100644 --- a/apps/emqx_auth_mysql/test/emqx_authn_mysql_SUITE.erl +++ b/apps/emqx_auth_mysql/test/emqx_authn_mysql_SUITE.erl @@ -58,7 +58,6 @@ init_per_suite(Config) -> work_dir => ?config(priv_dir, Config) }), {ok, _} = emqx_resource:create_local( - mysql, ?MYSQL_RESOURCE, ?AUTHN_RESOURCE_GROUP, emqx_mysql, diff --git a/apps/emqx_auth_mysql/test/emqx_authz_mysql_SUITE.erl b/apps/emqx_auth_mysql/test/emqx_authz_mysql_SUITE.erl index 5fe020f1b..ce30e203e 100644 --- a/apps/emqx_auth_mysql/test/emqx_authz_mysql_SUITE.erl +++ b/apps/emqx_auth_mysql/test/emqx_authz_mysql_SUITE.erl @@ -446,7 +446,6 @@ stop_apps(Apps) -> create_mysql_resource() -> {ok, _} = emqx_resource:create_local( - mysql, ?MYSQL_RESOURCE, ?AUTHZ_RESOURCE_GROUP, emqx_mysql, diff --git a/apps/emqx_auth_postgresql/src/emqx_authn_postgresql.erl b/apps/emqx_auth_postgresql/src/emqx_authn_postgresql.erl index 95e8ac7ce..980c2ddd8 100644 --- a/apps/emqx_auth_postgresql/src/emqx_authn_postgresql.erl +++ b/apps/emqx_auth_postgresql/src/emqx_authn_postgresql.erl @@ -45,7 +45,6 @@ create(Config0) -> ResourceId = emqx_authn_utils:make_resource_id(?MODULE), {Config, State} = parse_config(Config0, ResourceId), {ok, _Data} = emqx_authn_utils:create_resource( - postgresql, ResourceId, emqx_postgresql, Config @@ -54,7 +53,7 @@ create(Config0) -> update(Config0, #{resource_id := ResourceId} = _State) -> {Config, NState} = parse_config(Config0, ResourceId), - case emqx_authn_utils:update_resource(postgresql, emqx_postgresql, Config, ResourceId) of + case emqx_authn_utils:update_resource(emqx_postgresql, Config, ResourceId) of {error, Reason} -> error({load_config_error, Reason}); {ok, _} -> diff --git a/apps/emqx_auth_postgresql/src/emqx_authz_postgresql.erl b/apps/emqx_auth_postgresql/src/emqx_authz_postgresql.erl index e0fc9e032..d1a0b32ea 100644 --- a/apps/emqx_auth_postgresql/src/emqx_authz_postgresql.erl +++ b/apps/emqx_auth_postgresql/src/emqx_authz_postgresql.erl @@ -53,7 +53,6 @@ create(#{query := SQL0} = Source) -> {SQL, PlaceHolders} = emqx_authz_utils:parse_sql(SQL0, '$n', ?ALLOWED_VARS), ResourceID = emqx_authz_utils:make_resource_id(emqx_postgresql), {ok, _Data} = emqx_authz_utils:create_resource( - postgresql, ResourceID, emqx_postgresql, Source#{prepare_statement => #{ResourceID => SQL}} @@ -64,7 +63,6 @@ update(#{query := SQL0, annotations := #{id := ResourceID}} = Source) -> {SQL, PlaceHolders} = emqx_authz_utils:parse_sql(SQL0, '$n', ?ALLOWED_VARS), case emqx_authz_utils:update_resource( - postgresql, emqx_postgresql, Source#{prepare_statement => #{ResourceID => SQL}} ) diff --git a/apps/emqx_auth_postgresql/test/emqx_authn_postgresql_SUITE.erl b/apps/emqx_auth_postgresql/test/emqx_authn_postgresql_SUITE.erl index dde5a184a..1dfd30899 100644 --- a/apps/emqx_auth_postgresql/test/emqx_authn_postgresql_SUITE.erl +++ b/apps/emqx_auth_postgresql/test/emqx_authn_postgresql_SUITE.erl @@ -79,7 +79,6 @@ init_per_suite(Config) -> work_dir => ?config(priv_dir, Config) }), {ok, _} = emqx_resource:create_local( - postgresql, ?PGSQL_RESOURCE, ?AUTHN_RESOURCE_GROUP, emqx_postgresql, @@ -199,13 +198,9 @@ test_user_auth(#{ t_authenticate_disabled_prepared_statements(_Config) -> ResConfig = maps:merge(pgsql_config(), #{disable_prepared_statements => true}), - {ok, _} = emqx_resource:recreate_local( - postgresql, ?PGSQL_RESOURCE, emqx_postgresql, ResConfig, #{} - ), + {ok, _} = emqx_resource:recreate_local(?PGSQL_RESOURCE, emqx_postgresql, ResConfig, #{}), on_exit(fun() -> - emqx_resource:recreate_local( - postgresql, ?PGSQL_RESOURCE, emqx_postgresql, pgsql_config(), #{} - ) + emqx_resource:recreate_local(?PGSQL_RESOURCE, emqx_postgresql, pgsql_config(), #{}) end), ok = lists:foreach( fun(Sample0) -> diff --git a/apps/emqx_auth_postgresql/test/emqx_authz_postgresql_SUITE.erl b/apps/emqx_auth_postgresql/test/emqx_authz_postgresql_SUITE.erl index 9346bf863..78b1e17a8 100644 --- a/apps/emqx_auth_postgresql/test/emqx_authz_postgresql_SUITE.erl +++ b/apps/emqx_auth_postgresql/test/emqx_authz_postgresql_SUITE.erl @@ -437,7 +437,6 @@ pgsql_config() -> create_pgsql_resource() -> emqx_resource:create_local( - postgresql, ?PGSQL_RESOURCE, ?AUTHZ_RESOURCE_GROUP, emqx_postgresql, diff --git a/apps/emqx_auth_redis/src/emqx_authn_redis.erl b/apps/emqx_auth_redis/src/emqx_authn_redis.erl index 4eb5a36a3..779c58e39 100644 --- a/apps/emqx_auth_redis/src/emqx_authn_redis.erl +++ b/apps/emqx_auth_redis/src/emqx_authn_redis.erl @@ -42,7 +42,6 @@ create(Config0) -> Res; {Config, State} -> {ok, _Data} = emqx_authn_utils:create_resource( - redis, ResourceId, emqx_redis, Config @@ -52,7 +51,7 @@ create(Config0) -> update(Config0, #{resource_id := ResourceId} = _State) -> {Config, NState} = parse_config(Config0), - case emqx_authn_utils:update_resource(redis, emqx_redis, Config, ResourceId) of + case emqx_authn_utils:update_resource(emqx_redis, Config, ResourceId) of {error, Reason} -> error({load_config_error, Reason}); {ok, _} -> diff --git a/apps/emqx_auth_redis/src/emqx_authz_redis.erl b/apps/emqx_auth_redis/src/emqx_authz_redis.erl index b83d1abad..8ce975033 100644 --- a/apps/emqx_auth_redis/src/emqx_authz_redis.erl +++ b/apps/emqx_auth_redis/src/emqx_authz_redis.erl @@ -50,12 +50,12 @@ description() -> create(#{cmd := CmdStr} = Source) -> CmdTemplate = parse_cmd(CmdStr), ResourceId = emqx_authz_utils:make_resource_id(?MODULE), - {ok, _Data} = emqx_authz_utils:create_resource(redis, ResourceId, emqx_redis, Source), + {ok, _Data} = emqx_authz_utils:create_resource(ResourceId, emqx_redis, Source), Source#{annotations => #{id => ResourceId}, cmd_template => CmdTemplate}. update(#{cmd := CmdStr} = Source) -> CmdTemplate = parse_cmd(CmdStr), - case emqx_authz_utils:update_resource(redis, emqx_redis, Source) of + case emqx_authz_utils:update_resource(emqx_redis, Source) of {error, Reason} -> error({load_config_error, Reason}); {ok, Id} -> diff --git a/apps/emqx_auth_redis/test/emqx_authn_redis_SUITE.erl b/apps/emqx_auth_redis/test/emqx_authn_redis_SUITE.erl index 1e9d825d2..e8c8760de 100644 --- a/apps/emqx_auth_redis/test/emqx_authn_redis_SUITE.erl +++ b/apps/emqx_auth_redis/test/emqx_authn_redis_SUITE.erl @@ -63,7 +63,6 @@ init_per_suite(Config) -> work_dir => ?config(priv_dir, Config) }), {ok, _} = emqx_resource:create_local( - redis, ?REDIS_RESOURCE, ?AUTHN_RESOURCE_GROUP, emqx_redis, diff --git a/apps/emqx_auth_redis/test/emqx_authz_redis_SUITE.erl b/apps/emqx_auth_redis/test/emqx_authz_redis_SUITE.erl index d0c695c73..5818eea07 100644 --- a/apps/emqx_auth_redis/test/emqx_authz_redis_SUITE.erl +++ b/apps/emqx_auth_redis/test/emqx_authz_redis_SUITE.erl @@ -384,7 +384,6 @@ stop_apps(Apps) -> create_redis_resource() -> {ok, _} = emqx_resource:create_local( - redis, ?REDIS_RESOURCE, ?AUTHZ_RESOURCE_GROUP, emqx_redis, diff --git a/apps/emqx_bridge/src/emqx_bridge_resource.erl b/apps/emqx_bridge/src/emqx_bridge_resource.erl index 7e9f5300a..40db2aee9 100644 --- a/apps/emqx_bridge/src/emqx_bridge_resource.erl +++ b/apps/emqx_bridge/src/emqx_bridge_resource.erl @@ -195,7 +195,6 @@ create(Type, Name, Conf0, Opts) -> TypeBin = bin(Type), Conf = Conf0#{bridge_type => TypeBin, bridge_name => Name}, {ok, _Data} = emqx_resource:create_local( - Type, resource_id(Type, Name), <<"bridge">>, bridge_to_resource_type(Type), @@ -265,7 +264,6 @@ recreate(Type, Name, Conf0, Opts) -> TypeBin = bin(Type), Conf = Conf0#{bridge_type => TypeBin, bridge_name => Name}, emqx_resource:recreate_local( - Type, resource_id(Type, Name), bridge_to_resource_type(Type), parse_confs(TypeBin, Name, Conf), @@ -284,7 +282,7 @@ create_dry_run(Type0, Conf0) -> create_dry_run_bridge_v1(Type, Conf0) -> TmpName = iolist_to_binary([?TEST_ID_PREFIX, emqx_utils:gen_id(8)]), TmpPath = emqx_utils:safe_filename(TmpName), - %% Already typechecked, no need to catch errors + %% Already type checked, no need to catch errors TypeBin = bin(Type), TypeAtom = safe_atom(Type), Conf1 = maps:without([<<"name">>], Conf0), @@ -302,7 +300,7 @@ create_dry_run_bridge_v1(Type, Conf0) -> {error, Reason}; {ok, ConfNew} -> ParseConf = parse_confs(TypeBin, TmpName, ConfNew), - emqx_resource:create_dry_run_local(Type, bridge_to_resource_type(Type), ParseConf) + emqx_resource:create_dry_run_local(bridge_to_resource_type(Type), ParseConf) end catch %% validation errors diff --git a/apps/emqx_bridge/test/emqx_bridge_v2_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_v2_SUITE.erl index dc2e8f275..d81d710ff 100644 --- a/apps/emqx_bridge/test/emqx_bridge_v2_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_v2_SUITE.erl @@ -1110,6 +1110,7 @@ t_query_uses_action_query_mode(_Config) -> %% ... now we use a quite different query mode for the action meck:expect(con_mod(), query_mode, 1, simple_async_internal_buffer), + meck:expect(con_mod(), resource_type, 0, dummy), meck:expect(con_mod(), callback_mode, 0, async_if_possible), {ok, _} = emqx_bridge_v2:create(bridge_type(), ActionName, ActionConfig), diff --git a/apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl index 039402738..e5d13f452 100644 --- a/apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl @@ -293,6 +293,7 @@ init_mocks() -> meck:new(emqx_connector_resource, [passthrough, no_link]), meck:expect(emqx_connector_resource, connector_to_resource_type, 1, ?CONNECTOR_IMPL), meck:new(?CONNECTOR_IMPL, [non_strict, no_link]), + meck:expect(?CONNECTOR_IMPL, resource_type, 0, dummy), meck:expect(?CONNECTOR_IMPL, callback_mode, 0, async_if_possible), meck:expect( ?CONNECTOR_IMPL, diff --git a/apps/emqx_bridge/test/emqx_bridge_v2_dummy_connector.erl b/apps/emqx_bridge/test/emqx_bridge_v2_dummy_connector.erl index 6b4001b6b..1bb7fd37f 100644 --- a/apps/emqx_bridge/test/emqx_bridge_v2_dummy_connector.erl +++ b/apps/emqx_bridge/test/emqx_bridge_v2_dummy_connector.erl @@ -15,15 +15,17 @@ %% this module is only intended to be mocked -module(emqx_bridge_v2_dummy_connector). +-behavior(emqx_resource). -export([ + resource_type/0, callback_mode/0, on_start/2, on_stop/2, on_add_channel/4, on_get_channel_status/3 ]). - +resource_type() -> dummy. callback_mode() -> error(unexpected). on_start(_, _) -> error(unexpected). on_stop(_, _) -> error(unexpected). diff --git a/apps/emqx_bridge/test/emqx_bridge_v2_test_connector.erl b/apps/emqx_bridge/test/emqx_bridge_v2_test_connector.erl index c528d097c..7daedf19a 100644 --- a/apps/emqx_bridge/test/emqx_bridge_v2_test_connector.erl +++ b/apps/emqx_bridge/test/emqx_bridge_v2_test_connector.erl @@ -19,6 +19,7 @@ -export([ query_mode/1, + resource_type/0, callback_mode/0, on_start/2, on_stop/2, @@ -34,6 +35,8 @@ query_mode(_Config) -> sync. +resource_type() -> test_connector. + callback_mode() -> always_sync. diff --git a/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.app.src b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.app.src index 946ca591a..3e7422112 100644 --- a/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.app.src +++ b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_cassandra, [ {description, "EMQX Enterprise Cassandra Bridge"}, - {vsn, "0.3.1"}, + {vsn, "0.3.2"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_connector.erl b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_connector.erl index df278b791..9f830eb69 100644 --- a/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_connector.erl +++ b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_connector.erl @@ -19,6 +19,7 @@ %% callbacks of behaviour emqx_resource -export([ + resource_type/0, callback_mode/0, on_start/2, on_stop/2, @@ -94,6 +95,7 @@ desc("connector") -> %%-------------------------------------------------------------------- %% callbacks for emqx_resource +resource_type() -> cassandra. callback_mode() -> async_if_possible. diff --git a/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_connector_SUITE.erl b/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_connector_SUITE.erl index 02c102832..b784d36c0 100644 --- a/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_connector_SUITE.erl +++ b/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_connector_SUITE.erl @@ -212,7 +212,6 @@ check_config(Config) -> create_local_resource(ResourceId, CheckedConfig) -> {ok, Bridge} = emqx_resource:create_local( - cassandra, ResourceId, ?CONNECTOR_RESOURCE_GROUP, ?CASSANDRA_RESOURCE_MOD, diff --git a/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse.app.src b/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse.app.src index f38036b83..794d067bd 100644 --- a/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse.app.src +++ b/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_clickhouse, [ {description, "EMQX Enterprise ClickHouse Bridge"}, - {vsn, "0.4.1"}, + {vsn, "0.4.2"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse_connector.erl b/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse_connector.erl index f6888cad5..c5b82122a 100644 --- a/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse_connector.erl +++ b/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse_connector.erl @@ -29,6 +29,7 @@ %% callbacks for behaviour emqx_resource -export([ + resource_type/0, callback_mode/0, on_start/2, on_stop/2, @@ -128,6 +129,7 @@ values(_) -> %% =================================================================== %% Callbacks defined in emqx_resource %% =================================================================== +resource_type() -> clickhouse. callback_mode() -> always_sync. diff --git a/apps/emqx_bridge_clickhouse/test/emqx_bridge_clickhouse_connector_SUITE.erl b/apps/emqx_bridge_clickhouse/test/emqx_bridge_clickhouse_connector_SUITE.erl index 0b0acd78a..1c83961a5 100644 --- a/apps/emqx_bridge_clickhouse/test/emqx_bridge_clickhouse_connector_SUITE.erl +++ b/apps/emqx_bridge_clickhouse/test/emqx_bridge_clickhouse_connector_SUITE.erl @@ -109,7 +109,6 @@ t_start_passfile(Config) -> ?assertMatch( {ok, #{status := connected}}, emqx_resource:create_local( - clickhouse, ResourceID, ?CONNECTOR_RESOURCE_GROUP, ?CLICKHOUSE_RESOURCE_MOD, @@ -139,7 +138,6 @@ perform_lifecycle_check(ResourceID, InitialConfig) -> status := InitialStatus }} = emqx_resource:create_local( - clickhouse, ResourceID, ?CONNECTOR_RESOURCE_GROUP, ?CLICKHOUSE_RESOURCE_MOD, diff --git a/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo.app.src b/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo.app.src index ac71e04e7..b8fee4dee 100644 --- a/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo.app.src +++ b/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_dynamo, [ {description, "EMQX Enterprise Dynamo Bridge"}, - {vsn, "0.2.2"}, + {vsn, "0.2.3"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector.erl b/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector.erl index 82f5fb18d..181de34a8 100644 --- a/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector.erl +++ b/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector.erl @@ -17,6 +17,7 @@ %% `emqx_resource' API -export([ + resource_type/0, callback_mode/0, on_start/2, on_stop/2, @@ -68,6 +69,7 @@ fields(config) -> %%======================================================================================== %% `emqx_resource' API %%======================================================================================== +resource_type() -> dynamo. callback_mode() -> always_sync. diff --git a/apps/emqx_bridge_es/src/emqx_bridge_es.app.src b/apps/emqx_bridge_es/src/emqx_bridge_es.app.src index 262ac84bd..8f3dc3a7e 100644 --- a/apps/emqx_bridge_es/src/emqx_bridge_es.app.src +++ b/apps/emqx_bridge_es/src/emqx_bridge_es.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_bridge_es, [ {description, "EMQX Enterprise Elastic Search Bridge"}, - {vsn, "0.1.3"}, + {vsn, "0.1.4"}, {modules, [ emqx_bridge_es, emqx_bridge_es_connector diff --git a/apps/emqx_bridge_es/src/emqx_bridge_es_connector.erl b/apps/emqx_bridge_es/src/emqx_bridge_es_connector.erl index 20de92e6e..feccd42f1 100644 --- a/apps/emqx_bridge_es/src/emqx_bridge_es_connector.erl +++ b/apps/emqx_bridge_es/src/emqx_bridge_es_connector.erl @@ -14,6 +14,7 @@ %% `emqx_resource' API -export([ + resource_type/0, callback_mode/0, on_start/2, on_stop/2, @@ -207,6 +208,8 @@ base_url(#{server := Server}) -> "http://" ++ Server. %%------------------------------------------------------------------------------------- %% `emqx_resource' API %%------------------------------------------------------------------------------------- +resource_type() -> elastic_search. + callback_mode() -> async_if_possible. -spec on_start(manager_id(), config()) -> {ok, state()} | no_return(). diff --git a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.app.src b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.app.src index d98355a90..eff7847f2 100644 --- a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.app.src +++ b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_gcp_pubsub, [ {description, "EMQX Enterprise GCP Pub/Sub Bridge"}, - {vsn, "0.3.1"}, + {vsn, "0.3.2"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_impl_consumer.erl b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_impl_consumer.erl index 5c51cd2d9..344fc05c6 100644 --- a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_impl_consumer.erl +++ b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_impl_consumer.erl @@ -8,6 +8,7 @@ %% `emqx_resource' API -export([ + resource_type/0, callback_mode/0, query_mode/1, on_start/2, @@ -84,6 +85,8 @@ %%------------------------------------------------------------------------------------------------- %% `emqx_resource' API %%------------------------------------------------------------------------------------------------- +-spec resource_type() -> resource_type(). +resource_type() -> gcp_pubsub_consumer. -spec callback_mode() -> callback_mode(). callback_mode() -> async_if_possible. diff --git a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_impl_producer.erl b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_impl_producer.erl index 48e50c416..aec78f74c 100644 --- a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_impl_producer.erl +++ b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_impl_producer.erl @@ -41,6 +41,7 @@ %% `emqx_resource' API -export([ + resource_type/0, callback_mode/0, query_mode/1, on_start/2, @@ -62,6 +63,7 @@ %%------------------------------------------------------------------------------------------------- %% `emqx_resource' API %%------------------------------------------------------------------------------------------------- +resource_type() -> gcp_pubsub. callback_mode() -> async_if_possible. diff --git a/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector.erl b/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector.erl index e4cc0aa31..a2ffd6219 100644 --- a/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector.erl +++ b/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector.erl @@ -16,6 +16,7 @@ %% callbacks of behaviour emqx_resource -export([ + resource_type/0, callback_mode/0, on_start/2, on_stop/2, @@ -67,6 +68,8 @@ %% ------------------------------------------------------------------------------------------------- %% resource callback +resource_type() -> greptimedb. + callback_mode() -> async_if_possible. on_add_channel( diff --git a/apps/emqx_bridge_greptimedb/test/emqx_bridge_greptimedb_connector_SUITE.erl b/apps/emqx_bridge_greptimedb/test/emqx_bridge_greptimedb_connector_SUITE.erl index 68a32e9c2..be36cb167 100644 --- a/apps/emqx_bridge_greptimedb/test/emqx_bridge_greptimedb_connector_SUITE.erl +++ b/apps/emqx_bridge_greptimedb/test/emqx_bridge_greptimedb_connector_SUITE.erl @@ -83,7 +83,6 @@ perform_lifecycle_check(PoolName, InitialConfig) -> state := #{client := #{pool := ReturnedPoolName}} = State, status := InitialStatus }} = emqx_resource:create_local( - greptimedb, PoolName, ?CONNECTOR_RESOURCE_GROUP, ?GREPTIMEDB_RESOURCE_MOD, diff --git a/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb.app.src b/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb.app.src index af232accc..7ae86bba0 100644 --- a/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb.app.src +++ b/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_hstreamdb, [ {description, "EMQX Enterprise HStreamDB Bridge"}, - {vsn, "0.2.1"}, + {vsn, "0.2.2"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb_connector.erl b/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb_connector.erl index cf53291b2..154e43b3d 100644 --- a/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb_connector.erl +++ b/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb_connector.erl @@ -16,6 +16,7 @@ %% callbacks of behaviour emqx_resource -export([ + resource_type/0, callback_mode/0, on_start/2, on_stop/2, @@ -44,6 +45,8 @@ %% ------------------------------------------------------------------------------------------------- %% resource callback +resource_type() -> hstreamsdb. + callback_mode() -> always_sync. on_start(InstId, Config) -> diff --git a/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl b/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl index 91a0878c3..616b42e75 100644 --- a/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl +++ b/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl @@ -26,6 +26,7 @@ %% callbacks of behaviour emqx_resource -export([ + resource_type/0, callback_mode/0, on_start/2, on_stop/2, @@ -183,6 +184,7 @@ sc(Type, Meta) -> hoconsc:mk(Type, Meta). ref(Field) -> hoconsc:ref(?MODULE, Field). %% =================================================================== +resource_type() -> webhook. callback_mode() -> async_if_possible. diff --git a/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.app.src b/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.app.src index a8314541a..eae5028c6 100644 --- a/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.app.src +++ b/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_influxdb, [ {description, "EMQX Enterprise InfluxDB Bridge"}, - {vsn, "0.2.3"}, + {vsn, "0.2.4"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb_connector.erl b/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb_connector.erl index 852f78485..bf93309f8 100644 --- a/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb_connector.erl +++ b/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb_connector.erl @@ -16,6 +16,7 @@ %% callbacks of behaviour emqx_resource -export([ + resource_type/0, callback_mode/0, on_start/2, on_stop/2, @@ -70,6 +71,8 @@ %% ------------------------------------------------------------------------------------------------- %% resource callback +resource_type() -> influxdb. + callback_mode() -> async_if_possible. on_add_channel( diff --git a/apps/emqx_bridge_influxdb/test/emqx_bridge_influxdb_connector_SUITE.erl b/apps/emqx_bridge_influxdb/test/emqx_bridge_influxdb_connector_SUITE.erl index a7f78f253..0ca693171 100644 --- a/apps/emqx_bridge_influxdb/test/emqx_bridge_influxdb_connector_SUITE.erl +++ b/apps/emqx_bridge_influxdb/test/emqx_bridge_influxdb_connector_SUITE.erl @@ -86,7 +86,6 @@ perform_lifecycle_check(PoolName, InitialConfig) -> state := #{client := #{pool := ReturnedPoolName}} = State, status := InitialStatus }} = emqx_resource:create_local( - influxdb, PoolName, ?CONNECTOR_RESOURCE_GROUP, ?INFLUXDB_RESOURCE_MOD, @@ -198,7 +197,6 @@ perform_tls_opts_check(PoolName, InitialConfig, VerifyReturn) -> config := #{ssl := #{enable := SslEnabled}}, status := Status }} = emqx_resource:create_local( - influxdb, PoolName, ?CONNECTOR_RESOURCE_GROUP, ?INFLUXDB_RESOURCE_MOD, diff --git a/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.app.src b/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.app.src index 691778cfd..88ac09da9 100644 --- a/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.app.src +++ b/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_bridge_iotdb, [ {description, "EMQX Enterprise Apache IoTDB Bridge"}, - {vsn, "0.2.2"}, + {vsn, "0.2.3"}, {modules, [ emqx_bridge_iotdb, emqx_bridge_iotdb_connector diff --git a/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb_connector.erl b/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb_connector.erl index 78866ef79..ec880e785 100644 --- a/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb_connector.erl +++ b/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb_connector.erl @@ -15,6 +15,7 @@ %% `emqx_resource' API -export([ + resource_type/0, callback_mode/0, on_start/2, on_stop/2, @@ -206,6 +207,8 @@ proplists_without(Keys, List) -> %%------------------------------------------------------------------------------------- %% `emqx_resource' API %%------------------------------------------------------------------------------------- +resource_type() -> iotdb. + callback_mode() -> async_if_possible. -spec on_start(manager_id(), config()) -> {ok, state()} | no_return(). diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_consumer.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_consumer.erl index 35ffbc90b..44e007415 100644 --- a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_consumer.erl +++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_consumer.erl @@ -7,6 +7,7 @@ %% `emqx_resource' API -export([ + resource_type/0, callback_mode/0, query_mode/1, on_start/2, @@ -125,6 +126,7 @@ %%------------------------------------------------------------------------------------- %% `emqx_resource' API %%------------------------------------------------------------------------------------- +resource_type() -> kafka_consumer. callback_mode() -> async_if_possible. diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl index 8b6326545..6d88a329e 100644 --- a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl +++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl @@ -10,6 +10,7 @@ %% callbacks of behaviour emqx_resource -export([ + resource_type/0, query_mode/1, callback_mode/0, on_start/2, @@ -35,6 +36,8 @@ -define(kafka_client_id, kafka_client_id). -define(kafka_producers, kafka_producers). +resource_type() -> kafka_producer. + query_mode(#{parameters := #{query_mode := sync}}) -> simple_sync_internal_buffer; query_mode(_) -> diff --git a/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis.app.src b/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis.app.src index f411b95fb..a85121905 100644 --- a/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis.app.src +++ b/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_kinesis, [ {description, "EMQX Enterprise Amazon Kinesis Bridge"}, - {vsn, "0.2.1"}, + {vsn, "0.2.2"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis_impl_producer.erl b/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis_impl_producer.erl index 95d193d92..3143cf904 100644 --- a/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis_impl_producer.erl +++ b/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis_impl_producer.erl @@ -30,6 +30,7 @@ %% `emqx_resource' API -export([ + resource_type/0, callback_mode/0, on_start/2, on_stop/2, @@ -50,6 +51,7 @@ %%------------------------------------------------------------------------------------------------- %% `emqx_resource' API %%------------------------------------------------------------------------------------------------- +resource_type() -> kinesis_producer. callback_mode() -> always_sync. diff --git a/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.app.src b/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.app.src index df9935dbb..5bb7e396d 100644 --- a/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.app.src +++ b/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_mongodb, [ {description, "EMQX Enterprise MongoDB Bridge"}, - {vsn, "0.3.2"}, + {vsn, "0.3.3"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb_connector.erl b/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb_connector.erl index 6b6db358a..dac9bef57 100644 --- a/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb_connector.erl +++ b/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb_connector.erl @@ -11,6 +11,7 @@ %% `emqx_resource' API -export([ on_remove_channel/3, + resource_type/0, callback_mode/0, on_add_channel/4, on_get_channel_status/3, @@ -25,6 +26,7 @@ %%======================================================================================== %% `emqx_resource' API %%======================================================================================== +resource_type() -> emqx_mongodb:resource_type(). callback_mode() -> emqx_mongodb:callback_mode(). diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl index 9c2506bab..118542356 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl @@ -30,6 +30,7 @@ %% callbacks of behaviour emqx_resource -export([ + resource_type/0, callback_mode/0, on_start/2, on_stop/2, @@ -76,6 +77,8 @@ on_message_received(Msg, HookPoints, ResId) -> ok. %% =================================================================== +resource_type() -> mqtt. + callback_mode() -> async_if_possible. on_start(ResourceId, #{server := Server} = Conf) -> diff --git a/apps/emqx_bridge_mysql/src/emqx_bridge_mysql.app.src b/apps/emqx_bridge_mysql/src/emqx_bridge_mysql.app.src index 63bc61e62..fb670e072 100644 --- a/apps/emqx_bridge_mysql/src/emqx_bridge_mysql.app.src +++ b/apps/emqx_bridge_mysql/src/emqx_bridge_mysql.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_mysql, [ {description, "EMQX Enterprise MySQL Bridge"}, - {vsn, "0.1.7"}, + {vsn, "0.1.8"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_mysql/src/emqx_bridge_mysql_connector.erl b/apps/emqx_bridge_mysql/src/emqx_bridge_mysql_connector.erl index da9377814..6905c86eb 100644 --- a/apps/emqx_bridge_mysql/src/emqx_bridge_mysql_connector.erl +++ b/apps/emqx_bridge_mysql/src/emqx_bridge_mysql_connector.erl @@ -10,6 +10,7 @@ %% `emqx_resource' API -export([ on_remove_channel/3, + resource_type/0, callback_mode/0, on_add_channel/4, on_batch_query/3, @@ -24,6 +25,7 @@ %%======================================================================================== %% `emqx_resource' API %%======================================================================================== +resource_type() -> emqx_mysql:resource_type(). callback_mode() -> emqx_mysql:callback_mode(). diff --git a/apps/emqx_bridge_opents/src/emqx_bridge_opents.app.src b/apps/emqx_bridge_opents/src/emqx_bridge_opents.app.src index 65cc97e4c..a27791853 100644 --- a/apps/emqx_bridge_opents/src/emqx_bridge_opents.app.src +++ b/apps/emqx_bridge_opents/src/emqx_bridge_opents.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_opents, [ {description, "EMQX Enterprise OpenTSDB Bridge"}, - {vsn, "0.2.1"}, + {vsn, "0.2.2"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_opents/src/emqx_bridge_opents_connector.erl b/apps/emqx_bridge_opents/src/emqx_bridge_opents_connector.erl index 19e117a0d..a970bb374 100644 --- a/apps/emqx_bridge_opents/src/emqx_bridge_opents_connector.erl +++ b/apps/emqx_bridge_opents/src/emqx_bridge_opents_connector.erl @@ -18,6 +18,7 @@ %% `emqx_resource' API -export([ + resource_type/0, callback_mode/0, on_start/2, on_stop/2, @@ -114,6 +115,8 @@ connector_example_values() -> -define(HTTP_CONNECT_TIMEOUT, 1000). +resource_type() -> opents. + callback_mode() -> always_sync. on_start( diff --git a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_connector.erl b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_connector.erl index 835536bda..64dde77fb 100644 --- a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_connector.erl +++ b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_connector.erl @@ -10,6 +10,7 @@ %% `emqx_resource' API -export([ + resource_type/0, callback_mode/0, query_mode/1, on_start/2, @@ -55,6 +56,7 @@ %%------------------------------------------------------------------------------------- %% `emqx_resource' API %%------------------------------------------------------------------------------------- +resource_type() -> pulsar. callback_mode() -> async_if_possible. diff --git a/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq.app.src b/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq.app.src index 27a4fedc4..c178b1f5e 100644 --- a/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq.app.src +++ b/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_rabbitmq, [ {description, "EMQX Enterprise RabbitMQ Bridge"}, - {vsn, "0.2.1"}, + {vsn, "0.2.2"}, {registered, []}, {mod, {emqx_bridge_rabbitmq_app, []}}, {applications, [ diff --git a/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq_connector.erl b/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq_connector.erl index dacb47a57..7e3e18e5f 100644 --- a/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq_connector.erl +++ b/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq_connector.erl @@ -31,6 +31,7 @@ on_remove_channel/3, on_get_channels/1, on_stop/2, + resource_type/0, callback_mode/0, on_get_status/2, on_get_channel_status/3, @@ -60,6 +61,7 @@ fields(config) -> %% =================================================================== %% emqx_resource callback +resource_type() -> rabbitmq. callback_mode() -> always_sync. diff --git a/apps/emqx_bridge_rabbitmq/test/emqx_bridge_rabbitmq_connector_SUITE.erl b/apps/emqx_bridge_rabbitmq/test/emqx_bridge_rabbitmq_connector_SUITE.erl index fe288a185..77482ae0f 100644 --- a/apps/emqx_bridge_rabbitmq/test/emqx_bridge_rabbitmq_connector_SUITE.erl +++ b/apps/emqx_bridge_rabbitmq/test/emqx_bridge_rabbitmq_connector_SUITE.erl @@ -135,7 +135,6 @@ check_config(Config) -> create_local_resource(ResourceID, CheckedConfig) -> {ok, Bridge} = emqx_resource:create_local( - rabbitmq, ResourceID, ?CONNECTOR_RESOURCE_GROUP, emqx_bridge_rabbitmq_connector, diff --git a/apps/emqx_bridge_redis/src/emqx_bridge_redis.app.src b/apps/emqx_bridge_redis/src/emqx_bridge_redis.app.src index 2cd037ed5..61cd837bb 100644 --- a/apps/emqx_bridge_redis/src/emqx_bridge_redis.app.src +++ b/apps/emqx_bridge_redis/src/emqx_bridge_redis.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_redis, [ {description, "EMQX Enterprise Redis Bridge"}, - {vsn, "0.1.8"}, + {vsn, "0.1.9"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_redis/src/emqx_bridge_redis_connector.erl b/apps/emqx_bridge_redis/src/emqx_bridge_redis_connector.erl index f117c4e7a..162c38368 100644 --- a/apps/emqx_bridge_redis/src/emqx_bridge_redis_connector.erl +++ b/apps/emqx_bridge_redis/src/emqx_bridge_redis_connector.erl @@ -12,6 +12,7 @@ %% callbacks of behaviour emqx_resource -export([ + resource_type/0, callback_mode/0, on_add_channel/4, on_remove_channel/3, @@ -29,7 +30,9 @@ %% resource callbacks %% ------------------------------------------------------------------------------------------------- -callback_mode() -> always_sync. +resource_type() -> emqx_redis:resource_type(). + +callback_mode() -> emqx_redis:callback_mode(). on_add_channel( _InstanceId, diff --git a/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq.app.src b/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq.app.src index fc59aeeca..9657ac115 100644 --- a/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq.app.src +++ b/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_rocketmq, [ {description, "EMQX Enterprise RocketMQ Bridge"}, - {vsn, "0.2.2"}, + {vsn, "0.2.3"}, {registered, []}, {applications, [kernel, stdlib, emqx_resource, rocketmq]}, {env, [ diff --git a/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq_connector.erl b/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq_connector.erl index 4fe3ea4c4..b03602bd2 100644 --- a/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq_connector.erl +++ b/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq_connector.erl @@ -16,6 +16,7 @@ %% `emqx_resource' API -export([ + resource_type/0, callback_mode/0, on_start/2, on_stop/2, @@ -90,6 +91,8 @@ servers() -> %% `emqx_resource' API %%======================================================================================== +resource_type() -> rocketmq. + callback_mode() -> always_sync. on_start( diff --git a/apps/emqx_bridge_s3/src/emqx_bridge_s3_connector.erl b/apps/emqx_bridge_s3/src/emqx_bridge_s3_connector.erl index fdc6d255b..a7aa7eae7 100644 --- a/apps/emqx_bridge_s3/src/emqx_bridge_s3_connector.erl +++ b/apps/emqx_bridge_s3/src/emqx_bridge_s3_connector.erl @@ -13,6 +13,7 @@ -behaviour(emqx_resource). -export([ + resource_type/0, callback_mode/0, on_start/2, on_stop/2, @@ -92,6 +93,8 @@ -define(AGGREG_SUP, emqx_bridge_s3_sup). %% +-spec resource_type() -> resource_type(). +resource_type() -> s3. -spec callback_mode() -> callback_mode(). callback_mode() -> diff --git a/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver.app.src b/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver.app.src index 3bc62734c..009a8d16b 100644 --- a/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver.app.src +++ b/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_sqlserver, [ {description, "EMQX Enterprise SQL Server Bridge"}, - {vsn, "0.2.2"}, + {vsn, "0.2.3"}, {registered, []}, {applications, [kernel, stdlib, emqx_resource, odbc]}, {env, [ diff --git a/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver_connector.erl b/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver_connector.erl index 603ef18d0..e14e395c0 100644 --- a/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver_connector.erl +++ b/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver_connector.erl @@ -30,6 +30,7 @@ %% callbacks for behaviour emqx_resource -export([ + resource_type/0, callback_mode/0, on_start/2, on_stop/2, @@ -173,6 +174,7 @@ server() -> %%==================================================================== %% Callbacks defined in emqx_resource %%==================================================================== +resource_type() -> sqlserver. callback_mode() -> always_sync. diff --git a/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper.app.src b/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper.app.src index 5ae95ca67..cd1d51b01 100644 --- a/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper.app.src +++ b/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_syskeeper, [ {description, "EMQX Enterprise Data bridge for Syskeeper"}, - {vsn, "0.1.3"}, + {vsn, "0.1.4"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_connector.erl b/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_connector.erl index 898915f56..c277faa4f 100644 --- a/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_connector.erl +++ b/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_connector.erl @@ -18,6 +18,7 @@ %% `emqx_resource' API -export([ + resource_type/0, callback_mode/0, query_mode/1, on_start/2, @@ -147,6 +148,7 @@ server() -> %% ------------------------------------------------------------------------------------------------- %% `emqx_resource' API +resource_type() -> syskeeper. callback_mode() -> always_sync. diff --git a/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_proxy_server.erl b/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_proxy_server.erl index e26ae43c1..d0aa44cd3 100644 --- a/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_proxy_server.erl +++ b/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_proxy_server.erl @@ -12,6 +12,7 @@ %% `emqx_resource' API -export([ + resource_type/0, query_mode/1, on_start/2, on_stop/2, @@ -40,6 +41,8 @@ %% ------------------------------------------------------------------------------------------------- %% emqx_resource +resource_type() -> + syskeeper_proxy_server. query_mode(_) -> no_queries. diff --git a/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine.app.src b/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine.app.src index d358ba8fa..5fe325b38 100644 --- a/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine.app.src +++ b/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_tdengine, [ {description, "EMQX Enterprise TDEngine Bridge"}, - {vsn, "0.2.1"}, + {vsn, "0.2.2"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine_connector.erl b/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine_connector.erl index 324694edc..46980e768 100644 --- a/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine_connector.erl +++ b/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine_connector.erl @@ -19,6 +19,7 @@ %% `emqx_resource' API -export([ + resource_type/0, callback_mode/0, on_start/2, on_stop/2, @@ -140,6 +141,7 @@ connector_example_values() -> %%======================================================================================== %% `emqx_resource' API %%======================================================================================== +resource_type() -> tdengine. callback_mode() -> always_sync. diff --git a/apps/emqx_connector/src/emqx_connector_resource.erl b/apps/emqx_connector/src/emqx_connector_resource.erl index 2a5b3bcfc..50b2132e2 100644 --- a/apps/emqx_connector/src/emqx_connector_resource.erl +++ b/apps/emqx_connector/src/emqx_connector_resource.erl @@ -126,7 +126,6 @@ create(Type, Name, Conf0, Opts) -> ResourceId = resource_id(Type, Name), Conf = Conf0#{connector_type => TypeBin, connector_name => Name}, {ok, _Data} = emqx_resource:create_local( - Type, ResourceId, ?CONNECTOR_RESOURCE_GROUP, ?MODULE:connector_to_resource_type(Type), @@ -200,7 +199,6 @@ recreate(Type, Name, Conf) -> recreate(Type, Name, Conf, Opts) -> TypeBin = bin(Type), emqx_resource:recreate_local( - Type, resource_id(Type, Name), ?MODULE:connector_to_resource_type(Type), parse_confs(TypeBin, Name, Conf), @@ -211,7 +209,7 @@ create_dry_run(Type, Conf) -> create_dry_run(Type, Conf, fun(_) -> ok end). create_dry_run(Type, Conf0, Callback) -> - %% Already typechecked, no need to catch errors + %% Already type checked, no need to catch errors TypeBin = bin(Type), TypeAtom = safe_atom(Type), %% We use a fixed name here to avoid creating an atom @@ -237,7 +235,7 @@ create_dry_run(Type, Conf0, Callback) -> {ok, ConfNew} -> ParseConf = parse_confs(bin(Type), TmpName, ConfNew), emqx_resource:create_dry_run_local( - Type, TmpName, ?MODULE:connector_to_resource_type(Type), ParseConf, Callback + TmpName, ?MODULE:connector_to_resource_type(Type), ParseConf, Callback ) end catch diff --git a/apps/emqx_connector/test/emqx_connector_SUITE.erl b/apps/emqx_connector/test/emqx_connector_SUITE.erl index fbdece6ff..8e5a6d288 100644 --- a/apps/emqx_connector/test/emqx_connector_SUITE.erl +++ b/apps/emqx_connector/test/emqx_connector_SUITE.erl @@ -50,6 +50,7 @@ t_connector_lifecycle({init, Config}) -> meck:new(emqx_connector_resource, [passthrough]), meck:expect(emqx_connector_resource, connector_to_resource_type, 1, ?CONNECTOR), meck:new(?CONNECTOR, [non_strict]), + meck:expect(?CONNECTOR, resource_type, 0, dummy), meck:expect(?CONNECTOR, callback_mode, 0, async_if_possible), meck:expect(?CONNECTOR, on_start, 2, {ok, connector_state}), meck:expect(?CONNECTOR, on_stop, 2, ok), @@ -171,6 +172,7 @@ t_remove_fail({'init', Config}) -> meck:expect(emqx_connector_resource, connector_to_resource_type, 1, ?CONNECTOR), meck:new(?CONNECTOR, [non_strict]), meck:expect(?CONNECTOR, callback_mode, 0, async_if_possible), + meck:expect(?CONNECTOR, resource_type, 0, dummy), meck:expect(?CONNECTOR, on_start, 2, {ok, connector_state}), meck:expect(?CONNECTOR, on_get_channels, 1, [{<<"my_channel">>, #{enable => true}}]), meck:expect(?CONNECTOR, on_add_channel, 4, {ok, connector_state}), @@ -234,6 +236,7 @@ t_create_with_bad_name_direct_path({init, Config}) -> meck:new(emqx_connector_resource, [passthrough]), meck:expect(emqx_connector_resource, connector_to_resource_type, 1, ?CONNECTOR), meck:new(?CONNECTOR, [non_strict]), + meck:expect(?CONNECTOR, resource_type, 0, dummy), meck:expect(?CONNECTOR, callback_mode, 0, async_if_possible), meck:expect(?CONNECTOR, on_start, 2, {ok, connector_state}), meck:expect(?CONNECTOR, on_stop, 2, ok), @@ -265,6 +268,7 @@ t_create_with_bad_name_root_path({init, Config}) -> meck:new(emqx_connector_resource, [passthrough]), meck:expect(emqx_connector_resource, connector_to_resource_type, 1, ?CONNECTOR), meck:new(?CONNECTOR, [non_strict]), + meck:expect(?CONNECTOR, resource_type, 0, dummy), meck:expect(?CONNECTOR, callback_mode, 0, async_if_possible), meck:expect(?CONNECTOR, on_start, 2, {ok, connector_state}), meck:expect(?CONNECTOR, on_stop, 2, ok), @@ -299,6 +303,7 @@ t_no_buffer_workers({'init', Config}) -> meck:new(emqx_connector_resource, [passthrough]), meck:expect(emqx_connector_resource, connector_to_resource_type, 1, ?CONNECTOR), meck:new(?CONNECTOR, [non_strict]), + meck:expect(?CONNECTOR, resource_type, 0, dummy), meck:expect(?CONNECTOR, callback_mode, 0, async_if_possible), meck:expect(?CONNECTOR, on_start, 2, {ok, connector_state}), meck:expect(?CONNECTOR, on_get_channels, 1, []), diff --git a/apps/emqx_connector/test/emqx_connector_api_SUITE.erl b/apps/emqx_connector/test/emqx_connector_api_SUITE.erl index f3e91ef12..01f4fd188 100644 --- a/apps/emqx_connector/test/emqx_connector_api_SUITE.erl +++ b/apps/emqx_connector/test/emqx_connector_api_SUITE.erl @@ -224,6 +224,7 @@ init_mocks(_TestCase) -> meck:new(emqx_connector_resource, [passthrough, no_link]), meck:expect(emqx_connector_resource, connector_to_resource_type, 1, ?CONNECTOR_IMPL), meck:new(?CONNECTOR_IMPL, [non_strict, no_link]), + meck:expect(?CONNECTOR_IMPL, resource_type, 0, dummy), meck:expect(?CONNECTOR_IMPL, callback_mode, 0, async_if_possible), meck:expect( ?CONNECTOR_IMPL, diff --git a/apps/emqx_connector/test/emqx_connector_dummy_impl.erl b/apps/emqx_connector/test/emqx_connector_dummy_impl.erl index c5d9e4f83..d506c9633 100644 --- a/apps/emqx_connector/test/emqx_connector_dummy_impl.erl +++ b/apps/emqx_connector/test/emqx_connector_dummy_impl.erl @@ -15,8 +15,10 @@ %% this module is only intended to be mocked -module(emqx_connector_dummy_impl). +-behavior(emqx_resource). -export([ + resource_type/0, query_mode/1, callback_mode/0, on_start/2, @@ -25,6 +27,7 @@ on_get_channel_status/3 ]). +resource_type() -> dummy. query_mode(_) -> error(unexpected). callback_mode() -> error(unexpected). on_start(_, _) -> error(unexpected). diff --git a/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_manager.erl b/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_manager.erl index 6ac02efc6..60fec5171 100644 --- a/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_manager.erl +++ b/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_manager.erl @@ -132,7 +132,6 @@ make_resource_id(Backend) -> create_resource(ResourceId, Module, Config) -> Result = emqx_resource:create_local( - dashboard_sso, ResourceId, ?RESOURCE_GROUP, Module, @@ -143,7 +142,7 @@ create_resource(ResourceId, Module, Config) -> update_resource(ResourceId, Module, Config) -> Result = emqx_resource:recreate_local( - dashboard_sso, ResourceId, Module, Config, ?DEFAULT_RESOURCE_OPTS + ResourceId, Module, Config, ?DEFAULT_RESOURCE_OPTS ), start_resource_if_enabled(ResourceId, Result, Config). diff --git a/apps/emqx_ldap/src/emqx_ldap.app.src b/apps/emqx_ldap/src/emqx_ldap.app.src index b0d8ec59c..1b1b667cc 100644 --- a/apps/emqx_ldap/src/emqx_ldap.app.src +++ b/apps/emqx_ldap/src/emqx_ldap.app.src @@ -1,6 +1,6 @@ {application, emqx_ldap, [ {description, "EMQX LDAP Connector"}, - {vsn, "0.1.8"}, + {vsn, "0.1.9"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_ldap/src/emqx_ldap.erl b/apps/emqx_ldap/src/emqx_ldap.erl index d04be5d68..67b250420 100644 --- a/apps/emqx_ldap/src/emqx_ldap.erl +++ b/apps/emqx_ldap/src/emqx_ldap.erl @@ -27,6 +27,7 @@ %% callbacks of behaviour emqx_resource -export([ + resource_type/0, callback_mode/0, on_start/2, on_stop/2, @@ -129,6 +130,8 @@ ensure_username(Field) -> emqx_connector_schema_lib:username(Field). %% =================================================================== +resource_type() -> ldap. + callback_mode() -> always_sync. -spec on_start(binary(), hocon:config()) -> {ok, state()} | {error, _}. diff --git a/apps/emqx_ldap/test/emqx_ldap_SUITE.erl b/apps/emqx_ldap/test/emqx_ldap_SUITE.erl index 413cbc3a5..a15ff2775 100644 --- a/apps/emqx_ldap/test/emqx_ldap_SUITE.erl +++ b/apps/emqx_ldap/test/emqx_ldap_SUITE.erl @@ -96,7 +96,6 @@ perform_lifecycle_check(ResourceId, InitialConfig) -> state := #{pool_name := PoolName} = State, status := InitialStatus }} = emqx_resource:create_local( - ldap, ResourceId, ?CONNECTOR_RESOURCE_GROUP, ?LDAP_RESOURCE_MOD, @@ -172,7 +171,6 @@ t_get_status(Config) -> ?LDAP_RESOURCE_MOD, ldap_config(Config) ), {ok, _} = emqx_resource:create_local( - ldap, ResourceId, ?CONNECTOR_RESOURCE_GROUP, ?LDAP_RESOURCE_MOD, diff --git a/apps/emqx_mongodb/src/emqx_mongodb.app.src b/apps/emqx_mongodb/src/emqx_mongodb.app.src index 92d7026cc..51230fd47 100644 --- a/apps/emqx_mongodb/src/emqx_mongodb.app.src +++ b/apps/emqx_mongodb/src/emqx_mongodb.app.src @@ -1,6 +1,6 @@ {application, emqx_mongodb, [ {description, "EMQX MongoDB Connector"}, - {vsn, "0.1.6"}, + {vsn, "0.1.7"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_mongodb/src/emqx_mongodb.erl b/apps/emqx_mongodb/src/emqx_mongodb.erl index e262f5ccd..8d6fad89f 100644 --- a/apps/emqx_mongodb/src/emqx_mongodb.erl +++ b/apps/emqx_mongodb/src/emqx_mongodb.erl @@ -26,6 +26,7 @@ %% callbacks of behaviour emqx_resource -export([ + resource_type/0, callback_mode/0, on_start/2, on_stop/2, @@ -172,6 +173,7 @@ desc(_) -> undefined. %% =================================================================== +resource_type() -> mongodb. callback_mode() -> always_sync. diff --git a/apps/emqx_mongodb/test/emqx_mongodb_SUITE.erl b/apps/emqx_mongodb/test/emqx_mongodb_SUITE.erl index 850683d99..8af05e0d3 100644 --- a/apps/emqx_mongodb/test/emqx_mongodb_SUITE.erl +++ b/apps/emqx_mongodb/test/emqx_mongodb_SUITE.erl @@ -143,7 +143,6 @@ check_config(Config) -> create_local_resource(ResourceId, CheckedConfig) -> {ok, Bridge} = emqx_resource:create_local( - mongodb, ResourceId, ?CONNECTOR_RESOURCE_GROUP, ?MONGO_RESOURCE_MOD, diff --git a/apps/emqx_mysql/src/emqx_mysql.app.src b/apps/emqx_mysql/src/emqx_mysql.app.src index 9637cc473..c7fcb0975 100644 --- a/apps/emqx_mysql/src/emqx_mysql.app.src +++ b/apps/emqx_mysql/src/emqx_mysql.app.src @@ -1,6 +1,6 @@ {application, emqx_mysql, [ {description, "EMQX MySQL Database Connector"}, - {vsn, "0.1.9"}, + {vsn, "0.2.0"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_mysql/src/emqx_mysql.erl b/apps/emqx_mysql/src/emqx_mysql.erl index 6311d66f2..197e33d75 100644 --- a/apps/emqx_mysql/src/emqx_mysql.erl +++ b/apps/emqx_mysql/src/emqx_mysql.erl @@ -25,6 +25,7 @@ %% callbacks of behaviour emqx_resource -export([ + resource_type/0, callback_mode/0, on_start/2, on_stop/2, @@ -91,6 +92,8 @@ server() -> emqx_schema:servers_sc(Meta, ?MYSQL_HOST_OPTIONS). %% =================================================================== +resource_type() -> mysql. + callback_mode() -> always_sync. -spec on_start(binary(), hocon:config()) -> {ok, state()} | {error, _}. diff --git a/apps/emqx_mysql/test/emqx_mysql_SUITE.erl b/apps/emqx_mysql/test/emqx_mysql_SUITE.erl index be69140fc..03e6c6797 100644 --- a/apps/emqx_mysql/test/emqx_mysql_SUITE.erl +++ b/apps/emqx_mysql/test/emqx_mysql_SUITE.erl @@ -67,7 +67,6 @@ perform_lifecycle_check(ResourceId, InitialConfig) -> state := #{pool_name := PoolName} = State, status := InitialStatus }} = emqx_resource:create_local( - mysql, ResourceId, ?CONNECTOR_RESOURCE_GROUP, ?MYSQL_RESOURCE_MOD, diff --git a/apps/emqx_oracle/src/emqx_oracle.app.src b/apps/emqx_oracle/src/emqx_oracle.app.src index 3f238ae9c..80ff8da09 100644 --- a/apps/emqx_oracle/src/emqx_oracle.app.src +++ b/apps/emqx_oracle/src/emqx_oracle.app.src @@ -1,6 +1,6 @@ {application, emqx_oracle, [ {description, "EMQX Enterprise Oracle Database Connector"}, - {vsn, "0.2.2"}, + {vsn, "0.2.3"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_oracle/src/emqx_oracle.erl b/apps/emqx_oracle/src/emqx_oracle.erl index 5b25e049a..6e2a40f95 100644 --- a/apps/emqx_oracle/src/emqx_oracle.erl +++ b/apps/emqx_oracle/src/emqx_oracle.erl @@ -21,6 +21,7 @@ %% callbacks for behaviour emqx_resource -export([ + resource_type/0, callback_mode/0, on_start/2, on_stop/2, @@ -67,6 +68,8 @@ batch_params_tokens := params_tokens() }. +resource_type() -> oracle. + % As ecpool is not monitoring the worker's PID when doing a handover_async, the % request can be lost if worker crashes. Thus, it's better to force requests to % be sync for now. diff --git a/apps/emqx_postgresql/src/emqx_postgresql.app.src b/apps/emqx_postgresql/src/emqx_postgresql.app.src index 7aaf42e71..e1bd67325 100644 --- a/apps/emqx_postgresql/src/emqx_postgresql.app.src +++ b/apps/emqx_postgresql/src/emqx_postgresql.app.src @@ -1,6 +1,6 @@ {application, emqx_postgresql, [ {description, "EMQX PostgreSQL Database Connector"}, - {vsn, "0.2.2"}, + {vsn, "0.2.3"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_postgresql/src/emqx_postgresql.erl b/apps/emqx_postgresql/src/emqx_postgresql.erl index 7e64a3e83..4df8e0af1 100644 --- a/apps/emqx_postgresql/src/emqx_postgresql.erl +++ b/apps/emqx_postgresql/src/emqx_postgresql.erl @@ -29,6 +29,7 @@ %% callbacks of behaviour emqx_resource -export([ + resource_type/0, callback_mode/0, on_start/2, on_stop/2, @@ -120,6 +121,8 @@ adjust_fields(Fields) -> ). %% =================================================================== +resource_type() -> postgresql. + callback_mode() -> always_sync. -spec on_start(binary(), hocon:config()) -> {ok, state()} | {error, _}. diff --git a/apps/emqx_postgresql/test/emqx_postgresql_SUITE.erl b/apps/emqx_postgresql/test/emqx_postgresql_SUITE.erl index 06210be86..d771d80d8 100644 --- a/apps/emqx_postgresql/test/emqx_postgresql_SUITE.erl +++ b/apps/emqx_postgresql/test/emqx_postgresql_SUITE.erl @@ -75,7 +75,6 @@ perform_lifecycle_check(ResourceId, InitialConfig) -> status := InitialStatus }} = emqx_resource:create_local( - postgresql, ResourceId, ?CONNECTOR_RESOURCE_GROUP, ?PGSQL_RESOURCE_MOD, diff --git a/apps/emqx_redis/src/emqx_redis.erl b/apps/emqx_redis/src/emqx_redis.erl index 059e9aa23..9507913ed 100644 --- a/apps/emqx_redis/src/emqx_redis.erl +++ b/apps/emqx_redis/src/emqx_redis.erl @@ -28,6 +28,7 @@ %% callbacks of behaviour emqx_resource -export([ + resource_type/0, callback_mode/0, on_start/2, on_stop/2, @@ -119,6 +120,8 @@ redis_type(Type) -> desc => ?DESC(Type) }}. +resource_type() -> redis. + callback_mode() -> always_sync. on_start(InstId, Config0) -> diff --git a/apps/emqx_redis/test/emqx_redis_SUITE.erl b/apps/emqx_redis/test/emqx_redis_SUITE.erl index 06ac82143..a9064f184 100644 --- a/apps/emqx_redis/test/emqx_redis_SUITE.erl +++ b/apps/emqx_redis/test/emqx_redis_SUITE.erl @@ -115,7 +115,6 @@ perform_lifecycle_check(ResourceId, InitialConfig, RedisCommand) -> state := #{pool_name := PoolName} = State, status := InitialStatus }} = emqx_resource:create_local( - redis, ResourceId, ?CONNECTOR_RESOURCE_GROUP, ?REDIS_RESOURCE_MOD, diff --git a/apps/emqx_resource/include/emqx_resource.hrl b/apps/emqx_resource/include/emqx_resource.hrl index 0c0cc8f6b..8c2bb39a1 100644 --- a/apps/emqx_resource/include/emqx_resource.hrl +++ b/apps/emqx_resource/include/emqx_resource.hrl @@ -23,8 +23,8 @@ %% remind us of that. -define(rm_status_stopped, stopped). --type type() :: atom() | binary(). --type resource_type() :: module(). +-type resource_type() :: atom(). +-type resource_module() :: module(). -type resource_id() :: binary(). -type channel_id() :: binary(). -type raw_resource_config() :: binary() | raw_term_resource_config(). diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl index e0a2e9343..721df9690 100644 --- a/apps/emqx_resource/src/emqx_resource.erl +++ b/apps/emqx_resource/src/emqx_resource.erl @@ -28,9 +28,9 @@ -export([ check_config/2, + check_and_create_local/4, check_and_create_local/5, - check_and_create_local/6, - check_and_recreate_local/5 + check_and_recreate_local/4 ]). %% Sync resource instances and files @@ -39,11 +39,11 @@ -export([ %% store the config and start the instance - create_local/6, + create_local/5, + create_dry_run_local/2, create_dry_run_local/3, create_dry_run_local/4, - create_dry_run_local/5, - recreate_local/5, + recreate_local/4, %% remove the config and stop the instance remove_local/1, reset_metrics/1, @@ -96,6 +96,7 @@ -export([ %% get the callback mode of a specific module get_callback_mode/1, + get_resource_type/1, %% start the instance call_start/3, %% verify if the resource is working normally @@ -140,9 +141,6 @@ -export([is_dry_run/1]). -%% For emqx_resource_proto_v1 rpc only --export([create_local/5, recreate_local/4, create_dry_run_local/2]). - -export_type([ query_mode/0, resource_id/0, @@ -283,42 +281,42 @@ is_resource_mod(Module) -> %% APIs for resource instances %% ================================================================================= -spec create_local( - type(), resource_id(), resource_group(), - resource_type(), + resource_module(), resource_config(), creation_opts() ) -> {ok, resource_data()}. -create_local(Type, ResId, Group, ResourceType, Config, Opts) -> - emqx_resource_manager:ensure_resource(Type, ResId, Group, ResourceType, Config, Opts). +create_local(ResId, Group, ResourceType, Config, Opts) -> + emqx_resource_manager:ensure_resource(ResId, Group, ResourceType, Config, Opts). --spec create_dry_run_local(type(), resource_type(), resource_config()) -> +-spec create_dry_run_local(resource_module(), resource_config()) -> ok | {error, Reason :: term()}. -create_dry_run_local(Type, ResourceType, Config) -> - emqx_resource_manager:create_dry_run(Type, ResourceType, Config). +create_dry_run_local(ResourceType, Config) -> + emqx_resource_manager:create_dry_run(ResourceType, Config). -create_dry_run_local(Type, ResId, ResourceType, Config) -> - emqx_resource_manager:create_dry_run(Type, ResId, ResourceType, Config). +create_dry_run_local(ResId, ResourceType, Config) -> + emqx_resource_manager:create_dry_run(ResId, ResourceType, Config). -spec create_dry_run_local( - type(), resource_id(), - resource_type(), + resource_module(), resource_config(), OnReadyCallback ) -> ok | {error, Reason :: term()} when OnReadyCallback :: fun((resource_id()) -> ok | {error, Reason :: term()}). -create_dry_run_local(Type, ResId, ResourceType, Config, OnReadyCallback) -> - emqx_resource_manager:create_dry_run(Type, ResId, ResourceType, Config, OnReadyCallback). +create_dry_run_local(ResId, ResourceType, Config, OnReadyCallback) -> + emqx_resource_manager:create_dry_run(ResId, ResourceType, Config, OnReadyCallback). --spec recreate_local(type(), resource_id(), resource_type(), resource_config(), creation_opts()) -> +-spec recreate_local( + resource_id(), resource_module(), resource_config(), creation_opts() +) -> {ok, resource_data()} | {error, Reason :: term()}. -recreate_local(Type, ResId, ResourceType, Config, Opts) -> - emqx_resource_manager:recreate(Type, ResId, ResourceType, Config, Opts). +recreate_local(ResId, ResourceType, Config, Opts) -> + emqx_resource_manager:recreate(ResId, ResourceType, Config, Opts). -spec remove_local(resource_id()) -> ok. remove_local(ResId) -> @@ -490,6 +488,10 @@ list_group_instances(Group) -> emqx_resource_manager:list_group(Group). get_callback_mode(Mod) -> Mod:callback_mode(). +-spec get_resource_type(module()) -> resource_type(). +get_resource_type(Mod) -> + Mod:resource_type(). + -spec call_start(resource_id(), module(), resource_config()) -> {ok, resource_state()} | {error, Reason :: term()}. call_start(ResId, Mod, Config) -> @@ -602,50 +604,47 @@ query_mode(Mod, Config, Opts) -> maps:get(query_mode, Opts, sync) end. --spec check_config(resource_type(), raw_resource_config()) -> +-spec check_config(resource_module(), raw_resource_config()) -> {ok, resource_config()} | {error, term()}. check_config(ResourceType, Conf) -> emqx_hocon:check(ResourceType, Conf). -spec check_and_create_local( - type(), resource_id(), resource_group(), - resource_type(), + resource_module(), raw_resource_config() ) -> {ok, resource_data()} | {error, term()}. -check_and_create_local(Type, ResId, Group, ResourceType, RawConfig) -> - check_and_create_local(Type, ResId, Group, ResourceType, RawConfig, #{}). +check_and_create_local(ResId, Group, ResourceType, RawConfig) -> + check_and_create_local(ResId, Group, ResourceType, RawConfig, #{}). -spec check_and_create_local( - type(), resource_id(), resource_group(), - resource_type(), + resource_module(), raw_resource_config(), creation_opts() ) -> {ok, resource_data()} | {error, term()}. -check_and_create_local(Type, ResId, Group, ResourceType, RawConfig, Opts) -> +check_and_create_local(ResId, Group, ResourceType, RawConfig, Opts) -> check_and_do( ResourceType, RawConfig, - fun(ResConf) -> create_local(Type, ResId, Group, ResourceType, ResConf, Opts) end + fun(ResConf) -> create_local(ResId, Group, ResourceType, ResConf, Opts) end ). -spec check_and_recreate_local( - type(), resource_id(), - resource_type(), + resource_module(), raw_resource_config(), creation_opts() ) -> {ok, resource_data()} | {error, term()}. -check_and_recreate_local(Type, ResId, ResourceType, RawConfig, Opts) -> +check_and_recreate_local(ResId, ResourceType, RawConfig, Opts) -> check_and_do( ResourceType, RawConfig, - fun(ResConf) -> recreate_local(Type, ResId, ResourceType, ResConf, Opts) end + fun(ResConf) -> recreate_local(ResId, ResourceType, ResConf, Opts) end ). check_and_do(ResourceType, RawConfig, Do) when is_function(Do) -> @@ -809,18 +808,3 @@ validate_name(Name, Opts) -> -spec invalid_data(binary()) -> no_return(). invalid_data(Reason) -> throw(#{kind => validation_error, reason => Reason}). - -%% Those functions is only used in the emqx_resource_proto_v1 -%% for versions that are less than version 5.6.0. -%% begin --spec create_local( - resource_id(), resource_group(), resource_type(), resource_config(), creation_opts() -) -> - {ok, resource_data() | 'already_created'} | {error, Reason :: term()}. -create_local(ResId, Group, ResourceType, Config, Opts) -> - create_local(deprecated, ResId, Group, ResourceType, Config, Opts). -create_dry_run_local(ResourceType, Config) -> - create_dry_run_local(deprecated, ResourceType, Config). -recreate_local(ResId, ResourceType, Config, Opts) -> - recreate_local(deprecated, ResId, ResourceType, Config, Opts). -%% end diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl index a742c4486..fe674630c 100644 --- a/apps/emqx_resource/src/emqx_resource_manager.erl +++ b/apps/emqx_resource/src/emqx_resource_manager.erl @@ -25,12 +25,12 @@ % API -export([ - ensure_resource/6, - recreate/5, + ensure_resource/5, + recreate/4, remove/1, + create_dry_run/2, create_dry_run/3, create_dry_run/4, - create_dry_run/5, restart/2, start/2, stop/1, @@ -59,7 +59,7 @@ ]). % Server --export([start_link/6]). +-export([start_link/5]). % Behaviour -export([init/1, callback_mode/0, handle_event/4, terminate/3]). @@ -162,44 +162,45 @@ %% Triggers the emqx_resource_manager_sup supervisor to actually create %% and link the process itself if not already started. -spec ensure_resource( - type(), resource_id(), resource_group(), - resource_type(), + resource_module(), resource_config(), creation_opts() ) -> {ok, resource_data()}. -ensure_resource(Type, ResId, Group, ResourceType, Config, Opts) -> +ensure_resource(ResId, Group, ResourceType, Config, Opts) -> case lookup(ResId) of {ok, _Group, Data} -> {ok, Data}; {error, not_found} -> - create_and_return_data(Type, ResId, Group, ResourceType, Config, Opts) + create_and_return_data(ResId, Group, ResourceType, Config, Opts) end. %% @doc Called from emqx_resource when recreating a resource which may or may not exist --spec recreate(type(), resource_id(), resource_type(), resource_config(), creation_opts()) -> +-spec recreate( + resource_id(), resource_module(), resource_config(), creation_opts() +) -> {ok, resource_data()} | {error, not_found} | {error, updating_to_incorrect_resource_type}. -recreate(Type, ResId, ResourceType, NewConfig, Opts) -> +recreate(ResId, ResourceType, NewConfig, Opts) -> case lookup(ResId) of {ok, Group, #{mod := ResourceType, status := _} = _Data} -> _ = remove(ResId, false), - create_and_return_data(Type, ResId, Group, ResourceType, NewConfig, Opts); + create_and_return_data(ResId, Group, ResourceType, NewConfig, Opts); {ok, _, #{mod := Mod}} when Mod =/= ResourceType -> {error, updating_to_incorrect_resource_type}; {error, not_found} -> {error, not_found} end. -create_and_return_data(Type, ResId, Group, ResourceType, Config, Opts) -> - _ = create(Type, ResId, Group, ResourceType, Config, Opts), +create_and_return_data(ResId, Group, ResourceType, Config, Opts) -> + _ = create(ResId, Group, ResourceType, Config, Opts), {ok, _Group, Data} = lookup(ResId), {ok, Data}. %% @doc Create a resource_manager and wait until it is running -create(Type, ResId, Group, ResourceType, Config, Opts) -> +create(ResId, Group, ResourceType, Config, Opts) -> % The state machine will make the actual call to the callback/resource module after init - ok = emqx_resource_manager_sup:ensure_child(Type, ResId, Group, ResourceType, Config, Opts), + ok = emqx_resource_manager_sup:ensure_child(ResId, Group, ResourceType, Config, Opts), % Create metrics for the resource ok = emqx_resource:create_metrics(ResId), QueryMode = emqx_resource:query_mode(ResourceType, Config, Opts), @@ -222,30 +223,32 @@ create(Type, ResId, Group, ResourceType, Config, Opts) -> %% %% Triggers the `emqx_resource_manager_sup` supervisor to actually create %% and link the process itself if not already started, and then immediately stops. --spec create_dry_run(type(), resource_type(), resource_config()) -> +-spec create_dry_run(resource_module(), resource_config()) -> ok | {error, Reason :: term()}. -create_dry_run(Type, ResourceType, Config) -> +create_dry_run(ResourceType, Config) -> ResId = make_test_id(), - create_dry_run(Type, ResId, ResourceType, Config). + create_dry_run(ResId, ResourceType, Config). -create_dry_run(Type, ResId, ResourceType, Config) -> - create_dry_run(Type, ResId, ResourceType, Config, fun do_nothing_on_ready/1). +create_dry_run(ResId, ResourceType, Config) -> + create_dry_run(ResId, ResourceType, Config, fun do_nothing_on_ready/1). do_nothing_on_ready(_ResId) -> ok. --spec create_dry_run(type(), resource_id(), resource_type(), resource_config(), OnReadyCallback) -> +-spec create_dry_run( + resource_id(), resource_module(), resource_config(), OnReadyCallback +) -> ok | {error, Reason :: term()} when OnReadyCallback :: fun((resource_id()) -> ok | {error, Reason :: term()}). -create_dry_run(Type, ResId, ResourceType, Config, OnReadyCallback) -> +create_dry_run(ResId, ResourceType, Config, OnReadyCallback) -> Opts = case is_map(Config) of true -> maps:get(resource_opts, Config, #{}); false -> #{} end, ok = emqx_resource_manager_sup:ensure_child( - Type, ResId, <<"dry_run">>, ResourceType, Config, Opts + ResId, <<"dry_run">>, ResourceType, Config, Opts ), HealthCheckInterval = maps:get(health_check_interval, Opts, ?HEALTHCHECK_INTERVAL), Timeout = emqx_utils:clamp(HealthCheckInterval, 5_000, 60_000), @@ -495,7 +498,7 @@ try_clean_allocated_resources(ResId) -> %% Server start/stop callbacks %% @doc Function called from the supervisor to actually start the server -start_link(Type, ResId, Group, ResourceType, Config, Opts) -> +start_link(ResId, Group, ResourceType, Config, Opts) -> QueryMode = emqx_resource:query_mode( ResourceType, Config, @@ -503,7 +506,7 @@ start_link(Type, ResId, Group, ResourceType, Config, Opts) -> ), Data = #data{ id = ResId, - type = Type, + type = emqx_resource:get_resource_type(ResourceType), group = Group, mod = ResourceType, callback_mode = emqx_resource:get_callback_mode(ResourceType), diff --git a/apps/emqx_resource/src/emqx_resource_manager_sup.erl b/apps/emqx_resource/src/emqx_resource_manager_sup.erl index c14b08f94..8542eec1c 100644 --- a/apps/emqx_resource/src/emqx_resource_manager_sup.erl +++ b/apps/emqx_resource/src/emqx_resource_manager_sup.erl @@ -19,16 +19,14 @@ -include("emqx_resource.hrl"). --export([ensure_child/6, delete_child/1]). +-export([ensure_child/5, delete_child/1]). -export([start_link/0]). -export([init/1]). -ensure_child(Type, ResId, Group, ResourceType, Config, Opts) -> - case - supervisor:start_child(?MODULE, child_spec(Type, ResId, Group, ResourceType, Config, Opts)) - of +ensure_child(ResId, Group, ResourceType, Config, Opts) -> + case supervisor:start_child(?MODULE, child_spec(ResId, Group, ResourceType, Config, Opts)) of {error, Reason} -> %% This should not happen in production but it can be a huge time sink in %% development environments if the error is just silently ignored. @@ -57,11 +55,11 @@ init([]) -> SupFlags = #{strategy => one_for_one, intensity => 10, period => 10}, {ok, {SupFlags, ChildSpecs}}. -child_spec(Type, ResId, Group, ResourceType, Config, Opts) -> +child_spec(ResId, Group, ResourceType, Config, Opts) -> #{ id => ResId, start => - {emqx_resource_manager, start_link, [Type, ResId, Group, ResourceType, Config, Opts]}, + {emqx_resource_manager, start_link, [ResId, Group, ResourceType, Config, Opts]}, restart => transient, %% never force kill a resource manager. %% because otherwise it may lead to release leak, diff --git a/apps/emqx_resource/src/proto/emqx_resource_proto_v1.erl b/apps/emqx_resource/src/proto/emqx_resource_proto_v1.erl index 859b9fa52..47b724271 100644 --- a/apps/emqx_resource/src/proto/emqx_resource_proto_v1.erl +++ b/apps/emqx_resource/src/proto/emqx_resource_proto_v1.erl @@ -40,7 +40,7 @@ deprecated_since() -> -spec create( resource_id(), resource_group(), - resource_type(), + resource_module(), resource_config(), creation_opts() ) -> @@ -51,7 +51,7 @@ create(ResId, Group, ResourceType, Config, Opts) -> ]). -spec create_dry_run( - resource_type(), + resource_module(), resource_config() ) -> ok | {error, Reason :: term()}. @@ -60,7 +60,7 @@ create_dry_run(ResourceType, Config) -> -spec recreate( resource_id(), - resource_type(), + resource_module(), resource_config(), creation_opts() ) -> diff --git a/apps/emqx_resource/test/emqx_connector_demo.erl b/apps/emqx_resource/test/emqx_connector_demo.erl index 0fc11cc66..e068defb1 100644 --- a/apps/emqx_resource/test/emqx_connector_demo.erl +++ b/apps/emqx_resource/test/emqx_connector_demo.erl @@ -24,6 +24,7 @@ %% callbacks of behaviour emqx_resource -export([ + resource_type/0, callback_mode/0, on_start/2, on_stop/2, @@ -62,6 +63,8 @@ register(required) -> true; register(default) -> false; register(_) -> undefined. +resource_type() -> demo. + callback_mode() -> persistent_term:get(?CM_KEY). diff --git a/apps/emqx_resource/test/emqx_resource_SUITE.erl b/apps/emqx_resource/test/emqx_resource_SUITE.erl index 8e16ec26a..b8fc66c10 100644 --- a/apps/emqx_resource/test/emqx_resource_SUITE.erl +++ b/apps/emqx_resource/test/emqx_resource_SUITE.erl @@ -23,7 +23,6 @@ -include_lib("snabbkaffe/include/snabbkaffe.hrl"). -define(TEST_RESOURCE, emqx_connector_demo). --define(TYPE, test). -define(ID, <<"id">>). -define(ID1, <<"id1">>). -define(DEFAULT_RESOURCE_GROUP, <<"default">>). @@ -91,7 +90,6 @@ t_create_remove(_) -> ?assertMatch( {error, _}, emqx_resource:check_and_create_local( - ?TYPE, ?ID, ?DEFAULT_RESOURCE_GROUP, ?TEST_RESOURCE, @@ -112,7 +110,6 @@ t_create_remove(_) -> ?assertMatch( {ok, _}, emqx_resource:recreate_local( - ?TYPE, ?ID, ?TEST_RESOURCE, #{name => test_resource}, @@ -138,7 +135,6 @@ t_create_remove_local(_) -> ?assertMatch( {error, _}, emqx_resource:check_and_create_local( - ?TYPE, ?ID, ?DEFAULT_RESOURCE_GROUP, ?TEST_RESOURCE, @@ -157,7 +153,6 @@ t_create_remove_local(_) -> ), emqx_resource:recreate_local( - ?TYPE, ?ID, ?TEST_RESOURCE, #{name => test_resource}, @@ -171,7 +166,6 @@ t_create_remove_local(_) -> emqx_resource:set_resource_status_connecting(?ID), emqx_resource:recreate_local( - ?TYPE, ?ID, ?TEST_RESOURCE, #{name => test_resource}, @@ -943,7 +937,6 @@ t_stop_start(_) -> ?assertMatch( {error, _}, emqx_resource:check_and_create_local( - ?TYPE, ?ID, ?DEFAULT_RESOURCE_GROUP, ?TEST_RESOURCE, @@ -954,7 +947,6 @@ t_stop_start(_) -> ?assertMatch( {ok, _}, emqx_resource:check_and_create_local( - ?TYPE, ?ID, ?DEFAULT_RESOURCE_GROUP, ?TEST_RESOURCE, @@ -972,7 +964,6 @@ t_stop_start(_) -> ?assertMatch( {ok, _}, emqx_resource:check_and_recreate_local( - ?TYPE, ?ID, ?TEST_RESOURCE, #{<<"name">> => <<"test_resource">>}, @@ -1022,7 +1013,6 @@ t_stop_start_local(_) -> ?assertMatch( {error, _}, emqx_resource:check_and_create_local( - ?TYPE, ?ID, ?DEFAULT_RESOURCE_GROUP, ?TEST_RESOURCE, @@ -1033,7 +1023,6 @@ t_stop_start_local(_) -> ?assertMatch( {ok, _}, emqx_resource:check_and_create_local( - ?TYPE, ?ID, ?DEFAULT_RESOURCE_GROUP, ?TEST_RESOURCE, @@ -1044,7 +1033,6 @@ t_stop_start_local(_) -> ?assertMatch( {ok, _}, emqx_resource:check_and_recreate_local( - ?TYPE, ?ID, ?TEST_RESOURCE, #{<<"name">> => <<"test_resource">>}, @@ -1120,7 +1108,6 @@ create_dry_run_local_succ() -> ?assertEqual( ok, emqx_resource:create_dry_run_local( - test, ?TEST_RESOURCE, #{name => test_resource, register => true} ) @@ -1131,7 +1118,6 @@ t_create_dry_run_local_failed(_) -> ct:timetrap({seconds, 120}), ct:pal("creating with creation error"), Res1 = emqx_resource:create_dry_run_local( - test, ?TEST_RESOURCE, #{create_error => true} ), @@ -1139,7 +1125,6 @@ t_create_dry_run_local_failed(_) -> ct:pal("creating with health check error"), Res2 = emqx_resource:create_dry_run_local( - test, ?TEST_RESOURCE, #{name => test_resource, health_check_error => true} ), @@ -1147,7 +1132,6 @@ t_create_dry_run_local_failed(_) -> ct:pal("creating with stop error"), Res3 = emqx_resource:create_dry_run_local( - test, ?TEST_RESOURCE, #{name => test_resource, stop_error => true} ), @@ -3506,10 +3490,10 @@ gauge_metric_set_fns() -> ]. create(Id, Group, Type, Config) -> - emqx_resource:create_local(test, Id, Group, Type, Config, #{}). + emqx_resource:create_local(Id, Group, Type, Config, #{}). create(Id, Group, Type, Config, Opts) -> - emqx_resource:create_local(test, Id, Group, Type, Config, Opts). + emqx_resource:create_local(Id, Group, Type, Config, Opts). log_consistency_prop() -> {"check state and cache consistency", fun ?MODULE:log_consistency_prop/1}. diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_test_connector.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_test_connector.erl index 6a0d6b3ec..cf5d3afbe 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_test_connector.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_test_connector.erl @@ -25,6 +25,7 @@ %% callbacks of behaviour emqx_resource -export([ + resource_type/0, callback_mode/0, on_start/2, on_stop/2, @@ -40,6 +41,8 @@ ]). %% =================================================================== +resource_type() -> test_connector. + callback_mode() -> always_sync. on_start( From 7374123c5cf61e4aec39e452ca96ae232d1f5095 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 23 Jul 2024 16:10:55 -0300 Subject: [PATCH 022/131] fix(jwt): fix grace period for renewal check --- apps/emqx_connector/src/emqx_connector_jwt.erl | 2 +- apps/emqx_connector/test/emqx_connector_jwt_SUITE.erl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_connector/src/emqx_connector_jwt.erl b/apps/emqx_connector/src/emqx_connector_jwt.erl index dd74754ba..60b35ddbb 100644 --- a/apps/emqx_connector/src/emqx_connector_jwt.erl +++ b/apps/emqx_connector/src/emqx_connector_jwt.erl @@ -141,5 +141,5 @@ store_jwt(#{resource_id := ResourceId, table := TId}, JWT) -> is_about_to_expire(JWT) -> #jose_jwt{fields = #{<<"exp">> := Exp}} = jose_jwt:peek(JWT), Now = erlang:system_time(seconds), - GraceExp = Exp - timer:seconds(5), + GraceExp = Exp - 5, Now >= GraceExp. diff --git a/apps/emqx_connector/test/emqx_connector_jwt_SUITE.erl b/apps/emqx_connector/test/emqx_connector_jwt_SUITE.erl index 6469614f8..aef0e660c 100644 --- a/apps/emqx_connector/test/emqx_connector_jwt_SUITE.erl +++ b/apps/emqx_connector/test/emqx_connector_jwt_SUITE.erl @@ -125,7 +125,7 @@ t_ensure_jwt(_Config) -> JWT0 = emqx_connector_jwt:ensure_jwt(JWTConfig), ?assertNot(is_expired(JWT0)), %% should refresh 5 s before expiration - ct:sleep(Expiration - 5500), + ct:sleep(Expiration - 3000), JWT1 = emqx_connector_jwt:ensure_jwt(JWTConfig), ?assertNot(is_expired(JWT1)), %% fully expired From 4d7535df2df8a2c59607e5bdc97e5e0b3b273f24 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 24 Jul 2024 09:50:01 +0800 Subject: [PATCH 023/131] chore: use pgsqsl replace postgresql --- .../src/emqx_bridge_hstreamdb_connector.erl | 2 +- apps/emqx_postgresql/src/emqx_postgresql.erl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb_connector.erl b/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb_connector.erl index 154e43b3d..2d061e455 100644 --- a/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb_connector.erl +++ b/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb_connector.erl @@ -45,7 +45,7 @@ %% ------------------------------------------------------------------------------------------------- %% resource callback -resource_type() -> hstreamsdb. +resource_type() -> hstreamdb. callback_mode() -> always_sync. diff --git a/apps/emqx_postgresql/src/emqx_postgresql.erl b/apps/emqx_postgresql/src/emqx_postgresql.erl index 4df8e0af1..a061fc15e 100644 --- a/apps/emqx_postgresql/src/emqx_postgresql.erl +++ b/apps/emqx_postgresql/src/emqx_postgresql.erl @@ -121,7 +121,7 @@ adjust_fields(Fields) -> ). %% =================================================================== -resource_type() -> postgresql. +resource_type() -> pgsql. callback_mode() -> always_sync. From ca2d4ad2a0ac86230226713e927c0fede081664a Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 24 Jul 2024 10:04:27 -0300 Subject: [PATCH 024/131] refactor: move metrics logic to separate module --- .../src/emqx_cluster_link.erl | 57 ---------------- .../src/emqx_cluster_link_api.erl | 2 +- .../src/emqx_cluster_link_app.erl | 2 +- .../src/emqx_cluster_link_config.erl | 4 +- .../src/emqx_cluster_link_extrouter.erl | 4 +- .../src/emqx_cluster_link_metrics.erl | 67 +++++++++++++++++++ 6 files changed, 73 insertions(+), 63 deletions(-) create mode 100644 apps/emqx_cluster_link/src/emqx_cluster_link_metrics.erl diff --git a/apps/emqx_cluster_link/src/emqx_cluster_link.erl b/apps/emqx_cluster_link/src/emqx_cluster_link.erl index e3bc04a29..76228c052 100644 --- a/apps/emqx_cluster_link/src/emqx_cluster_link.erl +++ b/apps/emqx_cluster_link/src/emqx_cluster_link.erl @@ -26,29 +26,11 @@ on_message_publish/1 ]). -%% metrics API --export([ - maybe_create_metrics/1, - drop_metrics/1, - - get_metrics/1, - routes_inc/2 -]). - -include("emqx_cluster_link.hrl"). -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/emqx_hooks.hrl"). -include_lib("emqx/include/logger.hrl"). -%%-------------------------------------------------------------------- -%% Type definitions -%%-------------------------------------------------------------------- - --define(METRICS, [ - ?route_metric -]). --define(RATE_METRICS, []). - %%-------------------------------------------------------------------- %% emqx_external_broker API %%-------------------------------------------------------------------- @@ -150,32 +132,6 @@ put_hook() -> delete_hook() -> emqx_hooks:del('message.publish', {?MODULE, on_message_publish, []}). -%%-------------------------------------------------------------------- -%% metrics API -%%-------------------------------------------------------------------- - -get_metrics(ClusterName) -> - Nodes = emqx:running_nodes(), - Timeout = 15_000, - Results = emqx_metrics_proto_v2:get_metrics(Nodes, ?METRIC_NAME, ClusterName, Timeout), - sequence_multicall_results(Nodes, Results). - -maybe_create_metrics(ClusterName) -> - case emqx_metrics_worker:has_metrics(?METRIC_NAME, ClusterName) of - true -> - ok = emqx_metrics_worker:reset_metrics(?METRIC_NAME, ClusterName); - false -> - ok = emqx_metrics_worker:create_metrics( - ?METRIC_NAME, ClusterName, ?METRICS, ?RATE_METRICS - ) - end. - -drop_metrics(ClusterName) -> - ok = emqx_metrics_worker:clear_metrics(?METRIC_NAME, ClusterName). - -routes_inc(ClusterName, Val) -> - catch emqx_metrics_worker:inc(?METRIC_NAME, ClusterName, ?route_metric, Val). - %%-------------------------------------------------------------------- %% Internal functions %%-------------------------------------------------------------------- @@ -297,16 +253,3 @@ maybe_filter_incomming_msg(#message{topic = T} = Msg, ClusterName) -> true -> with_sender_name(Msg, ClusterName); false -> [] end. - --spec sequence_multicall_results([node()], emqx_rpc:erpc_multicall(term())) -> - {ok, [{node(), term()}]} | {error, [term()]}. -sequence_multicall_results(Nodes, Results) -> - case lists:partition(fun is_ok/1, lists:zip(Nodes, Results)) of - {OkResults, []} -> - {ok, [{Node, Res} || {Node, {ok, Res}} <- OkResults]}; - {_OkResults, BadResults} -> - {error, BadResults} - end. - -is_ok({_Node, {ok, _}}) -> true; -is_ok(_) -> false. diff --git a/apps/emqx_cluster_link/src/emqx_cluster_link_api.erl b/apps/emqx_cluster_link/src/emqx_cluster_link_api.erl index 7e70a9ccc..0d748a267 100644 --- a/apps/emqx_cluster_link/src/emqx_cluster_link_api.erl +++ b/apps/emqx_cluster_link/src/emqx_cluster_link_api.erl @@ -227,7 +227,7 @@ handle_lookup(Name, Link) -> ?OK(add_status(Name, Link)). handle_metrics(Name) -> - case emqx_cluster_link:get_metrics(Name) of + case emqx_cluster_link_metrics:get_metrics(Name) of {error, BadResults} -> ?SLOG(warning, #{ msg => "cluster_link_api_metrics_bad_erpc_results", diff --git a/apps/emqx_cluster_link/src/emqx_cluster_link_app.erl b/apps/emqx_cluster_link/src/emqx_cluster_link_app.erl index f9625fae4..9502ad1c3 100644 --- a/apps/emqx_cluster_link/src/emqx_cluster_link_app.erl +++ b/apps/emqx_cluster_link/src/emqx_cluster_link_app.erl @@ -59,7 +59,7 @@ remove_msg_fwd_resources(LinksConf) -> create_metrics(LinksConf) -> lists:foreach( fun(#{name := ClusterName}) -> - ok = emqx_cluster_link:maybe_create_metrics(ClusterName) + ok = emqx_cluster_link_metrics:maybe_create_metrics(ClusterName) end, LinksConf ). diff --git a/apps/emqx_cluster_link/src/emqx_cluster_link_config.erl b/apps/emqx_cluster_link/src/emqx_cluster_link_config.erl index 0455ab21c..5e257a247 100644 --- a/apps/emqx_cluster_link/src/emqx_cluster_link_config.erl +++ b/apps/emqx_cluster_link/src/emqx_cluster_link_config.erl @@ -280,7 +280,7 @@ add_links(LinksConf) -> add_link(#{name := ClusterName, enable := true} = LinkConf) -> {ok, _Pid} = emqx_cluster_link_sup:ensure_actor(LinkConf), {ok, _} = emqx_cluster_link_mqtt:ensure_msg_fwd_resource(LinkConf), - ok = emqx_cluster_link:maybe_create_metrics(ClusterName), + ok = emqx_cluster_link_metrics:maybe_create_metrics(ClusterName), ok; add_link(_DisabledLinkConf) -> ok. @@ -291,7 +291,7 @@ remove_links(LinksConf) -> remove_link(Name) -> _ = emqx_cluster_link_mqtt:remove_msg_fwd_resource(Name), _ = ensure_actor_stopped(Name), - emqx_cluster_link:drop_metrics(Name). + emqx_cluster_link_metrics:drop_metrics(Name). update_links(LinksConf) -> [update_link(Link) || Link <- LinksConf]. diff --git a/apps/emqx_cluster_link/src/emqx_cluster_link_extrouter.erl b/apps/emqx_cluster_link/src/emqx_cluster_link_extrouter.erl index c45b12ae0..3e2ff1804 100644 --- a/apps/emqx_cluster_link/src/emqx_cluster_link_extrouter.erl +++ b/apps/emqx_cluster_link/src/emqx_cluster_link_extrouter.erl @@ -282,7 +282,7 @@ apply_operation(ActorID, Entry, MCounter, OpName, Lane) -> case MCounter band Marker of 0 when OpName =:= add -> Res = mria:dirty_update_counter(?EXTROUTE_TAB, Entry, Marker), - _ = emqx_cluster_link:routes_inc(ClusterName, 1), + _ = emqx_cluster_link_metrics:routes_inc(ClusterName, 1), ?tp("cluster_link_extrouter_route_added", #{}), Res; Marker when OpName =:= add -> @@ -293,7 +293,7 @@ apply_operation(ActorID, Entry, MCounter, OpName, Lane) -> 0 -> Record = #extroute{entry = Entry, mcounter = 0}, ok = mria:dirty_delete_object(?EXTROUTE_TAB, Record), - _ = emqx_cluster_link:routes_inc(ClusterName, -1), + _ = emqx_cluster_link_metrics:routes_inc(ClusterName, -1), ?tp("cluster_link_extrouter_route_deleted", #{}), 0; C -> diff --git a/apps/emqx_cluster_link/src/emqx_cluster_link_metrics.erl b/apps/emqx_cluster_link/src/emqx_cluster_link_metrics.erl new file mode 100644 index 000000000..3d6f1edc8 --- /dev/null +++ b/apps/emqx_cluster_link/src/emqx_cluster_link_metrics.erl @@ -0,0 +1,67 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_cluster_link_metrics). + +-include("emqx_cluster_link.hrl"). + +%% API +-export([ + maybe_create_metrics/1, + drop_metrics/1, + + get_metrics/1, + routes_inc/2 +]). + +%%-------------------------------------------------------------------- +%% Type definitions +%%-------------------------------------------------------------------- + +-define(METRICS, [ + ?route_metric +]). +-define(RATE_METRICS, []). + +%%-------------------------------------------------------------------- +%% metrics API +%%-------------------------------------------------------------------- + +get_metrics(ClusterName) -> + Nodes = emqx:running_nodes(), + Timeout = 15_000, + Results = emqx_metrics_proto_v2:get_metrics(Nodes, ?METRIC_NAME, ClusterName, Timeout), + sequence_multicall_results(Nodes, Results). + +maybe_create_metrics(ClusterName) -> + case emqx_metrics_worker:has_metrics(?METRIC_NAME, ClusterName) of + true -> + ok = emqx_metrics_worker:reset_metrics(?METRIC_NAME, ClusterName); + false -> + ok = emqx_metrics_worker:create_metrics( + ?METRIC_NAME, ClusterName, ?METRICS, ?RATE_METRICS + ) + end. + +drop_metrics(ClusterName) -> + ok = emqx_metrics_worker:clear_metrics(?METRIC_NAME, ClusterName). + +routes_inc(ClusterName, Val) -> + catch emqx_metrics_worker:inc(?METRIC_NAME, ClusterName, ?route_metric, Val). + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- + +-spec sequence_multicall_results([node()], emqx_rpc:erpc_multicall(term())) -> + {ok, [{node(), term()}]} | {error, [term()]}. +sequence_multicall_results(Nodes, Results) -> + case lists:partition(fun is_ok/1, lists:zip(Nodes, Results)) of + {OkResults, []} -> + {ok, [{Node, Res} || {Node, {ok, Res}} <- OkResults]}; + {_OkResults, BadResults} -> + {error, BadResults} + end. + +is_ok({_Node, {ok, _}}) -> true; +is_ok(_) -> false. From 216a6abed9e8d4d04f4b4df23b4958f2649ede8f Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 24 Jul 2024 10:10:43 -0300 Subject: [PATCH 025/131] refactor: rename CRUD functions --- .../src/emqx_cluster_link_api.erl | 6 +++--- .../src/emqx_cluster_link_config.erl | 18 +++++++++--------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/apps/emqx_cluster_link/src/emqx_cluster_link_api.erl b/apps/emqx_cluster_link/src/emqx_cluster_link_api.erl index 0d748a267..552e7d6d3 100644 --- a/apps/emqx_cluster_link/src/emqx_cluster_link_api.erl +++ b/apps/emqx_cluster_link/src/emqx_cluster_link_api.erl @@ -181,7 +181,7 @@ fields(node_metrics) -> with_link( Name, fun() -> - case emqx_cluster_link_config:delete(Name) of + case emqx_cluster_link_config:delete_link(Name) of ok -> ?NO_CONTENT; {error, Reason} -> @@ -215,7 +215,7 @@ handle_list() -> ?OK(Response). handle_create(Name, Params) -> - case emqx_cluster_link_config:create(Params) of + case emqx_cluster_link_config:create_link(Params) of {ok, Link} -> ?CREATED(add_status(Name, Link)); {error, Reason} -> @@ -273,7 +273,7 @@ add_status(Name, Link) -> handle_update(Name, Params0) -> Params = Params0#{<<"name">> => Name}, - case emqx_cluster_link_config:update_one_link(Params) of + case emqx_cluster_link_config:update_link(Params) of {ok, Link} -> ?OK(add_status(Name, Link)); {error, Reason} -> diff --git a/apps/emqx_cluster_link/src/emqx_cluster_link_config.erl b/apps/emqx_cluster_link/src/emqx_cluster_link_config.erl index 5e257a247..c35b218db 100644 --- a/apps/emqx_cluster_link/src/emqx_cluster_link_config.erl +++ b/apps/emqx_cluster_link/src/emqx_cluster_link_config.erl @@ -30,9 +30,9 @@ -export([ %% General - create/1, - delete/1, - update_one_link/1, + create_link/1, + delete_link/1, + update_link/1, update/1, cluster/0, enabled_links/0, @@ -61,7 +61,7 @@ %% -create(LinkConfig) -> +create_link(LinkConfig) -> #{<<"name">> := Name} = LinkConfig, case emqx_conf:update( @@ -77,7 +77,7 @@ create(LinkConfig) -> {error, Reason} end. -delete(Name) -> +delete_link(Name) -> case emqx_conf:update( ?LINKS_PATH, @@ -91,7 +91,7 @@ delete(Name) -> {error, Reason} end. -update_one_link(LinkConfig) -> +update_link(LinkConfig) -> #{<<"name">> := Name} = LinkConfig, case emqx_conf:update( @@ -294,9 +294,9 @@ remove_link(Name) -> emqx_cluster_link_metrics:drop_metrics(Name). update_links(LinksConf) -> - [update_link(Link) || Link <- LinksConf]. + [do_update_link(Link) || Link <- LinksConf]. -update_link({OldLinkConf, #{enable := true, name := Name} = NewLinkConf}) -> +do_update_link({OldLinkConf, #{enable := true, name := Name} = NewLinkConf}) -> case what_is_changed(OldLinkConf, NewLinkConf) of both -> _ = ensure_actor_stopped(Name), @@ -309,7 +309,7 @@ update_link({OldLinkConf, #{enable := true, name := Name} = NewLinkConf}) -> msg_resource -> ok = update_msg_fwd_resource(OldLinkConf, NewLinkConf) end; -update_link({_OldLinkConf, #{enable := false, name := Name} = _NewLinkConf}) -> +do_update_link({_OldLinkConf, #{enable := false, name := Name} = _NewLinkConf}) -> _ = emqx_cluster_link_mqtt:remove_msg_fwd_resource(Name), ensure_actor_stopped(Name). From 2d507146ab3ff5669ff45363d69a09d21526ec82 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 24 Jul 2024 10:13:48 -0300 Subject: [PATCH 026/131] refactor: change style of case clause --- apps/emqx_cluster_link/src/emqx_cluster_link_api.erl | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/apps/emqx_cluster_link/src/emqx_cluster_link_api.erl b/apps/emqx_cluster_link/src/emqx_cluster_link_api.erl index 552e7d6d3..8e84b376b 100644 --- a/apps/emqx_cluster_link/src/emqx_cluster_link_api.erl +++ b/apps/emqx_cluster_link/src/emqx_cluster_link_api.erl @@ -498,13 +498,11 @@ with_link(Name, FoundFn, NotFoundFn) -> case emqx_cluster_link_config:link_raw(Name) of undefined -> NotFoundFn(); - Link0 = #{} -> + Link0 = #{} when is_function(FoundFn, 1) -> Link = fill_defaults_single(Link0), - {arity, Arity} = erlang:fun_info(FoundFn, arity), - case Arity of - 1 -> FoundFn(Link); - 0 -> FoundFn() - end + FoundFn(Link); + _Link = #{} when is_function(FoundFn, 0) -> + FoundFn() end. fill_defaults_single(Link0) -> From 82bb876de060811de030c34d565b197289ae0980 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 24 Jul 2024 10:15:01 -0300 Subject: [PATCH 027/131] docs: improve descriptions Co-authored-by: Andrew Mayorov --- apps/emqx_cluster_link/src/emqx_cluster_link_api.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_cluster_link/src/emqx_cluster_link_api.erl b/apps/emqx_cluster_link/src/emqx_cluster_link_api.erl index 8e84b376b..5eca0a944 100644 --- a/apps/emqx_cluster_link/src/emqx_cluster_link_api.erl +++ b/apps/emqx_cluster_link/src/emqx_cluster_link_api.erl @@ -55,7 +55,7 @@ schema("/cluster/links") -> }, post => #{ - description => "Create a cluster link configuration", + description => "Create a cluster link", tags => ?TAGS, 'requestBody' => link_config_schema(), responses => @@ -87,7 +87,7 @@ schema("/cluster/links/link/:name") -> }, delete => #{ - description => "Delete a cluster link configuration", + description => "Delete a cluster link", tags => ?TAGS, parameters => [param_path_name()], responses => From 76e51fa532c1b81db9a6bdbb5d157e7426b42077 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 24 Jul 2024 10:17:45 -0300 Subject: [PATCH 028/131] fix: correctly use maybe match clause --- apps/emqx_cluster_link/src/emqx_cluster_link_config.erl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx_cluster_link/src/emqx_cluster_link_config.erl b/apps/emqx_cluster_link/src/emqx_cluster_link_config.erl index c35b218db..2a97f2d69 100644 --- a/apps/emqx_cluster_link/src/emqx_cluster_link_config.erl +++ b/apps/emqx_cluster_link/src/emqx_cluster_link_config.erl @@ -208,7 +208,7 @@ pre_config_update(?LINKS_PATH, {create, LinkRawConf}, OldRawConf) -> pre_config_update(?LINKS_PATH, {update, LinkRawConf}, OldRawConf) -> #{<<"name">> := Name} = LinkRawConf, maybe - {ok, {_Found, Front, Rear}} = safe_take(Name, OldRawConf), + {_Found, Front, Rear} ?= safe_take(Name, OldRawConf), NewRawConf0 = Front ++ [LinkRawConf] ++ Rear, NewRawConf = convert_certs(maybe_increment_ps_actor_incr(NewRawConf0, OldRawConf)), {ok, NewRawConf} @@ -218,7 +218,7 @@ pre_config_update(?LINKS_PATH, {update, LinkRawConf}, OldRawConf) -> end; pre_config_update(?LINKS_PATH, {delete, Name}, OldRawConf) -> maybe - {ok, {_Found, Front, Rear}} = safe_take(Name, OldRawConf), + {_Found, Front, Rear} ?= safe_take(Name, OldRawConf), NewRawConf = Front ++ Rear, {ok, NewRawConf} else @@ -420,5 +420,5 @@ safe_take(Name, Transformations) -> {_Front, []} -> not_found; {Front, [Found | Rear]} -> - {ok, {Found, Front, Rear}} + {Found, Front, Rear} end. From d2da3114163998da951251c202f77e122b717905 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 24 Jul 2024 09:38:09 -0300 Subject: [PATCH 029/131] fix(resource): create undocumented callback Created by https://github.com/emqx/emqx/pull/13449 but not added as a callback. --- apps/emqx_resource/src/emqx_resource.erl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl index 721df9690..bf47c83ab 100644 --- a/apps/emqx_resource/src/emqx_resource.erl +++ b/apps/emqx_resource/src/emqx_resource.erl @@ -244,6 +244,9 @@ QueryResult :: term() ) -> term(). +%% Used for tagging log entries. +-callback resource_type() -> atom(). + -define(SAFE_CALL(EXPR), (fun() -> try From 3e4eeddb78002653fd0f6fc3bdaa7d0b6f70ee5b Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 24 Jul 2024 09:43:58 -0300 Subject: [PATCH 030/131] fix: add missing `resource_type` callback implementations --- .../src/emqx_bridge_azure_blob_storage_connector.erl | 5 +++++ .../src/emqx_bridge_couchbase_connector.erl | 5 +++++ apps/emqx_cluster_link/src/emqx_cluster_link_mqtt.erl | 5 +++++ 3 files changed, 15 insertions(+) diff --git a/apps/emqx_bridge_azure_blob_storage/src/emqx_bridge_azure_blob_storage_connector.erl b/apps/emqx_bridge_azure_blob_storage/src/emqx_bridge_azure_blob_storage_connector.erl index cdf16a8cb..740493f49 100644 --- a/apps/emqx_bridge_azure_blob_storage/src/emqx_bridge_azure_blob_storage_connector.erl +++ b/apps/emqx_bridge_azure_blob_storage/src/emqx_bridge_azure_blob_storage_connector.erl @@ -18,6 +18,7 @@ %% `emqx_resource' API -export([ callback_mode/0, + resource_type/0, on_start/2, on_stop/2, @@ -148,6 +149,10 @@ callback_mode() -> always_sync. +-spec resource_type() -> atom(). +resource_type() -> + azure_blob_storage. + -spec on_start(connector_resource_id(), connector_config()) -> {ok, connector_state()} | {error, _Reason}. on_start(_ConnResId, ConnConfig) -> diff --git a/apps/emqx_bridge_couchbase/src/emqx_bridge_couchbase_connector.erl b/apps/emqx_bridge_couchbase/src/emqx_bridge_couchbase_connector.erl index 1e7122800..2c104ee16 100644 --- a/apps/emqx_bridge_couchbase/src/emqx_bridge_couchbase_connector.erl +++ b/apps/emqx_bridge_couchbase/src/emqx_bridge_couchbase_connector.erl @@ -15,6 +15,7 @@ %% `emqx_resource' API -export([ callback_mode/0, + resource_type/0, on_start/2, on_stop/2, @@ -84,6 +85,10 @@ callback_mode() -> always_sync. +-spec resource_type() -> atom(). +resource_type() -> + couchbase. + -spec on_start(connector_resource_id(), connector_config()) -> {ok, connector_state()} | {error, _Reason}. on_start(ConnResId, ConnConfig) -> diff --git a/apps/emqx_cluster_link/src/emqx_cluster_link_mqtt.erl b/apps/emqx_cluster_link/src/emqx_cluster_link_mqtt.erl index 5185803b6..3a6411cbe 100644 --- a/apps/emqx_cluster_link/src/emqx_cluster_link_mqtt.erl +++ b/apps/emqx_cluster_link/src/emqx_cluster_link_mqtt.erl @@ -19,6 +19,7 @@ %% callbacks of behaviour emqx_resource -export([ callback_mode/0, + resource_type/0, on_start/2, on_stop/2, on_query/3, @@ -99,6 +100,10 @@ remove_msg_fwd_resource(ClusterName) -> callback_mode() -> async_if_possible. +-spec resource_type() -> atom(). +resource_type() -> + cluster_link_mqtt. + on_start(ResourceId, #{pool_size := PoolSize} = ClusterConf) -> PoolName = ResourceId, Options = [ From 79db2e6d7faaa9a96e9da4acb9d6f9ddfba93f1a Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 24 Jul 2024 10:28:11 -0300 Subject: [PATCH 031/131] test: fix flaky test --- .../test/emqx_cluster_link_api_SUITE.erl | 38 ++++++++++--------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/apps/emqx_cluster_link/test/emqx_cluster_link_api_SUITE.erl b/apps/emqx_cluster_link/test/emqx_cluster_link_api_SUITE.erl index 27cbe7532..0e5143865 100644 --- a/apps/emqx_cluster_link/test/emqx_cluster_link_api_SUITE.erl +++ b/apps/emqx_cluster_link/test/emqx_cluster_link_api_SUITE.erl @@ -303,23 +303,27 @@ t_crud(_Config) -> t_status(Config) -> [SN1 | _] = ?config(source_nodes, Config), Name = <<"cl.target">>, - ?assertMatch( - {200, [ - #{ - <<"status">> := <<"connected">>, - <<"node_status">> := [ - #{ - <<"node">> := _, - <<"status">> := <<"connected">> - }, - #{ - <<"node">> := _, - <<"status">> := <<"connected">> - } - ] - } - ]}, - list() + ?retry( + 100, + 10, + ?assertMatch( + {200, [ + #{ + <<"status">> := <<"connected">>, + <<"node_status">> := [ + #{ + <<"node">> := _, + <<"status">> := <<"connected">> + }, + #{ + <<"node">> := _, + <<"status">> := <<"connected">> + } + ] + } + ]}, + list() + ) ), ?assertMatch( {200, #{ From 34f5a886cea0caa0d9bfa5631ac36fdcf261b9ec Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 24 Jul 2024 12:06:48 -0300 Subject: [PATCH 032/131] refactor(cluster link api): return erpc errors in status and metrics responses --- .../src/emqx_cluster_link_api.erl | 151 ++++++++++-------- .../src/emqx_cluster_link_metrics.erl | 15 +- .../src/emqx_cluster_link_mqtt.erl | 23 +-- .../test/emqx_cluster_link_api_SUITE.erl | 24 ++- 4 files changed, 109 insertions(+), 104 deletions(-) diff --git a/apps/emqx_cluster_link/src/emqx_cluster_link_api.erl b/apps/emqx_cluster_link/src/emqx_cluster_link_api.erl index 5eca0a944..257a19854 100644 --- a/apps/emqx_cluster_link/src/emqx_cluster_link_api.erl +++ b/apps/emqx_cluster_link/src/emqx_cluster_link_api.erl @@ -201,9 +201,15 @@ fields(node_metrics) -> handle_list() -> Links = get_raw(), - NodeResults = get_all_link_status_cluster(), - NameToStatus = collect_all_status(NodeResults), - EmptyStatus = #{status => inconsistent, node_status => []}, + NodeRPCResults = emqx_cluster_link_mqtt:get_all_resources_cluster(), + {NameToStatus, Errors} = collect_all_status(NodeRPCResults), + NodeErrors = lists:map( + fun({Node, Error}) -> + #{node => Node, status => inconsistent, reason => Error} + end, + Errors + ), + EmptyStatus = #{status => inconsistent, node_status => NodeErrors}, Response = lists:map( fun(#{<<"name">> := Name} = Link) -> @@ -227,25 +233,32 @@ handle_lookup(Name, Link) -> ?OK(add_status(Name, Link)). handle_metrics(Name) -> - case emqx_cluster_link_metrics:get_metrics(Name) of - {error, BadResults} -> + Results = emqx_cluster_link_metrics:get_metrics(Name), + {NodeMetrics0, NodeErrors} = + lists:foldl( + fun + ({Node, {ok, Metrics}}, {OkAccIn, ErrAccIn}) -> + {[format_metrics(Node, Metrics) | OkAccIn], ErrAccIn}; + ({Node, Error}, {OkAccIn, ErrAccIn}) -> + {OkAccIn, [{Node, Error} | ErrAccIn]} + end, + {[], []}, + Results + ), + case NodeErrors of + [] -> + ok; + [_ | _] -> ?SLOG(warning, #{ msg => "cluster_link_api_metrics_bad_erpc_results", - results => BadResults - }), - ?OK(#{metrics => #{}, node_metrics => []}); - {ok, NodeResults} -> - NodeMetrics = - lists:map( - fun({Node, Metrics}) -> - format_metrics(Node, Metrics) - end, - NodeResults - ), - AggregatedMetrics = aggregate_metrics(NodeMetrics), - Response = #{metrics => AggregatedMetrics, node_metrics => NodeMetrics}, - ?OK(Response) - end. + errors => maps:from_list(NodeErrors) + }) + end, + NodeMetrics1 = lists:map(fun({Node, _Error}) -> format_metrics(Node, #{}) end, NodeErrors), + NodeMetrics = NodeMetrics1 ++ NodeMetrics0, + AggregatedMetrics = aggregate_metrics(NodeMetrics), + Response = #{metrics => AggregatedMetrics, node_metrics => NodeMetrics}, + ?OK(Response). aggregate_metrics(NodeMetrics) -> ErrorLogger = fun(_) -> ok end, @@ -267,8 +280,8 @@ format_metrics(Node, Metrics) -> }. add_status(Name, Link) -> - NodeResults = get_link_status_cluster(Name), - Status = collect_single_status(NodeResults), + NodeRPCResults = emqx_cluster_link_mqtt:get_resource_cluster(Name), + Status = collect_single_status(NodeRPCResults), maps:merge(Link, Status). handle_update(Name, Params0) -> @@ -289,64 +302,62 @@ get_raw() -> ), Links. -get_all_link_status_cluster() -> - case emqx_cluster_link_mqtt:get_all_resources_cluster() of - {error, BadResults} -> - ?SLOG(warning, #{ - msg => "cluster_link_api_all_status_bad_erpc_results", - results => BadResults - }), - []; - {ok, NodeResults} -> - NodeResults - end. - -get_link_status_cluster(Name) -> - case emqx_cluster_link_mqtt:get_resource_cluster(Name) of - {error, BadResults} -> - ?SLOG(warning, #{ - msg => "cluster_link_api_lookup_status_bad_erpc_results", - results => BadResults - }), - []; - {ok, NodeResults} -> - NodeResults - end. - --spec collect_all_status([{node(), #{cluster_name() => _}}]) -> - #{ +-spec collect_all_status([{node(), {ok, #{cluster_name() => _}} | _Error}]) -> + {ClusterToStatus, Errors} +when + ClusterToStatus :: #{ cluster_name() => #{ node := node(), status := emqx_resource:resource_status() | inconsistent } - }. + }, + Errors :: [{node(), term()}]. collect_all_status(NodeResults) -> - Reindexed = lists:foldl( - fun({Node, AllLinkData}, Acc) -> - maps:fold( - fun(Name, Data, AccIn) -> - collect_all_status1(Node, Name, Data, AccIn) - end, - Acc, - AllLinkData - ) + {Reindexed, Errors} = lists:foldl( + fun + ({Node, {ok, AllLinkData}}, {OkAccIn, ErrAccIn}) -> + OkAcc = maps:fold( + fun(Name, Data, AccIn) -> + collect_all_status1(Node, Name, Data, AccIn) + end, + OkAccIn, + AllLinkData + ), + {OkAcc, ErrAccIn}; + ({Node, Error}, {OkAccIn, ErrAccIn}) -> + {OkAccIn, [{Node, Error} | ErrAccIn]} end, - #{}, + {#{}, []}, NodeResults ), - maps:fold( + NoErrors = + case Errors of + [] -> + true; + [_ | _] -> + ?SLOG(warning, #{ + msg => "cluster_link_api_lookup_status_bad_erpc_results", + errors => Errors + }), + false + end, + ClusterToStatus = maps:fold( fun(Name, NodeToData, Acc) -> OnlyStatus = [S || #{status := S} <- maps:values(NodeToData)], SummaryStatus = case lists:usort(OnlyStatus) of - [SameStatus] -> SameStatus; + [SameStatus] when NoErrors -> SameStatus; _ -> inconsistent end, NodeStatus = lists:map( - fun({Node, #{status := S}}) -> - #{node => Node, status => S} + fun + ({Node, #{status := S}}) -> + #{node => Node, status => S}; + ({Node, Error0}) -> + Error = emqx_logger_jsonfmt:best_effort_json(Error0), + #{node => Node, status => inconsistent, reason => Error} end, - maps:to_list(NodeToData) + maps:to_list(NodeToData) ++ Errors ), Acc#{ Name => #{ @@ -357,7 +368,8 @@ collect_all_status(NodeResults) -> end, #{}, Reindexed - ). + ), + {ClusterToStatus, Errors}. collect_all_status1(Node, Name, Data, Acc) -> maps:update_with( @@ -371,12 +383,13 @@ collect_single_status(NodeResults) -> NodeStatus = lists:map( fun - ({Node, {ok, #{status := S}}}) -> + ({Node, {ok, {ok, #{status := S}}}}) -> #{node => Node, status => S}; - ({Node, {error, _}}) -> + ({Node, {ok, {error, _}}}) -> #{node => Node, status => ?status_disconnected}; - ({Node, _}) -> - #{node => Node, status => inconsistent} + ({Node, Error0}) -> + Error = emqx_logger_jsonfmt:best_effort_json(Error0), + #{node => Node, status => inconsistent, reason => Error} end, NodeResults ), diff --git a/apps/emqx_cluster_link/src/emqx_cluster_link_metrics.erl b/apps/emqx_cluster_link/src/emqx_cluster_link_metrics.erl index 3d6f1edc8..695419c50 100644 --- a/apps/emqx_cluster_link/src/emqx_cluster_link_metrics.erl +++ b/apps/emqx_cluster_link/src/emqx_cluster_link_metrics.erl @@ -31,7 +31,7 @@ get_metrics(ClusterName) -> Nodes = emqx:running_nodes(), Timeout = 15_000, Results = emqx_metrics_proto_v2:get_metrics(Nodes, ?METRIC_NAME, ClusterName, Timeout), - sequence_multicall_results(Nodes, Results). + lists:zip(Nodes, Results). maybe_create_metrics(ClusterName) -> case emqx_metrics_worker:has_metrics(?METRIC_NAME, ClusterName) of @@ -52,16 +52,3 @@ routes_inc(ClusterName, Val) -> %%-------------------------------------------------------------------- %% Internal functions %%-------------------------------------------------------------------- - --spec sequence_multicall_results([node()], emqx_rpc:erpc_multicall(term())) -> - {ok, [{node(), term()}]} | {error, [term()]}. -sequence_multicall_results(Nodes, Results) -> - case lists:partition(fun is_ok/1, lists:zip(Nodes, Results)) of - {OkResults, []} -> - {ok, [{Node, Res} || {Node, {ok, Res}} <- OkResults]}; - {_OkResults, BadResults} -> - {error, BadResults} - end. - -is_ok({_Node, {ok, _}}) -> true; -is_ok(_) -> false. diff --git a/apps/emqx_cluster_link/src/emqx_cluster_link_mqtt.erl b/apps/emqx_cluster_link/src/emqx_cluster_link_mqtt.erl index 3b37a304e..65e02f53e 100644 --- a/apps/emqx_cluster_link/src/emqx_cluster_link_mqtt.erl +++ b/apps/emqx_cluster_link/src/emqx_cluster_link_mqtt.erl @@ -106,20 +106,18 @@ remove_msg_fwd_resource(ClusterName) -> emqx_resource:remove_local(?MSG_RES_ID(ClusterName)). -spec get_all_resources_cluster() -> - {ok, [{node(), #{cluster_name() => emqx_resource:resource_data()}}]} - | {error, [term()]}. + [{node(), emqx_rpc:erpc(#{cluster_name() => emqx_resource:resource_data()})}]. get_all_resources_cluster() -> Nodes = emqx:running_nodes(), Results = emqx_cluster_link_proto_v1:get_all_resources(Nodes), - sequence_multicall_results(Nodes, Results). + lists:zip(Nodes, Results). -spec get_resource_cluster(cluster_name()) -> - {ok, [{node(), {ok, emqx_resource:resource_data()} | {error, not_found}}]} - | {error, [term()]}. + [{node(), {ok, {ok, emqx_resource:resource_data()} | {error, not_found}} | _Error}]. get_resource_cluster(ClusterName) -> Nodes = emqx:running_nodes(), Results = emqx_cluster_link_proto_v1:get_resource(Nodes, ClusterName), - sequence_multicall_results(Nodes, Results). + lists:zip(Nodes, Results). %% RPC Target in `emqx_cluster_link_proto_v1'. -spec get_resource_local_v1(cluster_name()) -> @@ -478,16 +476,3 @@ emqtt_client_opts(ClientIdSuffix, ClusterConf) -> #{clientid := BaseClientId} = Opts = emqx_cluster_link_config:mk_emqtt_options(ClusterConf), ClientId = emqx_bridge_mqtt_lib:clientid_base([BaseClientId, ClientIdSuffix]), Opts#{clientid => ClientId}. - --spec sequence_multicall_results([node()], emqx_rpc:erpc_multicall(term())) -> - {ok, [{node(), term()}]} | {error, [term()]}. -sequence_multicall_results(Nodes, Results) -> - case lists:partition(fun is_ok/1, lists:zip(Nodes, Results)) of - {OkResults, []} -> - {ok, [{Node, Res} || {Node, {ok, Res}} <- OkResults]}; - {_OkResults, BadResults} -> - {error, BadResults} - end. - -is_ok({_Node, {ok, _}}) -> true; -is_ok(_) -> false. diff --git a/apps/emqx_cluster_link/test/emqx_cluster_link_api_SUITE.erl b/apps/emqx_cluster_link/test/emqx_cluster_link_api_SUITE.erl index 0e5143865..6b469272a 100644 --- a/apps/emqx_cluster_link/test/emqx_cluster_link_api_SUITE.erl +++ b/apps/emqx_cluster_link/test/emqx_cluster_link_api_SUITE.erl @@ -409,7 +409,17 @@ t_status(Config) -> {200, [ #{ <<"status">> := <<"inconsistent">>, - <<"node_status">> := [] + <<"node_status">> := [ + #{ + <<"node">> := _, + <<"status">> := <<"connected">> + }, + #{ + <<"node">> := _, + <<"status">> := <<"inconsistent">>, + <<"reason">> := _ + } + ] } ]}, list() @@ -417,7 +427,17 @@ t_status(Config) -> ?assertMatch( {200, #{ <<"status">> := <<"inconsistent">>, - <<"node_status">> := [] + <<"node_status">> := [ + #{ + <<"node">> := _, + <<"status">> := <<"connected">> + }, + #{ + <<"node">> := _, + <<"status">> := <<"inconsistent">>, + <<"reason">> := _ + } + ] }}, get_link(Name) ), From 80e035f115777e5ab259ef5856173f8ead021cf6 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 22 Jul 2024 14:32:28 -0300 Subject: [PATCH 033/131] feat(rule engine api): add filters options for action and source ids Fixes https://emqx.atlassian.net/browse/EMQX-12654 (requirement 2) --- .../src/emqx_rule_engine_api.erl | 47 +++++++- .../test/emqx_rule_engine_api_2_SUITE.erl | 109 +++++++++++++++++- changes/ce/feat-13505.en.md | 1 + rel/i18n/emqx_rule_engine_api.hocon | 6 + 4 files changed, 159 insertions(+), 4 deletions(-) create mode 100644 changes/ce/feat-13505.en.md diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl index 0ac59b36c..8af78f67f 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl @@ -131,6 +131,8 @@ end). {<<"like_id">>, binary}, {<<"like_from">>, binary}, {<<"match_from">>, binary}, + {<<"action">>, binary}, + {<<"source">>, binary}, {<<"like_description">>, binary} ]). @@ -194,6 +196,10 @@ schema("/rules") -> })}, {match_from, mk(binary(), #{desc => ?DESC("api1_match_from"), in => query, required => false})}, + {action, + mk(hoconsc:array(binary()), #{in => query, desc => ?DESC("api1_qs_action")})}, + {source, + mk(hoconsc:array(binary()), #{in => query, desc => ?DESC("api1_qs_source")})}, ref(emqx_dashboard_swagger, page), ref(emqx_dashboard_swagger, limit) ], @@ -731,7 +737,8 @@ filter_out_request_body(Conf) -> maps:without(ExtraConfs, Conf). -spec qs2ms(atom(), {list(), list()}) -> emqx_mgmt_api:match_spec_and_filter(). -qs2ms(_Tab, {Qs, Fuzzy}) -> +qs2ms(_Tab, {Qs0, Fuzzy0}) -> + {Qs, Fuzzy} = adapt_custom_filters(Qs0, Fuzzy0), case lists:keytake(from, 1, Qs) of false -> #{match_spec => generate_match_spec(Qs), fuzzy_fun => fuzzy_match_fun(Fuzzy)}; @@ -742,6 +749,38 @@ qs2ms(_Tab, {Qs, Fuzzy}) -> } end. +%% Some filters are run as fuzzy filters because they cannot be expressed as simple ETS +%% match specs. +-spec adapt_custom_filters(Qs, Fuzzy) -> {Qs, Fuzzy}. +adapt_custom_filters(Qs, Fuzzy) -> + lists:foldl( + fun + ({action, '=:=', X}, {QsAcc, FuzzyAcc}) -> + ActionIds = wrap(X), + Parsed = lists:map(fun emqx_rule_actions:parse_action/1, ActionIds), + {QsAcc, [{action, in, Parsed} | FuzzyAcc]}; + ({source, '=:=', X}, {QsAcc, FuzzyAcc}) -> + SourceIds = wrap(X), + Parsed = lists:flatmap( + fun(SourceId) -> + [ + emqx_bridge_resource:bridge_hookpoint(SourceId), + emqx_bridge_v2:source_hookpoint(SourceId) + ] + end, + SourceIds + ), + {QsAcc, [{source, in, Parsed} | FuzzyAcc]}; + (Clause, {QsAcc, FuzzyAcc}) -> + {[Clause | QsAcc], FuzzyAcc} + end, + {[], Fuzzy}, + Qs + ). + +wrap(Xs) when is_list(Xs) -> Xs; +wrap(X) -> [X]. + generate_match_spec(Qs) -> {MtchHead, Conds} = generate_match_spec(Qs, 2, {#{}, []}), [{{'_', MtchHead}, Conds, ['$_']}]. @@ -779,6 +818,12 @@ run_fuzzy_match(E = {_Id, #{from := Topics}}, [{from, match, Pattern} | Fuzzy]) run_fuzzy_match(E = {_Id, #{from := Topics}}, [{from, like, Pattern} | Fuzzy]) -> lists:any(fun(For) -> binary:match(For, Pattern) /= nomatch end, Topics) andalso run_fuzzy_match(E, Fuzzy); +run_fuzzy_match(E = {_Id, #{actions := Actions}}, [{action, in, ActionIds} | Fuzzy]) -> + lists:any(fun(AId) -> lists:member(AId, Actions) end, ActionIds) andalso + run_fuzzy_match(E, Fuzzy); +run_fuzzy_match(E = {_Id, #{from := Froms}}, [{source, in, SourceIds} | Fuzzy]) -> + lists:any(fun(SId) -> lists:member(SId, Froms) end, SourceIds) andalso + run_fuzzy_match(E, Fuzzy); run_fuzzy_match(E, [_ | Fuzzy]) -> run_fuzzy_match(E, Fuzzy). diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_api_2_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_api_2_SUITE.erl index 31a094055..fee06edd1 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_api_2_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_api_2_SUITE.erl @@ -33,7 +33,6 @@ init_per_suite(Config) -> app_specs(), #{work_dir => emqx_cth_suite:work_dir(Config)} ), - emqx_common_test_http:create_default_app(), [{apps, Apps} | Config]. end_per_suite(Config) -> @@ -46,7 +45,7 @@ app_specs() -> emqx_conf, emqx_rule_engine, emqx_management, - {emqx_dashboard, "dashboard.listeners.http { enable = true, bind = 18083 }"} + emqx_mgmt_api_test_util:emqx_dashboard() ]. %%------------------------------------------------------------------------------ @@ -64,8 +63,12 @@ request(Method, Path, Params) -> request(Method, Path, Params, Opts). request(Method, Path, Params, Opts) -> + request(Method, Path, Params, _QueryParams = [], Opts). + +request(Method, Path, Params, QueryParams0, Opts) when is_list(QueryParams0) -> AuthHeader = emqx_mgmt_api_test_util:auth_header_(), - case emqx_mgmt_api_test_util:request_api(Method, Path, "", AuthHeader, Params, Opts) of + QueryParams = uri_string:compose_query(QueryParams0, [{encoding, utf8}]), + case emqx_mgmt_api_test_util:request_api(Method, Path, QueryParams, AuthHeader, Params, Opts) of {ok, {Status, Headers, Body0}} -> Body = maybe_json_decode(Body0), {ok, {Status, Headers, Body}}; @@ -93,6 +96,45 @@ sql_test_api(Params) -> ct:pal("sql test (http) result:\n ~p", [Res]), Res. +list_rules(QueryParams) when is_list(QueryParams) -> + Method = get, + Path = emqx_mgmt_api_test_util:api_path(["rules"]), + Opts = #{return_all => true}, + Res = request(Method, Path, _Body = [], QueryParams, Opts), + emqx_mgmt_api_test_util:simplify_result(Res). + +list_rules_just_ids(QueryParams) when is_list(QueryParams) -> + case list_rules(QueryParams) of + {200, #{<<"data">> := Results0}} -> + Results = lists:sort([Id || #{<<"id">> := Id} <- Results0]), + {200, Results}; + Res -> + Res + end. + +create_rule() -> + create_rule(_Overrides = #{}). + +create_rule(Overrides) -> + Params0 = #{ + <<"enable">> => true, + <<"sql">> => <<"select true from t">> + }, + Params = emqx_utils_maps:deep_merge(Params0, Overrides), + Method = post, + Path = emqx_mgmt_api_test_util:api_path(["rules"]), + Res = request(Method, Path, Params), + emqx_mgmt_api_test_util:simplify_result(Res). + +sources_sql(Sources) -> + Froms = iolist_to_binary(lists:join(<<", ">>, lists:map(fun source_from/1, Sources))), + <<"select * from ", Froms/binary>>. + +source_from({v1, Id}) -> + <<"\"$bridges/", Id/binary, "\" ">>; +source_from({v2, Id}) -> + <<"\"$sources/", Id/binary, "\" ">>. + %%------------------------------------------------------------------------------ %% Test cases %%------------------------------------------------------------------------------ @@ -450,3 +492,64 @@ do_t_rule_test_smoke(#{input := Input, expected := #{code := ExpectedCode}} = Ca resp_body => Body }} end. + +%% Tests filtering the rule list by used actions and/or sources. +t_filter_by_source_and_action(_Config) -> + ?assertMatch( + {200, #{<<"data">> := []}}, + list_rules([]) + ), + + ActionId1 = <<"mqtt:a1">>, + ActionId2 = <<"mqtt:a2">>, + SourceId1 = <<"mqtt:s1">>, + SourceId2 = <<"mqtt:s2">>, + {201, #{<<"id">> := Id1}} = create_rule(#{<<"actions">> => [ActionId1]}), + {201, #{<<"id">> := Id2}} = create_rule(#{<<"actions">> => [ActionId2]}), + {201, #{<<"id">> := Id3}} = create_rule(#{<<"actions">> => [ActionId2, ActionId1]}), + {201, #{<<"id">> := Id4}} = create_rule(#{<<"sql">> => sources_sql([{v1, SourceId1}])}), + {201, #{<<"id">> := Id5}} = create_rule(#{<<"sql">> => sources_sql([{v2, SourceId2}])}), + {201, #{<<"id">> := Id6}} = create_rule(#{ + <<"sql">> => sources_sql([{v2, SourceId1}, {v2, SourceId1}]) + }), + {201, #{<<"id">> := Id7}} = create_rule(#{ + <<"sql">> => sources_sql([{v2, SourceId1}]), + <<"actions">> => [ActionId1] + }), + + ?assertMatch( + {200, [_, _, _, _, _, _, _]}, + list_rules_just_ids([]) + ), + + ?assertEqual( + {200, lists:sort([Id1, Id3, Id7])}, + list_rules_just_ids([{<<"action">>, ActionId1}]) + ), + + ?assertEqual( + {200, lists:sort([Id1, Id2, Id3, Id7])}, + list_rules_just_ids([{<<"action">>, ActionId1}, {<<"action">>, ActionId2}]) + ), + + ?assertEqual( + {200, lists:sort([Id4, Id6, Id7])}, + list_rules_just_ids([{<<"source">>, SourceId1}]) + ), + + ?assertEqual( + {200, lists:sort([Id4, Id5, Id6, Id7])}, + list_rules_just_ids([{<<"source">>, SourceId1}, {<<"source">>, SourceId2}]) + ), + + %% When mixing source and action id filters, we use AND. + ?assertEqual( + {200, lists:sort([])}, + list_rules_just_ids([{<<"source">>, SourceId2}, {<<"action">>, ActionId2}]) + ), + ?assertEqual( + {200, lists:sort([Id7])}, + list_rules_just_ids([{<<"source">>, SourceId1}, {<<"action">>, ActionId1}]) + ), + + ok. diff --git a/changes/ce/feat-13505.en.md b/changes/ce/feat-13505.en.md new file mode 100644 index 000000000..6b9ba8b94 --- /dev/null +++ b/changes/ce/feat-13505.en.md @@ -0,0 +1 @@ +Now, it's possible to filter rules in the HTTP API by the IDs of used data integration actions/sources. diff --git a/rel/i18n/emqx_rule_engine_api.hocon b/rel/i18n/emqx_rule_engine_api.hocon index 0745a108d..fd78d7ca1 100644 --- a/rel/i18n/emqx_rule_engine_api.hocon +++ b/rel/i18n/emqx_rule_engine_api.hocon @@ -96,4 +96,10 @@ api11.desc: api11.label: """Apply Rule""" +api1_qs_action.desc: +"""Filters rules that contain any of the given action id(s). When used in conjunction with source id filtering, the rules must contain sources *and* actions that match some of the criteria.""" + +api1_qs_source.desc: +"""Filters rules that contain any of the given source id(s). When used in conjunction with action id filtering, the rules must contain sources *and* actions that match some of the criteria.""" + } From 7829838dc5f9083bdbc41bbf61b534fb6d0c0ef9 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 24 Jul 2024 12:26:01 -0300 Subject: [PATCH 034/131] feat(cluster link api): add forwarding resource metrics to response --- .../src/emqx_cluster_link_api.erl | 40 +++++++++++++---- .../src/emqx_cluster_link_metrics.erl | 8 +++- .../src/emqx_cluster_link_mqtt.erl | 6 +++ .../test/emqx_cluster_link_api_SUITE.erl | 45 +++++++++++++++++-- 4 files changed, 86 insertions(+), 13 deletions(-) diff --git a/apps/emqx_cluster_link/src/emqx_cluster_link_api.erl b/apps/emqx_cluster_link/src/emqx_cluster_link_api.erl index 257a19854..6511cf395 100644 --- a/apps/emqx_cluster_link/src/emqx_cluster_link_api.erl +++ b/apps/emqx_cluster_link/src/emqx_cluster_link_api.erl @@ -237,10 +237,19 @@ handle_metrics(Name) -> {NodeMetrics0, NodeErrors} = lists:foldl( fun - ({Node, {ok, Metrics}}, {OkAccIn, ErrAccIn}) -> - {[format_metrics(Node, Metrics) | OkAccIn], ErrAccIn}; - ({Node, Error}, {OkAccIn, ErrAccIn}) -> - {OkAccIn, [{Node, Error} | ErrAccIn]} + ({Node, {ok, RouterMetrics}, {ok, ResourceMetrics}}, {OkAccIn, ErrAccIn}) -> + OkAcc = [format_metrics(Node, RouterMetrics, ResourceMetrics) | OkAccIn], + {OkAcc, ErrAccIn}; + ({Node, {ok, RouterMetrics}, ResError}, {OkAccIn, ErrAccIn}) -> + OkAcc = [format_metrics(Node, RouterMetrics, _ResourceMetrics = #{}) | OkAccIn], + {OkAcc, [{Node, #{resource => ResError}} | ErrAccIn]}; + ({Node, RouterError, {ok, ResourceMetrics}}, {OkAccIn, ErrAccIn}) -> + OkAcc = [format_metrics(Node, _RouterMetrics = #{}, ResourceMetrics) | OkAccIn], + {OkAcc, [{Node, #{router => RouterError}} | ErrAccIn]}; + ({Node, RouterError, ResourceError}, {OkAccIn, ErrAccIn}) -> + {OkAccIn, [ + {Node, #{router => RouterError, resource => ResourceError}} | ErrAccIn + ]} end, {[], []}, Results @@ -254,7 +263,7 @@ handle_metrics(Name) -> errors => maps:from_list(NodeErrors) }) end, - NodeMetrics1 = lists:map(fun({Node, _Error}) -> format_metrics(Node, #{}) end, NodeErrors), + NodeMetrics1 = lists:map(fun({Node, _Error}) -> format_metrics(Node, #{}, #{}) end, NodeErrors), NodeMetrics = NodeMetrics1 ++ NodeMetrics0, AggregatedMetrics = aggregate_metrics(NodeMetrics), Response = #{metrics => AggregatedMetrics, node_metrics => NodeMetrics}, @@ -270,12 +279,27 @@ aggregate_metrics(NodeMetrics) -> NodeMetrics ). -format_metrics(Node, Metrics) -> - Routes = emqx_utils_maps:deep_get([counters, ?route_metric], Metrics, 0), +format_metrics(Node, RouterMetrics, ResourceMetrics) -> + Get = fun(Path, Map) -> emqx_utils_maps:deep_get(Path, Map, 0) end, + Routes = Get([counters, ?route_metric], RouterMetrics), #{ node => Node, metrics => #{ - ?route_metric => Routes + ?route_metric => Routes, + + 'matched' => Get([counters, 'matched'], ResourceMetrics), + 'success' => Get([counters, 'success'], ResourceMetrics), + 'failed' => Get([counters, 'failed'], ResourceMetrics), + 'dropped' => Get([counters, 'dropped'], ResourceMetrics), + 'retried' => Get([counters, 'retried'], ResourceMetrics), + 'received' => Get([counters, 'received'], ResourceMetrics), + + 'queuing' => Get([gauges, 'queuing'], ResourceMetrics), + 'inflight' => Get([gauges, 'inflight'], ResourceMetrics), + + 'rate' => Get([rate, 'matched', current], ResourceMetrics), + 'rate_last5m' => Get([rate, 'matched', last5m], ResourceMetrics), + 'rate_max' => Get([rate, 'matched', max], ResourceMetrics) } }. diff --git a/apps/emqx_cluster_link/src/emqx_cluster_link_metrics.erl b/apps/emqx_cluster_link/src/emqx_cluster_link_metrics.erl index 695419c50..61e5fc9ce 100644 --- a/apps/emqx_cluster_link/src/emqx_cluster_link_metrics.erl +++ b/apps/emqx_cluster_link/src/emqx_cluster_link_metrics.erl @@ -30,8 +30,12 @@ get_metrics(ClusterName) -> Nodes = emqx:running_nodes(), Timeout = 15_000, - Results = emqx_metrics_proto_v2:get_metrics(Nodes, ?METRIC_NAME, ClusterName, Timeout), - lists:zip(Nodes, Results). + RouterResults = emqx_metrics_proto_v2:get_metrics(Nodes, ?METRIC_NAME, ClusterName, Timeout), + ResourceId = emqx_cluster_link_mqtt:resource_id(ClusterName), + ResourceResults = emqx_metrics_proto_v2:get_metrics( + Nodes, resource_metrics, ResourceId, Timeout + ), + lists:zip3(Nodes, RouterResults, ResourceResults). maybe_create_metrics(ClusterName) -> case emqx_metrics_worker:has_metrics(?METRIC_NAME, ClusterName) of diff --git a/apps/emqx_cluster_link/src/emqx_cluster_link_mqtt.erl b/apps/emqx_cluster_link/src/emqx_cluster_link_mqtt.erl index 65e02f53e..5a0f6db9c 100644 --- a/apps/emqx_cluster_link/src/emqx_cluster_link_mqtt.erl +++ b/apps/emqx_cluster_link/src/emqx_cluster_link_mqtt.erl @@ -9,6 +9,7 @@ -include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("emqx/include/logger.hrl"). -include_lib("snabbkaffe/include/trace.hrl"). +-include_lib("emqx_resource/include/emqx_resource.hrl"). -behaviour(emqx_resource). -behaviour(ecpool_worker). @@ -27,6 +28,7 @@ ]). -export([ + resource_id/1, ensure_msg_fwd_resource/1, remove_msg_fwd_resource/1, decode_route_op/1, @@ -92,6 +94,10 @@ -type cluster_name() :: binary(). +-spec resource_id(cluster_name()) -> resource_id(). +resource_id(ClusterName) -> + ?MSG_RES_ID(ClusterName). + -spec ensure_msg_fwd_resource(map()) -> {ok, emqx_resource:resource_data() | already_started} | {error, Reason :: term()}. ensure_msg_fwd_resource(#{name := Name, resource_opts := ResOpts} = ClusterConf) -> diff --git a/apps/emqx_cluster_link/test/emqx_cluster_link_api_SUITE.erl b/apps/emqx_cluster_link/test/emqx_cluster_link_api_SUITE.erl index 6b469272a..e8e8f345e 100644 --- a/apps/emqx_cluster_link/test/emqx_cluster_link_api_SUITE.erl +++ b/apps/emqx_cluster_link/test/emqx_cluster_link_api_SUITE.erl @@ -477,15 +477,54 @@ t_metrics(Config) -> ?assertMatch( {200, #{ - <<"metrics">> := #{<<"routes">> := 0}, + <<"metrics">> := #{ + <<"routes">> := 0, + <<"matched">> := _, + <<"success">> := _, + <<"failed">> := _, + <<"dropped">> := _, + <<"retried">> := _, + <<"received">> := _, + <<"queuing">> := _, + <<"inflight">> := _, + <<"rate">> := _, + <<"rate_last5m">> := _, + <<"rate_max">> := _ + }, <<"node_metrics">> := [ #{ <<"node">> := _, - <<"metrics">> := #{<<"routes">> := 0} + <<"metrics">> := #{ + <<"routes">> := 0, + <<"matched">> := _, + <<"success">> := _, + <<"failed">> := _, + <<"dropped">> := _, + <<"retried">> := _, + <<"received">> := _, + <<"queuing">> := _, + <<"inflight">> := _, + <<"rate">> := _, + <<"rate_last5m">> := _, + <<"rate_max">> := _ + } }, #{ <<"node">> := _, - <<"metrics">> := #{<<"routes">> := 0} + <<"metrics">> := #{ + <<"routes">> := 0, + <<"matched">> := _, + <<"success">> := _, + <<"failed">> := _, + <<"dropped">> := _, + <<"retried">> := _, + <<"received">> := _, + <<"queuing">> := _, + <<"inflight">> := _, + <<"rate">> := _, + <<"rate_last5m">> := _, + <<"rate_max">> := _ + } } ] }}, From dda73651c58cb03ed7b53164041bf768fb30e3f1 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 24 Jul 2024 15:29:27 -0300 Subject: [PATCH 035/131] fix(cluster link metrics): use periodic full table scan and gauge to count routes --- .../src/emqx_cluster_link_api.erl | 2 +- .../src/emqx_cluster_link_bookkeeper.erl | 84 +++++++++++++++++++ .../src/emqx_cluster_link_config.erl | 8 +- .../src/emqx_cluster_link_extrouter.erl | 34 +++++--- .../src/emqx_cluster_link_metrics.erl | 8 +- .../src/emqx_cluster_link_schema.erl | 13 ++- .../src/emqx_cluster_link_sup.erl | 12 ++- .../test/emqx_cluster_link_SUITE.erl | 2 + .../test/emqx_cluster_link_api_SUITE.erl | 61 +++++++++----- 9 files changed, 185 insertions(+), 39 deletions(-) create mode 100644 apps/emqx_cluster_link/src/emqx_cluster_link_bookkeeper.erl diff --git a/apps/emqx_cluster_link/src/emqx_cluster_link_api.erl b/apps/emqx_cluster_link/src/emqx_cluster_link_api.erl index 6511cf395..21c435a0f 100644 --- a/apps/emqx_cluster_link/src/emqx_cluster_link_api.erl +++ b/apps/emqx_cluster_link/src/emqx_cluster_link_api.erl @@ -281,7 +281,7 @@ aggregate_metrics(NodeMetrics) -> format_metrics(Node, RouterMetrics, ResourceMetrics) -> Get = fun(Path, Map) -> emqx_utils_maps:deep_get(Path, Map, 0) end, - Routes = Get([counters, ?route_metric], RouterMetrics), + Routes = Get([gauges, ?route_metric], RouterMetrics), #{ node => Node, metrics => #{ diff --git a/apps/emqx_cluster_link/src/emqx_cluster_link_bookkeeper.erl b/apps/emqx_cluster_link/src/emqx_cluster_link_bookkeeper.erl new file mode 100644 index 000000000..992fc7bf1 --- /dev/null +++ b/apps/emqx_cluster_link/src/emqx_cluster_link_bookkeeper.erl @@ -0,0 +1,84 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_cluster_link_bookkeeper). + +%% API +-export([ + start_link/0 +]). + +%% `gen_server' API +-export([ + init/1, + handle_continue/2, + handle_call/3, + handle_cast/2, + handle_info/2 +]). + +%%------------------------------------------------------------------------------ +%% Type declarations +%%------------------------------------------------------------------------------ + +%% call/cast/info events +-record(tally_routes, {}). + +%%------------------------------------------------------------------------------ +%% API +%%------------------------------------------------------------------------------ + +-spec start_link() -> gen_server:start_ret(). +start_link() -> + gen_server:start_link({local, ?MODULE}, ?MODULE, _InitOpts = #{}, _Opts = []). + +%%------------------------------------------------------------------------------ +%% `gen_server' API +%%------------------------------------------------------------------------------ + +init(_Opts) -> + State = #{}, + {ok, State, {continue, #tally_routes{}}}. + +handle_continue(#tally_routes{}, State) -> + handle_tally_routes(), + {noreply, State}. + +handle_call(_Call, _From, State) -> + {reply, {error, bad_call}, State}. + +handle_cast(_Cast, State) -> + {noreply, State}. + +handle_info(#tally_routes{}, State) -> + handle_tally_routes(), + {noreply, State}; +handle_info(_Info, State) -> + {noreply, State}. + +%%------------------------------------------------------------------------------ +%% Internal fns +%%------------------------------------------------------------------------------ + +cluster_names() -> + Links = emqx_cluster_link_config:links(), + lists:map(fun(#{name := Name}) -> Name end, Links). + +ensure_timer(Event, Timeout) -> + _ = erlang:send_after(Timeout, self(), Event), + ok. + +handle_tally_routes() -> + ClusterNames = cluster_names(), + tally_routes(ClusterNames), + ensure_timer(#tally_routes{}, emqx_cluster_link_config:tally_routes_interval()), + ok. + +tally_routes([ClusterName | ClusterNames]) -> + Tab = emqx_cluster_link_extrouter:extroute_tab(), + Pat = emqx_cluster_link_extrouter:cluster_routes_ms(ClusterName), + NumRoutes = ets:select_count(Tab, Pat), + emqx_cluster_link_metrics:routes_set(ClusterName, NumRoutes), + tally_routes(ClusterNames); +tally_routes([]) -> + ok. diff --git a/apps/emqx_cluster_link/src/emqx_cluster_link_config.erl b/apps/emqx_cluster_link/src/emqx_cluster_link_config.erl index 2a97f2d69..fd84a5b7f 100644 --- a/apps/emqx_cluster_link/src/emqx_cluster_link_config.erl +++ b/apps/emqx_cluster_link/src/emqx_cluster_link_config.erl @@ -46,7 +46,9 @@ %% Actor Lifecycle actor_ttl/0, actor_gc_interval/0, - actor_heartbeat_interval/0 + actor_heartbeat_interval/0, + %% Metrics + tally_routes_interval/0 ]). -export([ @@ -163,6 +165,10 @@ actor_gc_interval() -> actor_heartbeat_interval() -> actor_ttl() div 3. +-spec tally_routes_interval() -> _Milliseconds :: timeout(). +tally_routes_interval() -> + emqx_config:get([cluster, tally_routes_interval]). + %% mk_emqtt_options(#{server := Server, ssl := #{enable := EnableSsl} = Ssl} = LinkConf) -> diff --git a/apps/emqx_cluster_link/src/emqx_cluster_link_extrouter.erl b/apps/emqx_cluster_link/src/emqx_cluster_link_extrouter.erl index 3e2ff1804..e76b24d79 100644 --- a/apps/emqx_cluster_link/src/emqx_cluster_link_extrouter.erl +++ b/apps/emqx_cluster_link/src/emqx_cluster_link_extrouter.erl @@ -34,6 +34,9 @@ apply_actor_operation/5 ]). +%% Internal export for bookkeeping +-export([cluster_routes_ms/1, extroute_tab/0]). + %% Strictly monotonically increasing integer. -type smint() :: integer(). @@ -147,6 +150,18 @@ make_extroute_rec_pat(Entry) -> [{1, extroute}, {#extroute.entry, Entry}] ). +%% Internal exports for bookkeeping +cluster_routes_ms(ClusterName) -> + TopicPat = '_', + RouteIDPat = '_', + Pat = make_extroute_rec_pat( + emqx_trie_search:make_pat(TopicPat, ?ROUTE_ID(ClusterName, RouteIDPat)) + ), + [{Pat, [], [true]}]. + +extroute_tab() -> + ?EXTROUTE_TAB. + %% -record(state, { @@ -256,33 +271,31 @@ actor_apply_operation( apply_actor_operation(ActorID, Incarnation, Entry, OpName, Lane) -> _ = assert_current_incarnation(ActorID, Incarnation), - apply_operation(ActorID, Entry, OpName, Lane). + apply_operation(Entry, OpName, Lane). -apply_operation(ActorID, Entry, OpName, Lane) -> +apply_operation(Entry, OpName, Lane) -> %% NOTE %% This is safe sequence of operations only on core nodes. On replicants, %% `mria:dirty_update_counter/3` will be replicated asynchronously, which %% means this read can be stale. case mnesia:dirty_read(?EXTROUTE_TAB, Entry) of [#extroute{mcounter = MCounter}] -> - apply_operation(ActorID, Entry, MCounter, OpName, Lane); + apply_operation(Entry, MCounter, OpName, Lane); [] -> - apply_operation(ActorID, Entry, 0, OpName, Lane) + apply_operation(Entry, 0, OpName, Lane) end. -apply_operation(ActorID, Entry, MCounter, OpName, Lane) -> +apply_operation(Entry, MCounter, OpName, Lane) -> %% NOTE %% We are relying on the fact that changes to each individual lane of this %% multi-counter are synchronized. Without this, such counter updates would %% be unsafe. Instead, we would have to use another, more complex approach, %% that runs `ets:lookup/2` + `ets:select_replace/2` in a loop until the %% counter is updated accordingly. - ?ACTOR_ID(ClusterName, _Actor) = ActorID, Marker = 1 bsl Lane, case MCounter band Marker of 0 when OpName =:= add -> Res = mria:dirty_update_counter(?EXTROUTE_TAB, Entry, Marker), - _ = emqx_cluster_link_metrics:routes_inc(ClusterName, 1), ?tp("cluster_link_extrouter_route_added", #{}), Res; Marker when OpName =:= add -> @@ -293,7 +306,6 @@ apply_operation(ActorID, Entry, MCounter, OpName, Lane) -> 0 -> Record = #extroute{entry = Entry, mcounter = 0}, ok = mria:dirty_delete_object(?EXTROUTE_TAB, Record), - _ = emqx_cluster_link_metrics:routes_inc(ClusterName, -1), ?tp("cluster_link_extrouter_route_deleted", #{}), 0; C -> @@ -368,16 +380,16 @@ clean_incarnation(Rec = #actor{id = {Cluster, Actor}}) -> mnesia_clean_incarnation(#actor{id = Actor, incarnation = Incarnation, lane = Lane}) -> case mnesia:read(?EXTROUTE_ACTOR_TAB, Actor, write) of [#actor{incarnation = Incarnation}] -> - _ = clean_lane(Actor, Lane), + _ = clean_lane(Lane), mnesia:delete(?EXTROUTE_ACTOR_TAB, Actor, write); _Renewed -> stale end. -clean_lane(ActorID, Lane) -> +clean_lane(Lane) -> ets:foldl( fun(#extroute{entry = Entry, mcounter = MCounter}, _) -> - apply_operation(ActorID, Entry, MCounter, delete, Lane) + apply_operation(Entry, MCounter, delete, Lane) end, 0, ?EXTROUTE_TAB diff --git a/apps/emqx_cluster_link/src/emqx_cluster_link_metrics.erl b/apps/emqx_cluster_link/src/emqx_cluster_link_metrics.erl index 61e5fc9ce..36c5e791d 100644 --- a/apps/emqx_cluster_link/src/emqx_cluster_link_metrics.erl +++ b/apps/emqx_cluster_link/src/emqx_cluster_link_metrics.erl @@ -11,7 +11,7 @@ drop_metrics/1, get_metrics/1, - routes_inc/2 + routes_set/2 ]). %%-------------------------------------------------------------------- @@ -50,8 +50,10 @@ maybe_create_metrics(ClusterName) -> drop_metrics(ClusterName) -> ok = emqx_metrics_worker:clear_metrics(?METRIC_NAME, ClusterName). -routes_inc(ClusterName, Val) -> - catch emqx_metrics_worker:inc(?METRIC_NAME, ClusterName, ?route_metric, Val). +routes_set(ClusterName, Val) -> + catch emqx_metrics_worker:set_gauge( + ?METRIC_NAME, ClusterName, <<"singleton">>, ?route_metric, Val + ). %%-------------------------------------------------------------------- %% Internal functions diff --git a/apps/emqx_cluster_link/src/emqx_cluster_link_schema.erl b/apps/emqx_cluster_link/src/emqx_cluster_link_schema.erl index a369429d5..dd5ab66f6 100644 --- a/apps/emqx_cluster_link/src/emqx_cluster_link_schema.erl +++ b/apps/emqx_cluster_link/src/emqx_cluster_link_schema.erl @@ -30,7 +30,18 @@ namespace() -> "cluster". roots() -> []. injected_fields() -> - #{cluster => [{links, links_schema(#{})}]}. + #{ + cluster => [ + {links, links_schema(#{})}, + {tally_routes_interval, + hoconsc:mk( + emqx_schema:timeout_duration(), #{ + default => <<"15s">>, + importance => ?IMPORTANCE_HIDDEN + } + )} + ] + }. links_schema(Meta) -> ?HOCON(?ARRAY(?R_REF("link")), Meta#{ diff --git a/apps/emqx_cluster_link/src/emqx_cluster_link_sup.erl b/apps/emqx_cluster_link/src/emqx_cluster_link_sup.erl index 81b5afb4c..42f195cf7 100644 --- a/apps/emqx_cluster_link/src/emqx_cluster_link_sup.erl +++ b/apps/emqx_cluster_link/src/emqx_cluster_link_sup.erl @@ -30,12 +30,13 @@ init(LinksConf) -> period => 5 }, Metrics = emqx_metrics_worker:child_spec(metrics, ?METRIC_NAME), + BookKeeper = bookkeeper_spec(), ExtrouterGC = extrouter_gc_spec(), RouteActors = [ sup_spec(Name, ?ACTOR_MODULE, [LinkConf]) || #{name := Name} = LinkConf <- LinksConf ], - {ok, {SupFlags, [Metrics, ExtrouterGC | RouteActors]}}. + {ok, {SupFlags, [Metrics, BookKeeper, ExtrouterGC | RouteActors]}}. extrouter_gc_spec() -> %% NOTE: This one is currently global, not per-link. @@ -56,6 +57,15 @@ sup_spec(Id, Mod, Args) -> modules => [Mod] }. +bookkeeper_spec() -> + #{ + id => bookkeeper, + start => {emqx_cluster_link_bookkeeper, start_link, []}, + restart => permanent, + type => worker, + shutdown => 5_000 + }. + ensure_actor(#{name := Name} = LinkConf) -> case supervisor:start_child(?SERVER, sup_spec(Name, ?ACTOR_MODULE, [LinkConf])) of {ok, Pid} -> diff --git a/apps/emqx_cluster_link/test/emqx_cluster_link_SUITE.erl b/apps/emqx_cluster_link/test/emqx_cluster_link_SUITE.erl index e023aacab..21aa1a70e 100644 --- a/apps/emqx_cluster_link/test/emqx_cluster_link_SUITE.erl +++ b/apps/emqx_cluster_link/test/emqx_cluster_link_SUITE.erl @@ -53,6 +53,7 @@ mk_source_cluster(BaseName, Config) -> SourceConf = "cluster {" "\n name = cl.source" + "\n tally_routes_interval = 300ms" "\n links = [" "\n { enable = true" "\n name = cl.target" @@ -75,6 +76,7 @@ mk_target_cluster(BaseName, Config) -> TargetConf = "cluster {" "\n name = cl.target" + "\n tally_routes_interval = 300ms" "\n links = [" "\n { enable = true" "\n name = cl.source" diff --git a/apps/emqx_cluster_link/test/emqx_cluster_link_api_SUITE.erl b/apps/emqx_cluster_link/test/emqx_cluster_link_api_SUITE.erl index e8e8f345e..486179af1 100644 --- a/apps/emqx_cluster_link/test/emqx_cluster_link_api_SUITE.erl +++ b/apps/emqx_cluster_link/test/emqx_cluster_link_api_SUITE.erl @@ -600,14 +600,22 @@ t_metrics(Config) -> #{?snk_kind := clink_route_sync_complete} ), - %% Routes = 2 in source cluster, because the target cluster has some topic filters - %% configured and subscribers to them, which were replicated to the source cluster. - ?assertMatch( - {200, #{ - <<"metrics">> := #{<<"routes">> := 2}, - <<"node_metrics">> := _ - }}, - get_metrics(source, SourceName) + %% Routes = 4 in source cluster, because the target cluster has some topic filters + %% configured and subscribers to them, which were replicated to the source cluster, + %% and we have 2 nodes with 2 routes each. + ?retry( + 300, + 10, + ?assertMatch( + {200, #{ + <<"metrics">> := #{<<"routes">> := 4}, + <<"node_metrics">> := [ + #{<<"metrics">> := #{<<"routes">> := 2}}, + #{<<"metrics">> := #{<<"routes">> := 2}} + ] + }}, + get_metrics(source, SourceName) + ) ), ?assertMatch( {200, #{ @@ -627,12 +635,19 @@ t_metrics(Config) -> #{?snk_kind := clink_route_sync_complete} ), - ?assertMatch( - {200, #{ - <<"metrics">> := #{<<"routes">> := 1}, - <<"node_metrics">> := _ - }}, - get_metrics(source, SourceName) + ?retry( + 300, + 10, + ?assertMatch( + {200, #{ + <<"metrics">> := #{<<"routes">> := 2}, + <<"node_metrics">> := [ + #{<<"metrics">> := #{<<"routes">> := 1}}, + #{<<"metrics">> := #{<<"routes">> := 1}} + ] + }}, + get_metrics(source, SourceName) + ) ), %% Disabling the link should remove the routes. @@ -658,12 +673,16 @@ t_metrics(Config) -> #{?snk_kind := "cluster_link_extrouter_route_deleted"} ), - ?assertMatch( - {200, #{ - <<"metrics">> := #{<<"routes">> := 0}, - <<"node_metrics">> := _ - }}, - get_metrics(source, SourceName) + ?retry( + 300, + 10, + ?assertMatch( + {200, #{ + <<"metrics">> := #{<<"routes">> := 0}, + <<"node_metrics">> := _ + }}, + get_metrics(source, SourceName) + ) ), %% Enabling again @@ -678,7 +697,7 @@ t_metrics(Config) -> ?assertMatch( {200, #{ - <<"metrics">> := #{<<"routes">> := 1}, + <<"metrics">> := #{<<"routes">> := 2}, <<"node_metrics">> := _ }}, get_metrics(source, SourceName) From 117c8197d72ff16b8b5cc0e8e71a29e5d8ad8849 Mon Sep 17 00:00:00 2001 From: firest Date: Wed, 24 Jul 2024 18:33:15 +0800 Subject: [PATCH 036/131] fix(exclusive): allow the same client to resubscribe to an existing exclusive topic --- apps/emqx/src/emqx_exclusive_subscription.erl | 7 +++++++ apps/emqx/test/emqx_exclusive_sub_SUITE.erl | 2 ++ 2 files changed, 9 insertions(+) diff --git a/apps/emqx/src/emqx_exclusive_subscription.erl b/apps/emqx/src/emqx_exclusive_subscription.erl index 3a9dc8014..095233fbb 100644 --- a/apps/emqx/src/emqx_exclusive_subscription.erl +++ b/apps/emqx/src/emqx_exclusive_subscription.erl @@ -117,6 +117,13 @@ try_subscribe(ClientId, Topic) -> write ), allow; + [#exclusive_subscription{clientid = ClientId, topic = Topic}] -> + %% Fixed the issue-13476 + %% In this feature, the user must manually call `unsubscribe` to release the lock, + %% but sometimes the node may go down for some reason, + %% then the client will reconnect to this node and resubscribe. + %% We need to allow resubscription, otherwise the lock will never be released. + allow; [_] -> deny end. diff --git a/apps/emqx/test/emqx_exclusive_sub_SUITE.erl b/apps/emqx/test/emqx_exclusive_sub_SUITE.erl index abbdb5f44..a859612b2 100644 --- a/apps/emqx/test/emqx_exclusive_sub_SUITE.erl +++ b/apps/emqx/test/emqx_exclusive_sub_SUITE.erl @@ -56,6 +56,8 @@ t_exclusive_sub(_) -> {ok, _} = emqtt:connect(C1), ?CHECK_SUB(C1, 0), + ?CHECK_SUB(C1, 0), + {ok, C2} = emqtt:start_link([ {clientid, <<"client2">>}, {clean_start, false}, From 4f21594707432a5f93dbe428d263b1585fc3dd54 Mon Sep 17 00:00:00 2001 From: firest Date: Wed, 24 Jul 2024 18:59:06 +0800 Subject: [PATCH 037/131] chore: update changes --- changes/ce/fix-13515.en.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/ce/fix-13515.en.md diff --git a/changes/ce/fix-13515.en.md b/changes/ce/fix-13515.en.md new file mode 100644 index 000000000..775c21848 --- /dev/null +++ b/changes/ce/fix-13515.en.md @@ -0,0 +1 @@ +Fixed an issue where the same client could not subscribe to the same exclusive topic when the node was down for some reason. From 141d8144e415ae1fe0d49033eb74b1a1a9356bae Mon Sep 17 00:00:00 2001 From: firest Date: Thu, 25 Jul 2024 10:52:26 +0800 Subject: [PATCH 038/131] fix(scram): change the name from `scram_http` to `scram_restapi` --- apps/emqx_auth_http/src/emqx_auth_http_app.erl | 2 +- ...ram_http.erl => emqx_authn_scram_restapi.erl} | 12 +++++++++--- ...a.erl => emqx_authn_scram_restapi_schema.erl} | 16 ++++++++-------- ...TE.erl => emqx_authn_scram_restapi_SUITE.erl} | 16 +++++++++------- ... => emqx_authn_scram_restapi_test_server.erl} | 2 +- apps/emqx_conf/src/emqx_conf_schema_inject.erl | 2 +- apps/emqx_gateway/src/emqx_gateway_api_authn.erl | 2 +- changes/ee/feat-13504.en.md | 4 ++++ 8 files changed, 34 insertions(+), 22 deletions(-) rename apps/emqx_auth_http/src/{emqx_authn_scram_http.erl => emqx_authn_scram_restapi.erl} (93%) rename apps/emqx_auth_http/src/{emqx_authn_scram_http_schema.erl => emqx_authn_scram_restapi_schema.erl} (88%) rename apps/emqx_auth_http/test/{emqx_authn_scram_http_SUITE.erl => emqx_authn_scram_restapi_SUITE.erl} (96%) rename apps/emqx_auth_http/test/{emqx_authn_scram_http_test_server.erl => emqx_authn_scram_restapi_test_server.erl} (98%) diff --git a/apps/emqx_auth_http/src/emqx_auth_http_app.erl b/apps/emqx_auth_http/src/emqx_auth_http_app.erl index 3d8ae0dad..8b7d08c4e 100644 --- a/apps/emqx_auth_http/src/emqx_auth_http_app.erl +++ b/apps/emqx_auth_http/src/emqx_auth_http_app.erl @@ -25,7 +25,7 @@ start(_StartType, _StartArgs) -> ok = emqx_authz:register_source(?AUTHZ_TYPE, emqx_authz_http), ok = emqx_authn:register_provider(?AUTHN_TYPE, emqx_authn_http), - ok = emqx_authn:register_provider(?AUTHN_TYPE_SCRAM, emqx_authn_scram_http), + ok = emqx_authn:register_provider(?AUTHN_TYPE_SCRAM, emqx_authn_scram_restapi), {ok, Sup} = emqx_auth_http_sup:start_link(), {ok, Sup}. diff --git a/apps/emqx_auth_http/src/emqx_authn_scram_http.erl b/apps/emqx_auth_http/src/emqx_authn_scram_restapi.erl similarity index 93% rename from apps/emqx_auth_http/src/emqx_authn_scram_http.erl rename to apps/emqx_auth_http/src/emqx_authn_scram_restapi.erl index 0e6190b4b..f1cca5da2 100644 --- a/apps/emqx_auth_http/src/emqx_authn_scram_http.erl +++ b/apps/emqx_auth_http/src/emqx_authn_scram_restapi.erl @@ -2,7 +2,13 @@ %% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_authn_scram_http). +%% Note: +%% This is not an implementation of the RFC 7804: +%% Salted Challenge Response HTTP Authentication Mechanism. +%% This backend is an implementation of scram, +%% which uses an external web resource as a source of user information. + +-module(emqx_authn_scram_restapi). -include_lib("emqx_auth/include/emqx_authn.hrl"). -include_lib("emqx/include/logger.hrl"). @@ -95,7 +101,7 @@ retrieve( ) -> Request = emqx_authn_http:generate_request(Credential#{username := Username}, State), Response = emqx_resource:simple_sync_query(ResourceId, {Method, Request, RequestTimeout}), - ?TRACE_AUTHN_PROVIDER("scram_http_response", #{ + ?TRACE_AUTHN_PROVIDER("scram_restapi_response", #{ request => emqx_authn_http:request_for_log(Credential, State), response => emqx_authn_http:response_for_log(Response), resource => ResourceId @@ -119,7 +125,7 @@ handle_response(Headers, Body) -> {error, Reason} = Error -> ?TRACE_AUTHN_PROVIDER( error, - "parse_scram_http_response_failed", + "parse_scram_restapi_response_failed", #{content_type => ContentType, body => Body, reason => Reason} ), Error diff --git a/apps/emqx_auth_http/src/emqx_authn_scram_http_schema.erl b/apps/emqx_auth_http/src/emqx_authn_scram_restapi_schema.erl similarity index 88% rename from apps/emqx_auth_http/src/emqx_authn_scram_http_schema.erl rename to apps/emqx_auth_http/src/emqx_authn_scram_restapi_schema.erl index ca43fe3a6..bf3398abb 100644 --- a/apps/emqx_auth_http/src/emqx_authn_scram_http_schema.erl +++ b/apps/emqx_auth_http/src/emqx_authn_scram_restapi_schema.erl @@ -2,7 +2,7 @@ %% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_authn_scram_http_schema). +-module(emqx_authn_scram_restapi_schema). -behaviour(emqx_authn_schema). @@ -22,16 +22,16 @@ namespace() -> "authn". refs() -> - [?R_REF(scram_http_get), ?R_REF(scram_http_post)]. + [?R_REF(scram_restapi_get), ?R_REF(scram_restapi_post)]. select_union_member( #{<<"mechanism">> := ?AUTHN_MECHANISM_SCRAM_BIN, <<"backend">> := ?AUTHN_BACKEND_BIN} = Value ) -> case maps:get(<<"method">>, Value, undefined) of <<"get">> -> - [?R_REF(scram_http_get)]; + [?R_REF(scram_restapi_get)]; <<"post">> -> - [?R_REF(scramm_http_post)]; + [?R_REF(scram_restapi_post)]; Else -> throw(#{ reason => "unknown_http_method", @@ -43,20 +43,20 @@ select_union_member( select_union_member(_Value) -> undefined. -fields(scram_http_get) -> +fields(scram_restapi_get) -> [ {method, #{type => get, required => true, desc => ?DESC(emqx_authn_http_schema, method)}}, {headers, fun emqx_authn_http_schema:headers_no_content_type/1} ] ++ common_fields(); -fields(scram_http_post) -> +fields(scram_restapi_post) -> [ {method, #{type => post, required => true, desc => ?DESC(emqx_authn_http_schema, method)}}, {headers, fun emqx_authn_http_schema:headers/1} ] ++ common_fields(). -desc(scram_http_get) -> +desc(scram_restapi_get) -> ?DESC(emqx_authn_http_schema, get); -desc(scram_http_post) -> +desc(scram_restapi_post) -> ?DESC(emqx_authn_http_schema, post); desc(_) -> undefined. diff --git a/apps/emqx_auth_http/test/emqx_authn_scram_http_SUITE.erl b/apps/emqx_auth_http/test/emqx_authn_scram_restapi_SUITE.erl similarity index 96% rename from apps/emqx_auth_http/test/emqx_authn_scram_http_SUITE.erl rename to apps/emqx_auth_http/test/emqx_authn_scram_restapi_SUITE.erl index b00212cb1..8cd83f973 100644 --- a/apps/emqx_auth_http/test/emqx_authn_scram_http_SUITE.erl +++ b/apps/emqx_auth_http/test/emqx_authn_scram_restapi_SUITE.erl @@ -2,7 +2,7 @@ %% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_authn_scram_http_SUITE). +-module(emqx_authn_scram_restapi_SUITE). -compile(export_all). -compile(nowarn_export_all). @@ -54,11 +54,11 @@ init_per_testcase(_Case, Config) -> [authentication], ?GLOBAL ), - {ok, _} = emqx_authn_scram_http_test_server:start_link(?HTTP_PORT, ?HTTP_PATH), + {ok, _} = emqx_authn_scram_restapi_test_server:start_link(?HTTP_PORT, ?HTTP_PATH), Config. end_per_testcase(_Case, _Config) -> - ok = emqx_authn_scram_http_test_server:stop(). + ok = emqx_authn_scram_restapi_test_server:stop(). %%------------------------------------------------------------------------------ %% Tests @@ -72,7 +72,9 @@ t_create(_Config) -> {create_authenticator, ?GLOBAL, AuthConfig} ), - {ok, [#{provider := emqx_authn_scram_http}]} = emqx_authn_chains:list_authenticators(?GLOBAL). + {ok, [#{provider := emqx_authn_scram_restapi}]} = emqx_authn_chains:list_authenticators( + ?GLOBAL + ). t_create_invalid(_Config) -> AuthConfig = raw_config(), @@ -329,7 +331,7 @@ test_is_superuser(State, ExpectedIsSuperuser) -> ClientFirstMessage = esasl_scram:client_first_message(Username), {continue, ServerFirstMessage, ServerCache} = - emqx_authn_scram_http:authenticate( + emqx_authn_scram_restapi:authenticate( #{ auth_method => <<"SCRAM-SHA-512">>, auth_data => ClientFirstMessage, @@ -349,7 +351,7 @@ test_is_superuser(State, ExpectedIsSuperuser) -> ), {ok, UserInfo1, ServerFinalMessage} = - emqx_authn_scram_http:authenticate( + emqx_authn_scram_restapi:authenticate( #{ auth_method => <<"SCRAM-SHA-512">>, auth_data => ClientFinalMessage, @@ -399,7 +401,7 @@ set_user_handler(Username, Password, IsSuperuser) -> ), {ok, Req, State} end, - ok = emqx_authn_scram_http_test_server:set_handler(Handler). + ok = emqx_authn_scram_restapi_test_server:set_handler(Handler). init_auth() -> init_auth(raw_config()). diff --git a/apps/emqx_auth_http/test/emqx_authn_scram_http_test_server.erl b/apps/emqx_auth_http/test/emqx_authn_scram_restapi_test_server.erl similarity index 98% rename from apps/emqx_auth_http/test/emqx_authn_scram_http_test_server.erl rename to apps/emqx_auth_http/test/emqx_authn_scram_restapi_test_server.erl index 5467df621..1e1432e0b 100644 --- a/apps/emqx_auth_http/test/emqx_authn_scram_http_test_server.erl +++ b/apps/emqx_auth_http/test/emqx_authn_scram_restapi_test_server.erl @@ -2,7 +2,7 @@ %% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_authn_scram_http_test_server). +-module(emqx_authn_scram_restapi_test_server). -behaviour(supervisor). -behaviour(cowboy_handler). diff --git a/apps/emqx_conf/src/emqx_conf_schema_inject.erl b/apps/emqx_conf/src/emqx_conf_schema_inject.erl index 5c155bbf5..d94657325 100644 --- a/apps/emqx_conf/src/emqx_conf_schema_inject.erl +++ b/apps/emqx_conf/src/emqx_conf_schema_inject.erl @@ -51,7 +51,7 @@ authn_mods(ee) -> authn_mods(ce) ++ [ emqx_gcp_device_authn_schema, - emqx_authn_scram_http_schema + emqx_authn_scram_restapi_schema ]. authz() -> diff --git a/apps/emqx_gateway/src/emqx_gateway_api_authn.erl b/apps/emqx_gateway/src/emqx_gateway_api_authn.erl index 0707c12aa..c79bc8e61 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api_authn.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api_authn.erl @@ -383,7 +383,7 @@ schema_authn() -> emqx_dashboard_swagger:schema_with_examples( emqx_authn_schema:authenticator_type_without([ emqx_authn_scram_mnesia_schema, - emqx_authn_scram_http_schema + emqx_authn_scram_restapi_schema ]), emqx_authn_api:authenticator_examples() ). diff --git a/changes/ee/feat-13504.en.md b/changes/ee/feat-13504.en.md index c9b22f403..20b3aa1e2 100644 --- a/changes/ee/feat-13504.en.md +++ b/changes/ee/feat-13504.en.md @@ -1 +1,5 @@ Added a HTTP backend for the authentication mechanism `scram`. + +Note: This is not an implementation of the RFC 7804: Salted Challenge Response HTTP Authentication Mechanism. + +This backend is an implementation of scram that uses an external web resource as a source of user information. From 79020b2436df6e40a09450ec72401234c8dc89bc Mon Sep 17 00:00:00 2001 From: firest Date: Tue, 23 Jul 2024 15:06:06 +0800 Subject: [PATCH 039/131] feat(variform): add a builtin function to get env vars --- apps/emqx_rule_engine/src/emqx_rule_funcs.erl | 9 +++++++ apps/emqx_utils/src/emqx_variform_bif.erl | 27 +++++++++++++++++++ .../test/emqx_variform_bif_tests.erl | 7 +++++ 3 files changed, 43 insertions(+) diff --git a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl index 3f7f24604..ee6a83ab1 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl @@ -252,6 +252,9 @@ timezone_to_offset_seconds/1 ]). +%% System functions +-export([getenv/1]). + %% See extra_functions_module/0 and set_extra_functions_module/1 in the %% emqx_rule_engine module -callback handle_rule_function(atom(), list()) -> any() | {error, no_match_for_function}. @@ -1262,3 +1265,9 @@ convert_timestamp(MillisecondsTimestamp) -> uuid_str(UUID, DisplayOpt) -> uuid:uuid_to_string(UUID, DisplayOpt). + +%%------------------------------------------------------------------------------ +%% System Funcs +%%------------------------------------------------------------------------------ +getenv(Env) -> + emqx_variform_bif:getenv(Env). diff --git a/apps/emqx_utils/src/emqx_variform_bif.erl b/apps/emqx_utils/src/emqx_variform_bif.erl index f30db8f7a..e66b8e47d 100644 --- a/apps/emqx_utils/src/emqx_variform_bif.erl +++ b/apps/emqx_utils/src/emqx_variform_bif.erl @@ -79,6 +79,12 @@ %% Number compare functions -export([num_comp/2, num_eq/2, num_lt/2, num_lte/2, num_gt/2, num_gte/2]). +%% System +-export([getenv/1]). + +-define(CACHE(Key), {?MODULE, Key}). +-define(ENV_CACHE(Env), ?CACHE({env, Env})). + %%------------------------------------------------------------------------------ %% String Funcs %%------------------------------------------------------------------------------ @@ -569,3 +575,24 @@ num_lte(A, B) -> num_gte(A, B) -> R = num_comp(A, B), R =:= gt orelse R =:= eq. + +%%------------------------------------------------------------------------------ +%% System +%%------------------------------------------------------------------------------ +getenv(Bin) when is_binary(Bin) -> + EnvKey = ?ENV_CACHE(Bin), + case persistent_term:get(EnvKey, undefined) of + undefined -> + Name = erlang:binary_to_list(Bin), + Result = + case os:getenv(Name) of + false -> + <<>>; + Value -> + erlang:list_to_binary(Value) + end, + persistent_term:put(EnvKey, Result), + Result; + Result -> + Result + end. diff --git a/apps/emqx_utils/test/emqx_variform_bif_tests.erl b/apps/emqx_utils/test/emqx_variform_bif_tests.erl index 92144ff43..36235be40 100644 --- a/apps/emqx_utils/test/emqx_variform_bif_tests.erl +++ b/apps/emqx_utils/test/emqx_variform_bif_tests.erl @@ -72,3 +72,10 @@ base64_encode_decode_test() -> RandBytes = crypto:strong_rand_bytes(100), Encoded = emqx_variform_bif:base64_encode(RandBytes), ?assertEqual(RandBytes, emqx_variform_bif:base64_decode(Encoded)). + +system_test() -> + EnvName = erlang:atom_to_list(?MODULE), + EnvVal = erlang:atom_to_list(?FUNCTION_NAME), + EnvNameBin = erlang:list_to_binary(EnvName), + os:putenv(EnvName, EnvVal), + ?assertEqual(erlang:list_to_binary(EnvVal), emqx_variform_bif:getenv(EnvNameBin)). From 57959ac7d482401f4895ddabe52541da659371d8 Mon Sep 17 00:00:00 2001 From: firest Date: Thu, 25 Jul 2024 18:59:40 +0800 Subject: [PATCH 040/131] chore: update changes --- changes/ce/feat-13507.en.md | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 changes/ce/feat-13507.en.md diff --git a/changes/ce/feat-13507.en.md b/changes/ce/feat-13507.en.md new file mode 100644 index 000000000..115fa49a9 --- /dev/null +++ b/changes/ce/feat-13507.en.md @@ -0,0 +1,2 @@ +Added a new builtin function `getenv` in the rule engine and variform expression to access the environment variables. +Note this value is immutable once loaded from the environment. From 87e4e2340d80f95537cfcc61bc7db62e325c6302 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 25 Jul 2024 11:10:54 -0300 Subject: [PATCH 041/131] refactor: better metric and error fold --- .../src/emqx_cluster_link_api.erl | 31 ++++++++++--------- .../test/emqx_cluster_link_api_SUITE.erl | 16 ++++++---- 2 files changed, 27 insertions(+), 20 deletions(-) diff --git a/apps/emqx_cluster_link/src/emqx_cluster_link_api.erl b/apps/emqx_cluster_link/src/emqx_cluster_link_api.erl index 21c435a0f..4ff1ee7ef 100644 --- a/apps/emqx_cluster_link/src/emqx_cluster_link_api.erl +++ b/apps/emqx_cluster_link/src/emqx_cluster_link_api.erl @@ -236,20 +236,11 @@ handle_metrics(Name) -> Results = emqx_cluster_link_metrics:get_metrics(Name), {NodeMetrics0, NodeErrors} = lists:foldl( - fun - ({Node, {ok, RouterMetrics}, {ok, ResourceMetrics}}, {OkAccIn, ErrAccIn}) -> - OkAcc = [format_metrics(Node, RouterMetrics, ResourceMetrics) | OkAccIn], - {OkAcc, ErrAccIn}; - ({Node, {ok, RouterMetrics}, ResError}, {OkAccIn, ErrAccIn}) -> - OkAcc = [format_metrics(Node, RouterMetrics, _ResourceMetrics = #{}) | OkAccIn], - {OkAcc, [{Node, #{resource => ResError}} | ErrAccIn]}; - ({Node, RouterError, {ok, ResourceMetrics}}, {OkAccIn, ErrAccIn}) -> - OkAcc = [format_metrics(Node, _RouterMetrics = #{}, ResourceMetrics) | OkAccIn], - {OkAcc, [{Node, #{router => RouterError}} | ErrAccIn]}; - ({Node, RouterError, ResourceError}, {OkAccIn, ErrAccIn}) -> - {OkAccIn, [ - {Node, #{router => RouterError, resource => ResourceError}} | ErrAccIn - ]} + fun({Node, RouterMetrics0, ResourceMetrics0}, {OkAccIn, ErrAccIn}) -> + {RouterMetrics, RouterError} = get_metrics_or_errors(RouterMetrics0), + {ResourceMetrics, ResourceError} = get_metrics_or_errors(ResourceMetrics0), + ErrAcc = append_errors(RouterError, ResourceError, Node, ErrAccIn), + {[format_metrics(Node, RouterMetrics, ResourceMetrics) | OkAccIn], ErrAcc} end, {[], []}, Results @@ -269,6 +260,18 @@ handle_metrics(Name) -> Response = #{metrics => AggregatedMetrics, node_metrics => NodeMetrics}, ?OK(Response). +get_metrics_or_errors({ok, Metrics}) -> + {Metrics, undefined}; +get_metrics_or_errors(Error) -> + {#{}, Error}. + +append_errors(undefined, undefined, _Node, Acc) -> + Acc; +append_errors(RouterError, ResourceError, Node, Acc) -> + Err0 = emqx_utils_maps:put_if(#{}, router, RouterError, RouterError =/= undefined), + Err = emqx_utils_maps:put_if(Err0, resource, ResourceError, ResourceError =/= undefined), + [{Node, Err} | Acc]. + aggregate_metrics(NodeMetrics) -> ErrorLogger = fun(_) -> ok end, lists:foldl( diff --git a/apps/emqx_cluster_link/test/emqx_cluster_link_api_SUITE.erl b/apps/emqx_cluster_link/test/emqx_cluster_link_api_SUITE.erl index 486179af1..5e3ae5d50 100644 --- a/apps/emqx_cluster_link/test/emqx_cluster_link_api_SUITE.erl +++ b/apps/emqx_cluster_link/test/emqx_cluster_link_api_SUITE.erl @@ -695,12 +695,16 @@ t_metrics(Config) -> #{?snk_kind := "cluster_link_extrouter_route_added"} ), - ?assertMatch( - {200, #{ - <<"metrics">> := #{<<"routes">> := 2}, - <<"node_metrics">> := _ - }}, - get_metrics(source, SourceName) + ?retry( + 300, + 10, + ?assertMatch( + {200, #{ + <<"metrics">> := #{<<"routes">> := 2}, + <<"node_metrics">> := _ + }}, + get_metrics(source, SourceName) + ) ), ok. From 30259284d148c6ec306ee8fe60894b3eb147b5f3 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 25 Jul 2024 11:17:29 -0300 Subject: [PATCH 042/131] chore: namespace metrics by type --- .../src/emqx_cluster_link_api.erl | 29 +++-- .../test/emqx_cluster_link_api_SUITE.erl | 120 ++++++++++-------- 2 files changed, 82 insertions(+), 67 deletions(-) diff --git a/apps/emqx_cluster_link/src/emqx_cluster_link_api.erl b/apps/emqx_cluster_link/src/emqx_cluster_link_api.erl index 4ff1ee7ef..45b97e8f7 100644 --- a/apps/emqx_cluster_link/src/emqx_cluster_link_api.erl +++ b/apps/emqx_cluster_link/src/emqx_cluster_link_api.erl @@ -288,21 +288,24 @@ format_metrics(Node, RouterMetrics, ResourceMetrics) -> #{ node => Node, metrics => #{ - ?route_metric => Routes, + router => #{ + ?route_metric => Routes + }, + forwarding => #{ + 'matched' => Get([counters, 'matched'], ResourceMetrics), + 'success' => Get([counters, 'success'], ResourceMetrics), + 'failed' => Get([counters, 'failed'], ResourceMetrics), + 'dropped' => Get([counters, 'dropped'], ResourceMetrics), + 'retried' => Get([counters, 'retried'], ResourceMetrics), + 'received' => Get([counters, 'received'], ResourceMetrics), - 'matched' => Get([counters, 'matched'], ResourceMetrics), - 'success' => Get([counters, 'success'], ResourceMetrics), - 'failed' => Get([counters, 'failed'], ResourceMetrics), - 'dropped' => Get([counters, 'dropped'], ResourceMetrics), - 'retried' => Get([counters, 'retried'], ResourceMetrics), - 'received' => Get([counters, 'received'], ResourceMetrics), + 'queuing' => Get([gauges, 'queuing'], ResourceMetrics), + 'inflight' => Get([gauges, 'inflight'], ResourceMetrics), - 'queuing' => Get([gauges, 'queuing'], ResourceMetrics), - 'inflight' => Get([gauges, 'inflight'], ResourceMetrics), - - 'rate' => Get([rate, 'matched', current], ResourceMetrics), - 'rate_last5m' => Get([rate, 'matched', last5m], ResourceMetrics), - 'rate_max' => Get([rate, 'matched', max], ResourceMetrics) + 'rate' => Get([rate, 'matched', current], ResourceMetrics), + 'rate_last5m' => Get([rate, 'matched', last5m], ResourceMetrics), + 'rate_max' => Get([rate, 'matched', max], ResourceMetrics) + } } }. diff --git a/apps/emqx_cluster_link/test/emqx_cluster_link_api_SUITE.erl b/apps/emqx_cluster_link/test/emqx_cluster_link_api_SUITE.erl index 5e3ae5d50..6fd54e228 100644 --- a/apps/emqx_cluster_link/test/emqx_cluster_link_api_SUITE.erl +++ b/apps/emqx_cluster_link/test/emqx_cluster_link_api_SUITE.erl @@ -478,52 +478,64 @@ t_metrics(Config) -> ?assertMatch( {200, #{ <<"metrics">> := #{ - <<"routes">> := 0, - <<"matched">> := _, - <<"success">> := _, - <<"failed">> := _, - <<"dropped">> := _, - <<"retried">> := _, - <<"received">> := _, - <<"queuing">> := _, - <<"inflight">> := _, - <<"rate">> := _, - <<"rate_last5m">> := _, - <<"rate_max">> := _ + <<"router">> := #{ + <<"routes">> := 0 + }, + <<"forwarding">> := #{ + <<"matched">> := _, + <<"success">> := _, + <<"failed">> := _, + <<"dropped">> := _, + <<"retried">> := _, + <<"received">> := _, + <<"queuing">> := _, + <<"inflight">> := _, + <<"rate">> := _, + <<"rate_last5m">> := _, + <<"rate_max">> := _ + } }, <<"node_metrics">> := [ #{ <<"node">> := _, <<"metrics">> := #{ - <<"routes">> := 0, - <<"matched">> := _, - <<"success">> := _, - <<"failed">> := _, - <<"dropped">> := _, - <<"retried">> := _, - <<"received">> := _, - <<"queuing">> := _, - <<"inflight">> := _, - <<"rate">> := _, - <<"rate_last5m">> := _, - <<"rate_max">> := _ + <<"router">> := #{ + <<"routes">> := 0 + }, + <<"forwarding">> := #{ + <<"matched">> := _, + <<"success">> := _, + <<"failed">> := _, + <<"dropped">> := _, + <<"retried">> := _, + <<"received">> := _, + <<"queuing">> := _, + <<"inflight">> := _, + <<"rate">> := _, + <<"rate_last5m">> := _, + <<"rate_max">> := _ + } } }, #{ <<"node">> := _, <<"metrics">> := #{ - <<"routes">> := 0, - <<"matched">> := _, - <<"success">> := _, - <<"failed">> := _, - <<"dropped">> := _, - <<"retried">> := _, - <<"received">> := _, - <<"queuing">> := _, - <<"inflight">> := _, - <<"rate">> := _, - <<"rate_last5m">> := _, - <<"rate_max">> := _ + <<"router">> := #{ + <<"routes">> := 0 + }, + <<"forwarding">> := #{ + <<"matched">> := _, + <<"success">> := _, + <<"failed">> := _, + <<"dropped">> := _, + <<"retried">> := _, + <<"received">> := _, + <<"queuing">> := _, + <<"inflight">> := _, + <<"rate">> := _, + <<"rate_last5m">> := _, + <<"rate_max">> := _ + } } } ] @@ -532,15 +544,15 @@ t_metrics(Config) -> ), ?assertMatch( {200, #{ - <<"metrics">> := #{<<"routes">> := 0}, + <<"metrics">> := #{<<"router">> := #{<<"routes">> := 0}}, <<"node_metrics">> := [ #{ <<"node">> := _, - <<"metrics">> := #{<<"routes">> := 0} + <<"metrics">> := #{<<"router">> := #{<<"routes">> := 0}} }, #{ <<"node">> := _, - <<"metrics">> := #{<<"routes">> := 0} + <<"metrics">> := #{<<"router">> := #{<<"routes">> := 0}} } ] }}, @@ -556,15 +568,15 @@ t_metrics(Config) -> %% cluster. ?assertMatch( {200, #{ - <<"metrics">> := #{<<"routes">> := 0}, + <<"metrics">> := #{<<"router">> := #{<<"routes">> := 0}}, <<"node_metrics">> := [ #{ <<"node">> := _, - <<"metrics">> := #{<<"routes">> := 0} + <<"metrics">> := #{<<"router">> := #{<<"routes">> := 0}} }, #{ <<"node">> := _, - <<"metrics">> := #{<<"routes">> := 0} + <<"metrics">> := #{<<"router">> := #{<<"routes">> := 0}} } ] }}, @@ -572,15 +584,15 @@ t_metrics(Config) -> ), ?assertMatch( {200, #{ - <<"metrics">> := #{<<"routes">> := 0}, + <<"metrics">> := #{<<"router">> := #{<<"routes">> := 0}}, <<"node_metrics">> := [ #{ <<"node">> := _, - <<"metrics">> := #{<<"routes">> := 0} + <<"metrics">> := #{<<"router">> := #{<<"routes">> := 0}} }, #{ <<"node">> := _, - <<"metrics">> := #{<<"routes">> := 0} + <<"metrics">> := #{<<"router">> := #{<<"routes">> := 0}} } ] }}, @@ -608,10 +620,10 @@ t_metrics(Config) -> 10, ?assertMatch( {200, #{ - <<"metrics">> := #{<<"routes">> := 4}, + <<"metrics">> := #{<<"router">> := #{<<"routes">> := 4}}, <<"node_metrics">> := [ - #{<<"metrics">> := #{<<"routes">> := 2}}, - #{<<"metrics">> := #{<<"routes">> := 2}} + #{<<"metrics">> := #{<<"router">> := #{<<"routes">> := 2}}}, + #{<<"metrics">> := #{<<"router">> := #{<<"routes">> := 2}}} ] }}, get_metrics(source, SourceName) @@ -619,7 +631,7 @@ t_metrics(Config) -> ), ?assertMatch( {200, #{ - <<"metrics">> := #{<<"routes">> := 0}, + <<"metrics">> := #{<<"router">> := #{<<"routes">> := 0}}, <<"node_metrics">> := _ }}, get_metrics(target, TargetName) @@ -640,10 +652,10 @@ t_metrics(Config) -> 10, ?assertMatch( {200, #{ - <<"metrics">> := #{<<"routes">> := 2}, + <<"metrics">> := #{<<"router">> := #{<<"routes">> := 2}}, <<"node_metrics">> := [ - #{<<"metrics">> := #{<<"routes">> := 1}}, - #{<<"metrics">> := #{<<"routes">> := 1}} + #{<<"metrics">> := #{<<"router">> := #{<<"routes">> := 1}}}, + #{<<"metrics">> := #{<<"router">> := #{<<"routes">> := 1}}} ] }}, get_metrics(source, SourceName) @@ -678,7 +690,7 @@ t_metrics(Config) -> 10, ?assertMatch( {200, #{ - <<"metrics">> := #{<<"routes">> := 0}, + <<"metrics">> := #{<<"router">> := #{<<"routes">> := 0}}, <<"node_metrics">> := _ }}, get_metrics(source, SourceName) @@ -700,7 +712,7 @@ t_metrics(Config) -> 10, ?assertMatch( {200, #{ - <<"metrics">> := #{<<"routes">> := 2}, + <<"metrics">> := #{<<"router">> := #{<<"routes">> := 2}}, <<"node_metrics">> := _ }}, get_metrics(source, SourceName) From 6dbf015c932d46dc1d660fde4b9d1f3a7a8e79e3 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 25 Jul 2024 11:24:15 -0300 Subject: [PATCH 043/131] refactor: demote hidden config to hardcoded value --- .../src/emqx_cluster_link_bookkeeper.erl | 10 +++++++++- .../emqx_cluster_link/src/emqx_cluster_link_config.erl | 8 +------- .../emqx_cluster_link/src/emqx_cluster_link_schema.erl | 9 +-------- .../emqx_cluster_link/test/emqx_cluster_link_SUITE.erl | 2 -- 4 files changed, 11 insertions(+), 18 deletions(-) diff --git a/apps/emqx_cluster_link/src/emqx_cluster_link_bookkeeper.erl b/apps/emqx_cluster_link/src/emqx_cluster_link_bookkeeper.erl index 992fc7bf1..da5d7eaae 100644 --- a/apps/emqx_cluster_link/src/emqx_cluster_link_bookkeeper.erl +++ b/apps/emqx_cluster_link/src/emqx_cluster_link_bookkeeper.erl @@ -21,6 +21,14 @@ %% Type declarations %%------------------------------------------------------------------------------ +-ifdef(TEST). +%% ms +-define(TALLY_ROUTES_INTERVAL, 300). +-else. +%% ms +-define(TALLY_ROUTES_INTERVAL, 15_000). +-endif. + %% call/cast/info events -record(tally_routes, {}). @@ -71,7 +79,7 @@ ensure_timer(Event, Timeout) -> handle_tally_routes() -> ClusterNames = cluster_names(), tally_routes(ClusterNames), - ensure_timer(#tally_routes{}, emqx_cluster_link_config:tally_routes_interval()), + ensure_timer(#tally_routes{}, ?TALLY_ROUTES_INTERVAL), ok. tally_routes([ClusterName | ClusterNames]) -> diff --git a/apps/emqx_cluster_link/src/emqx_cluster_link_config.erl b/apps/emqx_cluster_link/src/emqx_cluster_link_config.erl index fd84a5b7f..2a97f2d69 100644 --- a/apps/emqx_cluster_link/src/emqx_cluster_link_config.erl +++ b/apps/emqx_cluster_link/src/emqx_cluster_link_config.erl @@ -46,9 +46,7 @@ %% Actor Lifecycle actor_ttl/0, actor_gc_interval/0, - actor_heartbeat_interval/0, - %% Metrics - tally_routes_interval/0 + actor_heartbeat_interval/0 ]). -export([ @@ -165,10 +163,6 @@ actor_gc_interval() -> actor_heartbeat_interval() -> actor_ttl() div 3. --spec tally_routes_interval() -> _Milliseconds :: timeout(). -tally_routes_interval() -> - emqx_config:get([cluster, tally_routes_interval]). - %% mk_emqtt_options(#{server := Server, ssl := #{enable := EnableSsl} = Ssl} = LinkConf) -> diff --git a/apps/emqx_cluster_link/src/emqx_cluster_link_schema.erl b/apps/emqx_cluster_link/src/emqx_cluster_link_schema.erl index dd5ab66f6..9b08510b9 100644 --- a/apps/emqx_cluster_link/src/emqx_cluster_link_schema.erl +++ b/apps/emqx_cluster_link/src/emqx_cluster_link_schema.erl @@ -32,14 +32,7 @@ roots() -> []. injected_fields() -> #{ cluster => [ - {links, links_schema(#{})}, - {tally_routes_interval, - hoconsc:mk( - emqx_schema:timeout_duration(), #{ - default => <<"15s">>, - importance => ?IMPORTANCE_HIDDEN - } - )} + {links, links_schema(#{})} ] }. diff --git a/apps/emqx_cluster_link/test/emqx_cluster_link_SUITE.erl b/apps/emqx_cluster_link/test/emqx_cluster_link_SUITE.erl index 21aa1a70e..e023aacab 100644 --- a/apps/emqx_cluster_link/test/emqx_cluster_link_SUITE.erl +++ b/apps/emqx_cluster_link/test/emqx_cluster_link_SUITE.erl @@ -53,7 +53,6 @@ mk_source_cluster(BaseName, Config) -> SourceConf = "cluster {" "\n name = cl.source" - "\n tally_routes_interval = 300ms" "\n links = [" "\n { enable = true" "\n name = cl.target" @@ -76,7 +75,6 @@ mk_target_cluster(BaseName, Config) -> TargetConf = "cluster {" "\n name = cl.target" - "\n tally_routes_interval = 300ms" "\n links = [" "\n { enable = true" "\n name = cl.source" From 6da71200f3c4dbaba25072d8199fc5157d04b339 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 25 Jul 2024 11:26:44 -0300 Subject: [PATCH 044/131] refactor: improve bookkeeping api --- .../src/emqx_cluster_link_bookkeeper.erl | 4 +--- .../src/emqx_cluster_link_extrouter.erl | 10 ++++------ 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/apps/emqx_cluster_link/src/emqx_cluster_link_bookkeeper.erl b/apps/emqx_cluster_link/src/emqx_cluster_link_bookkeeper.erl index da5d7eaae..826d4f0db 100644 --- a/apps/emqx_cluster_link/src/emqx_cluster_link_bookkeeper.erl +++ b/apps/emqx_cluster_link/src/emqx_cluster_link_bookkeeper.erl @@ -83,9 +83,7 @@ handle_tally_routes() -> ok. tally_routes([ClusterName | ClusterNames]) -> - Tab = emqx_cluster_link_extrouter:extroute_tab(), - Pat = emqx_cluster_link_extrouter:cluster_routes_ms(ClusterName), - NumRoutes = ets:select_count(Tab, Pat), + NumRoutes = emqx_cluster_link_extrouter:count(ClusterName), emqx_cluster_link_metrics:routes_set(ClusterName, NumRoutes), tally_routes(ClusterNames); tally_routes([]) -> diff --git a/apps/emqx_cluster_link/src/emqx_cluster_link_extrouter.erl b/apps/emqx_cluster_link/src/emqx_cluster_link_extrouter.erl index e76b24d79..44b147454 100644 --- a/apps/emqx_cluster_link/src/emqx_cluster_link_extrouter.erl +++ b/apps/emqx_cluster_link/src/emqx_cluster_link_extrouter.erl @@ -35,7 +35,7 @@ ]). %% Internal export for bookkeeping --export([cluster_routes_ms/1, extroute_tab/0]). +-export([count/1]). %% Strictly monotonically increasing integer. -type smint() :: integer(). @@ -151,16 +151,14 @@ make_extroute_rec_pat(Entry) -> ). %% Internal exports for bookkeeping -cluster_routes_ms(ClusterName) -> +count(ClusterName) -> TopicPat = '_', RouteIDPat = '_', Pat = make_extroute_rec_pat( emqx_trie_search:make_pat(TopicPat, ?ROUTE_ID(ClusterName, RouteIDPat)) ), - [{Pat, [], [true]}]. - -extroute_tab() -> - ?EXTROUTE_TAB. + MS = [{Pat, [], [true]}], + ets:select_count(?EXTROUTE_TAB, MS). %% From 03821c7b497e8fa1e3128d6e06832162b40fd94e Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 25 Jul 2024 11:41:37 -0300 Subject: [PATCH 045/131] fix(cluster link metrics): route count metric is cluster-wide --- .../src/emqx_cluster_link_api.erl | 38 ++++++++++++++++--- .../test/emqx_cluster_link_api_SUITE.erl | 12 +++--- 2 files changed, 39 insertions(+), 11 deletions(-) diff --git a/apps/emqx_cluster_link/src/emqx_cluster_link_api.erl b/apps/emqx_cluster_link/src/emqx_cluster_link_api.erl index 45b97e8f7..77f613e45 100644 --- a/apps/emqx_cluster_link/src/emqx_cluster_link_api.erl +++ b/apps/emqx_cluster_link/src/emqx_cluster_link_api.erl @@ -274,13 +274,41 @@ append_errors(RouterError, ResourceError, Node, Acc) -> aggregate_metrics(NodeMetrics) -> ErrorLogger = fun(_) -> ok end, - lists:foldl( - fun(#{metrics := Metrics}, Acc) -> - emqx_utils_maps:best_effort_recursive_sum(Metrics, Acc, ErrorLogger) + #{metrics := #{router := EmptyRouterMetrics}} = format_metrics(node(), #{}, #{}), + {RouterMetrics, ResourceMetrics} = lists:foldl( + fun( + #{metrics := #{router := RMetrics, forwarding := FMetrics}}, + {RouterAccIn, ResourceAccIn} + ) -> + ResourceAcc = + emqx_utils_maps:best_effort_recursive_sum(FMetrics, ResourceAccIn, ErrorLogger), + RouterAcc = merge_cluster_wide_metrics(RMetrics, RouterAccIn), + {RouterAcc, ResourceAcc} end, - #{}, + {EmptyRouterMetrics, #{}}, NodeMetrics - ). + ), + #{router => RouterMetrics, forwarding => ResourceMetrics}. + +merge_cluster_wide_metrics(Metrics, Acc) -> + %% For cluster-wide metrics, all nodes should report the same values, except if the + %% RPC to fetch a node's metrics failed, in which case all values will be 0. + F = + fun(_Key, V1, V2) -> + case {erlang:is_map(V1), erlang:is_map(V2)} of + {true, true} -> + merge_cluster_wide_metrics(V1, V2); + {true, false} -> + merge_cluster_wide_metrics(V1, #{}); + {false, true} -> + merge_cluster_wide_metrics(V2, #{}); + {false, false} -> + true = is_number(V1), + true = is_number(V2), + max(V1, V2) + end + end, + maps:merge_with(F, Acc, Metrics). format_metrics(Node, RouterMetrics, ResourceMetrics) -> Get = fun(Path, Map) -> emqx_utils_maps:deep_get(Path, Map, 0) end, diff --git a/apps/emqx_cluster_link/test/emqx_cluster_link_api_SUITE.erl b/apps/emqx_cluster_link/test/emqx_cluster_link_api_SUITE.erl index 6fd54e228..8157c86d6 100644 --- a/apps/emqx_cluster_link/test/emqx_cluster_link_api_SUITE.erl +++ b/apps/emqx_cluster_link/test/emqx_cluster_link_api_SUITE.erl @@ -612,15 +612,15 @@ t_metrics(Config) -> #{?snk_kind := clink_route_sync_complete} ), - %% Routes = 4 in source cluster, because the target cluster has some topic filters - %% configured and subscribers to them, which were replicated to the source cluster, - %% and we have 2 nodes with 2 routes each. + %% Routes = 2 in source cluster, because the target cluster has some topic filters + %% configured and subscribers to them, which were replicated to the source cluster. + %% This metric is global (cluster-wide). ?retry( 300, 10, ?assertMatch( {200, #{ - <<"metrics">> := #{<<"router">> := #{<<"routes">> := 4}}, + <<"metrics">> := #{<<"router">> := #{<<"routes">> := 2}}, <<"node_metrics">> := [ #{<<"metrics">> := #{<<"router">> := #{<<"routes">> := 2}}}, #{<<"metrics">> := #{<<"router">> := #{<<"routes">> := 2}}} @@ -652,7 +652,7 @@ t_metrics(Config) -> 10, ?assertMatch( {200, #{ - <<"metrics">> := #{<<"router">> := #{<<"routes">> := 2}}, + <<"metrics">> := #{<<"router">> := #{<<"routes">> := 1}}, <<"node_metrics">> := [ #{<<"metrics">> := #{<<"router">> := #{<<"routes">> := 1}}}, #{<<"metrics">> := #{<<"router">> := #{<<"routes">> := 1}}} @@ -712,7 +712,7 @@ t_metrics(Config) -> 10, ?assertMatch( {200, #{ - <<"metrics">> := #{<<"router">> := #{<<"routes">> := 2}}, + <<"metrics">> := #{<<"router">> := #{<<"routes">> := 1}}, <<"node_metrics">> := _ }}, get_metrics(source, SourceName) From 43f799508a10ecb4bbe7482e74a1213b507623eb Mon Sep 17 00:00:00 2001 From: zmstone Date: Thu, 25 Jul 2024 10:35:36 +0200 Subject: [PATCH 046/131] chore: add ldap test doc --- .ci/docker-compose-file/openldap/README.md | 61 ++++++++++++++++++++++ rel/config/ee-examples/ldap-authn.conf | 19 +++++++ 2 files changed, 80 insertions(+) create mode 100644 .ci/docker-compose-file/openldap/README.md create mode 100644 rel/config/ee-examples/ldap-authn.conf diff --git a/.ci/docker-compose-file/openldap/README.md b/.ci/docker-compose-file/openldap/README.md new file mode 100644 index 000000000..c91b5c1dc --- /dev/null +++ b/.ci/docker-compose-file/openldap/README.md @@ -0,0 +1,61 @@ +# LDAP authentication + +To run manual tests with the default docker-compose files. + +Expose openldap container port by uncommenting the `ports` config in `docker-compose-ldap.yaml ` + +To start openldap: + +``` +docker-compose -f ./.ci/docker-compose-file/docker-compose.yaml -f ./.ci/docker-compose-file/docker-compose-ldap.yaml up -docker +``` + +## LDAP database + +LDAP database is populated from below files: +``` +apps/emqx_ldap/test/data/emqx.io.ldif /usr/local/etc/openldap/schema/emqx.io.ldif +apps/emqx_ldap/test/data/emqx.schema /usr/local/etc/openldap/schema/emqx.schema +``` + +## Minimal EMQX config + +``` +authentication = [ + { + backend = ldap + base_dn = "uid=${username},ou=testdevice,dc=emqx,dc=io" + filter = "(& (objectClass=mqttUser) (uid=${username}))" + mechanism = password_based + method { + is_superuser_attribute = isSuperuser + password_attribute = userPassword + type = hash + } + password = public + pool_size = 8 + query_timeout = "5s" + request_timeout = "10s" + server = "localhost:1389" + username = "cn=root,dc=emqx,dc=io" + } +] +``` + +## Example ldapsearch command + +``` +ldapsearch -x -H ldap://localhost:389 -D "cn=root,dc=emqx,dc=io" -W -b "uid=mqttuser0007,ou=testdevice,dc=emqx,dc=io" "(&(objectClass=mqttUser)(uid=mqttuser0007))" +``` + +## Example mqttx command + +The client password hashes are generated from their username. + +``` +# disabled user +mqttx pub -t 't/1' -h localhost -p 1883 -m x -u mqttuser0006 -P mqttuser0006 + +# enabled super-user +mqttx pub -t 't/1' -h localhost -p 1883 -m x -u mqttuser0007 -P mqttuser0007 +``` diff --git a/rel/config/ee-examples/ldap-authn.conf b/rel/config/ee-examples/ldap-authn.conf new file mode 100644 index 000000000..633a5cc7b --- /dev/null +++ b/rel/config/ee-examples/ldap-authn.conf @@ -0,0 +1,19 @@ +authentication = [ + { + backend = ldap + base_dn = "uid=${username},ou=testdevice,dc=emqx,dc=io" + filter = "(& (objectClass=mqttUser) (uid=${username}))" + mechanism = password_based + method { + is_superuser_attribute = isSuperuser + password_attribute = userPassword + type = hash + } + password = public + pool_size = 8 + query_timeout = "5s" + request_timeout = "10s" + server = "localhost:1389" + username = "cn=root,dc=emqx,dc=io" + } +] From 8f94e9684c38ff8095f2f44dedee43743c0e622c Mon Sep 17 00:00:00 2001 From: zmstone Date: Thu, 25 Jul 2024 14:27:11 +0200 Subject: [PATCH 047/131] fix: handle ldap seqrch error --- apps/emqx_ldap/src/emqx_ldap.erl | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/apps/emqx_ldap/src/emqx_ldap.erl b/apps/emqx_ldap/src/emqx_ldap.erl index 67b250420..a2e12db69 100644 --- a/apps/emqx_ldap/src/emqx_ldap.erl +++ b/apps/emqx_ldap/src/emqx_ldap.erl @@ -41,6 +41,7 @@ -export([namespace/0, roots/0, fields/1, desc/1]). -export([do_get_status/1, get_status_with_poolname/1]). +-export([search/2]). -define(LDAP_HOST_OPTIONS, #{ default_port => 389 @@ -273,6 +274,21 @@ on_query( Error end. +search(Pid, SearchOptions) -> + case eldap:search(Pid, SearchOptions) of + {error, ldap_closed} -> + %% ldap server closing the socket does not result in + %% process restart, so we need to kill it and reconnect + _ = exit(Pid, kill), + {error, ldap_closed}; + {error, {gen_tcp_error, timeout}} -> + %% kill the process to trigger reconnect + _ = exit(Pid, kill), + {error, timeout_cause_reconnect}; + Result -> + Result + end. + do_ldap_query( InstId, SearchOptions, @@ -283,7 +299,7 @@ do_ldap_query( case ecpool:pick_and_do( PoolName, - {eldap, search, [SearchOptions]}, + {?MODULE, search, [SearchOptions]}, handover ) of @@ -319,7 +335,7 @@ do_ldap_query( ?SLOG( error, LogMeta#{ - msg => "ldap_connector_do_query_failed", + msg => "ldap_connector_query_failed", reason => emqx_utils:redact(Reason) } ), From 7631420eefeebb2d1c9ebb93cee621d123281a53 Mon Sep 17 00:00:00 2001 From: zmstone Date: Thu, 25 Jul 2024 16:42:44 +0200 Subject: [PATCH 048/131] test: add test case to cover ldap search timeout --- .../test/emqx_authn_ldap_SUITE.erl | 93 ++++++++++++++++--- .../test/emqx_authz_ldap_SUITE.erl | 14 --- apps/emqx_ldap/src/emqx_ldap.erl | 7 +- 3 files changed, 86 insertions(+), 28 deletions(-) diff --git a/apps/emqx_auth_ldap/test/emqx_authn_ldap_SUITE.erl b/apps/emqx_auth_ldap/test/emqx_authn_ldap_SUITE.erl index ac941f268..af8100c23 100644 --- a/apps/emqx_auth_ldap/test/emqx_authn_ldap_SUITE.erl +++ b/apps/emqx_auth_ldap/test/emqx_authn_ldap_SUITE.erl @@ -21,6 +21,7 @@ -include_lib("emqx_auth/include/emqx_authn.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). -define(LDAP_HOST, "ldap"). -define(LDAP_DEFAULT_PORT, 389). @@ -46,13 +47,6 @@ init_per_suite(Config) -> Apps = emqx_cth_suite:start([emqx, emqx_conf, emqx_auth, emqx_auth_ldap], #{ work_dir => ?config(priv_dir, Config) }), - {ok, _} = emqx_resource:create_local( - ?LDAP_RESOURCE, - ?AUTHN_RESOURCE_GROUP, - emqx_ldap, - ldap_config(), - #{} - ), [{apps, Apps} | Config]; false -> {skip, no_ldap} @@ -63,7 +57,6 @@ end_per_suite(Config) -> [authentication], ?GLOBAL ), - ok = emqx_resource:remove_local(?LDAP_RESOURCE), ok = emqx_cth_suite:stop(?config(apps, Config)). %%------------------------------------------------------------------------------ @@ -128,6 +121,87 @@ t_create_invalid(_Config) -> InvalidConfigs ). +t_authenticate_timeout_cause_reconnect(_Config) -> + TestPid = self(), + meck:new(eldap, [non_strict, no_link, passthrough]), + try + %% cause eldap process to be killed + meck:expect( + eldap, + search, + fun + (Pid, [{base, <<"uid=mqttuser0007", _/binary>>} | _]) -> + TestPid ! {eldap_pid, Pid}, + {error, {gen_tcp_error, timeout}}; + (Pid, Args) -> + meck:passthrough([Pid, Args]) + end + ), + + Credentials = fun(Username) -> + #{ + username => Username, + password => Username, + listener => 'tcp:default', + protocol => mqtt + } + end, + + SpecificConfigParams = #{}, + Result = {ok, #{is_superuser => true}}, + + Timeout = 1000, + Config0 = raw_ldap_auth_config(), + Config = Config0#{ + <<"pool_size">> => 1, + <<"request_timeout">> => Timeout + }, + AuthConfig = maps:merge(Config, SpecificConfigParams), + {ok, _} = emqx:update_config( + ?PATH, + {create_authenticator, ?GLOBAL, AuthConfig} + ), + + %% 0006 is a disabled user + ?assertEqual( + {error, user_disabled}, + emqx_access_control:authenticate(Credentials(<<"mqttuser0006">>)) + ), + ?assertEqual( + {error, not_authorized}, + emqx_access_control:authenticate(Credentials(<<"mqttuser0007">>)) + ), + ok = wait_for_ldap_pid(1000), + [#{id := ResourceID}] = emqx_resource_manager:list_all(), + ?retry(1_000, 10, {ok, connected} = emqx_resource_manager:health_check(ResourceID)), + %% turn back to normal + meck:expect( + eldap, + search, + 2, + fun(Pid2, Query) -> + meck:passthrough([Pid2, Query]) + end + ), + %% expect eldap process to be restarted + ?assertEqual(Result, emqx_access_control:authenticate(Credentials(<<"mqttuser0007">>))), + emqx_authn_test_lib:delete_authenticators( + [authentication], + ?GLOBAL + ) + after + meck:unload(eldap) + end. + +wait_for_ldap_pid(After) -> + receive + {eldap_pid, Pid} -> + ?assertNot(is_process_alive(Pid)), + ok + after After -> + error(timeout) + end. + t_authenticate(_Config) -> ok = lists:foreach( fun(Sample) -> @@ -300,6 +374,3 @@ user_seeds() -> ldap_server() -> iolist_to_binary(io_lib:format("~s:~B", [?LDAP_HOST, ?LDAP_DEFAULT_PORT])). - -ldap_config() -> - emqx_ldap_SUITE:ldap_config([]). diff --git a/apps/emqx_auth_ldap/test/emqx_authz_ldap_SUITE.erl b/apps/emqx_auth_ldap/test/emqx_authz_ldap_SUITE.erl index 09875a3fa..7ff6fdebe 100644 --- a/apps/emqx_auth_ldap/test/emqx_authz_ldap_SUITE.erl +++ b/apps/emqx_auth_ldap/test/emqx_authz_ldap_SUITE.erl @@ -44,7 +44,6 @@ init_per_suite(Config) -> ], #{work_dir => emqx_cth_suite:work_dir(Config)} ), - ok = create_ldap_resource(), [{apps, Apps} | Config]; false -> {skip, no_ldap} @@ -167,21 +166,8 @@ setup_config(SpecialParams) -> ldap_server() -> iolist_to_binary(io_lib:format("~s:~B", [?LDAP_HOST, ?LDAP_DEFAULT_PORT])). -ldap_config() -> - emqx_ldap_SUITE:ldap_config([]). - start_apps(Apps) -> lists:foreach(fun application:ensure_all_started/1, Apps). stop_apps(Apps) -> lists:foreach(fun application:stop/1, Apps). - -create_ldap_resource() -> - {ok, _} = emqx_resource:create_local( - ?LDAP_RESOURCE, - ?AUTHZ_RESOURCE_GROUP, - emqx_ldap, - ldap_config(), - #{} - ), - ok. diff --git a/apps/emqx_ldap/src/emqx_ldap.erl b/apps/emqx_ldap/src/emqx_ldap.erl index a2e12db69..a2b09ccda 100644 --- a/apps/emqx_ldap/src/emqx_ldap.erl +++ b/apps/emqx_ldap/src/emqx_ldap.erl @@ -278,13 +278,14 @@ search(Pid, SearchOptions) -> case eldap:search(Pid, SearchOptions) of {error, ldap_closed} -> %% ldap server closing the socket does not result in - %% process restart, so we need to kill it and reconnect + %% process restart, so we need to kill it to trigger a quick reconnect + %% instead of waiting for the next health-check _ = exit(Pid, kill), {error, ldap_closed}; - {error, {gen_tcp_error, timeout}} -> + {error, {gen_tcp_error, _} = Reason} -> %% kill the process to trigger reconnect _ = exit(Pid, kill), - {error, timeout_cause_reconnect}; + {error, Reason}; Result -> Result end. From f6a0f5677181b30f28927c8f4a9759b2403c2b0c Mon Sep 17 00:00:00 2001 From: zmstone Date: Thu, 25 Jul 2024 19:24:52 +0200 Subject: [PATCH 049/131] docs: add changelog for PR 13521 --- changes/ce/feat-13521.en.md | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 changes/ce/feat-13521.en.md diff --git a/changes/ce/feat-13521.en.md b/changes/ce/feat-13521.en.md new file mode 100644 index 000000000..6d57eee23 --- /dev/null +++ b/changes/ce/feat-13521.en.md @@ -0,0 +1,4 @@ +Fix LDAP query timeout issue. + +Previously, LDAP query timeout may cause the underlying connection to be unusable. +Fixed to always reconnect if timeout happens. From 33eccb35dacde59a549d9310a2ed2e70cfa6b6f3 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 24 Jul 2024 15:49:58 -0300 Subject: [PATCH 050/131] chore: update wolff -> 3.0.2 --- apps/emqx_bridge_azure_event_hub/mix.exs | 2 +- apps/emqx_bridge_azure_event_hub/rebar.config | 2 +- apps/emqx_bridge_confluent/mix.exs | 2 +- apps/emqx_bridge_confluent/rebar.config | 2 +- apps/emqx_bridge_kafka/mix.exs | 2 +- apps/emqx_bridge_kafka/rebar.config | 2 +- .../test/emqx_bridge_kafka_impl_consumer_SUITE.erl | 6 +++--- mix.exs | 2 +- 8 files changed, 10 insertions(+), 10 deletions(-) diff --git a/apps/emqx_bridge_azure_event_hub/mix.exs b/apps/emqx_bridge_azure_event_hub/mix.exs index 42edddbbe..8f5068d0e 100644 --- a/apps/emqx_bridge_azure_event_hub/mix.exs +++ b/apps/emqx_bridge_azure_event_hub/mix.exs @@ -23,7 +23,7 @@ defmodule EMQXBridgeAzureEventHub.MixProject do def deps() do [ - {:wolff, github: "kafka4beam/wolff", tag: "2.0.0"}, + {:wolff, github: "kafka4beam/wolff", tag: "3.0.2"}, {:kafka_protocol, github: "kafka4beam/kafka_protocol", tag: "4.1.5", override: true}, {:brod_gssapi, github: "kafka4beam/brod_gssapi", tag: "v0.1.1"}, {:brod, github: "kafka4beam/brod", tag: "3.18.0"}, diff --git a/apps/emqx_bridge_azure_event_hub/rebar.config b/apps/emqx_bridge_azure_event_hub/rebar.config index 76ea7fa6c..c8be2a6a3 100644 --- a/apps/emqx_bridge_azure_event_hub/rebar.config +++ b/apps/emqx_bridge_azure_event_hub/rebar.config @@ -2,7 +2,7 @@ {erl_opts, [debug_info]}. {deps, [ - {wolff, {git, "https://github.com/kafka4beam/wolff.git", {tag, "2.0.0"}}}, + {wolff, {git, "https://github.com/kafka4beam/wolff.git", {tag, "3.0.2"}}}, {kafka_protocol, {git, "https://github.com/kafka4beam/kafka_protocol.git", {tag, "4.1.5"}}}, {brod_gssapi, {git, "https://github.com/kafka4beam/brod_gssapi.git", {tag, "v0.1.1"}}}, {brod, {git, "https://github.com/kafka4beam/brod.git", {tag, "3.18.0"}}}, diff --git a/apps/emqx_bridge_confluent/mix.exs b/apps/emqx_bridge_confluent/mix.exs index 46cbe9a02..134e924fc 100644 --- a/apps/emqx_bridge_confluent/mix.exs +++ b/apps/emqx_bridge_confluent/mix.exs @@ -23,7 +23,7 @@ defmodule EMQXBridgeConfluent.MixProject do def deps() do [ - {:wolff, github: "kafka4beam/wolff", tag: "2.0.0"}, + {:wolff, github: "kafka4beam/wolff", tag: "3.0.2"}, {:kafka_protocol, github: "kafka4beam/kafka_protocol", tag: "4.1.5", override: true}, {:brod_gssapi, github: "kafka4beam/brod_gssapi", tag: "v0.1.1"}, {:brod, github: "kafka4beam/brod", tag: "3.18.0"}, diff --git a/apps/emqx_bridge_confluent/rebar.config b/apps/emqx_bridge_confluent/rebar.config index 1a91f501d..786b1cf82 100644 --- a/apps/emqx_bridge_confluent/rebar.config +++ b/apps/emqx_bridge_confluent/rebar.config @@ -2,7 +2,7 @@ {erl_opts, [debug_info]}. {deps, [ - {wolff, {git, "https://github.com/kafka4beam/wolff.git", {tag, "2.0.0"}}}, + {wolff, {git, "https://github.com/kafka4beam/wolff.git", {tag, "3.0.2"}}}, {kafka_protocol, {git, "https://github.com/kafka4beam/kafka_protocol.git", {tag, "4.1.5"}}}, {brod_gssapi, {git, "https://github.com/kafka4beam/brod_gssapi.git", {tag, "v0.1.1"}}}, {brod, {git, "https://github.com/kafka4beam/brod.git", {tag, "3.18.0"}}}, diff --git a/apps/emqx_bridge_kafka/mix.exs b/apps/emqx_bridge_kafka/mix.exs index b74b1fdd0..a1a59cb08 100644 --- a/apps/emqx_bridge_kafka/mix.exs +++ b/apps/emqx_bridge_kafka/mix.exs @@ -23,7 +23,7 @@ defmodule EMQXBridgeKafka.MixProject do def deps() do [ - {:wolff, github: "kafka4beam/wolff", tag: "2.0.0"}, + {:wolff, github: "kafka4beam/wolff", tag: "3.0.2"}, {:kafka_protocol, github: "kafka4beam/kafka_protocol", tag: "4.1.5", override: true}, {:brod_gssapi, github: "kafka4beam/brod_gssapi", tag: "v0.1.1"}, {:brod, github: "kafka4beam/brod", tag: "3.18.0"}, diff --git a/apps/emqx_bridge_kafka/rebar.config b/apps/emqx_bridge_kafka/rebar.config index b89c9190f..77d9b95ef 100644 --- a/apps/emqx_bridge_kafka/rebar.config +++ b/apps/emqx_bridge_kafka/rebar.config @@ -2,7 +2,7 @@ {erl_opts, [debug_info]}. {deps, [ - {wolff, {git, "https://github.com/kafka4beam/wolff.git", {tag, "2.0.0"}}}, + {wolff, {git, "https://github.com/kafka4beam/wolff.git", {tag, "3.0.2"}}}, {kafka_protocol, {git, "https://github.com/kafka4beam/kafka_protocol.git", {tag, "4.1.5"}}}, {brod_gssapi, {git, "https://github.com/kafka4beam/brod_gssapi.git", {tag, "v0.1.1"}}}, {brod, {git, "https://github.com/kafka4beam/brod.git", {tag, "3.18.0"}}}, diff --git a/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl index 56aabb1c3..9119ee6c4 100644 --- a/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl +++ b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl @@ -477,7 +477,7 @@ do_start_producer(KafkaClientId, KafkaTopic) -> ProducerConfig = #{ name => Name, - partitioner => roundrobin, + partitioner => random, partition_count_refresh_interval_seconds => 1_000, replayq_max_total_bytes => 10_000, replayq_seg_bytes => 9_000, @@ -1520,7 +1520,7 @@ t_receive_after_recovery(Config) -> key => <<"commit", (integer_to_binary(N))/binary>>, value => <<"commit", (integer_to_binary(N))/binary>> } - || N <- lists:seq(1, NPartitions) + || N <- lists:seq(1, NPartitions * 10) ], %% we do distinct passes over this producing part so that %% wolff won't batch everything together. @@ -1933,7 +1933,7 @@ t_node_joins_existing_cluster(Config) -> Val = <<"v", (integer_to_binary(N))/binary>>, publish(Config, KafkaTopic, [#{key => Key, value => Val}]) end, - lists:seq(1, NPartitions) + lists:seq(1, 10 * NPartitions) ), {ok, _} = snabbkaffe:receive_events(SRef1), diff --git a/mix.exs b/mix.exs index 53e5b304f..399c996a6 100644 --- a/mix.exs +++ b/mix.exs @@ -361,7 +361,7 @@ defmodule EMQXUmbrella.MixProject do {:hstreamdb_erl, github: "hstreamdb/hstreamdb_erl", tag: "0.5.18+v0.18.1+ezstd-v1.0.5-emqx1"}, {:influxdb, github: "emqx/influxdb-client-erl", tag: "1.1.13", override: true}, - {:wolff, github: "kafka4beam/wolff", tag: "2.0.0"}, + {:wolff, github: "kafka4beam/wolff", tag: "3.0.2"}, {:kafka_protocol, github: "kafka4beam/kafka_protocol", tag: "4.1.5", override: true}, {:brod_gssapi, github: "kafka4beam/brod_gssapi", tag: "v0.1.1"}, {:brod, github: "kafka4beam/brod", tag: "3.18.0"}, From df1f4fad7009cc88990491b4b346ac15d07cf7ca Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 10 Jul 2024 18:20:05 -0300 Subject: [PATCH 051/131] feat(kafka producer): allow dynamic topics from pre-configured topics Fixes https://emqx.atlassian.net/browse/EMQX-12656 --- .../emqx_bridge_azure_event_hub_v2_SUITE.erl | 34 +++ .../emqx_bridge_confluent_producer_SUITE.erl | 34 +++ .../src/emqx_bridge_kafka.erl | 45 +++- .../src/emqx_bridge_kafka_impl_producer.erl | 178 +++++++++---- ...emqx_bridge_kafka_producer_action_info.erl | 13 +- .../emqx_bridge_v2_kafka_producer_SUITE.erl | 238 +++++++++++++++++- changes/ee/feat-13452.en.md | 1 + rel/i18n/emqx_bridge_azure_event_hub.hocon | 12 +- rel/i18n/emqx_bridge_confluent_producer.hocon | 14 +- rel/i18n/emqx_bridge_kafka.hocon | 11 +- 10 files changed, 518 insertions(+), 62 deletions(-) create mode 100644 changes/ee/feat-13452.en.md diff --git a/apps/emqx_bridge_azure_event_hub/test/emqx_bridge_azure_event_hub_v2_SUITE.erl b/apps/emqx_bridge_azure_event_hub/test/emqx_bridge_azure_event_hub_v2_SUITE.erl index 661b8819c..0136ec568 100644 --- a/apps/emqx_bridge_azure_event_hub/test/emqx_bridge_azure_event_hub_v2_SUITE.erl +++ b/apps/emqx_bridge_azure_event_hub/test/emqx_bridge_azure_event_hub_v2_SUITE.erl @@ -391,3 +391,37 @@ t_multiple_actions_sharing_topic(Config) -> ] ), ok. + +t_pre_configured_topics(Config) -> + ActionConfig0 = ?config(action_config, Config), + ActionConfig = + emqx_utils_maps:deep_merge( + ActionConfig0, + #{<<"parameters">> => #{<<"query_mode">> => <<"sync">>}} + ), + ok = emqx_bridge_v2_kafka_producer_SUITE:t_pre_configured_topics( + [ + {type, ?BRIDGE_TYPE_BIN}, + {connector_name, ?config(connector_name, Config)}, + {connector_config, ?config(connector_config, Config)}, + {action_config, ActionConfig} + ] + ), + ok. + +t_templated_topic_and_no_pre_configured_topics(Config) -> + ActionConfig0 = ?config(action_config, Config), + ActionConfig = + emqx_utils_maps:deep_merge( + ActionConfig0, + #{<<"parameters">> => #{<<"query_mode">> => <<"sync">>}} + ), + ok = emqx_bridge_v2_kafka_producer_SUITE:t_templated_topic_and_no_pre_configured_topics( + [ + {type, ?BRIDGE_TYPE_BIN}, + {connector_name, ?config(connector_name, Config)}, + {connector_config, ?config(connector_config, Config)}, + {action_config, ActionConfig} + ] + ), + ok. diff --git a/apps/emqx_bridge_confluent/test/emqx_bridge_confluent_producer_SUITE.erl b/apps/emqx_bridge_confluent/test/emqx_bridge_confluent_producer_SUITE.erl index 0b3a22a99..de92b9327 100644 --- a/apps/emqx_bridge_confluent/test/emqx_bridge_confluent_producer_SUITE.erl +++ b/apps/emqx_bridge_confluent/test/emqx_bridge_confluent_producer_SUITE.erl @@ -400,3 +400,37 @@ t_multiple_actions_sharing_topic(Config) -> ] ), ok. + +t_pre_configured_topics(Config) -> + ActionConfig0 = ?config(action_config, Config), + ActionConfig = + emqx_utils_maps:deep_merge( + ActionConfig0, + #{<<"parameters">> => #{<<"query_mode">> => <<"sync">>}} + ), + ok = emqx_bridge_v2_kafka_producer_SUITE:t_pre_configured_topics( + [ + {type, ?ACTION_TYPE_BIN}, + {connector_name, ?config(connector_name, Config)}, + {connector_config, ?config(connector_config, Config)}, + {action_config, ActionConfig} + ] + ), + ok. + +t_templated_topic_and_no_pre_configured_topics(Config) -> + ActionConfig0 = ?config(action_config, Config), + ActionConfig = + emqx_utils_maps:deep_merge( + ActionConfig0, + #{<<"parameters">> => #{<<"query_mode">> => <<"sync">>}} + ), + ok = emqx_bridge_v2_kafka_producer_SUITE:t_templated_topic_and_no_pre_configured_topics( + [ + {type, ?ACTION_TYPE_BIN}, + {connector_name, ?config(connector_name, Config)}, + {connector_config, ?config(connector_config, Config)}, + {action_config, ActionConfig} + ] + ), + ok. diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl index 83bc33266..9a2fa91cf 100644 --- a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl +++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl @@ -295,6 +295,7 @@ fields("config_producer") -> fields("config_consumer") -> fields(kafka_consumer); fields(kafka_producer) -> + %% Schema used by bridges V1. connector_config_fields() ++ producer_opts(v1); fields(kafka_producer_action) -> [ @@ -306,6 +307,10 @@ fields(kafka_producer_action) -> {tags, emqx_schema:tags_schema()}, {description, emqx_schema:description_schema()} ] ++ producer_opts(action); +fields(pre_configured_topic) -> + [ + {topic, mk(string(), #{required => true, desc => ?DESC(kafka_topic)})} + ]; fields(kafka_consumer) -> connector_config_fields() ++ fields(consumer_opts); fields(ssl_client_opts) -> @@ -364,9 +369,41 @@ fields(socket_opts) -> validator => fun emqx_schema:validate_tcp_keepalive/1 })} ]; +fields(v1_producer_kafka_opts) -> + OldSchemaFields = + [ + topic, + message, + max_batch_bytes, + compression, + partition_strategy, + required_acks, + kafka_headers, + kafka_ext_headers, + kafka_header_value_encode_mode, + partition_count_refresh_interval, + partitions_limit, + max_inflight, + buffer, + query_mode, + sync_query_timeout + ], + Fields = fields(producer_kafka_opts), + lists:filter( + fun({K, _V}) -> lists:member(K, OldSchemaFields) end, + Fields + ); fields(producer_kafka_opts) -> [ {topic, mk(string(), #{required => true, desc => ?DESC(kafka_topic)})}, + {pre_configured_topics, + mk( + hoconsc:array(ref(pre_configured_topic)), + #{ + default => [], + desc => ?DESC("producer_pre_configured_topics") + } + )}, {message, mk(ref(kafka_message), #{required => false, desc => ?DESC(kafka_message)})}, {max_batch_bytes, mk(emqx_schema:bytesize(), #{default => <<"896KB">>, desc => ?DESC(max_batch_bytes)})}, @@ -675,15 +712,15 @@ resource_opts() -> %% However we need to keep it backward compatible for generated schema json (version 0.1.0) %% since schema is data for the 'schemas' API. parameters_field(ActionOrBridgeV1) -> - {Name, Alias} = + {Name, Alias, Ref} = case ActionOrBridgeV1 of v1 -> - {kafka, parameters}; + {kafka, parameters, v1_producer_kafka_opts}; action -> - {parameters, kafka} + {parameters, kafka, producer_kafka_opts} end, {Name, - mk(ref(producer_kafka_opts), #{ + mk(ref(Ref), #{ required => true, aliases => [Alias], desc => ?DESC(producer_kafka_opts), diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl index 6d88a329e..80de98402 100644 --- a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl +++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl @@ -122,8 +122,8 @@ on_add_channel( {ok, NewState}. create_producers_for_bridge_v2( - InstId, - BridgeV2Id, + ConnResId, + ActionResId, ClientId, #{ bridge_type := BridgeType, @@ -132,33 +132,57 @@ create_producers_for_bridge_v2( ) -> #{ message := MessageTemplate, - topic := KafkaTopic, + pre_configured_topics := PreConfiguredTopics0, + topic := KafkaTopic0, sync_query_timeout := SyncQueryTimeout } = KafkaConfig, + TopicTemplate = {TopicType, KafkaTopic} = maybe_preproc_topic(KafkaTopic0), + PreConfiguredTopics = [T || #{topic := T} <- PreConfiguredTopics0], + KafkaTopics0 = + case TopicType of + fixed -> + [KafkaTopic | PreConfiguredTopics]; + dynamic -> + PreConfiguredTopics + end, + case KafkaTopics0 of + [] -> + throw(<< + "Either the Kafka topic must be fixed (not a template)," + " or at least one pre-defined topic must be set." + >>); + _ -> + ok + end, + KafkaTopics = lists:map(fun bin/1, KafkaTopics0), KafkaHeadersTokens = preproc_kafka_headers(maps:get(kafka_headers, KafkaConfig, undefined)), KafkaExtHeadersTokens = preproc_ext_headers(maps:get(kafka_ext_headers, KafkaConfig, [])), KafkaHeadersValEncodeMode = maps:get(kafka_header_value_encode_mode, KafkaConfig, none), MaxPartitions = maps:get(partitions_limit, KafkaConfig, all_partitions), - #{name := BridgeName} = emqx_bridge_v2:parse_id(BridgeV2Id), - IsDryRun = emqx_resource:is_dry_run(BridgeV2Id), - ok = check_topic_and_leader_connections(ClientId, KafkaTopic, MaxPartitions), + #{name := BridgeName} = emqx_bridge_v2:parse_id(ActionResId), + IsDryRun = emqx_resource:is_dry_run(ActionResId), + [AKafkaTopic | _] = KafkaTopics, + ok = check_topic_and_leader_connections(ActionResId, ClientId, AKafkaTopic, MaxPartitions), WolffProducerConfig = producers_config( - BridgeType, BridgeName, KafkaConfig, IsDryRun, BridgeV2Id + BridgeType, BridgeName, KafkaConfig, IsDryRun, ActionResId ), - case wolff:ensure_supervised_producers(ClientId, KafkaTopic, WolffProducerConfig) of + case wolff:ensure_supervised_dynamic_producers(ClientId, WolffProducerConfig) of {ok, Producers} -> - ok = emqx_resource:allocate_resource(InstId, {?kafka_producers, BridgeV2Id}, Producers), ok = emqx_resource:allocate_resource( - InstId, {?kafka_telemetry_id, BridgeV2Id}, BridgeV2Id + ConnResId, {?kafka_producers, ActionResId}, Producers ), - _ = maybe_install_wolff_telemetry_handlers(BridgeV2Id), + ok = emqx_resource:allocate_resource( + ConnResId, {?kafka_telemetry_id, ActionResId}, ActionResId + ), + _ = maybe_install_wolff_telemetry_handlers(ActionResId), {ok, #{ message_template => compile_message_template(MessageTemplate), kafka_client_id => ClientId, - kafka_topic => KafkaTopic, + topic_template => TopicTemplate, + pre_configured_topics => KafkaTopics, producers => Producers, - resource_id => BridgeV2Id, - connector_resource_id => InstId, + resource_id => ActionResId, + connector_resource_id => ConnResId, sync_query_timeout => SyncQueryTimeout, kafka_config => KafkaConfig, headers_tokens => KafkaHeadersTokens, @@ -169,7 +193,7 @@ create_producers_for_bridge_v2( {error, Reason2} -> ?SLOG(error, #{ msg => "failed_to_start_kafka_producer", - instance_id => InstId, + instance_id => ConnResId, kafka_client_id => ClientId, kafka_topic => KafkaTopic, reason => Reason2 @@ -264,7 +288,9 @@ remove_producers_for_bridge_v2( ClientId = maps:get(?kafka_client_id, AllocatedResources, no_client_id), maps:foreach( fun - ({?kafka_producers, BridgeV2IdCheck}, Producers) when BridgeV2IdCheck =:= BridgeV2Id -> + ({?kafka_producers, BridgeV2IdCheck}, Producers) when + BridgeV2IdCheck =:= BridgeV2Id + -> deallocate_producers(ClientId, Producers); ({?kafka_telemetry_id, BridgeV2IdCheck}, TelemetryId) when BridgeV2IdCheck =:= BridgeV2Id @@ -297,8 +323,10 @@ on_query( #{installed_bridge_v2s := BridgeV2Configs} = _ConnectorState ) -> #{ - message_template := Template, + message_template := MessageTemplate, + topic_template := TopicTemplate, producers := Producers, + pre_configured_topics := PreConfiguredTopics, sync_query_timeout := SyncTimeout, headers_tokens := KafkaHeadersTokens, ext_headers_tokens := KafkaExtHeadersTokens, @@ -310,7 +338,14 @@ on_query( headers_val_encode_mode => KafkaHeadersValEncodeMode }, try - KafkaMessage = render_message(Template, KafkaHeaders, Message), + KafkaTopic = render_topic(TopicTemplate, Message), + case lists:member(KafkaTopic, PreConfiguredTopics) of + false -> + throw({unknown_topic, KafkaTopic}); + true -> + ok + end, + KafkaMessage = render_message(MessageTemplate, KafkaHeaders, Message), ?tp( emqx_bridge_kafka_impl_producer_sync_query, #{headers_config => KafkaHeaders, instance_id => InstId} @@ -318,9 +353,15 @@ on_query( emqx_trace:rendered_action_template(MessageTag, #{ message => KafkaMessage }), - do_send_msg(sync, KafkaMessage, Producers, SyncTimeout) + do_send_msg(sync, KafkaTopic, KafkaMessage, Producers, SyncTimeout) catch - error:{invalid_partition_count, Count, _Partitioner} -> + throw:bad_topic -> + ?tp("kafka_producer_failed_to_render_topic", #{}), + {error, {unrecoverable_error, failed_to_render_topic}}; + throw:{unknown_topic, Topic} -> + ?tp("kafka_producer_resolved_to_unknown_topic", #{}), + {error, {unrecoverable_error, {resolved_to_unknown_topic, Topic}}}; + throw:#{cause := invalid_partition_count, count := Count} -> ?tp("kafka_producer_invalid_partition_count", #{ action_id => MessageTag, query_mode => sync @@ -365,7 +406,9 @@ on_query_async( ) -> #{ message_template := Template, + topic_template := TopicTemplate, producers := Producers, + pre_configured_topics := PreConfiguredTopics, headers_tokens := KafkaHeadersTokens, ext_headers_tokens := KafkaExtHeadersTokens, headers_val_encode_mode := KafkaHeadersValEncodeMode @@ -376,6 +419,13 @@ on_query_async( headers_val_encode_mode => KafkaHeadersValEncodeMode }, try + KafkaTopic = render_topic(TopicTemplate, Message), + case lists:member(KafkaTopic, PreConfiguredTopics) of + false -> + throw({unknown_topic, KafkaTopic}); + true -> + ok + end, KafkaMessage = render_message(Template, KafkaHeaders, Message), ?tp( emqx_bridge_kafka_impl_producer_async_query, @@ -384,9 +434,15 @@ on_query_async( emqx_trace:rendered_action_template(MessageTag, #{ message => KafkaMessage }), - do_send_msg(async, KafkaMessage, Producers, AsyncReplyFn) + do_send_msg(async, KafkaTopic, KafkaMessage, Producers, AsyncReplyFn) catch - error:{invalid_partition_count, Count, _Partitioner} -> + throw:bad_topic -> + ?tp("kafka_producer_failed_to_render_topic", #{}), + {error, {unrecoverable_error, failed_to_render_topic}}; + throw:{unknown_topic, Topic} -> + ?tp("kafka_producer_resolved_to_unknown_topic", #{}), + {error, {unrecoverable_error, {resolved_to_unknown_topic, Topic}}}; + throw:#{cause := invalid_partition_count, count := Count} -> ?tp("kafka_producer_invalid_partition_count", #{ action_id => MessageTag, query_mode => async @@ -424,9 +480,28 @@ compile_message_template(T) -> timestamp => preproc_tmpl(TimestampTemplate) }. +maybe_preproc_topic(Topic) -> + Template = emqx_template:parse(Topic), + case emqx_template:placeholders(Template) of + [] -> + {fixed, bin(Topic)}; + [_ | _] -> + {dynamic, Template} + end. + preproc_tmpl(Tmpl) -> emqx_placeholder:preproc_tmpl(Tmpl). +render_topic({fixed, KafkaTopic}, _Message) -> + KafkaTopic; +render_topic({dynamic, Template}, Message) -> + try + iolist_to_binary(emqx_template:render_strict(Template, Message)) + catch + error:_Errors -> + throw(bad_topic) + end. + render_message( #{key := KeyTemplate, value := ValueTemplate, timestamp := TimestampTemplate}, #{ @@ -468,9 +543,11 @@ render_timestamp(Template, Message) -> erlang:system_time(millisecond) end. -do_send_msg(sync, KafkaMessage, Producers, SyncTimeout) -> +do_send_msg(sync, KafkaTopic, KafkaMessage, Producers, SyncTimeout) -> try - {_Partition, _Offset} = wolff:send_sync(Producers, [KafkaMessage], SyncTimeout), + {_Partition, _Offset} = wolff:send_sync2( + Producers, KafkaTopic, [KafkaMessage], SyncTimeout + ), ok catch error:{producer_down, _} = Reason -> @@ -478,7 +555,7 @@ do_send_msg(sync, KafkaMessage, Producers, SyncTimeout) -> error:timeout -> {error, timeout} end; -do_send_msg(async, KafkaMessage, Producers, AsyncReplyFn) -> +do_send_msg(async, KafkaTopic, KafkaMessage, Producers, AsyncReplyFn) -> %% * Must be a batch because wolff:send and wolff:send_sync are batch APIs %% * Must be a single element batch because wolff books calls, but not batch sizes %% for counters and gauges. @@ -486,7 +563,9 @@ do_send_msg(async, KafkaMessage, Producers, AsyncReplyFn) -> %% The retuned information is discarded here. %% If the producer process is down when sending, this function would %% raise an error exception which is to be caught by the caller of this callback - {_Partition, Pid} = wolff:send(Producers, Batch, {fun ?MODULE:on_kafka_ack/3, [AsyncReplyFn]}), + {_Partition, Pid} = wolff:send2( + Producers, KafkaTopic, Batch, {fun ?MODULE:on_kafka_ack/3, [AsyncReplyFn]} + ), %% this Pid is so far never used because Kafka producer is by-passing the buffer worker {ok, Pid}. @@ -527,20 +606,24 @@ on_get_status( end. on_get_channel_status( - _ResId, - ChannelId, + _ConnResId, + ActionResId, #{ client_id := ClientId, installed_bridge_v2s := Channels - } = _State + } = _ConnState ) -> %% Note: we must avoid returning `?status_disconnected' here. Returning %% `?status_disconnected' will make resource manager try to restart the producers / %% connector, thus potentially dropping data held in wolff producer's replayq. The %% only exception is if the topic does not exist ("unhealthy target"). - #{kafka_topic := KafkaTopic, partitions_limit := MaxPartitions} = maps:get(ChannelId, Channels), + #{ + pre_configured_topics := PreConfiguredTopics, + partitions_limit := MaxPartitions + } = maps:get(ActionResId, Channels), + [KafkaTopic | _] = PreConfiguredTopics, try - ok = check_topic_and_leader_connections(ClientId, KafkaTopic, MaxPartitions), + ok = check_topic_and_leader_connections(ActionResId, ClientId, KafkaTopic, MaxPartitions), ?status_connected catch throw:{unhealthy_target, Msg} -> @@ -549,11 +632,11 @@ on_get_channel_status( {?status_connecting, {K, E}} end. -check_topic_and_leader_connections(ClientId, KafkaTopic, MaxPartitions) -> +check_topic_and_leader_connections(ActionResId, ClientId, KafkaTopic, MaxPartitions) -> case wolff_client_sup:find_client(ClientId) of {ok, Pid} -> ok = check_topic_status(ClientId, Pid, KafkaTopic), - ok = check_if_healthy_leaders(ClientId, Pid, KafkaTopic, MaxPartitions); + ok = check_if_healthy_leaders(ActionResId, ClientId, Pid, KafkaTopic, MaxPartitions); {error, #{reason := no_such_client}} -> throw(#{ reason => cannot_find_kafka_client, @@ -591,8 +674,10 @@ error_summary(Map, [Error]) -> error_summary(Map, [Error | More]) -> Map#{first_error => Error, total_errors => length(More) + 1}. -check_if_healthy_leaders(ClientId, ClientPid, KafkaTopic, MaxPartitions) when is_pid(ClientPid) -> - case wolff_client:get_leader_connections(ClientPid, KafkaTopic, MaxPartitions) of +check_if_healthy_leaders(ActionResId, ClientId, ClientPid, KafkaTopic, MaxPartitions) when + is_pid(ClientPid) +-> + case wolff_client:get_leader_connections(ClientPid, ActionResId, KafkaTopic, MaxPartitions) of {ok, Leaders} -> %% Kafka is considered healthy as long as any of the partition leader is reachable. case lists:partition(fun({_Partition, Pid}) -> is_alive(Pid) end, Leaders) of @@ -654,7 +739,7 @@ ssl(#{enable := true} = SSL) -> ssl(_) -> false. -producers_config(BridgeType, BridgeName, Input, IsDryRun, BridgeV2Id) -> +producers_config(BridgeType, BridgeName, Input, IsDryRun, ActionResId) -> #{ max_batch_bytes := MaxBatchBytes, compression := Compression, @@ -696,8 +781,8 @@ producers_config(BridgeType, BridgeName, Input, IsDryRun, BridgeV2Id) -> max_batch_bytes => MaxBatchBytes, max_send_ahead => MaxInflight - 1, compression => Compression, - alias => BridgeV2Id, - telemetry_meta_data => #{bridge_id => BridgeV2Id}, + group => ActionResId, + telemetry_meta_data => #{bridge_id => ActionResId}, max_partitions => MaxPartitions }. @@ -773,20 +858,19 @@ handle_telemetry_event(_EventId, _Metrics, _MetaData, _HandlerConfig) -> %% Note: don't use the instance/manager ID, as that changes everytime %% the bridge is recreated, and will lead to multiplication of %% metrics. --spec telemetry_handler_id(resource_id()) -> binary(). -telemetry_handler_id(ResourceID) -> - <<"emqx-bridge-kafka-producer-", ResourceID/binary>>. +-spec telemetry_handler_id(action_resource_id()) -> binary(). +telemetry_handler_id(ActionResId) -> + <<"emqx-bridge-kafka-producer-", ActionResId/binary>>. -uninstall_telemetry_handlers(ResourceID) -> - HandlerID = telemetry_handler_id(ResourceID), - telemetry:detach(HandlerID). +uninstall_telemetry_handlers(TelemetryId) -> + telemetry:detach(TelemetryId). -maybe_install_wolff_telemetry_handlers(ResourceID) -> +maybe_install_wolff_telemetry_handlers(TelemetryId) -> %% Attach event handlers for Kafka telemetry events. If a handler with the %% handler id already exists, the attach_many function does nothing telemetry:attach_many( %% unique handler id - telemetry_handler_id(ResourceID), + telemetry_handler_id(TelemetryId), [ [wolff, dropped_queue_full], [wolff, queuing], @@ -798,7 +882,7 @@ maybe_install_wolff_telemetry_handlers(ResourceID) -> %% wolff producers; otherwise, multiple kafka producer bridges %% will install multiple handlers to the same wolff events, %% multiplying the metric counts... - #{bridge_id => ResourceID} + #{bridge_id => TelemetryId} ). preproc_kafka_headers(HeadersTmpl) when HeadersTmpl =:= <<>>; HeadersTmpl =:= undefined -> diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_producer_action_info.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_producer_action_info.erl index d97e68ba6..b9e13e717 100644 --- a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_producer_action_info.erl +++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_producer_action_info.erl @@ -26,7 +26,12 @@ schema_module() -> emqx_bridge_kafka. connector_action_config_to_bridge_v1_config(ConnectorConfig, ActionConfig) -> BridgeV1Config1 = maps:remove(<<"connector">>, ActionConfig), BridgeV1Config2 = emqx_utils_maps:deep_merge(ConnectorConfig, BridgeV1Config1), - emqx_utils_maps:rename(<<"parameters">>, <<"kafka">>, BridgeV1Config2). + BridgeV1Config = emqx_utils_maps:rename(<<"parameters">>, <<"kafka">>, BridgeV1Config2), + maps:update_with( + <<"kafka">>, + fun(Params) -> maps:with(v1_parameters(), Params) end, + BridgeV1Config + ). bridge_v1_config_to_action_config(BridgeV1Conf0 = #{<<"producer">> := _}, ConnectorName) -> %% Ancient v1 config, when `kafka' key was wrapped by `producer' @@ -51,6 +56,12 @@ bridge_v1_config_to_action_config(BridgeV1Conf, ConnectorName) -> %% Internal helper functions %%------------------------------------------------------------------------------------------ +v1_parameters() -> + [ + to_bin(K) + || {K, _} <- emqx_bridge_kafka:fields(v1_producer_kafka_opts) + ]. + producer_action_field_keys() -> [ to_bin(K) diff --git a/apps/emqx_bridge_kafka/test/emqx_bridge_v2_kafka_producer_SUITE.erl b/apps/emqx_bridge_kafka/test/emqx_bridge_v2_kafka_producer_SUITE.erl index c26f5e94e..6246faaf1 100644 --- a/apps/emqx_bridge_kafka/test/emqx_bridge_v2_kafka_producer_SUITE.erl +++ b/apps/emqx_bridge_kafka/test/emqx_bridge_v2_kafka_producer_SUITE.erl @@ -23,6 +23,7 @@ -include_lib("snabbkaffe/include/snabbkaffe.hrl"). -include_lib("brod/include/brod.hrl"). -include_lib("emqx_resource/include/emqx_resource.hrl"). +-include_lib("emqx/include/asserts.hrl"). -import(emqx_common_test_helpers, [on_exit/1]). @@ -165,6 +166,9 @@ send_message(Type, ActionName) -> resolve_kafka_offset() -> KafkaTopic = emqx_bridge_kafka_impl_producer_SUITE:test_topic_one_partition(), + resolve_kafka_offset(KafkaTopic). + +resolve_kafka_offset(KafkaTopic) -> Partition = 0, Hosts = emqx_bridge_kafka_impl_producer_SUITE:kafka_hosts(), {ok, Offset0} = emqx_bridge_kafka_impl_producer_SUITE:resolve_kafka_offset( @@ -174,11 +178,32 @@ resolve_kafka_offset() -> check_kafka_message_payload(Offset, ExpectedPayload) -> KafkaTopic = emqx_bridge_kafka_impl_producer_SUITE:test_topic_one_partition(), + check_kafka_message_payload(KafkaTopic, Offset, ExpectedPayload). + +check_kafka_message_payload(KafkaTopic, Offset, ExpectedPayload) -> Partition = 0, Hosts = emqx_bridge_kafka_impl_producer_SUITE:kafka_hosts(), {ok, {_, [KafkaMsg0]}} = brod:fetch(Hosts, KafkaTopic, Partition, Offset), ?assertMatch(#kafka_message{value = ExpectedPayload}, KafkaMsg0). +ensure_kafka_topic(KafkaTopic) -> + TopicConfigs = [ + #{ + name => KafkaTopic, + num_partitions => 1, + replication_factor => 1, + assignments => [], + configs => [] + } + ], + RequestConfig = #{timeout => 5_000}, + ConnConfig = #{}, + Endpoints = emqx_bridge_kafka_impl_producer_SUITE:kafka_hosts(), + case brod:create_topics(Endpoints, TopicConfigs, RequestConfig, ConnConfig) of + ok -> ok; + {error, topic_already_exists} -> ok + end. + action_config(ConnectorName) -> action_config(ConnectorName, _Overrides = #{}). @@ -728,9 +753,13 @@ t_invalid_partition_count_metrics(Config) -> %% Simulate `invalid_partition_count' emqx_common_test_helpers:with_mock( wolff, - send_sync, - fun(_Producers, _Msgs, _Timeout) -> - error({invalid_partition_count, 0, partitioner}) + send_sync2, + fun(_Producers, _Topic, _Msgs, _Timeout) -> + throw(#{ + cause => invalid_partition_count, + count => 0, + partitioner => partitioner + }) end, fun() -> {{ok, _}, {ok, _}} = @@ -773,9 +802,13 @@ t_invalid_partition_count_metrics(Config) -> %% Simulate `invalid_partition_count' emqx_common_test_helpers:with_mock( wolff, - send, - fun(_Producers, _Msgs, _Timeout) -> - error({invalid_partition_count, 0, partitioner}) + send2, + fun(_Producers, _Topic, _Msgs, _AckCallback) -> + throw(#{ + cause => invalid_partition_count, + count => 0, + partitioner => partitioner + }) end, fun() -> {{ok, _}, {ok, _}} = @@ -881,3 +914,196 @@ t_multiple_actions_sharing_topic(Config) -> end ), ok. + +%% Smoke tests for using a templated topic and a list of pre-configured kafka topics. +t_pre_configured_topics(Config) -> + Type = proplists:get_value(type, Config, ?TYPE), + ConnectorName = proplists:get_value(connector_name, Config, <<"c">>), + ConnectorConfig = proplists:get_value(connector_config, Config, connector_config()), + ActionName = <<"pre_configured_topics">>, + ActionConfig1 = proplists:get_value(action_config, Config, action_config(ConnectorName)), + PreConfigureTopic1 = <<"pct1">>, + PreConfigureTopic2 = <<"pct2">>, + ensure_kafka_topic(PreConfigureTopic1), + ensure_kafka_topic(PreConfigureTopic2), + ActionConfig = emqx_bridge_v2_testlib:parse_and_check( + action, + Type, + ActionName, + emqx_utils_maps:deep_merge( + ActionConfig1, + #{ + <<"parameters">> => #{ + <<"topic">> => <<"pct${.payload.n}">>, + <<"message">> => #{ + <<"key">> => <<"${.clientid}">>, + <<"value">> => <<"${.payload.p}">> + }, + <<"pre_configured_topics">> => [ + #{<<"topic">> => PreConfigureTopic1}, + #{<<"topic">> => PreConfigureTopic2} + ] + } + } + ) + ), + ?check_trace( + #{timetrap => 7_000}, + begin + ConnectorParams = [ + {connector_config, ConnectorConfig}, + {connector_name, ConnectorName}, + {connector_type, Type} + ], + ActionParams = [ + {action_config, ActionConfig}, + {action_name, ActionName}, + {action_type, Type} + ], + {ok, {{_, 201, _}, _, #{}}} = + emqx_bridge_v2_testlib:create_connector_api(ConnectorParams), + + {ok, {{_, 201, _}, _, #{}}} = + emqx_bridge_v2_testlib:create_action_api(ActionParams), + RuleTopic = <<"pct">>, + {ok, _} = emqx_bridge_v2_testlib:create_rule_and_action_http( + Type, + RuleTopic, + [ + {bridge_name, ActionName} + ], + #{ + sql => + <<"select *, json_decode(payload) as payload from \"", RuleTopic/binary, + "\" ">> + } + ), + ?assertStatusAPI(Type, ActionName, <<"connected">>), + + HandlerId = ?FUNCTION_NAME, + TestPid = self(), + telemetry:attach_many( + HandlerId, + emqx_resource_metrics:events(), + fun(EventName, Measurements, Metadata, _Config) -> + Data = #{ + name => EventName, + measurements => Measurements, + metadata => Metadata + }, + TestPid ! {telemetry, Data}, + ok + end, + unused_config + ), + on_exit(fun() -> telemetry:detach(HandlerId) end), + + {ok, C} = emqtt:start_link(#{}), + {ok, _} = emqtt:connect(C), + Payload = fun(Map) -> emqx_utils_json:encode(Map) end, + Offset1 = resolve_kafka_offset(PreConfigureTopic1), + Offset2 = resolve_kafka_offset(PreConfigureTopic2), + {ok, _} = emqtt:publish(C, RuleTopic, Payload(#{n => 1, p => <<"p1">>}), [{qos, 1}]), + {ok, _} = emqtt:publish(C, RuleTopic, Payload(#{n => 2, p => <<"p2">>}), [{qos, 1}]), + + check_kafka_message_payload(PreConfigureTopic1, Offset1, <<"p1">>), + check_kafka_message_payload(PreConfigureTopic2, Offset2, <<"p2">>), + + ActionId = emqx_bridge_v2:id(Type, ActionName), + ?assertEqual(2, emqx_resource_metrics:matched_get(ActionId)), + ?assertEqual(2, emqx_resource_metrics:success_get(ActionId)), + ?assertEqual(0, emqx_resource_metrics:queuing_get(ActionId)), + + ?assertReceive( + {telemetry, #{ + measurements := #{gauge_set := _}, + metadata := #{worker_id := _, resource_id := ActionId} + }} + ), + + %% If there isn't enough information in the context to resolve to a topic, it + %% should be an unrecoverable error. + ?assertMatch( + {_, {ok, _}}, + ?wait_async_action( + emqtt:publish(C, RuleTopic, Payload(#{not_enough => <<"info">>}), [{qos, 1}]), + #{?snk_kind := "kafka_producer_failed_to_render_topic"} + ) + ), + + %% If it's possible to render the topic, but it isn't in the pre-configured + %% list, it should be an unrecoverable error. + ?assertMatch( + {_, {ok, _}}, + ?wait_async_action( + emqtt:publish(C, RuleTopic, Payload(#{n => 99}), [{qos, 1}]), + #{?snk_kind := "kafka_producer_resolved_to_unknown_topic"} + ) + ), + + ok + end, + [] + ), + ok. + +%% Checks that creating an action with templated topic and no pre-configured kafka topics +%% throws. +t_templated_topic_and_no_pre_configured_topics(Config) -> + Type = proplists:get_value(type, Config, ?TYPE), + ConnectorName = proplists:get_value(connector_name, Config, <<"c">>), + ConnectorConfig = proplists:get_value(connector_config, Config, connector_config()), + ActionName = <<"bad_pre_configured_topics">>, + ActionConfig1 = proplists:get_value(action_config, Config, action_config(ConnectorName)), + ActionConfig = emqx_bridge_v2_testlib:parse_and_check( + action, + Type, + ActionName, + emqx_utils_maps:deep_merge( + ActionConfig1, + #{ + <<"parameters">> => #{ + <<"topic">> => <<"pct${.payload.n}">>, + <<"pre_configured_topics">> => [] + } + } + ) + ), + ?check_trace( + #{timetrap => 7_000}, + begin + ConnectorParams = [ + {connector_config, ConnectorConfig}, + {connector_name, ConnectorName}, + {connector_type, Type} + ], + ActionParams = [ + {action_config, ActionConfig}, + {action_name, ActionName}, + {action_type, Type} + ], + {ok, {{_, 201, _}, _, #{}}} = + emqx_bridge_v2_testlib:create_connector_api(ConnectorParams), + + {ok, {{_, 201, _}, _, #{}}} = + emqx_bridge_v2_testlib:create_action_api(ActionParams), + + ?assertMatch( + {ok, + {{_, 200, _}, _, #{ + <<"status_reason">> := + << + "Either the Kafka topic must be fixed (not a template)," + " or at least one pre-defined topic must be set." + >>, + <<"status">> := <<"disconnected">>, + <<"node_status">> := [#{<<"status">> := <<"disconnected">>}] + }}}, + emqx_bridge_v2_testlib:get_bridge_api(Type, ActionName) + ), + + ok + end, + [] + ), + ok. diff --git a/changes/ee/feat-13452.en.md b/changes/ee/feat-13452.en.md new file mode 100644 index 000000000..95dae8d32 --- /dev/null +++ b/changes/ee/feat-13452.en.md @@ -0,0 +1 @@ +Added to possibility to configure a list of predefined Kafka topics to Kafka producer actions, and also to use templates to define the destination Kafka topic. diff --git a/rel/i18n/emqx_bridge_azure_event_hub.hocon b/rel/i18n/emqx_bridge_azure_event_hub.hocon index 3b96e23e6..e683bc9e9 100644 --- a/rel/i18n/emqx_bridge_azure_event_hub.hocon +++ b/rel/i18n/emqx_bridge_azure_event_hub.hocon @@ -69,7 +69,7 @@ producer_kafka_opts.label: """Azure Event Hubs Producer""" kafka_topic.desc: -"""Event Hubs name""" +"""Event Hubs name. Supports templates (e.g.: `t-${payload.t}`).""" kafka_topic.label: """Event Hubs Name""" @@ -350,4 +350,14 @@ Setting this to a value which is greater than the total number of partitions in partitions_limit.label: """Max Partitions""" +producer_pre_configured_topics.label: +"""Pre-configured Event Hubs""" +producer_pre_configured_topics.desc: +"""A list of pre-configured event hubs to be used when using templates to define outgoing topics. If the topic template fails to resolve to a value due to missing data in the incoming message, or if it resolves to a topic that is not contained in this list, then publishing will fail.""" + +pre_configured_topic.label: +"""Event Hubs Name""" +pre_configured_topic.desc: +"""Event Hubs name""" + } diff --git a/rel/i18n/emqx_bridge_confluent_producer.hocon b/rel/i18n/emqx_bridge_confluent_producer.hocon index 748373691..81c2c0a89 100644 --- a/rel/i18n/emqx_bridge_confluent_producer.hocon +++ b/rel/i18n/emqx_bridge_confluent_producer.hocon @@ -69,10 +69,10 @@ producer_kafka_opts.label: """Confluent Producer""" kafka_topic.desc: -"""Event Hub name""" +"""Kafka topic name. Supports templates (e.g.: `t-${payload.t}`).""" kafka_topic.label: -"""Event Hub Name""" +"""Kafka Topic Name""" kafka_message_timestamp.desc: """Which timestamp to use. The timestamp is expected to be a millisecond precision Unix epoch which can be in string format, e.g. 1661326462115 or '1661326462115'. When the desired data field for this template is not found, or if the found data is not a valid integer, the current system timestamp will be used.""" @@ -350,4 +350,14 @@ server_name_indication.desc: server_name_indication.label: """SNI""" +producer_pre_configured_topics.label: +"""Pre-configured Topics""" +producer_pre_configured_topics.desc: +"""A list of pre-configured topics to be used when using templates to define outgoing topics. If the topic template fails to resolve to a value due to missing data in the incoming message, or if it resolves to a topic that is not contained in this list, then publishing will fail.""" + +pre_configured_topic.label: +"""Kafka Topic Name""" +pre_configured_topic.desc: +"""Kafka topic name""" + } diff --git a/rel/i18n/emqx_bridge_kafka.hocon b/rel/i18n/emqx_bridge_kafka.hocon index 6e0074ddd..59896cc22 100644 --- a/rel/i18n/emqx_bridge_kafka.hocon +++ b/rel/i18n/emqx_bridge_kafka.hocon @@ -81,7 +81,7 @@ producer_kafka_opts.label: """Kafka Producer""" kafka_topic.desc: -"""Kafka topic name""" +"""Kafka topic name. Supports templates (e.g.: `t-${payload.t}`).""" kafka_topic.label: """Kafka Topic Name""" @@ -446,5 +446,14 @@ server_name_indication.desc: server_name_indication.label: """SNI""" +producer_pre_configured_topics.label: +"""Pre-configured Topics""" +producer_pre_configured_topics.desc: +"""A list of pre-configured topics to be used when using templates to define outgoing topics. If the topic template fails to resolve to a value due to missing data in the incoming message, or if it resolves to a topic that is not contained in this list, then publishing will fail.""" + +pre_configured_topic.label: +"""Kafka Topic Name""" +pre_configured_topic.desc: +"""Kafka topic name""" } From 7bf70aaab6feea69424c9df8fa40fc4fc276de02 Mon Sep 17 00:00:00 2001 From: firest Date: Thu, 25 Jul 2024 17:00:20 +0800 Subject: [PATCH 052/131] feat(scram): supports ACL rules in `scram_restapi` backend --- .../emqx_auth_http/include/emqx_auth_http.hrl | 2 + apps/emqx_auth_http/src/emqx_authn_http.erl | 67 +++--- .../src/emqx_authn_scram_restapi.erl | 54 ++--- .../test/emqx_authn_scram_restapi_SUITE.erl | 191 ++++++++++++------ .../src/emqx_authn_scram_mnesia.erl | 4 +- apps/emqx_utils/src/emqx_utils_scram.erl | 12 +- 6 files changed, 190 insertions(+), 140 deletions(-) diff --git a/apps/emqx_auth_http/include/emqx_auth_http.hrl b/apps/emqx_auth_http/include/emqx_auth_http.hrl index c0bfa2177..439087e9c 100644 --- a/apps/emqx_auth_http/include/emqx_auth_http.hrl +++ b/apps/emqx_auth_http/include/emqx_auth_http.hrl @@ -31,4 +31,6 @@ -define(AUTHN_TYPE, {?AUTHN_MECHANISM, ?AUTHN_BACKEND}). -define(AUTHN_TYPE_SCRAM, {?AUTHN_MECHANISM_SCRAM, ?AUTHN_BACKEND}). +-define(AUTHN_DATA_FIELDS, [is_superuser, client_attrs, expire_at, acl]). + -endif. diff --git a/apps/emqx_auth_http/src/emqx_authn_http.erl b/apps/emqx_auth_http/src/emqx_authn_http.erl index b294de24f..edaa0ee5a 100644 --- a/apps/emqx_auth_http/src/emqx_authn_http.erl +++ b/apps/emqx_auth_http/src/emqx_authn_http.erl @@ -32,7 +32,9 @@ with_validated_config/2, generate_request/2, request_for_log/2, - response_for_log/1 + response_for_log/1, + extract_auth_data/2, + safely_parse_body/2 ]). %%------------------------------------------------------------------------------ @@ -209,34 +211,14 @@ handle_response(Headers, Body) -> case safely_parse_body(ContentType, Body) of {ok, NBody} -> body_to_auth_data(NBody); - {error, Reason} -> - ?TRACE_AUTHN_PROVIDER( - error, - "parse_http_response_failed", - #{content_type => ContentType, body => Body, reason => Reason} - ), + {error, _Reason} -> ignore end. body_to_auth_data(Body) -> case maps:get(<<"result">>, Body, <<"ignore">>) of <<"allow">> -> - IsSuperuser = emqx_authn_utils:is_superuser(Body), - Attrs = emqx_authn_utils:client_attrs(Body), - try - ExpireAt = expire_at(Body), - ACL = acl(ExpireAt, Body), - Result = merge_maps([ExpireAt, IsSuperuser, ACL, Attrs]), - {ok, Result} - catch - throw:{bad_acl_rule, Reason} -> - %% it's a invalid token, so ok to log - ?TRACE_AUTHN_PROVIDER("bad_acl_rule", Reason#{http_body => Body}), - {error, bad_username_or_password}; - throw:Reason -> - ?TRACE_AUTHN_PROVIDER("bad_response_body", Reason#{http_body => Body}), - {error, bad_username_or_password} - end; + extract_auth_data(http, Body); <<"deny">> -> {error, not_authorized}; <<"ignore">> -> @@ -245,6 +227,24 @@ body_to_auth_data(Body) -> ignore end. +extract_auth_data(Source, Body) -> + IsSuperuser = emqx_authn_utils:is_superuser(Body), + Attrs = emqx_authn_utils:client_attrs(Body), + try + ExpireAt = expire_at(Body), + ACL = acl(ExpireAt, Source, Body), + Result = merge_maps([ExpireAt, IsSuperuser, ACL, Attrs]), + {ok, Result} + catch + throw:{bad_acl_rule, Reason} -> + %% it's a invalid token, so ok to log + ?TRACE_AUTHN_PROVIDER("bad_acl_rule", Reason#{http_body => Body}), + {error, bad_username_or_password}; + throw:Reason -> + ?TRACE_AUTHN_PROVIDER("bad_response_body", Reason#{http_body => Body}), + {error, bad_username_or_password} + end. + merge_maps([]) -> #{}; merge_maps([Map | Maps]) -> maps:merge(Map, merge_maps(Maps)). @@ -283,40 +283,43 @@ expire_sec(#{<<"expire_at">> := _}) -> expire_sec(_) -> undefined. -acl(#{expire_at := ExpireTimeMs}, #{<<"acl">> := Rules}) -> +acl(#{expire_at := ExpireTimeMs}, Source, #{<<"acl">> := Rules}) -> #{ acl => #{ - source_for_logging => http, + source_for_logging => Source, rules => emqx_authz_rule_raw:parse_and_compile_rules(Rules), %% It's seconds level precision (like JWT) for authz %% see emqx_authz_client_info:check/1 expire => erlang:convert_time_unit(ExpireTimeMs, millisecond, second) } }; -acl(_NoExpire, #{<<"acl">> := Rules}) -> +acl(_NoExpire, Source, #{<<"acl">> := Rules}) -> #{ acl => #{ - source_for_logging => http, + source_for_logging => Source, rules => emqx_authz_rule_raw:parse_and_compile_rules(Rules) } }; -acl(_, _) -> +acl(_, _, _) -> #{}. safely_parse_body(ContentType, Body) -> try parse_body(ContentType, Body) catch - _Class:_Reason -> + _Class:Reason -> + ?TRACE_AUTHN_PROVIDER( + error, + "parse_http_response_failed", + #{content_type => ContentType, body => Body, reason => Reason} + ), {error, invalid_body} end. parse_body(<<"application/json", _/binary>>, Body) -> {ok, emqx_utils_json:decode(Body, [return_maps])}; parse_body(<<"application/x-www-form-urlencoded", _/binary>>, Body) -> - Flags = [<<"result">>, <<"is_superuser">>], - RawMap = maps:from_list(cow_qs:parse_qs(Body)), - NBody = maps:with(Flags, RawMap), + NBody = maps:from_list(cow_qs:parse_qs(Body)), {ok, NBody}; parse_body(ContentType, _) -> {error, {unsupported_content_type, ContentType}}. diff --git a/apps/emqx_auth_http/src/emqx_authn_scram_restapi.erl b/apps/emqx_auth_http/src/emqx_authn_scram_restapi.erl index f1cca5da2..abb91f130 100644 --- a/apps/emqx_auth_http/src/emqx_authn_scram_restapi.erl +++ b/apps/emqx_auth_http/src/emqx_authn_scram_restapi.erl @@ -10,8 +10,11 @@ -module(emqx_authn_scram_restapi). --include_lib("emqx_auth/include/emqx_authn.hrl"). +-feature(maybe_expr, enable). + +-include("emqx_auth_http.hrl"). -include_lib("emqx/include/logger.hrl"). +-include_lib("emqx_auth/include/emqx_authn.hrl"). -behaviour(emqx_authn_provider). @@ -28,10 +31,6 @@ <<"salt">> ]). --define(OPTIONAL_USER_INFO_KEYS, [ - <<"is_superuser">> -]). - %%------------------------------------------------------------------------------ %% APIs %%------------------------------------------------------------------------------ @@ -78,7 +77,9 @@ authenticate( reason => Reason }) end, - emqx_utils_scram:authenticate(AuthMethod, AuthData, AuthCache, RetrieveFun, OnErrFun, State); + emqx_utils_scram:authenticate( + AuthMethod, AuthData, AuthCache, State, RetrieveFun, OnErrFun, ?AUTHN_DATA_FIELDS + ); authenticate(_Credential, _State) -> ignore. @@ -119,16 +120,11 @@ retrieve( handle_response(Headers, Body) -> ContentType = proplists:get_value(<<"content-type">>, Headers), - case safely_parse_body(ContentType, Body) of - {ok, NBody} -> - body_to_user_info(NBody); - {error, Reason} = Error -> - ?TRACE_AUTHN_PROVIDER( - error, - "parse_scram_restapi_response_failed", - #{content_type => ContentType, body => Body, reason => Reason} - ), - Error + maybe + {ok, NBody} ?= emqx_authn_http:safely_parse_body(ContentType, Body), + {ok, UserInfo} ?= body_to_user_info(NBody), + {ok, AuthData} ?= emqx_authn_http:extract_auth_data(scram_restapi, NBody), + {ok, maps:merge(AuthData, UserInfo)} end. body_to_user_info(Body) -> @@ -137,26 +133,16 @@ body_to_user_info(Body) -> true -> case safely_convert_hex(Required0) of {ok, Required} -> - UserInfo0 = maps:merge(Required, maps:with(?OPTIONAL_USER_INFO_KEYS, Body)), - UserInfo1 = emqx_utils_maps:safe_atom_key_map(UserInfo0), - UserInfo = maps:merge(#{is_superuser => false}, UserInfo1), - {ok, UserInfo}; + {ok, emqx_utils_maps:safe_atom_key_map(Required)}; Error -> + ?TRACE_AUTHN_PROVIDER("decode_keys_failed", #{http_body => Body}), Error end; _ -> - ?TRACE_AUTHN_PROVIDER("bad_response_body", #{http_body => Body}), + ?TRACE_AUTHN_PROVIDER("missing_requried_keys", #{http_body => Body}), {error, bad_response} end. -safely_parse_body(ContentType, Body) -> - try - parse_body(ContentType, Body) - catch - _Class:_Reason -> - {error, invalid_body} - end. - safely_convert_hex(Required) -> try {ok, @@ -171,15 +157,5 @@ safely_convert_hex(Required) -> {error, Reason} end. -parse_body(<<"application/json", _/binary>>, Body) -> - {ok, emqx_utils_json:decode(Body, [return_maps])}; -parse_body(<<"application/x-www-form-urlencoded", _/binary>>, Body) -> - Flags = ?REQUIRED_USER_INFO_KEYS ++ ?OPTIONAL_USER_INFO_KEYS, - RawMap = maps:from_list(cow_qs:parse_qs(Body)), - NBody = maps:with(Flags, RawMap), - {ok, NBody}; -parse_body(ContentType, _) -> - {error, {unsupported_content_type, ContentType}}. - merge_scram_conf(Conf, State) -> maps:merge(maps:with([algorithm, iteration_count], Conf), State). diff --git a/apps/emqx_auth_http/test/emqx_authn_scram_restapi_SUITE.erl b/apps/emqx_auth_http/test/emqx_authn_scram_restapi_SUITE.erl index 8cd83f973..7963cf1e3 100644 --- a/apps/emqx_auth_http/test/emqx_authn_scram_restapi_SUITE.erl +++ b/apps/emqx_auth_http/test/emqx_authn_scram_restapi_SUITE.erl @@ -21,6 +21,9 @@ -define(ALGORITHM_STR, <<"sha512">>). -define(ITERATION_COUNT, 4096). +-define(T_ACL_USERNAME, <<"username">>). +-define(T_ACL_PASSWORD, <<"password">>). + -include_lib("emqx/include/emqx_placeholder.hrl"). all() -> @@ -120,59 +123,8 @@ t_authenticate(_Config) -> ok = emqx_config:put([mqtt, idle_timeout], 500), - {ok, Pid} = emqx_authn_mqtt_test_client:start_link("127.0.0.1", 1883), - - ClientFirstMessage = esasl_scram:client_first_message(Username), - - ConnectPacket = ?CONNECT_PACKET( - #mqtt_packet_connect{ - proto_ver = ?MQTT_PROTO_V5, - properties = #{ - 'Authentication-Method' => <<"SCRAM-SHA-512">>, - 'Authentication-Data' => ClientFirstMessage - } - } - ), - - ok = emqx_authn_mqtt_test_client:send(Pid, ConnectPacket), - - %% Intentional sleep to trigger idle timeout for the connection not yet authenticated - ok = ct:sleep(1000), - - ?AUTH_PACKET( - ?RC_CONTINUE_AUTHENTICATION, - #{'Authentication-Data' := ServerFirstMessage} - ) = receive_packet(), - - {continue, ClientFinalMessage, ClientCache} = - esasl_scram:check_server_first_message( - ServerFirstMessage, - #{ - client_first_message => ClientFirstMessage, - password => Password, - algorithm => ?ALGORITHM - } - ), - - AuthContinuePacket = ?AUTH_PACKET( - ?RC_CONTINUE_AUTHENTICATION, - #{ - 'Authentication-Method' => <<"SCRAM-SHA-512">>, - 'Authentication-Data' => ClientFinalMessage - } - ), - - ok = emqx_authn_mqtt_test_client:send(Pid, AuthContinuePacket), - - ?CONNACK_PACKET( - ?RC_SUCCESS, - _, - #{'Authentication-Data' := ServerFinalMessage} - ) = receive_packet(), - - ok = esasl_scram:check_server_final_message( - ServerFinalMessage, ClientCache#{algorithm => ?ALGORITHM} - ). + {ok, Pid} = create_connection(Username, Password), + emqx_authn_mqtt_test_client:stop(Pid). t_authenticate_bad_props(_Config) -> Username = <<"u">>, @@ -316,6 +268,47 @@ t_destroy(_Config) -> _ ) = receive_packet(). +t_acl(_Config) -> + init_auth(), + + ACL = emqx_authn_http_SUITE:acl_rules(), + set_user_handler(?T_ACL_USERNAME, ?T_ACL_PASSWORD, #{acl => ACL}), + {ok, Pid} = create_connection(?T_ACL_USERNAME, ?T_ACL_PASSWORD), + + Cases = [ + {allow, <<"http-authn-acl/#">>}, + {deny, <<"http-authn-acl/1">>}, + {deny, <<"t/#">>} + ], + + try + lists:foreach( + fun(Case) -> + test_acl(Case, Pid) + end, + Cases + ) + after + ok = emqx_authn_mqtt_test_client:stop(Pid) + end. + +t_auth_expire(_Config) -> + init_auth(), + + ExpireSec = 3, + WaitTime = timer:seconds(ExpireSec + 1), + ACL = emqx_authn_http_SUITE:acl_rules(), + + set_user_handler(?T_ACL_USERNAME, ?T_ACL_PASSWORD, #{ + acl => ACL, + expire_at => + erlang:system_time(second) + ExpireSec + }), + {ok, Pid} = create_connection(?T_ACL_USERNAME, ?T_ACL_PASSWORD), + + timer:sleep(WaitTime), + ?assertEqual(false, erlang:is_process_alive(Pid)). + t_is_superuser() -> State = init_auth(), ok = test_is_superuser(State, false), @@ -326,7 +319,7 @@ test_is_superuser(State, ExpectedIsSuperuser) -> Username = <<"u">>, Password = <<"p">>, - set_user_handler(Username, Password, ExpectedIsSuperuser), + set_user_handler(Username, Password, #{is_superuser => ExpectedIsSuperuser}), ClientFirstMessage = esasl_scram:client_first_message(Username), @@ -384,19 +377,20 @@ raw_config() -> }. set_user_handler(Username, Password) -> - set_user_handler(Username, Password, false). -set_user_handler(Username, Password, IsSuperuser) -> + set_user_handler(Username, Password, #{is_superuser => false}). +set_user_handler(Username, Password, Extra0) -> %% HTTP Server Handler = fun(Req0, State) -> #{ username := Username } = cowboy_req:match_qs([username], Req0), - UserInfo = make_user_info(Password, ?ALGORITHM, ?ITERATION_COUNT, IsSuperuser), + UserInfo = make_user_info(Password, ?ALGORITHM, ?ITERATION_COUNT), + Extra = maps:merge(#{is_superuser => false}, Extra0), Req = cowboy_req:reply( 200, #{<<"content-type">> => <<"application/json">>}, - emqx_utils_json:encode(UserInfo), + emqx_utils_json:encode(maps:merge(Extra, UserInfo)), Req0 ), {ok, Req, State} @@ -415,7 +409,7 @@ init_auth(Config) -> {ok, [#{state := State}]} = emqx_authn_chains:list_authenticators(?GLOBAL), State. -make_user_info(Password, Algorithm, IterationCount, IsSuperuser) -> +make_user_info(Password, Algorithm, IterationCount) -> {StoredKey, ServerKey, Salt} = esasl_scram:generate_authentication_info( Password, #{ @@ -426,8 +420,7 @@ make_user_info(Password, Algorithm, IterationCount, IsSuperuser) -> #{ stored_key => binary:encode_hex(StoredKey), server_key => binary:encode_hex(ServerKey), - salt => binary:encode_hex(Salt), - is_superuser => IsSuperuser + salt => binary:encode_hex(Salt) }. receive_packet() -> @@ -438,3 +431,79 @@ receive_packet() -> after 1000 -> ct:fail("Deliver timeout") end. + +create_connection(Username, Password) -> + {ok, Pid} = emqx_authn_mqtt_test_client:start_link("127.0.0.1", 1883), + + ClientFirstMessage = esasl_scram:client_first_message(Username), + + ConnectPacket = ?CONNECT_PACKET( + #mqtt_packet_connect{ + proto_ver = ?MQTT_PROTO_V5, + properties = #{ + 'Authentication-Method' => <<"SCRAM-SHA-512">>, + 'Authentication-Data' => ClientFirstMessage + } + } + ), + + ok = emqx_authn_mqtt_test_client:send(Pid, ConnectPacket), + + %% Intentional sleep to trigger idle timeout for the connection not yet authenticated + ok = ct:sleep(1000), + + ?AUTH_PACKET( + ?RC_CONTINUE_AUTHENTICATION, + #{'Authentication-Data' := ServerFirstMessage} + ) = receive_packet(), + + {continue, ClientFinalMessage, ClientCache} = + esasl_scram:check_server_first_message( + ServerFirstMessage, + #{ + client_first_message => ClientFirstMessage, + password => Password, + algorithm => ?ALGORITHM + } + ), + + AuthContinuePacket = ?AUTH_PACKET( + ?RC_CONTINUE_AUTHENTICATION, + #{ + 'Authentication-Method' => <<"SCRAM-SHA-512">>, + 'Authentication-Data' => ClientFinalMessage + } + ), + + ok = emqx_authn_mqtt_test_client:send(Pid, AuthContinuePacket), + + ?CONNACK_PACKET( + ?RC_SUCCESS, + _, + #{'Authentication-Data' := ServerFinalMessage} + ) = receive_packet(), + + ok = esasl_scram:check_server_final_message( + ServerFinalMessage, ClientCache#{algorithm => ?ALGORITHM} + ), + {ok, Pid}. + +test_acl({allow, Topic}, C) -> + ?assertMatch( + [0], + send_subscribe(C, Topic) + ); +test_acl({deny, Topic}, C) -> + ?assertMatch( + [?RC_NOT_AUTHORIZED], + send_subscribe(C, Topic) + ). + +send_subscribe(Client, Topic) -> + TopicOpts = #{nl => 0, rap => 0, rh => 0, qos => 0}, + Packet = ?SUBSCRIBE_PACKET(1, [{Topic, TopicOpts}]), + emqx_authn_mqtt_test_client:send(Client, Packet), + timer:sleep(200), + + ?SUBACK_PACKET(1, ReasonCode) = receive_packet(), + ReasonCode. diff --git a/apps/emqx_auth_mnesia/src/emqx_authn_scram_mnesia.erl b/apps/emqx_auth_mnesia/src/emqx_authn_scram_mnesia.erl index d59afea28..9880b71ee 100644 --- a/apps/emqx_auth_mnesia/src/emqx_authn_scram_mnesia.erl +++ b/apps/emqx_auth_mnesia/src/emqx_authn_scram_mnesia.erl @@ -141,7 +141,9 @@ authenticate( reason => Reason }) end, - emqx_utils_scram:authenticate(AuthMethod, AuthData, AuthCache, RetrieveFun, OnErrFun, State); + emqx_utils_scram:authenticate( + AuthMethod, AuthData, AuthCache, State, RetrieveFun, OnErrFun, [is_superuser] + ); authenticate(_Credential, _State) -> ignore. diff --git a/apps/emqx_utils/src/emqx_utils_scram.erl b/apps/emqx_utils/src/emqx_utils_scram.erl index 9d0543703..cb11082fb 100644 --- a/apps/emqx_utils/src/emqx_utils_scram.erl +++ b/apps/emqx_utils/src/emqx_utils_scram.erl @@ -16,17 +16,17 @@ -module(emqx_utils_scram). --export([authenticate/6]). +-export([authenticate/7]). %%------------------------------------------------------------------------------ %% Authentication %%------------------------------------------------------------------------------ -authenticate(AuthMethod, AuthData, AuthCache, RetrieveFun, OnErrFun, Conf) -> +authenticate(AuthMethod, AuthData, AuthCache, Conf, RetrieveFun, OnErrFun, ResultKeys) -> case ensure_auth_method(AuthMethod, AuthData, Conf) of true -> case AuthCache of #{next_step := client_final} -> - check_client_final_message(AuthData, AuthCache, Conf, OnErrFun); + check_client_final_message(AuthData, AuthCache, Conf, OnErrFun, ResultKeys); _ -> check_client_first_message(AuthData, AuthCache, Conf, RetrieveFun, OnErrFun) end; @@ -64,9 +64,7 @@ check_client_first_message( {error, not_authorized} end. -check_client_final_message( - Bin, #{is_superuser := IsSuperuser} = Cache, #{algorithm := Alg}, OnErrFun -) -> +check_client_final_message(Bin, Cache, #{algorithm := Alg}, OnErrFun, ResultKeys) -> case esasl_scram:check_client_final_message( Bin, @@ -74,7 +72,7 @@ check_client_final_message( ) of {ok, ServerFinalMessage} -> - {ok, #{is_superuser => IsSuperuser}, ServerFinalMessage}; + {ok, maps:with(ResultKeys, Cache), ServerFinalMessage}; {error, Reason} -> OnErrFun("check_client_final_message_error", Reason), {error, not_authorized} From 49b24a304916a39809cd29cceda4c69990d80560 Mon Sep 17 00:00:00 2001 From: firest Date: Fri, 26 Jul 2024 15:41:22 +0800 Subject: [PATCH 053/131] fix(oidc): fixed update and callback errors for OIDC --- .../src/emqx_dashboard_sso_oidc.erl | 27 ++++++++++++++----- .../src/emqx_dashboard_sso_oidc_api.erl | 18 +++++++++---- 2 files changed, 34 insertions(+), 11 deletions(-) diff --git a/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_oidc.erl b/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_oidc.erl index 1d2520d0f..4d1fa9439 100644 --- a/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_oidc.erl +++ b/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_oidc.erl @@ -260,7 +260,15 @@ convert_certs(_Dir, Conf) -> %%------------------------------------------------------------------------------ save_jwks_file(Dir, Content) -> - Path = filename:join([emqx_tls_lib:pem_dir(Dir), "client_jwks"]), + case filelib:is_file(Content) of + true -> + {ok, Content}; + _ -> + Path = filename:join([emqx_tls_lib:pem_dir(Dir), "client_jwks"]), + write_jwks_file(Path, Content) + end. + +write_jwks_file(Path, Content) -> case filelib:ensure_dir(Path) of ok -> case file:write_file(Path, Content) of @@ -288,11 +296,18 @@ maybe_require_pkce(true, Opts) -> }. init_client_jwks(#{client_jwks := #{type := file, file := File}}) -> - case jose_jwk:from_file(File) of - {error, _} -> - none; - Jwks -> - Jwks + try + case jose_jwk:from_file(File) of + {error, Reason} -> + ?SLOG(error, #{msg => "failed_to_initialize_jwks", reason => Reason}), + none; + Jwks -> + Jwks + end + catch + _:CReason -> + ?SLOG(error, #{msg => "failed_to_initialize_jwks", reason => CReason}), + none end; init_client_jwks(_) -> none. diff --git a/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_oidc_api.erl b/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_oidc_api.erl index 3514b4fbb..eb887cce3 100644 --- a/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_oidc_api.erl +++ b/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_oidc_api.erl @@ -28,6 +28,7 @@ -export([code_callback/2, make_callback_url/1]). +-define(BAD_REQUEST, 'BAD_REQUEST'). -define(BAD_USERNAME_OR_PWD, 'BAD_USERNAME_OR_PWD'). -define(BACKEND_NOT_FOUND, 'BACKEND_NOT_FOUND'). @@ -62,6 +63,7 @@ schema("/sso/oidc/callback") -> desc => ?DESC(code_callback), responses => #{ 200 => emqx_dashboard_api:fields([token, version, license]), + 400 => response_schema(400), 401 => response_schema(401), 404 => response_schema(404) }, @@ -78,8 +80,9 @@ code_callback(get, #{query_string := QS}) -> ?SLOG(info, #{ msg => "dashboard_sso_login_successful" }), - {302, ?RESPHEADERS#{<<"location">> => Target}, ?REDIRECT_BODY}; + {error, invalid_query_string_param} -> + {400, #{code => ?BAD_REQUEST, message => <<"Invalid query string">>}}; {error, invalid_backend} -> {404, #{code => ?BACKEND_NOT_FOUND, message => <<"Backend not found">>}}; {error, Reason} -> @@ -93,11 +96,14 @@ code_callback(get, #{query_string := QS}) -> %%-------------------------------------------------------------------- %% internal %%-------------------------------------------------------------------- - +response_schema(400) -> + emqx_dashboard_swagger:error_codes([?BAD_REQUEST], <<"Bad Request">>); response_schema(401) -> - emqx_dashboard_swagger:error_codes([?BAD_USERNAME_OR_PWD], ?DESC(login_failed401)); + emqx_dashboard_swagger:error_codes( + [?BAD_USERNAME_OR_PWD], ?DESC(emqx_dashboard_api, login_failed401) + ); response_schema(404) -> - emqx_dashboard_swagger:error_codes([?BACKEND_NOT_FOUND], ?DESC(backend_not_found)). + emqx_dashboard_swagger:error_codes([?BACKEND_NOT_FOUND], <<"Backend not found">>). reason_to_message(Bin) when is_binary(Bin) -> Bin; @@ -119,7 +125,9 @@ ensure_oidc_state(#{<<"state">> := State} = QS, Cfg) -> retrieve_token(QS, Cfg, Data); _ -> {error, session_not_exists} - end. + end; +ensure_oidc_state(_, _Cfg) -> + {error, invalid_query_string_param}. retrieve_token( #{<<"code">> := Code}, From 3fae7049031e4b84e9c8ad8b21f56167c95fb07c Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 26 Jul 2024 09:24:31 -0300 Subject: [PATCH 054/131] fix(rule engine tester): fix message publish with bridge source in from clause Fixes https://emqx.atlassian.net/browse/EMQX-12762 --- .../src/emqx_rule_sqltester.erl | 8 +++-- .../test/emqx_rule_engine_api_2_SUITE.erl | 33 +++++++++++++++++++ changes/ce/fix-13527.en.md | 1 + 3 files changed, 39 insertions(+), 3 deletions(-) create mode 100644 changes/ce/fix-13527.en.md diff --git a/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl b/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl index a3d9d5ebe..c9be82127 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl @@ -115,11 +115,13 @@ test(#{sql := Sql, context := Context}) -> true -> %% test if the topic matches the topic filters in the rule case emqx_topic:match_any(InTopic, EventTopics) of - true -> test_rule(Sql, Select, Context, EventTopics); - false -> {error, nomatch} + true -> + test_rule(Sql, Select, Context, EventTopics); + false -> + {error, nomatch} end; false -> - case lists:member(InTopic, EventTopics) of + case emqx_topic:match_any(InTopic, EventTopics) of true -> %% the rule is for both publish and events, test it directly test_rule(Sql, Select, Context, EventTopics); diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_api_2_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_api_2_SUITE.erl index b2d2fdf86..b6f3eb307 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_api_2_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_api_2_SUITE.erl @@ -332,6 +332,38 @@ t_rule_test_smoke(_Config) -> } ], MultipleFrom = [ + #{ + expected => #{code => 200}, + input => + #{ + <<"context">> => + #{ + <<"clientid">> => <<"c_emqx">>, + <<"event_type">> => <<"message_publish">>, + <<"qos">> => 1, + <<"topic">> => <<"t/a">>, + <<"username">> => <<"u_emqx">> + }, + <<"sql">> => + <<"SELECT\n *\nFROM\n \"t/#\", \"$bridges/mqtt:source\" ">> + } + }, + #{ + expected => #{code => 200}, + input => + #{ + <<"context">> => + #{ + <<"clientid">> => <<"c_emqx">>, + <<"event_type">> => <<"message_publish">>, + <<"qos">> => 1, + <<"topic">> => <<"t/a">>, + <<"username">> => <<"u_emqx">> + }, + <<"sql">> => + <<"SELECT\n *\nFROM\n \"t/#\", \"$sources/mqtt:source\" ">> + } + }, #{ expected => #{code => 200}, input => @@ -395,6 +427,7 @@ do_t_rule_test_smoke(#{input := Input, expected := #{code := ExpectedCode}} = Ca {true, #{ expected => ExpectedCode, hint => maps:get(hint, Case, <<>>), + input => Input, got => Code, resp_body => Body }} diff --git a/changes/ce/fix-13527.en.md b/changes/ce/fix-13527.en.md new file mode 100644 index 000000000..0c3324e41 --- /dev/null +++ b/changes/ce/fix-13527.en.md @@ -0,0 +1 @@ +Fixed an issue where running a SQL test in Rule Engine for the Message Publish event when a `$bridges/...` source was included in the `FROM` clause would always yield no results. From 9da744c423b81e2a9c2295e2bc1e89b5c0a3ac97 Mon Sep 17 00:00:00 2001 From: ieQu1 <99872536+ieQu1@users.noreply.github.com> Date: Fri, 26 Jul 2024 12:17:00 +0200 Subject: [PATCH 055/131] fix(mria): Reserve replicant role for EE only --- apps/emqx_conf/src/emqx_conf_schema.erl | 14 +------------- apps/emqx_conf/src/emqx_conf_schema_inject.erl | 8 +++++++- apps/emqx_conf/test/emqx_conf_schema_tests.erl | 18 ++++++------------ .../src/emqx_enterprise.app.src | 2 +- .../src/emqx_enterprise_schema.erl | 7 +++++++ changes/ce/breaking-13526.en.md | 5 +++++ 6 files changed, 27 insertions(+), 27 deletions(-) create mode 100644 changes/ce/breaking-13526.en.md diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index df906911e..8bab4e777 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -194,18 +194,6 @@ fields("cluster") -> 'readOnly' => true } )}, - {"core_nodes", - sc( - node_array(), - #{ - %% This config is nerver needed (since 5.0.0) - importance => ?IMPORTANCE_HIDDEN, - mapping => "mria.core_nodes", - default => [], - 'readOnly' => true, - desc => ?DESC(db_core_nodes) - } - )}, {"autoclean", sc( emqx_schema:duration(), @@ -600,7 +588,7 @@ fields("node") -> )}, {"role", sc( - hoconsc:enum([core, replicant]), + hoconsc:enum([core] ++ emqx_schema_hooks:injection_point('node.role')), #{ mapping => "mria.node_role", default => core, diff --git a/apps/emqx_conf/src/emqx_conf_schema_inject.erl b/apps/emqx_conf/src/emqx_conf_schema_inject.erl index fb4fee4c7..5ffd3a5af 100644 --- a/apps/emqx_conf/src/emqx_conf_schema_inject.erl +++ b/apps/emqx_conf/src/emqx_conf_schema_inject.erl @@ -22,12 +22,18 @@ schemas() -> schemas(emqx_release:edition()). schemas(Edition) -> - auth_ext(Edition) ++ + mria(Edition) ++ + auth_ext(Edition) ++ cluster_linking(Edition) ++ authn(Edition) ++ authz() ++ customized(Edition). +mria(ce) -> + []; +mria(ee) -> + [emqx_enterprise_schema]. + auth_ext(ce) -> []; auth_ext(ee) -> diff --git a/apps/emqx_conf/test/emqx_conf_schema_tests.erl b/apps/emqx_conf/test/emqx_conf_schema_tests.erl index 72834f6d2..2bb56e5cc 100644 --- a/apps/emqx_conf/test/emqx_conf_schema_tests.erl +++ b/apps/emqx_conf/test/emqx_conf_schema_tests.erl @@ -32,7 +32,6 @@ name = emqxcl discovery_strategy = static static.seeds = ~p - core_nodes = ~p } "). @@ -41,7 +40,7 @@ array_nodes_test() -> ExpectNodes = ['emqx1@127.0.0.1', 'emqx2@127.0.0.1'], lists:foreach( fun(Nodes) -> - ConfFile = to_bin(?BASE_CONF, [Nodes, Nodes]), + ConfFile = to_bin(?BASE_CONF, [Nodes]), {ok, Conf} = hocon:binary(ConfFile, #{format => richmap}), ConfList = hocon_tconf:generate(emqx_conf_schema, Conf), VMArgs = proplists:get_value(vm_args, ConfList), @@ -57,11 +56,6 @@ array_nodes_test() -> {static, [{seeds, ExpectNodes}]}, ClusterDiscovery, Nodes - ), - ?assertEqual( - ExpectNodes, - proplists:get_value(core_nodes, proplists:get_value(mria, ConfList)), - Nodes ) end, [["emqx1@127.0.0.1", "emqx2@127.0.0.1"], "emqx1@127.0.0.1, emqx2@127.0.0.1"] @@ -158,7 +152,7 @@ outdated_log_test() -> validate_log(Conf) -> ensure_acl_conf(), - BaseConf = to_bin(?BASE_CONF, ["emqx1@127.0.0.1", "emqx1@127.0.0.1"]), + BaseConf = to_bin(?BASE_CONF, ["emqx1@127.0.0.1"]), Conf0 = <>, {ok, ConfMap0} = hocon:binary(Conf0, #{format => richmap}), ConfList = hocon_tconf:generate(emqx_conf_schema, ConfMap0), @@ -214,7 +208,7 @@ validate_log(Conf) -> file_log_infinity_rotation_size_test_() -> ensure_acl_conf(), - BaseConf = to_bin(?BASE_CONF, ["emqx1@127.0.0.1", "emqx1@127.0.0.1"]), + BaseConf = to_bin(?BASE_CONF, ["emqx1@127.0.0.1"]), Gen = fun(#{count := Count, size := Size}) -> Conf0 = to_bin(?FILE_LOG_BASE_CONF, [Count, Size]), Conf1 = [BaseConf, Conf0], @@ -292,7 +286,7 @@ log_rotation_count_limit_test() -> rotation_size = \"1024MB\" } ", - BaseConf = to_bin(?BASE_CONF, ["emqx1@127.0.0.1", "emqx1@127.0.0.1"]), + BaseConf = to_bin(?BASE_CONF, ["emqx1@127.0.0.1"]), lists:foreach(fun({Conf, Count}) -> Conf0 = <>, {ok, ConfMap0} = hocon:binary(Conf0, #{format => richmap}), @@ -352,7 +346,7 @@ log_rotation_count_limit_test() -> authn_validations_test() -> ensure_acl_conf(), - BaseConf = to_bin(?BASE_CONF, ["emqx1@127.0.0.1", "emqx1@127.0.0.1"]), + BaseConf = to_bin(?BASE_CONF, ["emqx1@127.0.0.1"]), OKHttps = to_bin(?BASE_AUTHN_ARRAY, [post, true, <<"https://127.0.0.1:8080">>]), Conf0 = <>, @@ -410,7 +404,7 @@ authn_validations_test() -> listeners_test() -> ensure_acl_conf(), - BaseConf = to_bin(?BASE_CONF, ["emqx1@127.0.0.1", "emqx1@127.0.0.1"]), + BaseConf = to_bin(?BASE_CONF, ["emqx1@127.0.0.1"]), Conf = <>, {ok, ConfMap0} = hocon:binary(Conf, #{format => richmap}), diff --git a/apps/emqx_enterprise/src/emqx_enterprise.app.src b/apps/emqx_enterprise/src/emqx_enterprise.app.src index 93fb02287..e79a6f3a3 100644 --- a/apps/emqx_enterprise/src/emqx_enterprise.app.src +++ b/apps/emqx_enterprise/src/emqx_enterprise.app.src @@ -1,6 +1,6 @@ {application, emqx_enterprise, [ {description, "EMQX Enterprise Edition"}, - {vsn, "0.2.2"}, + {vsn, "0.2.3"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_enterprise/src/emqx_enterprise_schema.erl b/apps/emqx_enterprise/src/emqx_enterprise_schema.erl index f593dc877..571757c75 100644 --- a/apps/emqx_enterprise/src/emqx_enterprise_schema.erl +++ b/apps/emqx_enterprise/src/emqx_enterprise_schema.erl @@ -5,12 +5,14 @@ -module(emqx_enterprise_schema). -behaviour(hocon_schema). +-behaviour(emqx_schema_hooks). -include_lib("typerefl/include/types.hrl"). -include_lib("hocon/include/hoconsc.hrl"). -export([namespace/0, roots/0, fields/1, translations/0, translation/1, desc/1, validations/0]). -export([upgrade_raw_conf/1]). +-export([injected_fields/0]). -define(EE_SCHEMA_MODULES, [ emqx_license_schema, @@ -127,6 +129,11 @@ desc(Name) -> validations() -> emqx_conf_schema:validations() ++ emqx_license_schema:validations(). +injected_fields() -> + #{ + 'node.role' => [replicant] + }. + %%------------------------------------------------------------------------------ %% helpers %%------------------------------------------------------------------------------ diff --git a/changes/ce/breaking-13526.en.md b/changes/ce/breaking-13526.en.md new file mode 100644 index 000000000..752e58ef3 --- /dev/null +++ b/changes/ce/breaking-13526.en.md @@ -0,0 +1,5 @@ +- Core-replicant feature has been removed from the Open-Source Edition. + Starting from release 5.8, all nodes running Open-Source Edition will assume Core role. + This change doesn't affect Enterprise Edition users. + +- Obsolete and unused `cluster.core_nodes` configuration parameter has been removed. From 1beda1cd118762107448491f092aa6b56326384e Mon Sep 17 00:00:00 2001 From: ieQu1 <99872536+ieQu1@users.noreply.github.com> Date: Fri, 26 Jul 2024 16:22:57 +0200 Subject: [PATCH 056/131] test(mria): Remove role from the example config --- apps/emqx/test/emqx_cth_suite.erl | 11 +++++++++++ apps/emqx_management/test/emqx_mgmt_cli_SUITE.erl | 8 +++++++- rel/config/examples/node.conf.example | 4 ++-- 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/apps/emqx/test/emqx_cth_suite.erl b/apps/emqx/test/emqx_cth_suite.erl index 5fe4dce66..75c54400b 100644 --- a/apps/emqx/test/emqx_cth_suite.erl +++ b/apps/emqx/test/emqx_cth_suite.erl @@ -519,3 +519,14 @@ upgrade_raw_conf(Conf) -> ce -> emqx_conf_schema:upgrade_raw_conf(Conf) end. + +skip_if_oss() -> + try emqx_release:edition() of + ee -> + false; + _ -> + {skip, not_supported_in_oss} + catch + error:undef -> + {skip, standalone_not_supported} + end. diff --git a/apps/emqx_management/test/emqx_mgmt_cli_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_cli_SUITE.erl index cb3451fc1..f85fdbe5b 100644 --- a/apps/emqx_management/test/emqx_mgmt_cli_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_cli_SUITE.erl @@ -22,7 +22,13 @@ -include_lib("common_test/include/ct.hrl"). all() -> - emqx_common_test_helpers:all(?MODULE). + All = emqx_common_test_helpers:all(?MODULE), + case emqx_cth_suite:skip_if_oss() of + false -> + All; + _ -> + All -- [t_autocluster_leave] + end. init_per_suite(Config) -> Apps = emqx_cth_suite:start( diff --git a/rel/config/examples/node.conf.example b/rel/config/examples/node.conf.example index 596e9884d..f4fd3288e 100644 --- a/rel/config/examples/node.conf.example +++ b/rel/config/examples/node.conf.example @@ -11,11 +11,11 @@ node { ## Secret cookie is a random string that should be the same on all nodes in the cluster, but unique per EMQX cluster cookie = "Yzc0NGExM2Rj" - ## Select a node role + ## Select a node role (Enterprise Edition feature) ## Possible values: ## - core: This is a core node which provides durability of the client states, and takes care of writes ## - replicant: This is a stateless worker node - role = core + ## role = core ## Maximum number of simultaneously existing processes for this Erlang system process_limit = 2097152 From 548bcceab7c0467e05ccb5c08a0c50f354bbd987 Mon Sep 17 00:00:00 2001 From: ieQu1 <99872536+ieQu1@users.noreply.github.com> Date: Fri, 26 Jul 2024 16:28:00 +0200 Subject: [PATCH 057/131] test(auth): Remove redundant config --- .../test/emqx_authz/emqx_authz_api_cluster_SUITE.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_auth/test/emqx_authz/emqx_authz_api_cluster_SUITE.erl b/apps/emqx_auth/test/emqx_authz/emqx_authz_api_cluster_SUITE.erl index a77b3df51..3884dd052 100644 --- a/apps/emqx_auth/test/emqx_authz/emqx_authz_api_cluster_SUITE.erl +++ b/apps/emqx_auth/test/emqx_authz/emqx_authz_api_cluster_SUITE.erl @@ -118,8 +118,8 @@ mk_cluster_spec(Opts) -> Node1Apps = Apps ++ [{emqx_dashboard, "dashboard.listeners.http {enable=true,bind=18083}"}], Node2Apps = Apps, [ - {emqx_authz_api_cluster_SUITE1, Opts#{role => core, apps => Node1Apps}}, - {emqx_authz_api_cluster_SUITE2, Opts#{role => core, apps => Node2Apps}} + {emqx_authz_api_cluster_SUITE1, Opts#{apps => Node1Apps}}, + {emqx_authz_api_cluster_SUITE2, Opts#{apps => Node2Apps}} ]. request(Method, URL, Body, Config) -> From 41bf5cd6caa701d05d0d7b89931ea4f9ababe4ea Mon Sep 17 00:00:00 2001 From: ieQu1 <99872536+ieQu1@users.noreply.github.com> Date: Fri, 26 Jul 2024 16:30:37 +0200 Subject: [PATCH 058/131] test(otel): Remove redundant config --- apps/emqx_opentelemetry/test/emqx_otel_trace_SUITE.erl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx_opentelemetry/test/emqx_otel_trace_SUITE.erl b/apps/emqx_opentelemetry/test/emqx_otel_trace_SUITE.erl index a02cec3ef..9cf545cdc 100644 --- a/apps/emqx_opentelemetry/test/emqx_otel_trace_SUITE.erl +++ b/apps/emqx_opentelemetry/test/emqx_otel_trace_SUITE.erl @@ -414,9 +414,9 @@ mqtt_host_port(Node) -> cluster(TC, Config) -> Nodes = emqx_cth_cluster:start( [ - {otel_trace_core1, #{role => core, apps => apps_spec()}}, - {otel_trace_core2, #{role => core, apps => apps_spec()}}, - {otel_trace_replicant, #{role => replicant, apps => apps_spec()}} + {otel_trace_node1, #{apps => apps_spec()}}, + {otel_trace_node2, #{apps => apps_spec()}}, + {otel_trace_node3, #{apps => apps_spec()}} ], #{work_dir => emqx_cth_suite:work_dir(TC, Config)} ), From 1b6494ab9a6bf4b13ad1a79260ad11194dbe3ae4 Mon Sep 17 00:00:00 2001 From: ieQu1 <99872536+ieQu1@users.noreply.github.com> Date: Fri, 26 Jul 2024 16:35:53 +0200 Subject: [PATCH 059/131] test(mgmt): Remove redundant config --- apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl index 398b48ab7..ef7b1fa54 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl @@ -107,8 +107,8 @@ init_per_group(persistent_sessions, Config) -> ], Dashboard = emqx_mgmt_api_test_util:emqx_dashboard(), Cluster = [ - {emqx_mgmt_api_clients_SUITE1, #{role => core, apps => AppSpecs ++ [Dashboard]}}, - {emqx_mgmt_api_clients_SUITE2, #{role => core, apps => AppSpecs}} + {emqx_mgmt_api_clients_SUITE1, #{apps => AppSpecs ++ [Dashboard]}}, + {emqx_mgmt_api_clients_SUITE2, #{apps => AppSpecs}} ], Nodes = [N1 | _] = emqx_cth_cluster:start( @@ -128,8 +128,8 @@ init_per_group(non_persistent_cluster, Config) -> ], Dashboard = emqx_mgmt_api_test_util:emqx_dashboard(), Cluster = [ - {mgmt_api_clients_SUITE1, #{role => core, apps => AppSpecs ++ [Dashboard]}}, - {mgmt_api_clients_SUITE2, #{role => core, apps => AppSpecs}} + {mgmt_api_clients_SUITE1, #{apps => AppSpecs ++ [Dashboard]}}, + {mgmt_api_clients_SUITE2, #{apps => AppSpecs}} ], Nodes = [N1 | _] = emqx_cth_cluster:start( From b7c424a13d0a5b9d2fa0fb5f034a8ad87f5e68d6 Mon Sep 17 00:00:00 2001 From: ieQu1 <99872536+ieQu1@users.noreply.github.com> Date: Fri, 26 Jul 2024 16:36:44 +0200 Subject: [PATCH 060/131] test(persmsg): Remove redundant config --- apps/emqx/test/emqx_persistent_messages_SUITE.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx/test/emqx_persistent_messages_SUITE.erl b/apps/emqx/test/emqx_persistent_messages_SUITE.erl index f225ba43d..19dac4575 100644 --- a/apps/emqx/test/emqx_persistent_messages_SUITE.erl +++ b/apps/emqx/test/emqx_persistent_messages_SUITE.erl @@ -573,7 +573,7 @@ app_specs(Opts) -> cluster() -> ExtraConf = "\n durable_storage.messages.n_sites = 2", - Spec = #{role => core, apps => app_specs(#{extra_emqx_conf => ExtraConf})}, + Spec = #{apps => app_specs(#{extra_emqx_conf => ExtraConf})}, [ {persistent_messages_SUITE1, Spec}, {persistent_messages_SUITE2, Spec} From b8a2a8ea18066760f67f4f63aede2af56d0b37de Mon Sep 17 00:00:00 2001 From: ieQu1 <99872536+ieQu1@users.noreply.github.com> Date: Fri, 26 Jul 2024 16:37:11 +0200 Subject: [PATCH 061/131] test(router): Skip certain tests on OSS --- apps/emqx/test/emqx_cth_suite.erl | 2 ++ apps/emqx/test/emqx_routing_SUITE.erl | 28 ++++++++++++++++++--------- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/apps/emqx/test/emqx_cth_suite.erl b/apps/emqx/test/emqx_cth_suite.erl index 75c54400b..8e7c84580 100644 --- a/apps/emqx/test/emqx_cth_suite.erl +++ b/apps/emqx/test/emqx_cth_suite.erl @@ -79,6 +79,8 @@ %% "Unofficial" `emqx_config_handler' and `emqx_conf' APIs -export([schema_module/0, upgrade_raw_conf/1]). +-export([skip_if_oss/0]). + -export_type([appspec/0]). -export_type([appspec_opts/0]). diff --git a/apps/emqx/test/emqx_routing_SUITE.erl b/apps/emqx/test/emqx_routing_SUITE.erl index 5112059ca..1e66a6ef7 100644 --- a/apps/emqx/test/emqx_routing_SUITE.erl +++ b/apps/emqx/test/emqx_routing_SUITE.erl @@ -64,18 +64,28 @@ init_per_group(routing_schema_v2, Config) -> init_per_group(batch_sync_on, Config) -> [{emqx_config, "broker.routing.batch_sync.enable_on = all"} | Config]; init_per_group(batch_sync_replicants, Config) -> - [{emqx_config, "broker.routing.batch_sync.enable_on = replicant"} | Config]; + case emqx_cth_suite:skip_if_oss() of + false -> + [{emqx_config, "broker.routing.batch_sync.enable_on = replicant"} | Config]; + True -> + True + end; init_per_group(batch_sync_off, Config) -> [{emqx_config, "broker.routing.batch_sync.enable_on = none"} | Config]; init_per_group(cluster, Config) -> - WorkDir = emqx_cth_suite:work_dir(Config), - NodeSpecs = [ - {emqx_routing_SUITE1, #{apps => [mk_emqx_appspec(1, Config)], role => core}}, - {emqx_routing_SUITE2, #{apps => [mk_emqx_appspec(2, Config)], role => core}}, - {emqx_routing_SUITE3, #{apps => [mk_emqx_appspec(3, Config)], role => replicant}} - ], - Nodes = emqx_cth_cluster:start(NodeSpecs, #{work_dir => WorkDir}), - [{cluster, Nodes} | Config]; + case emqx_cth_suite:skip_if_oss() of + false -> + WorkDir = emqx_cth_suite:work_dir(Config), + NodeSpecs = [ + {emqx_routing_SUITE1, #{apps => [mk_emqx_appspec(1, Config)], role => core}}, + {emqx_routing_SUITE2, #{apps => [mk_emqx_appspec(2, Config)], role => core}}, + {emqx_routing_SUITE3, #{apps => [mk_emqx_appspec(3, Config)], role => replicant}} + ], + Nodes = emqx_cth_cluster:start(NodeSpecs, #{work_dir => WorkDir}), + [{cluster, Nodes} | Config]; + True -> + True + end; init_per_group(GroupName, Config) when GroupName =:= single_batch_on; GroupName =:= single From 8c1302f455b96aa8b8c8709b6e311eab2680197f Mon Sep 17 00:00:00 2001 From: ieQu1 <99872536+ieQu1@users.noreply.github.com> Date: Fri, 26 Jul 2024 17:02:21 +0200 Subject: [PATCH 062/131] test(conf_app): Remove redundand config --- apps/emqx_conf/test/emqx_conf_app_SUITE.erl | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/apps/emqx_conf/test/emqx_conf_app_SUITE.erl b/apps/emqx_conf/test/emqx_conf_app_SUITE.erl index 5ba3f0b49..711274aaa 100644 --- a/apps/emqx_conf/test/emqx_conf_app_SUITE.erl +++ b/apps/emqx_conf/test/emqx_conf_app_SUITE.erl @@ -30,7 +30,7 @@ t_copy_conf_override_on_restarts(Config) -> ct:timetrap({seconds, 120}), Cluster = cluster( ?FUNCTION_NAME, - [cluster_spec({core, 1}), cluster_spec({core, 2}), cluster_spec({core, 3})], + [cluster_spec(1), cluster_spec(2), cluster_spec(3)], Config ), @@ -59,7 +59,7 @@ t_copy_new_data_dir(Config) -> ct:timetrap({seconds, 120}), Cluster = cluster( ?FUNCTION_NAME, - [cluster_spec({core, 4}), cluster_spec({core, 5}), cluster_spec({core, 6})], + [cluster_spec(4), cluster_spec(5), cluster_spec(6)], Config ), @@ -84,7 +84,7 @@ t_copy_deprecated_data_dir(Config) -> ct:timetrap({seconds, 120}), Cluster = cluster( ?FUNCTION_NAME, - [cluster_spec({core, 7}), cluster_spec({core, 8}), cluster_spec({core, 9})], + [cluster_spec(7), cluster_spec(8), cluster_spec(9)], Config ), @@ -109,7 +109,7 @@ t_no_copy_from_newer_version_node(Config) -> ct:timetrap({seconds, 120}), Cluster = cluster( ?FUNCTION_NAME, - [cluster_spec({core, 10}), cluster_spec({core, 11}), cluster_spec({core, 12})], + [cluster_spec(10), cluster_spec(11), cluster_spec(12)], Config ), OKs = [ok, ok, ok], @@ -242,12 +242,12 @@ cluster(TC, Specs, Config) -> {emqx_conf, #{}} ], emqx_cth_cluster:mk_nodespecs( - [{Name, #{role => Role, apps => Apps}} || {Role, Name} <- Specs], + [{Name, #{apps => Apps}} || Name <- Specs], #{work_dir => emqx_cth_suite:work_dir(TC, Config)} ). -cluster_spec({Type, Num}) -> - {Type, list_to_atom(atom_to_list(?MODULE) ++ integer_to_list(Num))}. +cluster_spec(Num) -> + list_to_atom(atom_to_list(?MODULE) ++ integer_to_list(Num)). sort_highest_uptime(Nodes) -> Ranking = lists:sort([{-get_node_uptime(N), N} || N <- Nodes]), From 4e0742c66facdf0f290157b832e5271ee045e42b Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 26 Jul 2024 14:25:20 -0300 Subject: [PATCH 063/131] feat: make kafka producer freely dynamic --- .../emqx_bridge_azure_event_hub_v2_SUITE.erl | 51 ++++------- .../emqx_bridge_confluent_producer_SUITE.erl | 51 ++++------- .../src/emqx_bridge_kafka.erl | 12 --- .../src/emqx_bridge_kafka_impl_producer.erl | 71 ++++++--------- .../emqx_bridge_v2_kafka_producer_SUITE.erl | 89 +++---------------- rel/i18n/emqx_bridge_azure_event_hub.hocon | 10 --- rel/i18n/emqx_bridge_confluent_producer.hocon | 10 --- rel/i18n/emqx_bridge_kafka.hocon | 10 --- 8 files changed, 73 insertions(+), 231 deletions(-) diff --git a/apps/emqx_bridge_azure_event_hub/test/emqx_bridge_azure_event_hub_v2_SUITE.erl b/apps/emqx_bridge_azure_event_hub/test/emqx_bridge_azure_event_hub_v2_SUITE.erl index 0136ec568..f2a06cf65 100644 --- a/apps/emqx_bridge_azure_event_hub/test/emqx_bridge_azure_event_hub_v2_SUITE.erl +++ b/apps/emqx_bridge_azure_event_hub/test/emqx_bridge_azure_event_hub_v2_SUITE.erl @@ -382,46 +382,31 @@ t_multiple_actions_sharing_topic(Config) -> ActionConfig0, #{<<"parameters">> => #{<<"query_mode">> => <<"sync">>}} ), - ok = emqx_bridge_v2_kafka_producer_SUITE:t_multiple_actions_sharing_topic( - [ - {type, ?BRIDGE_TYPE_BIN}, - {connector_name, ?config(connector_name, Config)}, - {connector_config, ?config(connector_config, Config)}, - {action_config, ActionConfig} - ] - ), + ok = + emqx_bridge_v2_kafka_producer_SUITE:?FUNCTION_NAME( + [ + {type, ?BRIDGE_TYPE_BIN}, + {connector_name, ?config(connector_name, Config)}, + {connector_config, ?config(connector_config, Config)}, + {action_config, ActionConfig} + ] + ), ok. -t_pre_configured_topics(Config) -> +t_dynamic_topics(Config) -> ActionConfig0 = ?config(action_config, Config), ActionConfig = emqx_utils_maps:deep_merge( ActionConfig0, #{<<"parameters">> => #{<<"query_mode">> => <<"sync">>}} ), - ok = emqx_bridge_v2_kafka_producer_SUITE:t_pre_configured_topics( - [ - {type, ?BRIDGE_TYPE_BIN}, - {connector_name, ?config(connector_name, Config)}, - {connector_config, ?config(connector_config, Config)}, - {action_config, ActionConfig} - ] - ), - ok. - -t_templated_topic_and_no_pre_configured_topics(Config) -> - ActionConfig0 = ?config(action_config, Config), - ActionConfig = - emqx_utils_maps:deep_merge( - ActionConfig0, - #{<<"parameters">> => #{<<"query_mode">> => <<"sync">>}} + ok = + emqx_bridge_v2_kafka_producer_SUITE:?FUNCTION_NAME( + [ + {type, ?BRIDGE_TYPE_BIN}, + {connector_name, ?config(connector_name, Config)}, + {connector_config, ?config(connector_config, Config)}, + {action_config, ActionConfig} + ] ), - ok = emqx_bridge_v2_kafka_producer_SUITE:t_templated_topic_and_no_pre_configured_topics( - [ - {type, ?BRIDGE_TYPE_BIN}, - {connector_name, ?config(connector_name, Config)}, - {connector_config, ?config(connector_config, Config)}, - {action_config, ActionConfig} - ] - ), ok. diff --git a/apps/emqx_bridge_confluent/test/emqx_bridge_confluent_producer_SUITE.erl b/apps/emqx_bridge_confluent/test/emqx_bridge_confluent_producer_SUITE.erl index de92b9327..f10e88463 100644 --- a/apps/emqx_bridge_confluent/test/emqx_bridge_confluent_producer_SUITE.erl +++ b/apps/emqx_bridge_confluent/test/emqx_bridge_confluent_producer_SUITE.erl @@ -391,46 +391,31 @@ t_multiple_actions_sharing_topic(Config) -> ActionConfig0, #{<<"parameters">> => #{<<"query_mode">> => <<"sync">>}} ), - ok = emqx_bridge_v2_kafka_producer_SUITE:t_multiple_actions_sharing_topic( - [ - {type, ?ACTION_TYPE_BIN}, - {connector_name, ?config(connector_name, Config)}, - {connector_config, ?config(connector_config, Config)}, - {action_config, ActionConfig} - ] - ), + ok = + emqx_bridge_v2_kafka_producer_SUITE:?FUNCTION_NAME( + [ + {type, ?ACTION_TYPE_BIN}, + {connector_name, ?config(connector_name, Config)}, + {connector_config, ?config(connector_config, Config)}, + {action_config, ActionConfig} + ] + ), ok. -t_pre_configured_topics(Config) -> +t_dynamic_topics(Config) -> ActionConfig0 = ?config(action_config, Config), ActionConfig = emqx_utils_maps:deep_merge( ActionConfig0, #{<<"parameters">> => #{<<"query_mode">> => <<"sync">>}} ), - ok = emqx_bridge_v2_kafka_producer_SUITE:t_pre_configured_topics( - [ - {type, ?ACTION_TYPE_BIN}, - {connector_name, ?config(connector_name, Config)}, - {connector_config, ?config(connector_config, Config)}, - {action_config, ActionConfig} - ] - ), - ok. - -t_templated_topic_and_no_pre_configured_topics(Config) -> - ActionConfig0 = ?config(action_config, Config), - ActionConfig = - emqx_utils_maps:deep_merge( - ActionConfig0, - #{<<"parameters">> => #{<<"query_mode">> => <<"sync">>}} + ok = + emqx_bridge_v2_kafka_producer_SUITE:?FUNCTION_NAME( + [ + {type, ?ACTION_TYPE_BIN}, + {connector_name, ?config(connector_name, Config)}, + {connector_config, ?config(connector_config, Config)}, + {action_config, ActionConfig} + ] ), - ok = emqx_bridge_v2_kafka_producer_SUITE:t_templated_topic_and_no_pre_configured_topics( - [ - {type, ?ACTION_TYPE_BIN}, - {connector_name, ?config(connector_name, Config)}, - {connector_config, ?config(connector_config, Config)}, - {action_config, ActionConfig} - ] - ), ok. diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl index 9a2fa91cf..8f72523b1 100644 --- a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl +++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl @@ -307,10 +307,6 @@ fields(kafka_producer_action) -> {tags, emqx_schema:tags_schema()}, {description, emqx_schema:description_schema()} ] ++ producer_opts(action); -fields(pre_configured_topic) -> - [ - {topic, mk(string(), #{required => true, desc => ?DESC(kafka_topic)})} - ]; fields(kafka_consumer) -> connector_config_fields() ++ fields(consumer_opts); fields(ssl_client_opts) -> @@ -396,14 +392,6 @@ fields(v1_producer_kafka_opts) -> fields(producer_kafka_opts) -> [ {topic, mk(string(), #{required => true, desc => ?DESC(kafka_topic)})}, - {pre_configured_topics, - mk( - hoconsc:array(ref(pre_configured_topic)), - #{ - default => [], - desc => ?DESC("producer_pre_configured_topics") - } - )}, {message, mk(ref(kafka_message), #{required => false, desc => ?DESC(kafka_message)})}, {max_batch_bytes, mk(emqx_schema:bytesize(), #{default => <<"896KB">>, desc => ?DESC(max_batch_bytes)})}, diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl index 80de98402..b358cd42b 100644 --- a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl +++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl @@ -3,6 +3,8 @@ %%-------------------------------------------------------------------- -module(emqx_bridge_kafka_impl_producer). +-feature(maybe_expr, enable). + -behaviour(emqx_resource). -include_lib("emqx_resource/include/emqx_resource.hrl"). @@ -132,37 +134,22 @@ create_producers_for_bridge_v2( ) -> #{ message := MessageTemplate, - pre_configured_topics := PreConfiguredTopics0, topic := KafkaTopic0, sync_query_timeout := SyncQueryTimeout } = KafkaConfig, - TopicTemplate = {TopicType, KafkaTopic} = maybe_preproc_topic(KafkaTopic0), - PreConfiguredTopics = [T || #{topic := T} <- PreConfiguredTopics0], - KafkaTopics0 = + TopicTemplate = {TopicType, TopicOrTemplate} = maybe_preproc_topic(KafkaTopic0), + MKafkaTopic = case TopicType of - fixed -> - [KafkaTopic | PreConfiguredTopics]; - dynamic -> - PreConfiguredTopics + fixed -> TopicOrTemplate; + dynamic -> dynamic end, - case KafkaTopics0 of - [] -> - throw(<< - "Either the Kafka topic must be fixed (not a template)," - " or at least one pre-defined topic must be set." - >>); - _ -> - ok - end, - KafkaTopics = lists:map(fun bin/1, KafkaTopics0), KafkaHeadersTokens = preproc_kafka_headers(maps:get(kafka_headers, KafkaConfig, undefined)), KafkaExtHeadersTokens = preproc_ext_headers(maps:get(kafka_ext_headers, KafkaConfig, [])), KafkaHeadersValEncodeMode = maps:get(kafka_header_value_encode_mode, KafkaConfig, none), MaxPartitions = maps:get(partitions_limit, KafkaConfig, all_partitions), #{name := BridgeName} = emqx_bridge_v2:parse_id(ActionResId), IsDryRun = emqx_resource:is_dry_run(ActionResId), - [AKafkaTopic | _] = KafkaTopics, - ok = check_topic_and_leader_connections(ActionResId, ClientId, AKafkaTopic, MaxPartitions), + ok = check_topic_and_leader_connections(ActionResId, ClientId, MKafkaTopic, MaxPartitions), WolffProducerConfig = producers_config( BridgeType, BridgeName, KafkaConfig, IsDryRun, ActionResId ), @@ -179,7 +166,7 @@ create_producers_for_bridge_v2( message_template => compile_message_template(MessageTemplate), kafka_client_id => ClientId, topic_template => TopicTemplate, - pre_configured_topics => KafkaTopics, + topic => MKafkaTopic, producers => Producers, resource_id => ActionResId, connector_resource_id => ConnResId, @@ -195,7 +182,7 @@ create_producers_for_bridge_v2( msg => "failed_to_start_kafka_producer", instance_id => ConnResId, kafka_client_id => ClientId, - kafka_topic => KafkaTopic, + kafka_topic => MKafkaTopic, reason => Reason2 }), throw( @@ -326,7 +313,6 @@ on_query( message_template := MessageTemplate, topic_template := TopicTemplate, producers := Producers, - pre_configured_topics := PreConfiguredTopics, sync_query_timeout := SyncTimeout, headers_tokens := KafkaHeadersTokens, ext_headers_tokens := KafkaExtHeadersTokens, @@ -339,12 +325,6 @@ on_query( }, try KafkaTopic = render_topic(TopicTemplate, Message), - case lists:member(KafkaTopic, PreConfiguredTopics) of - false -> - throw({unknown_topic, KafkaTopic}); - true -> - ok - end, KafkaMessage = render_message(MessageTemplate, KafkaHeaders, Message), ?tp( emqx_bridge_kafka_impl_producer_sync_query, @@ -358,7 +338,7 @@ on_query( throw:bad_topic -> ?tp("kafka_producer_failed_to_render_topic", #{}), {error, {unrecoverable_error, failed_to_render_topic}}; - throw:{unknown_topic, Topic} -> + throw:#{cause := unknown_topic_or_partition, topic := Topic} -> ?tp("kafka_producer_resolved_to_unknown_topic", #{}), {error, {unrecoverable_error, {resolved_to_unknown_topic, Topic}}}; throw:#{cause := invalid_partition_count, count := Count} -> @@ -408,7 +388,6 @@ on_query_async( message_template := Template, topic_template := TopicTemplate, producers := Producers, - pre_configured_topics := PreConfiguredTopics, headers_tokens := KafkaHeadersTokens, ext_headers_tokens := KafkaExtHeadersTokens, headers_val_encode_mode := KafkaHeadersValEncodeMode @@ -420,12 +399,6 @@ on_query_async( }, try KafkaTopic = render_topic(TopicTemplate, Message), - case lists:member(KafkaTopic, PreConfiguredTopics) of - false -> - throw({unknown_topic, KafkaTopic}); - true -> - ok - end, KafkaMessage = render_message(Template, KafkaHeaders, Message), ?tp( emqx_bridge_kafka_impl_producer_async_query, @@ -439,7 +412,7 @@ on_query_async( throw:bad_topic -> ?tp("kafka_producer_failed_to_render_topic", #{}), {error, {unrecoverable_error, failed_to_render_topic}}; - throw:{unknown_topic, Topic} -> + throw:#{cause := unknown_topic_or_partition, topic := Topic} -> ?tp("kafka_producer_resolved_to_unknown_topic", #{}), {error, {unrecoverable_error, {resolved_to_unknown_topic, Topic}}}; throw:#{cause := invalid_partition_count, count := Count} -> @@ -618,12 +591,11 @@ on_get_channel_status( %% connector, thus potentially dropping data held in wolff producer's replayq. The %% only exception is if the topic does not exist ("unhealthy target"). #{ - pre_configured_topics := PreConfiguredTopics, + topic := MKafkaTopic, partitions_limit := MaxPartitions } = maps:get(ActionResId, Channels), - [KafkaTopic | _] = PreConfiguredTopics, try - ok = check_topic_and_leader_connections(ActionResId, ClientId, KafkaTopic, MaxPartitions), + ok = check_topic_and_leader_connections(ActionResId, ClientId, MKafkaTopic, MaxPartitions), ?status_connected catch throw:{unhealthy_target, Msg} -> @@ -632,22 +604,29 @@ on_get_channel_status( {?status_connecting, {K, E}} end. -check_topic_and_leader_connections(ActionResId, ClientId, KafkaTopic, MaxPartitions) -> +check_topic_and_leader_connections(ActionResId, ClientId, MKafkaTopic, MaxPartitions) -> case wolff_client_sup:find_client(ClientId) of {ok, Pid} -> - ok = check_topic_status(ClientId, Pid, KafkaTopic), - ok = check_if_healthy_leaders(ActionResId, ClientId, Pid, KafkaTopic, MaxPartitions); + maybe + true ?= is_binary(MKafkaTopic), + ok = check_topic_status(ClientId, Pid, MKafkaTopic), + ok = check_if_healthy_leaders( + ActionResId, ClientId, Pid, MKafkaTopic, MaxPartitions + ) + else + false -> ok + end; {error, #{reason := no_such_client}} -> throw(#{ reason => cannot_find_kafka_client, kafka_client => ClientId, - kafka_topic => KafkaTopic + kafka_topic => MKafkaTopic }); {error, #{reason := client_supervisor_not_initialized}} -> throw(#{ reason => restarting, kafka_client => ClientId, - kafka_topic => KafkaTopic + kafka_topic => MKafkaTopic }) end. diff --git a/apps/emqx_bridge_kafka/test/emqx_bridge_v2_kafka_producer_SUITE.erl b/apps/emqx_bridge_kafka/test/emqx_bridge_v2_kafka_producer_SUITE.erl index 6246faaf1..08b2723e7 100644 --- a/apps/emqx_bridge_kafka/test/emqx_bridge_v2_kafka_producer_SUITE.erl +++ b/apps/emqx_bridge_kafka/test/emqx_bridge_v2_kafka_producer_SUITE.erl @@ -915,17 +915,17 @@ t_multiple_actions_sharing_topic(Config) -> ), ok. -%% Smoke tests for using a templated topic and a list of pre-configured kafka topics. -t_pre_configured_topics(Config) -> +%% Smoke tests for using a templated topic and adynamic kafka topics. +t_dynamic_topics(Config) -> Type = proplists:get_value(type, Config, ?TYPE), ConnectorName = proplists:get_value(connector_name, Config, <<"c">>), ConnectorConfig = proplists:get_value(connector_config, Config, connector_config()), - ActionName = <<"pre_configured_topics">>, + ActionName = <<"dynamic_topics">>, ActionConfig1 = proplists:get_value(action_config, Config, action_config(ConnectorName)), - PreConfigureTopic1 = <<"pct1">>, - PreConfigureTopic2 = <<"pct2">>, - ensure_kafka_topic(PreConfigureTopic1), - ensure_kafka_topic(PreConfigureTopic2), + PreConfiguredTopic1 = <<"pct1">>, + PreConfiguredTopic2 = <<"pct2">>, + ensure_kafka_topic(PreConfiguredTopic1), + ensure_kafka_topic(PreConfiguredTopic2), ActionConfig = emqx_bridge_v2_testlib:parse_and_check( action, Type, @@ -938,11 +938,7 @@ t_pre_configured_topics(Config) -> <<"message">> => #{ <<"key">> => <<"${.clientid}">>, <<"value">> => <<"${.payload.p}">> - }, - <<"pre_configured_topics">> => [ - #{<<"topic">> => PreConfigureTopic1}, - #{<<"topic">> => PreConfigureTopic2} - ] + } } } ) @@ -1001,13 +997,13 @@ t_pre_configured_topics(Config) -> {ok, C} = emqtt:start_link(#{}), {ok, _} = emqtt:connect(C), Payload = fun(Map) -> emqx_utils_json:encode(Map) end, - Offset1 = resolve_kafka_offset(PreConfigureTopic1), - Offset2 = resolve_kafka_offset(PreConfigureTopic2), + Offset1 = resolve_kafka_offset(PreConfiguredTopic1), + Offset2 = resolve_kafka_offset(PreConfiguredTopic2), {ok, _} = emqtt:publish(C, RuleTopic, Payload(#{n => 1, p => <<"p1">>}), [{qos, 1}]), {ok, _} = emqtt:publish(C, RuleTopic, Payload(#{n => 2, p => <<"p2">>}), [{qos, 1}]), - check_kafka_message_payload(PreConfigureTopic1, Offset1, <<"p1">>), - check_kafka_message_payload(PreConfigureTopic2, Offset2, <<"p2">>), + check_kafka_message_payload(PreConfiguredTopic1, Offset1, <<"p1">>), + check_kafka_message_payload(PreConfiguredTopic2, Offset2, <<"p2">>), ActionId = emqx_bridge_v2:id(Type, ActionName), ?assertEqual(2, emqx_resource_metrics:matched_get(ActionId)), @@ -1046,64 +1042,3 @@ t_pre_configured_topics(Config) -> [] ), ok. - -%% Checks that creating an action with templated topic and no pre-configured kafka topics -%% throws. -t_templated_topic_and_no_pre_configured_topics(Config) -> - Type = proplists:get_value(type, Config, ?TYPE), - ConnectorName = proplists:get_value(connector_name, Config, <<"c">>), - ConnectorConfig = proplists:get_value(connector_config, Config, connector_config()), - ActionName = <<"bad_pre_configured_topics">>, - ActionConfig1 = proplists:get_value(action_config, Config, action_config(ConnectorName)), - ActionConfig = emqx_bridge_v2_testlib:parse_and_check( - action, - Type, - ActionName, - emqx_utils_maps:deep_merge( - ActionConfig1, - #{ - <<"parameters">> => #{ - <<"topic">> => <<"pct${.payload.n}">>, - <<"pre_configured_topics">> => [] - } - } - ) - ), - ?check_trace( - #{timetrap => 7_000}, - begin - ConnectorParams = [ - {connector_config, ConnectorConfig}, - {connector_name, ConnectorName}, - {connector_type, Type} - ], - ActionParams = [ - {action_config, ActionConfig}, - {action_name, ActionName}, - {action_type, Type} - ], - {ok, {{_, 201, _}, _, #{}}} = - emqx_bridge_v2_testlib:create_connector_api(ConnectorParams), - - {ok, {{_, 201, _}, _, #{}}} = - emqx_bridge_v2_testlib:create_action_api(ActionParams), - - ?assertMatch( - {ok, - {{_, 200, _}, _, #{ - <<"status_reason">> := - << - "Either the Kafka topic must be fixed (not a template)," - " or at least one pre-defined topic must be set." - >>, - <<"status">> := <<"disconnected">>, - <<"node_status">> := [#{<<"status">> := <<"disconnected">>}] - }}}, - emqx_bridge_v2_testlib:get_bridge_api(Type, ActionName) - ), - - ok - end, - [] - ), - ok. diff --git a/rel/i18n/emqx_bridge_azure_event_hub.hocon b/rel/i18n/emqx_bridge_azure_event_hub.hocon index e683bc9e9..7e37d2e4c 100644 --- a/rel/i18n/emqx_bridge_azure_event_hub.hocon +++ b/rel/i18n/emqx_bridge_azure_event_hub.hocon @@ -350,14 +350,4 @@ Setting this to a value which is greater than the total number of partitions in partitions_limit.label: """Max Partitions""" -producer_pre_configured_topics.label: -"""Pre-configured Event Hubs""" -producer_pre_configured_topics.desc: -"""A list of pre-configured event hubs to be used when using templates to define outgoing topics. If the topic template fails to resolve to a value due to missing data in the incoming message, or if it resolves to a topic that is not contained in this list, then publishing will fail.""" - -pre_configured_topic.label: -"""Event Hubs Name""" -pre_configured_topic.desc: -"""Event Hubs name""" - } diff --git a/rel/i18n/emqx_bridge_confluent_producer.hocon b/rel/i18n/emqx_bridge_confluent_producer.hocon index 81c2c0a89..38623502e 100644 --- a/rel/i18n/emqx_bridge_confluent_producer.hocon +++ b/rel/i18n/emqx_bridge_confluent_producer.hocon @@ -350,14 +350,4 @@ server_name_indication.desc: server_name_indication.label: """SNI""" -producer_pre_configured_topics.label: -"""Pre-configured Topics""" -producer_pre_configured_topics.desc: -"""A list of pre-configured topics to be used when using templates to define outgoing topics. If the topic template fails to resolve to a value due to missing data in the incoming message, or if it resolves to a topic that is not contained in this list, then publishing will fail.""" - -pre_configured_topic.label: -"""Kafka Topic Name""" -pre_configured_topic.desc: -"""Kafka topic name""" - } diff --git a/rel/i18n/emqx_bridge_kafka.hocon b/rel/i18n/emqx_bridge_kafka.hocon index 59896cc22..a066d30fc 100644 --- a/rel/i18n/emqx_bridge_kafka.hocon +++ b/rel/i18n/emqx_bridge_kafka.hocon @@ -446,14 +446,4 @@ server_name_indication.desc: server_name_indication.label: """SNI""" -producer_pre_configured_topics.label: -"""Pre-configured Topics""" -producer_pre_configured_topics.desc: -"""A list of pre-configured topics to be used when using templates to define outgoing topics. If the topic template fails to resolve to a value due to missing data in the incoming message, or if it resolves to a topic that is not contained in this list, then publishing will fail.""" - -pre_configured_topic.label: -"""Kafka Topic Name""" -pre_configured_topic.desc: -"""Kafka topic name""" - } From 1d56ac6e5e87c21a3986f79c061f752d95f0ea48 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 26 Jul 2024 14:26:21 -0300 Subject: [PATCH 064/131] refactor: change topic schema type --- apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl index 8f72523b1..254e84036 100644 --- a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl +++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl @@ -391,7 +391,7 @@ fields(v1_producer_kafka_opts) -> ); fields(producer_kafka_opts) -> [ - {topic, mk(string(), #{required => true, desc => ?DESC(kafka_topic)})}, + {topic, mk(emqx_schema:template(), #{required => true, desc => ?DESC(kafka_topic)})}, {message, mk(ref(kafka_message), #{required => false, desc => ?DESC(kafka_message)})}, {max_batch_bytes, mk(emqx_schema:bytesize(), #{default => <<"896KB">>, desc => ?DESC(max_batch_bytes)})}, From 268f887700eb470403fdd7f44e6c6a71b36a349b Mon Sep 17 00:00:00 2001 From: ieQu1 <99872536+ieQu1@users.noreply.github.com> Date: Fri, 26 Jul 2024 20:24:53 +0200 Subject: [PATCH 065/131] test(mgmt): Disable certain tests on OSS --- apps/emqx_enterprise/src/emqx_enterprise_schema.erl | 1 - apps/emqx_management/test/emqx_mgmt_api_cluster_SUITE.erl | 7 ++++++- .../test/emqx_mgmt_api_data_backup_SUITE.erl | 7 ++++++- apps/emqx_management/test/emqx_mgmt_data_backup_SUITE.erl | 7 ++++++- 4 files changed, 18 insertions(+), 4 deletions(-) diff --git a/apps/emqx_enterprise/src/emqx_enterprise_schema.erl b/apps/emqx_enterprise/src/emqx_enterprise_schema.erl index 571757c75..2665afcf7 100644 --- a/apps/emqx_enterprise/src/emqx_enterprise_schema.erl +++ b/apps/emqx_enterprise/src/emqx_enterprise_schema.erl @@ -5,7 +5,6 @@ -module(emqx_enterprise_schema). -behaviour(hocon_schema). --behaviour(emqx_schema_hooks). -include_lib("typerefl/include/types.hrl"). -include_lib("hocon/include/hoconsc.hrl"). diff --git a/apps/emqx_management/test/emqx_mgmt_api_cluster_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_cluster_SUITE.erl index 113468f0b..135d2103a 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_cluster_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_cluster_SUITE.erl @@ -24,7 +24,12 @@ -define(APPS, [emqx_conf, emqx_management]). all() -> - emqx_common_test_helpers:all(?MODULE). + case emqx_cth_suite:skip_if_oss() of + false -> + emqx_common_test_helpers:all(?MODULE); + True -> + True + end. init_per_suite(Config) -> Config. diff --git a/apps/emqx_management/test/emqx_mgmt_api_data_backup_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_data_backup_SUITE.erl index 6a580fd57..994d34798 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_data_backup_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_data_backup_SUITE.erl @@ -36,7 +36,12 @@ ). all() -> - emqx_common_test_helpers:all(?MODULE). + case emqx_cth_suite:skip_if_oss() of + false -> + emqx_common_test_helpers:all(?MODULE); + True -> + True + end. init_per_suite(Config) -> Config. diff --git a/apps/emqx_management/test/emqx_mgmt_data_backup_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_data_backup_SUITE.erl index 9a3d3971c..80fac8c30 100644 --- a/apps/emqx_management/test/emqx_mgmt_data_backup_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_data_backup_SUITE.erl @@ -52,7 +52,12 @@ >>). all() -> - emqx_common_test_helpers:all(?MODULE). + case emqx_cth_suite:skip_if_oss() of + false -> + emqx_common_test_helpers:all(?MODULE); + True -> + True + end. init_per_suite(Config) -> Config. From 8036baf22cafe56c1cf94d20e16e119b1d027dca Mon Sep 17 00:00:00 2001 From: ieQu1 <99872536+ieQu1@users.noreply.github.com> Date: Fri, 26 Jul 2024 21:53:03 +0200 Subject: [PATCH 066/131] test(paho): Run RLOG paho test with replicants only on EE --- .github/workflows/run_docker_tests.yaml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/run_docker_tests.yaml b/.github/workflows/run_docker_tests.yaml index 9c695c4a6..63ea41416 100644 --- a/.github/workflows/run_docker_tests.yaml +++ b/.github/workflows/run_docker_tests.yaml @@ -69,7 +69,6 @@ jobs: shell: bash env: EMQX_NAME: ${{ matrix.profile }} - _EMQX_TEST_DB_BACKEND: ${{ matrix.cluster_db_backend }} strategy: fail-fast: false @@ -78,15 +77,17 @@ jobs: - emqx - emqx-enterprise - emqx-elixir - cluster_db_backend: - - mnesia - - rlog steps: - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - name: Set up environment id: env run: | source env.sh + if [ "$EMQX_NAME" = "emqx-interprise" ]; then + _EMQX_TEST_DB_BACKEND='rlog' + else + _EMQX_TEST_DB_BACKEND='mnesia' + fi PKG_VSN=$(docker run --rm -v $(pwd):$(pwd) -w $(pwd) -u $(id -u) "$EMQX_BUILDER" ./pkg-vsn.sh "$EMQX_NAME") echo "PKG_VSN=$PKG_VSN" >> "$GITHUB_ENV" - uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7 From 0b0a28ae442ab1e46ad00a24825101f755514dc2 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Mon, 29 Jul 2024 10:45:24 +0800 Subject: [PATCH 067/131] chore: update changes/ee/feat-13504.en.md Co-authored-by: zmstone --- changes/ee/feat-13504.en.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changes/ee/feat-13504.en.md b/changes/ee/feat-13504.en.md index 20b3aa1e2..acea1241a 100644 --- a/changes/ee/feat-13504.en.md +++ b/changes/ee/feat-13504.en.md @@ -2,4 +2,4 @@ Added a HTTP backend for the authentication mechanism `scram`. Note: This is not an implementation of the RFC 7804: Salted Challenge Response HTTP Authentication Mechanism. -This backend is an implementation of scram that uses an external web resource as a source of user information. +This backend is an implementation of scram that uses an external web resource as a source of SCRAM authentication data, including stored key of the client, server key, and the salt. It support other authentication and authorization extension fields like HTTP auth backend, namely: `is_superuser`, `client_attrs`, `expire_at` and `acl`. From 6786c9b51738f30ffd85ed73145ded9913bf1cd6 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 29 Jul 2024 09:45:52 -0300 Subject: [PATCH 068/131] refactor: improve descriptions and identifiers Co-authored-by: zmstone --- .../src/emqx_bridge_kafka_impl_producer.erl | 2 +- changes/ee/feat-13452.en.md | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl index b358cd42b..fb7fce63c 100644 --- a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl +++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl @@ -839,7 +839,7 @@ handle_telemetry_event(_EventId, _Metrics, _MetaData, _HandlerConfig) -> %% metrics. -spec telemetry_handler_id(action_resource_id()) -> binary(). telemetry_handler_id(ActionResId) -> - <<"emqx-bridge-kafka-producer-", ActionResId/binary>>. + ActionResId. uninstall_telemetry_handlers(TelemetryId) -> telemetry:detach(TelemetryId). diff --git a/changes/ee/feat-13452.en.md b/changes/ee/feat-13452.en.md index 95dae8d32..7b2427329 100644 --- a/changes/ee/feat-13452.en.md +++ b/changes/ee/feat-13452.en.md @@ -1 +1,5 @@ -Added to possibility to configure a list of predefined Kafka topics to Kafka producer actions, and also to use templates to define the destination Kafka topic. +Kafka producer action's `topic` config now supports templates. + +The topics must be already created in Kafka. If a message is rendered towards a non-existing topic in Kafka (given Kafka disabled topic auto-creation), the message will fail with an unrecoverable error. Also, if a message does not contain enough information to render to the configured template (e.g.: the template is `t-${t}` and the message context does not define `t`), this message will also fail with an unrecoverable error. + +This same feature is also available for Azure Event Hubs and Confluent Platform producer integrations. From b3074144cc81f1ac4e1557c1e9aeaa9eece96ce9 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 24 Jul 2024 16:40:30 -0300 Subject: [PATCH 069/131] chore: temporarily revert `NO_DOC` changes to fields with default value = false These will be dealt with in follow up PRs, by allowing the parent struct to be set to a special `disabled` value in such cases. --- apps/emqx/src/emqx_schema.erl | 8 ++++---- apps/emqx_conf/src/emqx_conf_schema.erl | 7 ++++++- apps/emqx_dashboard_sso/src/emqx_dashboard_sso_schema.erl | 2 +- apps/emqx_ft/src/emqx_ft_schema.erl | 2 +- apps/emqx_opentelemetry/src/emqx_otel_schema.erl | 8 +++++--- apps/emqx_prometheus/src/emqx_prometheus_schema.erl | 4 ++-- apps/emqx_psk/src/emqx_psk_schema.erl | 2 +- apps/emqx_slow_subs/src/emqx_slow_subs_schema.erl | 7 ++++--- rel/config/examples/gateway.exproto.conf.example | 1 + rel/config/examples/plugins.conf.example | 2 +- 10 files changed, 26 insertions(+), 17 deletions(-) diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index eb4c7cf95..827836540 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -388,7 +388,7 @@ fields("flapping_detect") -> boolean(), #{ default => false, - importance => ?IMPORTANCE_NO_DOC, + %% importance => ?IMPORTANCE_NO_DOC, desc => ?DESC(flapping_detect_enable) } )}, @@ -455,7 +455,7 @@ fields("overload_protection") -> boolean(), #{ desc => ?DESC(overload_protection_enable), - importance => ?IMPORTANCE_NO_DOC, + %% importance => ?IMPORTANCE_NO_DOC, default => false } )}, @@ -1673,7 +1673,7 @@ fields("durable_sessions") -> sc( boolean(), #{ desc => ?DESC(durable_sessions_enable), - importance => ?IMPORTANCE_NO_DOC, + %% importance => ?IMPORTANCE_NO_DOC, default => false } )}, @@ -2426,7 +2426,7 @@ client_ssl_opts_schema(Defaults) -> boolean(), #{ default => false, - importance => ?IMPORTANCE_NO_DOC, + %% importance => ?IMPORTANCE_NO_DOC, desc => ?DESC(client_ssl_opts_schema_enable) } )}, diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index a83349ff0..1ae32172b 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -1271,6 +1271,11 @@ log_handler_common_confs(Handler, Default) -> EnvValue = os:getenv("EMQX_DEFAULT_LOG_HANDLER"), Enable = lists:member(EnvValue, EnableValues), LevelDesc = maps:get(level_desc, Default, "common_handler_level"), + EnableImportance = + case Enable of + true -> ?IMPORTANCE_NO_DOC; + false -> ?IMPORTANCE_MEDIUM + end, [ {"level", sc( @@ -1287,7 +1292,7 @@ log_handler_common_confs(Handler, Default) -> #{ default => Enable, desc => ?DESC("common_handler_enable"), - importance => ?IMPORTANCE_NO_DOC + importance => EnableImportance } )}, {"formatter", diff --git a/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_schema.erl b/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_schema.erl index 314839ee4..f64c7a7df 100644 --- a/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_schema.erl +++ b/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_schema.erl @@ -47,7 +47,7 @@ common_backend_schema(Backend) -> mk( boolean(), #{ desc => ?DESC(backend_enable), - importance => ?IMPORTANCE_NO_DOC, + %% importance => ?IMPORTANCE_NO_DOC, required => false, default => false } diff --git a/apps/emqx_ft/src/emqx_ft_schema.erl b/apps/emqx_ft/src/emqx_ft_schema.erl index 770b0509c..f779121d3 100644 --- a/apps/emqx_ft/src/emqx_ft_schema.erl +++ b/apps/emqx_ft/src/emqx_ft_schema.erl @@ -66,7 +66,7 @@ fields(file_transfer) -> boolean(), #{ desc => ?DESC("enable"), - importance => ?IMPORTANCE_NO_DOC, + %% importance => ?IMPORTANCE_NO_DOC, required => false, default => false } diff --git a/apps/emqx_opentelemetry/src/emqx_otel_schema.erl b/apps/emqx_opentelemetry/src/emqx_otel_schema.erl index d4c6dca0a..a12efc7a9 100644 --- a/apps/emqx_opentelemetry/src/emqx_otel_schema.erl +++ b/apps/emqx_opentelemetry/src/emqx_otel_schema.erl @@ -72,7 +72,7 @@ fields("otel_metrics") -> boolean(), #{ default => false, - importance => ?IMPORTANCE_NO_DOC, + %% importance => ?IMPORTANCE_NO_DOC, required => true, desc => ?DESC(enable) } @@ -105,7 +105,8 @@ fields("otel_logs") -> #{ default => false, desc => ?DESC(enable), - importance => ?IMPORTANCE_NO_DOC + %% importance => ?IMPORTANCE_NO_DOC + importance => ?IMPORTANCE_HIGH } )}, {max_queue_size, @@ -144,7 +145,8 @@ fields("otel_traces") -> #{ default => false, desc => ?DESC(enable), - importance => ?IMPORTANCE_NO_DOC + %% importance => ?IMPORTANCE_NO_DOC + importance => ?IMPORTANCE_HIGH } )}, {max_queue_size, diff --git a/apps/emqx_prometheus/src/emqx_prometheus_schema.erl b/apps/emqx_prometheus/src/emqx_prometheus_schema.erl index 09f19bdb1..6d3503a82 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus_schema.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus_schema.erl @@ -78,7 +78,7 @@ fields(push_gateway) -> #{ default => false, required => true, - importance => ?IMPORTANCE_NO_DOC, + %% importance => ?IMPORTANCE_NO_DOC, desc => ?DESC(push_gateway_enable) } )}, @@ -230,7 +230,7 @@ fields(legacy_deprecated_setting) -> #{ default => false, required => true, - importance => ?IMPORTANCE_NO_DOC, + %% importance => ?IMPORTANCE_NO_DOC, desc => ?DESC(legacy_enable) } )}, diff --git a/apps/emqx_psk/src/emqx_psk_schema.erl b/apps/emqx_psk/src/emqx_psk_schema.erl index 286877d99..800845923 100644 --- a/apps/emqx_psk/src/emqx_psk_schema.erl +++ b/apps/emqx_psk/src/emqx_psk_schema.erl @@ -42,7 +42,7 @@ fields() -> [ {enable, ?HOCON(boolean(), #{ - importance => ?IMPORTANCE_NO_DOC, + %% importance => ?IMPORTANCE_NO_DOC, default => false, require => true, desc => ?DESC(enable) diff --git a/apps/emqx_slow_subs/src/emqx_slow_subs_schema.erl b/apps/emqx_slow_subs/src/emqx_slow_subs_schema.erl index febb0af0a..517457930 100644 --- a/apps/emqx_slow_subs/src/emqx_slow_subs_schema.erl +++ b/apps/emqx_slow_subs/src/emqx_slow_subs_schema.erl @@ -27,7 +27,8 @@ roots() -> fields("slow_subs") -> [ - {enable, sc(boolean(), false, enable, ?IMPORTANCE_NO_DOC)}, + %% {enable, sc(boolean(), false, enable, ?IMPORTANCE_NO_DOC)}, + {enable, sc(boolean(), false, enable)}, {threshold, sc( %% not used in a `receive ... after' block, just timestamp comparison @@ -67,5 +68,5 @@ desc(_) -> sc(Type, Default, Desc) -> ?HOCON(Type, #{default => Default, desc => ?DESC(Desc)}). -sc(Type, Default, Desc, Importance) -> - ?HOCON(Type, #{default => Default, desc => ?DESC(Desc), importance => Importance}). +%% sc(Type, Default, Desc, Importance) -> +%% ?HOCON(Type, #{default => Default, desc => ?DESC(Desc), importance => Importance}). diff --git a/rel/config/examples/gateway.exproto.conf.example b/rel/config/examples/gateway.exproto.conf.example index 303bd2872..fcedb944b 100644 --- a/rel/config/examples/gateway.exproto.conf.example +++ b/rel/config/examples/gateway.exproto.conf.example @@ -16,6 +16,7 @@ gateway.exproto { ## Configurations for request to ConnectionHandler service handler { address = "http://127.0.0.1:9001" + ssl_options {enable = false} } listeners.tcp.default { diff --git a/rel/config/examples/plugins.conf.example b/rel/config/examples/plugins.conf.example index 9388f4f24..b7673036e 100644 --- a/rel/config/examples/plugins.conf.example +++ b/rel/config/examples/plugins.conf.example @@ -10,7 +10,7 @@ plugins { ## Note: name and version should be what it is in the plugin application name_vsn = "my_acl-0.1.0", }, - {name_vsn = "my_rule-0.1.1"} + {name_vsn = "my_rule-0.1.1", enable = false} ] ## The installation directory for the external plugins From e80d43d14d8b7386e9ae6d6481ca5009bf4e391e Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Thu, 25 Jul 2024 09:39:36 -0300 Subject: [PATCH 070/131] test(fix): use ebin path without plugins Without the filtering that already exists in cth:ebin_path, the rebar3 plugins path may take priority over normal dependencies. Since we just updated hocon, and there seems to be an older hocon among the rebar3 plugins, it started to break the test because older hocon was getting loaded in the peer. --- apps/emqx/test/emqx_common_test_helpers.erl | 1 + apps/emqx/test/emqx_shared_sub_SUITE.erl | 5 +---- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index ce3c2543c..a33178d0a 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -78,6 +78,7 @@ start_epmd/0, start_peer/2, stop_peer/1, + ebin_path/0, listener_port/2 ]). diff --git a/apps/emqx/test/emqx_shared_sub_SUITE.erl b/apps/emqx/test/emqx_shared_sub_SUITE.erl index 040b3d295..e15aca6a1 100644 --- a/apps/emqx/test/emqx_shared_sub_SUITE.erl +++ b/apps/emqx/test/emqx_shared_sub_SUITE.erl @@ -1247,7 +1247,7 @@ recv_msgs(Count, Msgs) -> start_peer(Name, Port) -> {ok, Node} = emqx_cth_peer:start_link( Name, - ebin_path() + emqx_common_test_helpers:ebin_path() ), pong = net_adm:ping(Node), setup_node(Node, Port), @@ -1261,9 +1261,6 @@ host() -> [_, Host] = string:tokens(atom_to_list(node()), "@"), Host. -ebin_path() -> - ["-pa" | code:get_path()]. - setup_node(Node, Port) -> EnvHandler = fun(_) -> From f85db0a0e964496da36632e8a9824d066dce475c Mon Sep 17 00:00:00 2001 From: ieQu1 <99872536+ieQu1@users.noreply.github.com> Date: Mon, 29 Jul 2024 17:22:41 +0200 Subject: [PATCH 071/131] fix: Apply suggestions from code review Co-authored-by: Thales Macedo Garitezi --- .github/workflows/run_docker_tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run_docker_tests.yaml b/.github/workflows/run_docker_tests.yaml index 63ea41416..17d5395b2 100644 --- a/.github/workflows/run_docker_tests.yaml +++ b/.github/workflows/run_docker_tests.yaml @@ -83,7 +83,7 @@ jobs: id: env run: | source env.sh - if [ "$EMQX_NAME" = "emqx-interprise" ]; then + if [ "$EMQX_NAME" = "emqx-enterprise" ]; then _EMQX_TEST_DB_BACKEND='rlog' else _EMQX_TEST_DB_BACKEND='mnesia' From 693d5dd39425ad409f49d42d7c43406e1d3bf1c9 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Mon, 29 Jul 2024 11:37:33 -0300 Subject: [PATCH 072/131] feat: attempt to automatically decode `payload` similar to key and message templates --- .../src/emqx_bridge_kafka_impl_producer.erl | 2 +- .../test/emqx_bridge_v2_kafka_producer_SUITE.erl | 7 +------ 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl index fb7fce63c..a8e1b56dd 100644 --- a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl +++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl @@ -469,7 +469,7 @@ render_topic({fixed, KafkaTopic}, _Message) -> KafkaTopic; render_topic({dynamic, Template}, Message) -> try - iolist_to_binary(emqx_template:render_strict(Template, Message)) + iolist_to_binary(emqx_template:render_strict(Template, {emqx_jsonish, Message})) catch error:_Errors -> throw(bad_topic) diff --git a/apps/emqx_bridge_kafka/test/emqx_bridge_v2_kafka_producer_SUITE.erl b/apps/emqx_bridge_kafka/test/emqx_bridge_v2_kafka_producer_SUITE.erl index 08b2723e7..baa5368cf 100644 --- a/apps/emqx_bridge_kafka/test/emqx_bridge_v2_kafka_producer_SUITE.erl +++ b/apps/emqx_bridge_kafka/test/emqx_bridge_v2_kafka_producer_SUITE.erl @@ -967,12 +967,7 @@ t_dynamic_topics(Config) -> RuleTopic, [ {bridge_name, ActionName} - ], - #{ - sql => - <<"select *, json_decode(payload) as payload from \"", RuleTopic/binary, - "\" ">> - } + ] ), ?assertStatusAPI(Type, ActionName, <<"connected">>), From 8dc1d1424a704898a205f4056129f435c545d884 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 17 Jul 2024 16:05:16 +0800 Subject: [PATCH 073/131] chore: add resource tag for log --- apps/emqx_resource/include/emqx_resource.hrl | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/apps/emqx_resource/include/emqx_resource.hrl b/apps/emqx_resource/include/emqx_resource.hrl index 8c2bb39a1..6603f7708 100644 --- a/apps/emqx_resource/include/emqx_resource.hrl +++ b/apps/emqx_resource/include/emqx_resource.hrl @@ -167,4 +167,12 @@ ). -define(TAG, "RESOURCE"). +-define(LOG_LEVEL(_L_), + case _L_ of + true -> info; + false -> warning + end +). +-define(TAG, "RESOURCE"). + -define(RESOURCE_ALLOCATION_TAB, emqx_resource_allocations). From 2924ec582a111a0cd1018c18b54c4f47c0d2e676 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 17 Jul 2024 16:29:18 +0800 Subject: [PATCH 074/131] feat: add unrecoverable_resource_error throttle --- apps/emqx_conf/src/emqx_conf_schema.erl | 3 +- .../src/emqx_resource_buffer_worker.erl | 68 ++++++++++++------- 2 files changed, 46 insertions(+), 25 deletions(-) diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index 4c72b74b1..505aff3e3 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -80,7 +80,8 @@ cannot_publish_to_topic_due_to_not_authorized, cannot_publish_to_topic_due_to_quota_exceeded, connection_rejected_due_to_license_limit_reached, - dropped_msg_due_to_mqueue_is_full + dropped_msg_due_to_mqueue_is_full, + unrecoverable_resource_error ]). %% Callback to upgrade config after loaded from config file but before validation. diff --git a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl index 05d42ed1a..7419820f7 100644 --- a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl +++ b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl @@ -298,10 +298,10 @@ running(info, {flush_metrics, _Ref}, _Data) -> running(info, {'DOWN', _MRef, process, Pid, Reason}, Data0 = #{async_workers := AsyncWorkers0}) when is_map_key(Pid, AsyncWorkers0) -> - ?SLOG(info, #{msg => "async_worker_died", state => running, reason => Reason}), + ?SLOG(info, #{msg => "async_worker_died", state => running, reason => Reason}, #{tag => ?TAG}), handle_async_worker_down(Data0, Pid); running(info, Info, _St) -> - ?SLOG(error, #{msg => "unexpected_msg", state => running, info => Info}), + ?SLOG(error, #{msg => "unexpected_msg", state => running, info => Info}, #{tag => ?TAG}), keep_state_and_data. blocked(enter, _, #{resume_interval := ResumeT} = St0) -> @@ -331,10 +331,10 @@ blocked(info, {flush_metrics, _Ref}, _Data) -> blocked(info, {'DOWN', _MRef, process, Pid, Reason}, Data0 = #{async_workers := AsyncWorkers0}) when is_map_key(Pid, AsyncWorkers0) -> - ?SLOG(info, #{msg => "async_worker_died", state => blocked, reason => Reason}), + ?SLOG(info, #{msg => "async_worker_died", state => blocked, reason => Reason}, #{tag => ?TAG}), handle_async_worker_down(Data0, Pid); blocked(info, Info, _Data) -> - ?SLOG(error, #{msg => "unexpected_msg", state => blocked, info => Info}), + ?SLOG(error, #{msg => "unexpected_msg", state => blocked, info => Info}, #{tag => ?TAG}), keep_state_and_data. terminate(_Reason, #{id := Id, index := Index, queue := Q}) -> @@ -981,7 +981,11 @@ handle_query_result_pure(Id, {error, Reason} = Error, HasBeenSent, TraceCTX) -> true -> PostFn = fun() -> - ?SLOG(error, #{id => Id, msg => "unrecoverable_error", reason => Reason}), + ?SLOG_THROTTLE(error, #{ + resource_id => Id, + msg => unrecoverable_resource_error, + reason => Reason + }), ok end, Counters = @@ -1021,7 +1025,11 @@ handle_query_async_result_pure(Id, {error, Reason} = Error, HasBeenSent, TraceCT true -> PostFn = fun() -> - ?SLOG(error, #{id => Id, msg => "unrecoverable_error", reason => Reason}), + ?SLOG_THROTTLE(error, #{ + resource_id => Id, + msg => unrecoverable_resource_error, + reason => Reason + }), ok end, Counters = @@ -1141,12 +1149,16 @@ log_expired_message_count(_Data = #{id := Id, index := Index, counters := Counte false -> ok; true -> - ?SLOG(info, #{ - msg => "buffer_worker_dropped_expired_messages", - resource_id => Id, - worker_index => Index, - expired_count => ExpiredCount - }), + ?SLOG( + info, + #{ + msg => "buffer_worker_dropped_expired_messages", + resource_id => Id, + worker_index => Index, + expired_count => ExpiredCount + }, + #{tag => ?TAG} + ), ok end. @@ -1556,7 +1568,7 @@ handle_async_reply1( case is_expired(ExpireAt, Now) of true -> IsAcked = ack_inflight(InflightTID, Ref, BufferWorkerPid), - %% evalutate metrics call here since we're not inside + %% evaluate metrics call here since we're not inside %% buffer worker IsAcked andalso begin @@ -1797,12 +1809,16 @@ append_queue(Id, Index, Q, Queries) -> ok = replayq:ack(Q1, QAckRef), Dropped = length(Items2), Counters = #{dropped_queue_full => Dropped}, - ?SLOG(info, #{ - msg => "buffer_worker_overflow", - resource_id => Id, - worker_index => Index, - dropped => Dropped - }), + ?SLOG( + info, + #{ + msg => "buffer_worker_overflow", + resource_id => Id, + worker_index => Index, + dropped => Dropped + }, + #{tag => ?TAG} + ), {Items2, Q1, Counters} end, ?tp( @@ -2236,11 +2252,15 @@ adjust_batch_time(Id, RequestTTL, BatchTime0) -> BatchTime = max(0, min(BatchTime0, RequestTTL div 2)), case BatchTime =:= BatchTime0 of false -> - ?SLOG(info, #{ - id => Id, - msg => "adjusting_buffer_worker_batch_time", - new_batch_time => BatchTime - }); + ?SLOG( + info, + #{ + resource_id => Id, + msg => "adjusting_buffer_worker_batch_time", + new_batch_time => BatchTime + }, + #{tag => ?TAG} + ); true -> ok end, From f6f1d32da0ac8d7a21b95ae3d8eb616871bebb14 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Thu, 25 Jul 2024 13:06:35 +0800 Subject: [PATCH 075/131] feat: throttle with resource_id --- apps/emqx/include/logger.hrl | 8 +- apps/emqx/src/emqx_log_throttler.erl | 80 +++++++++++++++---- apps/emqx/test/emqx_log_throttler_SUITE.erl | 69 +++++++++++++--- apps/emqx_resource/include/emqx_resource.hrl | 8 -- .../src/emqx_resource_buffer_worker.erl | 30 ++++--- 5 files changed, 149 insertions(+), 46 deletions(-) diff --git a/apps/emqx/include/logger.hrl b/apps/emqx/include/logger.hrl index 31fe0e36a..a7455418d 100644 --- a/apps/emqx/include/logger.hrl +++ b/apps/emqx/include/logger.hrl @@ -38,16 +38,20 @@ ). %% NOTE: do not forget to use atom for msg and add every used msg to -%% the default value of `log.thorttling.msgs` list. +%% the default value of `log.throttling.msgs` list. -define(SLOG_THROTTLE(Level, Data), ?SLOG_THROTTLE(Level, Data, #{}) ). -define(SLOG_THROTTLE(Level, Data, Meta), + ?SLOG_THROTTLE(Level, undefined, Data, Meta) +). + +-define(SLOG_THROTTLE(Level, UniqueKey, Data, Meta), case logger:allow(Level, ?MODULE) of true -> (fun(#{msg := __Msg} = __Data) -> - case emqx_log_throttler:allow(__Msg) of + case emqx_log_throttler:allow(__Msg, UniqueKey) of true -> logger:log(Level, __Data, Meta); false -> diff --git a/apps/emqx/src/emqx_log_throttler.erl b/apps/emqx/src/emqx_log_throttler.erl index 3ebc268fa..008bd1663 100644 --- a/apps/emqx/src/emqx_log_throttler.erl +++ b/apps/emqx/src/emqx_log_throttler.erl @@ -25,7 +25,7 @@ -export([start_link/0]). %% throttler API --export([allow/1]). +-export([allow/2]). %% gen_server callbacks -export([ @@ -40,23 +40,22 @@ -define(SEQ_ID(Msg), {?MODULE, Msg}). -define(NEW_SEQ, atomics:new(1, [{signed, false}])). -define(GET_SEQ(Msg), persistent_term:get(?SEQ_ID(Msg), undefined)). +-define(ERASE_SEQ(Msg), persistent_term:erase(?SEQ_ID(Msg))). -define(RESET_SEQ(SeqRef), atomics:put(SeqRef, 1, 0)). -define(INC_SEQ(SeqRef), atomics:add(SeqRef, 1, 1)). -define(GET_DROPPED(SeqRef), atomics:get(SeqRef, 1) - 1). -define(IS_ALLOWED(SeqRef), atomics:add_get(SeqRef, 1, 1) =:= 1). --define(NEW_THROTTLE(Msg, SeqRef), persistent_term:put(?SEQ_ID(Msg), SeqRef)). - -define(MSGS_LIST, emqx:get_config([log, throttling, msgs], [])). -define(TIME_WINDOW_MS, timer:seconds(emqx:get_config([log, throttling, time_window], 60))). --spec allow(atom()) -> boolean(). -allow(Msg) when is_atom(Msg) -> +-spec allow(atom(), any()) -> boolean(). +allow(Msg, UniqueKey) when is_atom(Msg) -> case emqx_logger:get_primary_log_level() of debug -> true; _ -> - do_allow(Msg) + do_allow(Msg, UniqueKey) end. -spec start_link() -> startlink_ret(). @@ -68,7 +67,8 @@ start_link() -> %%-------------------------------------------------------------------- init([]) -> - ok = lists:foreach(fun(Msg) -> ?NEW_THROTTLE(Msg, ?NEW_SEQ) end, ?MSGS_LIST), + process_flag(trap_exit, true), + ok = lists:foreach(fun(Msg) -> new_throttler(Msg) end, ?MSGS_LIST), CurrentPeriodMs = ?TIME_WINDOW_MS, TimerRef = schedule_refresh(CurrentPeriodMs), {ok, #{timer_ref => TimerRef, current_period_ms => CurrentPeriodMs}}. @@ -88,14 +88,19 @@ handle_info(refresh, #{current_period_ms := PeriodMs} = State) -> case ?GET_SEQ(Msg) of %% Should not happen, unless the static ids list is updated at run-time. undefined -> - ?NEW_THROTTLE(Msg, ?NEW_SEQ), + new_throttler(Msg), ?tp(log_throttler_new_msg, #{throttled_msg => Msg}), Acc; + SeqMap when is_map(SeqMap) -> + maps:fold( + fun(Key, Ref, Acc0) -> + drop_stats(Ref, emqx_utils:format("~ts:~s", [Msg, Key]), Acc0) + end, + Acc, + SeqMap + ); SeqRef -> - Dropped = ?GET_DROPPED(SeqRef), - ok = ?RESET_SEQ(SeqRef), - ?tp(log_throttler_dropped, #{dropped_count => Dropped, throttled_msg => Msg}), - maybe_add_dropped(Msg, Dropped, Acc) + drop_stats(SeqRef, Msg, Acc) end end, #{}, @@ -112,7 +117,34 @@ handle_info(Info, State) -> ?SLOG(error, #{msg => "unxpected_info", info => Info}), {noreply, State}. +drop_stats(SeqRef, Msg, Acc) -> + Dropped = ?GET_DROPPED(SeqRef), + ok = ?RESET_SEQ(SeqRef), + ?tp(log_throttler_dropped, #{dropped_count => Dropped, throttled_msg => Msg}), + maybe_add_dropped(Msg, Dropped, Acc). + terminate(_Reason, _State) -> + lists:foreach( + fun(Msg) -> + case ?GET_SEQ(Msg) of + undefined -> + ok; + SeqMap when is_map(SeqMap) -> + maps:foreach( + fun(_, Ref) -> + ok = ?RESET_SEQ(Ref) + end, + SeqMap + ); + SeqRef -> + %% atomics don't have erase API... + %% (if nobody hold the ref, the atomics should erase automatically?) + ok = ?RESET_SEQ(SeqRef) + end, + ?ERASE_SEQ(Msg) + end, + ?MSGS_LIST + ), ok. code_change(_OldVsn, State, _Extra) -> @@ -122,17 +154,27 @@ code_change(_OldVsn, State, _Extra) -> %% internal functions %%-------------------------------------------------------------------- -do_allow(Msg) -> +do_allow(Msg, UniqueKey) -> case persistent_term:get(?SEQ_ID(Msg), undefined) of undefined -> %% This is either a race condition (emqx_log_throttler is not started yet) %% or a developer mistake (msg used in ?SLOG_THROTTLE/2,3 macro is %% not added to the default value of `log.throttling.msgs`. - ?SLOG(info, #{ - msg => "missing_log_throttle_sequence", + ?SLOG(debug, #{ + msg => "log_throttle_disabled", throttled_msg => Msg }), true; + %% e.g: unrecoverable msg throttle according resource_id + SeqMap when is_map(SeqMap) -> + case maps:find(UniqueKey, SeqMap) of + {ok, SeqRef} -> + ?IS_ALLOWED(SeqRef); + error -> + SeqRef = ?NEW_SEQ, + new_throttler(Msg, SeqMap#{UniqueKey => SeqRef}), + true + end; SeqRef -> ?IS_ALLOWED(SeqRef) end. @@ -154,3 +196,11 @@ maybe_log_dropped(_DroppedStats, _PeriodMs) -> schedule_refresh(PeriodMs) -> ?tp(log_throttler_sched_refresh, #{new_period_ms => PeriodMs}), erlang:send_after(PeriodMs, ?MODULE, refresh). + +new_throttler(unrecoverable_resource_error = Msg) -> + persistent_term:put(?SEQ_ID(Msg), #{}); +new_throttler(Msg) -> + persistent_term:put(?SEQ_ID(Msg), ?NEW_SEQ). + +new_throttler(Msg, Map) -> + persistent_term:put(?SEQ_ID(Msg), Map). diff --git a/apps/emqx/test/emqx_log_throttler_SUITE.erl b/apps/emqx/test/emqx_log_throttler_SUITE.erl index 8b3ac0207..23150a3b1 100644 --- a/apps/emqx/test/emqx_log_throttler_SUITE.erl +++ b/apps/emqx/test/emqx_log_throttler_SUITE.erl @@ -26,6 +26,7 @@ %% Have to use real msgs, as the schema is guarded by enum. -define(THROTTLE_MSG, authorization_permission_denied). -define(THROTTLE_MSG1, cannot_publish_to_topic_due_to_not_authorized). +-define(THROTTLE_UNRECOVERABLE_MSG, unrecoverable_resource_error). -define(TIME_WINDOW, <<"1s">>). all() -> emqx_common_test_helpers:all(?MODULE). @@ -59,6 +60,11 @@ end_per_suite(Config) -> emqx_cth_suite:stop(?config(suite_apps, Config)), emqx_config:delete_override_conf_files(). +init_per_testcase(t_throttle_recoverable_msg, Config) -> + ok = snabbkaffe:start_trace(), + [?THROTTLE_MSG] = Conf = emqx:get_config([log, throttling, msgs]), + {ok, _} = emqx_conf:update([log, throttling, msgs], [?THROTTLE_UNRECOVERABLE_MSG | Conf], #{}), + Config; init_per_testcase(t_throttle_add_new_msg, Config) -> ok = snabbkaffe:start_trace(), [?THROTTLE_MSG] = Conf = emqx:get_config([log, throttling, msgs]), @@ -72,6 +78,10 @@ init_per_testcase(_TC, Config) -> ok = snabbkaffe:start_trace(), Config. +end_per_testcase(t_throttle_recoverable_msg, _Config) -> + ok = snabbkaffe:stop(), + {ok, _} = emqx_conf:update([log, throttling, msgs], [?THROTTLE_MSG], #{}), + ok; end_per_testcase(t_throttle_add_new_msg, _Config) -> ok = snabbkaffe:stop(), {ok, _} = emqx_conf:update([log, throttling, msgs], [?THROTTLE_MSG], #{}), @@ -101,8 +111,8 @@ t_throttle(_Config) -> 5000 ), - ?assert(emqx_log_throttler:allow(?THROTTLE_MSG)), - ?assertNot(emqx_log_throttler:allow(?THROTTLE_MSG)), + ?assert(emqx_log_throttler:allow(?THROTTLE_MSG, undefined)), + ?assertNot(emqx_log_throttler:allow(?THROTTLE_MSG, undefined)), {ok, _} = ?block_until( #{ ?snk_kind := log_throttler_dropped, @@ -115,14 +125,48 @@ t_throttle(_Config) -> [] ). +t_throttle_recoverable_msg(_Config) -> + ResourceId = <<"resource_id">>, + ThrottledMsg = emqx_utils:format("~ts:~s", [?THROTTLE_UNRECOVERABLE_MSG, ResourceId]), + ?check_trace( + begin + %% Warm-up and block to increase the probability that next events + %% will be in the same throttling time window. + {ok, _} = ?block_until( + #{?snk_kind := log_throttler_new_msg, throttled_msg := ?THROTTLE_UNRECOVERABLE_MSG}, + 5000 + ), + {_, {ok, _}} = ?wait_async_action( + events(?THROTTLE_UNRECOVERABLE_MSG, ResourceId), + #{ + ?snk_kind := log_throttler_dropped, + throttled_msg := ThrottledMsg + }, + 5000 + ), + + ?assert(emqx_log_throttler:allow(?THROTTLE_UNRECOVERABLE_MSG, ResourceId)), + ?assertNot(emqx_log_throttler:allow(?THROTTLE_UNRECOVERABLE_MSG, ResourceId)), + {ok, _} = ?block_until( + #{ + ?snk_kind := log_throttler_dropped, + throttled_msg := ThrottledMsg, + dropped_count := 1 + }, + 3000 + ) + end, + [] + ). + t_throttle_add_new_msg(_Config) -> ?check_trace( begin {ok, _} = ?block_until( #{?snk_kind := log_throttler_new_msg, throttled_msg := ?THROTTLE_MSG1}, 5000 ), - ?assert(emqx_log_throttler:allow(?THROTTLE_MSG1)), - ?assertNot(emqx_log_throttler:allow(?THROTTLE_MSG1)), + ?assert(emqx_log_throttler:allow(?THROTTLE_MSG1, undefined)), + ?assertNot(emqx_log_throttler:allow(?THROTTLE_MSG1, undefined)), {ok, _} = ?block_until( #{ ?snk_kind := log_throttler_dropped, @@ -137,8 +181,8 @@ t_throttle_add_new_msg(_Config) -> t_throttle_no_msg(_Config) -> %% Must simply pass with no crashes - ?assert(emqx_log_throttler:allow(no_test_throttle_msg)), - ?assert(emqx_log_throttler:allow(no_test_throttle_msg)), + ?assert(emqx_log_throttler:allow(no_test_throttle_msg, undefined)), + ?assert(emqx_log_throttler:allow(no_test_throttle_msg, undefined)), timer:sleep(10), ?assert(erlang:is_process_alive(erlang:whereis(emqx_log_throttler))). @@ -168,8 +212,8 @@ t_throttle_debug_primary_level(_Config) -> #{?snk_kind := log_throttler_dropped, throttled_msg := ?THROTTLE_MSG}, 5000 ), - ?assert(emqx_log_throttler:allow(?THROTTLE_MSG)), - ?assertNot(emqx_log_throttler:allow(?THROTTLE_MSG)), + ?assert(emqx_log_throttler:allow(?THROTTLE_MSG, undefined)), + ?assertNot(emqx_log_throttler:allow(?THROTTLE_MSG, undefined)), {ok, _} = ?block_until( #{ ?snk_kind := log_throttler_dropped, @@ -187,10 +231,13 @@ t_throttle_debug_primary_level(_Config) -> %%-------------------------------------------------------------------- events(Msg) -> - events(100, Msg). + events(100, Msg, undefined). -events(N, Msg) -> - [emqx_log_throttler:allow(Msg) || _ <- lists:seq(1, N)]. +events(Msg, Id) -> + events(100, Msg, Id). + +events(N, Msg, Id) -> + [emqx_log_throttler:allow(Msg, Id) || _ <- lists:seq(1, N)]. module_exists(Mod) -> case erlang:module_loaded(Mod) of diff --git a/apps/emqx_resource/include/emqx_resource.hrl b/apps/emqx_resource/include/emqx_resource.hrl index 6603f7708..8c2bb39a1 100644 --- a/apps/emqx_resource/include/emqx_resource.hrl +++ b/apps/emqx_resource/include/emqx_resource.hrl @@ -167,12 +167,4 @@ ). -define(TAG, "RESOURCE"). --define(LOG_LEVEL(_L_), - case _L_ of - true -> info; - false -> warning - end -). --define(TAG, "RESOURCE"). - -define(RESOURCE_ALLOCATION_TAB, emqx_resource_allocations). diff --git a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl index 7419820f7..a203247e9 100644 --- a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl +++ b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl @@ -981,11 +981,16 @@ handle_query_result_pure(Id, {error, Reason} = Error, HasBeenSent, TraceCTX) -> true -> PostFn = fun() -> - ?SLOG_THROTTLE(error, #{ - resource_id => Id, - msg => unrecoverable_resource_error, - reason => Reason - }), + ?SLOG_THROTTLE( + error, + Id, + #{ + resource_id => Id, + msg => unrecoverable_resource_error, + reason => Reason + }, + #{tag => ?TAG} + ), ok end, Counters = @@ -1025,11 +1030,16 @@ handle_query_async_result_pure(Id, {error, Reason} = Error, HasBeenSent, TraceCT true -> PostFn = fun() -> - ?SLOG_THROTTLE(error, #{ - resource_id => Id, - msg => unrecoverable_resource_error, - reason => Reason - }), + ?SLOG_THROTTLE( + error, + Id, + #{ + resource_id => Id, + msg => unrecoverable_resource_error, + reason => Reason + }, + #{tag => ?TAG} + ), ok end, Counters = From e08425e67d82e357adb0250df96e0999baae0f4e Mon Sep 17 00:00:00 2001 From: zmstone Date: Fri, 26 Jul 2024 15:16:07 +0200 Subject: [PATCH 076/131] refactor(log-throttler): remove unnecessary code there is no need to reset counters before erasing --- apps/emqx/src/emqx_log_throttler.erl | 50 +++++++++------------ apps/emqx/test/emqx_log_throttler_SUITE.erl | 11 +++-- 2 files changed, 28 insertions(+), 33 deletions(-) diff --git a/apps/emqx/src/emqx_log_throttler.erl b/apps/emqx/src/emqx_log_throttler.erl index 008bd1663..928580e2b 100644 --- a/apps/emqx/src/emqx_log_throttler.erl +++ b/apps/emqx/src/emqx_log_throttler.erl @@ -49,8 +49,15 @@ -define(MSGS_LIST, emqx:get_config([log, throttling, msgs], [])). -define(TIME_WINDOW_MS, timer:seconds(emqx:get_config([log, throttling, time_window], 60))). --spec allow(atom(), any()) -> boolean(). -allow(Msg, UniqueKey) when is_atom(Msg) -> +%% @doc Check if a throttled log message is allowed to pass down to the logger this time. +%% The Msg has to be an atom, and the second argument `UniqueKey' should be `undefined' +%% for predefined message IDs. +%% For relatively static resources created from configurations such as data integration +%% resource IDs `UniqueKey' should be of `binary()' type. +-spec allow(atom(), undefined | binary()) -> boolean(). +allow(Msg, UniqueKey) when + is_atom(Msg) andalso (is_binary(UniqueKey) orelse UniqueKey =:= undefined) +-> case emqx_logger:get_primary_log_level() of debug -> true; @@ -68,7 +75,7 @@ start_link() -> init([]) -> process_flag(trap_exit, true), - ok = lists:foreach(fun(Msg) -> new_throttler(Msg) end, ?MSGS_LIST), + ok = lists:foreach(fun new_throttler/1, ?MSGS_LIST), CurrentPeriodMs = ?TIME_WINDOW_MS, TimerRef = schedule_refresh(CurrentPeriodMs), {ok, #{timer_ref => TimerRef, current_period_ms => CurrentPeriodMs}}. @@ -86,15 +93,16 @@ handle_info(refresh, #{current_period_ms := PeriodMs} = State) -> DroppedStats = lists:foldl( fun(Msg, Acc) -> case ?GET_SEQ(Msg) of - %% Should not happen, unless the static ids list is updated at run-time. undefined -> + %% Should not happen, unless the static ids list is updated at run-time. new_throttler(Msg), ?tp(log_throttler_new_msg, #{throttled_msg => Msg}), Acc; SeqMap when is_map(SeqMap) -> maps:fold( fun(Key, Ref, Acc0) -> - drop_stats(Ref, emqx_utils:format("~ts:~s", [Msg, Key]), Acc0) + ID = iolist_to_binary([atom_to_binary(Msg), $:, Key]), + drop_stats(Ref, ID, Acc0) end, Acc, SeqMap @@ -124,27 +132,9 @@ drop_stats(SeqRef, Msg, Acc) -> maybe_add_dropped(Msg, Dropped, Acc). terminate(_Reason, _State) -> - lists:foreach( - fun(Msg) -> - case ?GET_SEQ(Msg) of - undefined -> - ok; - SeqMap when is_map(SeqMap) -> - maps:foreach( - fun(_, Ref) -> - ok = ?RESET_SEQ(Ref) - end, - SeqMap - ); - SeqRef -> - %% atomics don't have erase API... - %% (if nobody hold the ref, the atomics should erase automatically?) - ok = ?RESET_SEQ(SeqRef) - end, - ?ERASE_SEQ(Msg) - end, - ?MSGS_LIST - ), + %% atomics do not have delete/remove/release/deallocate API + %% after the reference is garbage-collected the resource is released + lists:foreach(fun(Msg) -> ?ERASE_SEQ(Msg) end, ?MSGS_LIST), ok. code_change(_OldVsn, State, _Extra) -> @@ -198,9 +188,9 @@ schedule_refresh(PeriodMs) -> erlang:send_after(PeriodMs, ?MODULE, refresh). new_throttler(unrecoverable_resource_error = Msg) -> - persistent_term:put(?SEQ_ID(Msg), #{}); + new_throttler(Msg, #{}); new_throttler(Msg) -> - persistent_term:put(?SEQ_ID(Msg), ?NEW_SEQ). + new_throttler(Msg, ?NEW_SEQ). -new_throttler(Msg, Map) -> - persistent_term:put(?SEQ_ID(Msg), Map). +new_throttler(Msg, AtomicOrEmptyMap) -> + persistent_term:put(?SEQ_ID(Msg), AtomicOrEmptyMap). diff --git a/apps/emqx/test/emqx_log_throttler_SUITE.erl b/apps/emqx/test/emqx_log_throttler_SUITE.erl index 23150a3b1..f95d62969 100644 --- a/apps/emqx/test/emqx_log_throttler_SUITE.erl +++ b/apps/emqx/test/emqx_log_throttler_SUITE.erl @@ -127,7 +127,7 @@ t_throttle(_Config) -> t_throttle_recoverable_msg(_Config) -> ResourceId = <<"resource_id">>, - ThrottledMsg = emqx_utils:format("~ts:~s", [?THROTTLE_UNRECOVERABLE_MSG, ResourceId]), + ThrottledMsg = iolist_to_binary([atom_to_list(?THROTTLE_UNRECOVERABLE_MSG), ":", ResourceId]), ?check_trace( begin %% Warm-up and block to increase the probability that next events @@ -181,10 +181,15 @@ t_throttle_add_new_msg(_Config) -> t_throttle_no_msg(_Config) -> %% Must simply pass with no crashes + Pid = erlang:whereis(emqx_log_throttler), ?assert(emqx_log_throttler:allow(no_test_throttle_msg, undefined)), ?assert(emqx_log_throttler:allow(no_test_throttle_msg, undefined)), - timer:sleep(10), - ?assert(erlang:is_process_alive(erlang:whereis(emqx_log_throttler))). + %% assert process is not restarted + ?assertEqual(Pid, erlang:whereis(emqx_log_throttler)), + %% make a gen_call to ensure the process is alive + %% note: this call result in an 'unexpected_call' error log. + ?assertEqual(ignored, gen_server:call(Pid, probe)), + ok. t_update_time_window(_Config) -> ?check_trace( From eab440e0c16c41d3e8fe684f0db66455fb9830b1 Mon Sep 17 00:00:00 2001 From: zmstone Date: Fri, 26 Jul 2024 15:19:10 +0200 Subject: [PATCH 077/131] docs: add changelog for PR 13528 --- changes/ce/feat-13528.en.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/ce/feat-13528.en.md diff --git a/changes/ce/feat-13528.en.md b/changes/ce/feat-13528.en.md new file mode 100644 index 000000000..f761e9565 --- /dev/null +++ b/changes/ce/feat-13528.en.md @@ -0,0 +1 @@ +Add log throttling for data integration unrecoverable errors. From a49cd78aae3b85c8d97bf64420db3013e100c4db Mon Sep 17 00:00:00 2001 From: zmstone Date: Mon, 29 Jul 2024 23:54:00 +0200 Subject: [PATCH 078/131] refactor: force getenv to access only OS env with prefix EMQXVAR_ --- apps/emqx_utils/src/emqx_variform_bif.erl | 2 +- apps/emqx_utils/test/emqx_variform_bif_tests.erl | 2 +- changes/ce/feat-13507.en.md | 6 ++++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/apps/emqx_utils/src/emqx_variform_bif.erl b/apps/emqx_utils/src/emqx_variform_bif.erl index e66b8e47d..09048a697 100644 --- a/apps/emqx_utils/src/emqx_variform_bif.erl +++ b/apps/emqx_utils/src/emqx_variform_bif.erl @@ -583,7 +583,7 @@ getenv(Bin) when is_binary(Bin) -> EnvKey = ?ENV_CACHE(Bin), case persistent_term:get(EnvKey, undefined) of undefined -> - Name = erlang:binary_to_list(Bin), + Name = "EMQXVAR_" ++ erlang:binary_to_list(Bin), Result = case os:getenv(Name) of false -> diff --git a/apps/emqx_utils/test/emqx_variform_bif_tests.erl b/apps/emqx_utils/test/emqx_variform_bif_tests.erl index 36235be40..aa6724de5 100644 --- a/apps/emqx_utils/test/emqx_variform_bif_tests.erl +++ b/apps/emqx_utils/test/emqx_variform_bif_tests.erl @@ -77,5 +77,5 @@ system_test() -> EnvName = erlang:atom_to_list(?MODULE), EnvVal = erlang:atom_to_list(?FUNCTION_NAME), EnvNameBin = erlang:list_to_binary(EnvName), - os:putenv(EnvName, EnvVal), + os:putenv("EMQXVAR_" ++ EnvName, EnvVal), ?assertEqual(erlang:list_to_binary(EnvVal), emqx_variform_bif:getenv(EnvNameBin)). diff --git a/changes/ce/feat-13507.en.md b/changes/ce/feat-13507.en.md index 115fa49a9..026cf6bf4 100644 --- a/changes/ce/feat-13507.en.md +++ b/changes/ce/feat-13507.en.md @@ -1,2 +1,4 @@ -Added a new builtin function `getenv` in the rule engine and variform expression to access the environment variables. -Note this value is immutable once loaded from the environment. +Added a new builtin function `getenv` in the rule engine and variform expression to access the environment variables with below limitations. + +- Prefix `EMQXVAR_` is added before reading from OS environment variables. i.e. `getenv('FOO_BAR')` is to read `EMQXVAR_FOO_BAR`. +- The values are immutable once loaded from the OS environment. From c831f0772ff6fbf8acacdaf7314b8e16a7ff8982 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Fri, 21 Jun 2024 14:38:41 +0300 Subject: [PATCH 079/131] feat(queue): handle renew_lease_timeout --- .../src/emqx_ds_shared_sub_group_sm.erl | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_group_sm.erl b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_group_sm.erl index c6bdf9d93..5ddff8518 100644 --- a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_group_sm.erl +++ b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_group_sm.erl @@ -186,8 +186,8 @@ handle_renew_lease_timeout(GSM) -> %%----------------------------------------------------------------------- %% Updating state -% handle_updating(GSM) -> -% GSM. +handle_updating(GSM) -> + GSM. %%----------------------------------------------------------------------- %% Internal API @@ -277,6 +277,6 @@ cancel_timer(GSM, Name) -> run_enter_callback(#{state := ?connecting} = GSM) -> handle_connecting(GSM); run_enter_callback(#{state := ?replaying} = GSM) -> - handle_replaying(GSM). -% run_enter_callback(#{state := ?updating} = GSM) -> -% handle_updating(GSM). + handle_replaying(GSM); +run_enter_callback(#{state := ?updating} = GSM) -> + handle_updating(GSM). From 082514f557900dd93c1945c12f71254700417f50 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Wed, 26 Jun 2024 14:19:06 +0300 Subject: [PATCH 080/131] feat(queue): implement full protocol between agent and leader --- ...emqx_persistent_session_ds_shared_subs.erl | 7 +- .../src/emqx_ds_shared_sub_agent.erl | 72 +- .../src/emqx_ds_shared_sub_group_sm.erl | 285 ++++++-- .../src/emqx_ds_shared_sub_leader.erl | 634 ++++++++++++++---- .../src/emqx_ds_shared_sub_proto.erl | 55 +- .../src/emqx_ds_shared_sub_proto.hrl | 62 +- 6 files changed, 933 insertions(+), 182 deletions(-) diff --git a/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl index c4e929640..f3aaa146e 100644 --- a/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl +++ b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl @@ -123,10 +123,12 @@ on_streams_replayed(S, #{agent := Agent0} = SharedSubS0) -> Progress = fold_shared_stream_states( fun(TopicFilter, Stream, SRS, Acc) -> #srs{it_begin = BeginIt} = SRS, + StreamProgress = #{ topic_filter => TopicFilter, stream => Stream, - iterator => BeginIt + iterator => BeginIt, + use_finished => is_use_finished(S, SRS) }, [StreamProgress | Acc] end, @@ -336,3 +338,6 @@ agent_opts(#{session_id := SessionId}) -> -dialyzer({nowarn_function, now_ms/0}). now_ms() -> erlang:system_time(millisecond). + +is_use_finished(S, #srs{unsubscribed = Unsubscribed} = SRS) -> + Unsubscribed andalso emqx_persistent_session_ds_stream_scheduler:is_fully_acked(SRS, S). diff --git a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_agent.erl b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_agent.erl index 29745aa4a..6e43e0a65 100644 --- a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_agent.erl +++ b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_agent.erl @@ -56,20 +56,30 @@ on_unsubscribe(State, TopicFilter) -> renew_streams(#{} = State) -> fetch_stream_events(State). -on_stream_progress(State, _StreamProgress) -> - %% TODO https://emqx.atlassian.net/browse/EMQX-12572 - %% Send to leader - State. +on_stream_progress(State, StreamProgresses) -> + ProgressesByGroup = stream_progresses_by_group(StreamProgresses), + lists:foldl( + fun({Group, GroupProgresses}, StateAcc) -> + with_group_sm(StateAcc, Group, fun(GSM) -> + emqx_ds_shared_sub_group_sm:handle_stream_progress(GSM, GroupProgresses) + end) + end, + State, + maps:to_list(ProgressesByGroup) + ). -on_info(State, ?leader_lease_streams_match(Group, StreamProgresses, Version)) -> +on_info(State, ?leader_lease_streams_match(Group, Leader, StreamProgresses, Version)) -> ?SLOG(info, #{ msg => leader_lease_streams, group => Group, streams => StreamProgresses, - version => Version + version => Version, + leader => Leader }), with_group_sm(State, Group, fun(GSM) -> - emqx_ds_shared_sub_group_sm:handle_leader_lease_streams(GSM, StreamProgresses, Version) + emqx_ds_shared_sub_group_sm:handle_leader_lease_streams( + GSM, Leader, StreamProgresses, Version + ) end); on_info(State, ?leader_renew_stream_lease_match(Group, Version)) -> ?SLOG(info, #{ @@ -80,6 +90,37 @@ on_info(State, ?leader_renew_stream_lease_match(Group, Version)) -> with_group_sm(State, Group, fun(GSM) -> emqx_ds_shared_sub_group_sm:handle_leader_renew_stream_lease(GSM, Version) end); +on_info(State, ?leader_renew_stream_lease_match(Group, VersionOld, VersionNew)) -> + ?SLOG(info, #{ + msg => leader_renew_stream_lease, + group => Group, + version_old => VersionOld, + version_new => VersionNew + }), + with_group_sm(State, Group, fun(GSM) -> + emqx_ds_shared_sub_group_sm:handle_leader_renew_stream_lease(GSM, VersionOld, VersionNew) + end); +on_info(State, ?leader_update_streams_match(Group, VersionOld, VersionNew, StreamsNew)) -> + ?SLOG(info, #{ + msg => leader_update_streams, + group => Group, + version_old => VersionOld, + version_new => VersionNew, + streams_new => StreamsNew + }), + with_group_sm(State, Group, fun(GSM) -> + emqx_ds_shared_sub_group_sm:handle_leader_update_streams( + GSM, VersionOld, VersionNew, StreamsNew + ) + end); +on_info(State, ?leader_invalidate_match(Group)) -> + ?SLOG(info, #{ + msg => leader_invalidate, + group => Group + }), + with_group_sm(State, Group, fun(GSM) -> + emqx_ds_shared_sub_group_sm:handle_leader_invalidate(GSM) + end); %% Generic messages sent by group_sm's to themselves (timeouts). on_info(State, #message_to_group_sm{group = Group, message = Message}) -> with_group_sm(State, Group, fun(GSM) -> @@ -156,3 +197,20 @@ with_group_sm(State, Group, Fun) -> %% Error? State end. + +stream_progresses_by_group(StreamProgresses) -> + lists:foldl( + fun(#{topic_filter := #share{group = Group}} = Progress0, Acc) -> + Progress1 = maps:remove(topic_filter, Progress0), + maps:update_with( + Group, + fun(GroupStreams0) -> + [Progress1 | GroupStreams0] + end, + [Progress1], + Acc + ) + end, + #{}, + StreamProgresses + ). diff --git a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_group_sm.erl b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_group_sm.erl index 5ddff8518..eb13e7147 100644 --- a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_group_sm.erl +++ b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_group_sm.erl @@ -16,14 +16,24 @@ new/1, %% Leader messages - handle_leader_lease_streams/3, + handle_leader_lease_streams/4, handle_leader_renew_stream_lease/2, + handle_leader_renew_stream_lease/3, + handle_leader_update_streams/4, + handle_leader_invalidate/1, %% Self-initiated messages handle_info/2, %% API - fetch_stream_events/1 + fetch_stream_events/1, + handle_stream_progress/2 +]). + +-export_type([ + group_sm/0, + options/0, + state/0 ]). -type options() :: #{ @@ -32,7 +42,31 @@ send_after := fun((non_neg_integer(), term()) -> reference()) }. -%% Subscription states +-type stream_lease_event() :: + #{ + type => lease, + stream => emqx_ds:stream(), + iterator => emqx_ds:iterator() + } + | #{ + type => revoke, + stream => emqx_ds:stream() + }. + +-type external_lease_event() :: + #{ + type => lease, + stream => emqx_ds:stream(), + iterator => emqx_ds:iterator(), + topic_filter => emqx_persistent_session_ds:share_topic_filter() + } + | #{ + type => revoke, + stream => emqx_ds:stream(), + topic_filter => emqx_persistent_session_ds:share_topic_filter() + }. + +%% GroupSM States -define(connecting, connecting). -define(replaying, replaying). @@ -40,26 +74,47 @@ -type state() :: ?connecting | ?replaying | ?updating. --type group_sm() :: #{ - topic_filter => emqx_persistent_session_ds:share_topic_filter(), - agent => emqx_ds_shared_sub_proto:agent(), - send_after => fun((non_neg_integer(), term()) -> reference()), - - state => state(), - state_data => map(), - state_timers => map() +-type connecting_data() :: #{}. +-type replaying_data() :: #{ + leader => emqx_ds_shared_sub_proto:leader(), + streams => #{emqx_ds:stream() => emqx_ds:iterator()}, + version => emqx_ds_shared_sub_proto:version(), + prev_version => undefined }. +-type updating_data() :: #{ + leader => emqx_ds_shared_sub_proto:leader(), + streams => #{emqx_ds:stream() => emqx_ds:iterator()}, + version => emqx_ds_shared_sub_proto:version(), + prev_version => emqx_ds_shared_sub_proto:version() +}. + +-type state_data() :: connecting_data() | replaying_data() | updating_data(). -record(state_timeout, { id :: reference(), name :: atom(), message :: term() }). + -record(timer, { ref :: reference(), id :: reference() }). +-type timer_name() :: atom(). +-type timer() :: #timer{}. + +-type group_sm() :: #{ + topic_filter => emqx_persistent_session_ds:share_topic_filter(), + agent => emqx_ds_shared_sub_proto:agent(), + send_after => fun((non_neg_integer(), term()) -> reference()), + stream_lease_events => list(stream_lease_event()), + + state => state(), + state_data => state_data(), + state_timers => #{timer_name() => timer()} +}. + %%----------------------------------------------------------------------- %% Constants %%----------------------------------------------------------------------- @@ -94,11 +149,12 @@ new(#{ }, transition(GSM0, ?connecting, #{}). +-spec fetch_stream_events(group_sm()) -> {group_sm(), list(external_lease_event())}. fetch_stream_events( #{ - state := ?replaying, + state := _State, topic_filter := TopicFilter, - state_data := #{stream_lease_events := Events0} = Data + stream_lease_events := Events0 } = GSM ) -> Events1 = lists:map( @@ -107,14 +163,7 @@ fetch_stream_events( end, Events0 ), - { - GSM#{ - state_data => Data#{stream_lease_events => []} - }, - Events1 - }; -fetch_stream_events(GSM) -> - {GSM, []}. + {GSM#{stream_lease_events => []}, Events1}. %%----------------------------------------------------------------------- %% Event Handlers @@ -128,37 +177,23 @@ handle_connecting(#{agent := Agent, topic_filter := ShareTopicFilter} = GSM) -> ensure_state_timeout(GSM, find_leader_timeout, ?FIND_LEADER_TIMEOUT). handle_leader_lease_streams( - #{state := ?connecting, topic_filter := TopicFilter} = GSM0, StreamProgresses, Version + #{state := ?connecting, topic_filter := TopicFilter} = GSM0, Leader, StreamProgresses, Version ) -> ?tp(debug, leader_lease_streams, #{topic_filter => TopicFilter}), - Streams = lists:foldl( - fun(#{stream := Stream, iterator := It}, Acc) -> - Acc#{Stream => It} - end, - #{}, - StreamProgresses - ), - StreamLeaseEvents = lists:map( - fun(#{stream := Stream, iterator := It}) -> - #{ - type => lease, - stream => Stream, - iterator => It - } - end, - StreamProgresses - ), + Streams = progresses_to_map(StreamProgresses), + StreamLeaseEvents = progresses_to_lease_events(StreamProgresses), transition( GSM0, ?replaying, #{ + leader => Leader, streams => Streams, - stream_lease_events => StreamLeaseEvents, prev_version => undefined, version => Version - } + }, + StreamLeaseEvents ); -handle_leader_lease_streams(GSM, _StreamProgresses, _Version) -> +handle_leader_lease_streams(GSM, _Leader, _StreamProgresses, _Version) -> GSM. handle_find_leader_timeout(#{agent := Agent, topic_filter := TopicFilter} = GSM0) -> @@ -172,13 +207,6 @@ handle_find_leader_timeout(#{agent := Agent, topic_filter := TopicFilter} = GSM0 handle_replaying(GSM) -> ensure_state_timeout(GSM, renew_lease_timeout, ?RENEW_LEASE_TIMEOUT). -handle_leader_renew_stream_lease( - #{state := ?replaying, state_data := #{version := Version}} = GSM, Version -) -> - ensure_state_timeout(GSM, renew_lease_timeout, ?RENEW_LEASE_TIMEOUT); -handle_leader_renew_stream_lease(GSM, _Version) -> - GSM. - handle_renew_lease_timeout(GSM) -> ?tp(debug, renew_lease_timeout, #{}), transition(GSM, ?connecting, #{}). @@ -187,8 +215,140 @@ handle_renew_lease_timeout(GSM) -> %% Updating state handle_updating(GSM) -> + ensure_state_timeout(GSM, renew_lease_timeout, ?RENEW_LEASE_TIMEOUT). + +%%----------------------------------------------------------------------- +%% Common handlers + +handle_leader_update_streams( + #{ + state := ?replaying, + stream_data := #{streams := Streams0, version := VersionOld} = StateData + } = GSM, + VersionOld, + VersionNew, + StreamProgresses +) -> + {AddEvents, Streams1} = lists:foldl( + fun(#{stream := Stream, iterator := It}, {AddEventAcc, StreamsAcc}) -> + case maps:is_key(Stream, StreamsAcc) of + true -> + %% We prefer our own progress + {AddEventAcc, StreamsAcc}; + false -> + { + [#{type => lease, stream => Stream, iterator => It} | AddEventAcc], + StreamsAcc#{Stream => It} + } + end + end, + {[], Streams0}, + StreamProgresses + ), + NewStreamMap = progresses_to_map(StreamProgresses), + {RevokeEvents, Streams2} = lists:foldl( + fun(Stream, {RevokeEventAcc, StreamsAcc}) -> + case maps:is_key(Stream, NewStreamMap) of + true -> + {RevokeEventAcc, StreamsAcc}; + false -> + { + [#{type => revoke, stream => Stream} | RevokeEventAcc], + maps:remove(Stream, StreamsAcc) + } + end + end, + {[], Streams1}, + maps:keys(Streams1) + ), + StreamLeaseEvents = AddEvents ++ RevokeEvents, + transition( + GSM, + ?updating, + StateData#{ + streams => Streams2, + prev_version => VersionOld, + version => VersionNew + }, + StreamLeaseEvents + ); +handle_leader_update_streams( + #{ + state := ?updating, + stream_data := #{version := VersionNew} = _StreamData + } = GSM, + _VersionOld, + VersionNew, + _StreamProgresses +) -> + ensure_state_timeout(GSM, renew_lease_timeout, ?RENEW_LEASE_TIMEOUT); +handle_leader_update_streams(GSM, _VersionOld, _VersionNew, _StreamProgresses) -> + %% Unexpected versions or state + transition(GSM, ?connecting, #{}). + +handle_leader_renew_stream_lease( + #{state := ?replaying, state_data := #{version := Version}} = GSM, Version +) -> + ensure_state_timeout(GSM, renew_lease_timeout, ?RENEW_LEASE_TIMEOUT); +handle_leader_renew_stream_lease( + #{state := ?updating, state_data := #{version := Version} = StateData} = GSM, Version +) -> + transition( + GSM, + ?replaying, + StateData#{prev_version => undefined} + ); +handle_leader_renew_stream_lease(GSM, _Version) -> GSM. +handle_leader_renew_stream_lease( + #{state := ?replaying, state_data := #{version := Version}} = GSM, VersionOld, VersionNew +) when VersionOld =:= Version orelse VersionNew =:= Version -> + ensure_state_timeout(GSM, renew_lease_timeout, ?RENEW_LEASE_TIMEOUT); +handle_leader_renew_stream_lease( + #{state := ?updating, state_data := #{version := VersionNew, prev_version := VersionOld}} = GSM, + VersionOld, + VersionNew +) -> + ensure_state_timeout(GSM, renew_lease_timeout, ?RENEW_LEASE_TIMEOUT); +handle_leader_renew_stream_lease(GSM, _VersionOld, _VersionNew) -> + transition(GSM, ?connecting, #{}). + +handle_stream_progress(#{state := ?connecting} = GSM, _StreamProgresses) -> + GSM; +handle_stream_progress( + #{ + state := ?replaying, + state_data := #{ + agent := Agent, + leader := Leader, + version := Version + } + } = _GSM, + StreamProgresses +) -> + ok = emqx_ds_shared_sub_proto:agent_update_stream_states( + Leader, Agent, StreamProgresses, Version + ); +handle_stream_progress( + #{ + state := ?updating, + state_data := #{ + agent := Agent, + leader := Leader, + version := Version, + prev_version := PrevVersion + } + } = _GSM, + StreamProgresses +) -> + ok = emqx_ds_shared_sub_proto:agent_update_stream_states( + Leader, Agent, StreamProgresses, PrevVersion, Version + ). + +handle_leader_invalidate(GSM) -> + transition(GSM, ?connecting, #{}). + %%----------------------------------------------------------------------- %% Internal API %%----------------------------------------------------------------------- @@ -225,6 +385,9 @@ handle_info(GSM, _Info) -> %%-------------------------------------------------------------------- transition(GSM0, NewState, NewStateData) -> + transition(GSM0, NewState, NewStateData, []). + +transition(GSM0, NewState, NewStateData, LeaseEvents) -> Timers = maps:get(state_timers, GSM0, #{}), TimerNames = maps:keys(Timers), GSM1 = lists:foldl( @@ -237,7 +400,8 @@ transition(GSM0, NewState, NewStateData) -> GSM2 = GSM1#{ state => NewState, state_data => NewStateData, - state_timers => #{} + state_timers => #{}, + stream_lease_events => LeaseEvents }, run_enter_callback(GSM2). @@ -280,3 +444,24 @@ run_enter_callback(#{state := ?replaying} = GSM) -> handle_replaying(GSM); run_enter_callback(#{state := ?updating} = GSM) -> handle_updating(GSM). + +progresses_to_lease_events(StreamProgresses) -> + lists:map( + fun(#{stream := Stream, iterator := It}) -> + #{ + type => lease, + stream => Stream, + iterator => It + } + end, + StreamProgresses + ). + +progresses_to_map(StreamProgresses) -> + lists:foldl( + fun(#{stream := Stream, iterator := It}, Acc) -> + Acc#{Stream => It} + end, + #{}, + StreamProgresses + ). diff --git a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl index 5323595cf..3f2a85424 100644 --- a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl +++ b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl @@ -6,10 +6,12 @@ -behaviour(gen_statem). +-include("emqx_ds_shared_sub_proto.hrl"). + -include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("emqx/include/logger.hrl"). -include_lib("emqx/include/emqx_persistent_message.hrl"). --include("emqx_ds_shared_sub_proto.hrl"). +-include_lib("snabbkaffe/include/trace.hrl"). -export([ register/2, @@ -28,10 +30,21 @@ topic_filter := emqx_persistent_session_ds:share_topic_filter() }. --type stream_assignment() :: #{ +%% Agent states + +-define(waiting_replaying, waiting_replaying). +-define(replaying, replaying). +-define(waiting_updating, waiting_updating). +-define(updating, updating). + +-type agent_state() :: #{ + %% Our view of group gm's status + %% it lags the actual state + state := emqx_ds_shared_sub_agent:status(), prev_version := emqx_maybe:t(emqx_ds_shared_sub_proto:version()), version := emqx_ds_shared_sub_proto:version(), - streams := list(emqx_ds:stream()) + streams := list(emqx_ds:stream()), + revoked_streams := list(emqx_ds:stream()) }. -type data() :: #{ @@ -46,10 +59,10 @@ stream_progresses := #{ emqx_ds:stream() => emqx_ds:iterator() }, - agent_stream_assignments := #{ - emqx_ds_shared_sub_proto:agent() => stream_assignment() + agents := #{ + emqx_ds_shared_sub_proto:agent() => agent_state() }, - stream_assignments := #{ + stream_owners := #{ emqx_ds:stream() => emqx_ds_shared_sub_proto:agent() } }. @@ -61,8 +74,8 @@ %% States --define(waiting_registration, waiting_registration). --define(replaying, replaying). +-define(leader_waiting_registration, leader_waiting_registration). +-define(leader_replaying, leader_replaying). %% Events @@ -71,13 +84,17 @@ }). -record(renew_streams, {}). -record(renew_leases, {}). +-record(drop_timeout, {}). %% Constants %% TODO https://emqx.atlassian.net/browse/EMQX-12574 %% Move to settings --define(RENEW_LEASE_INTERVAL, 5000). --define(RENEW_STREAMS_INTERVAL, 5000). +-define(RENEW_LEASE_INTERVAL, 1000). +-define(RENEW_STREAMS_INTERVAL, 1000). +-define(DROP_TIMEOUT_INTERVAL, 1000). + +-define(AGENT_TIMEOUT, 5000). %%-------------------------------------------------------------------- %% API @@ -115,17 +132,17 @@ init([#{topic_filter := #share{group = Group, topic = Topic}} = _Options]) -> Data = #{ group => Group, topic => Topic, - router_id => router_id(), + router_id => gen_router_id(), stream_progresses => #{}, - stream_assignments => #{}, - agent_stream_assignments => #{} + stream_owners => #{}, + agents => #{} }, - {ok, ?waiting_registration, Data}. + {ok, ?leader_waiting_registration, Data}. %%-------------------------------------------------------------------- %% waiting_registration state -handle_event({call, From}, #register{register_fun = Fun}, ?waiting_registration, Data) -> +handle_event({call, From}, #register{register_fun = Fun}, ?leader_waiting_registration, Data) -> Self = self(), case Fun() of Self -> @@ -135,25 +152,44 @@ handle_event({call, From}, #register{register_fun = Fun}, ?waiting_registration, end; %%-------------------------------------------------------------------- %% repalying state -handle_event(enter, _OldState, ?replaying, #{topic := Topic, router_id := RouterId} = _Data) -> +handle_event(enter, _OldState, ?leader_replaying, #{topic := Topic, router_id := RouterId} = _Data) -> ok = emqx_persistent_session_ds_router:do_add_route(Topic, RouterId), {keep_state_and_data, [ {state_timeout, ?RENEW_LEASE_INTERVAL, #renew_leases{}}, + {state_timeout, ?DROP_TIMEOUT_INTERVAL, #drop_timeout{}}, {state_timeout, 0, #renew_streams{}} ]}; -handle_event(state_timeout, #renew_streams{}, ?replaying, Data0) -> +handle_event(state_timeout, #renew_streams{}, ?leader_replaying, Data0) -> Data1 = renew_streams(Data0), {keep_state, Data1, {state_timeout, ?RENEW_STREAMS_INTERVAL, #renew_streams{}}}; -handle_event(state_timeout, #renew_leases{}, ?replaying, Data0) -> +handle_event(state_timeout, #renew_leases{}, ?leader_replaying, Data0) -> Data1 = renew_leases(Data0), {keep_state, Data1, {state_timeout, ?RENEW_LEASE_INTERVAL, #renew_leases{}}}; -handle_event(info, ?agent_connect_leader_match(Agent, _TopicFilter), ?replaying, Data0) -> +handle_event(state_timeout, #drop_timeout{}, ?leader_replaying, Data0) -> + Data1 = drop_timeout_agents(Data0), + {keep_state, Data1, {state_timeout, ?DROP_TIMEOUT_INTERVAL, #drop_timeout{}}}; +handle_event(info, ?agent_connect_leader_match(Agent, _TopicFilter), ?leader_replaying, Data0) -> Data1 = connect_agent(Data0, Agent), {keep_state, Data1}; handle_event( - info, ?agent_update_stream_states_match(Agent, StreamProgresses, Version), ?replaying, Data0 + info, + ?agent_update_stream_states_match(Agent, StreamProgresses, Version), + ?leader_replaying, + Data0 ) -> - Data1 = update_agent_stream_states(Data0, Agent, StreamProgresses, Version), + Data1 = with_agent(Data0, Agent, fun() -> + update_agent_stream_states(Data0, Agent, StreamProgresses, Version) + end), + {keep_state, Data1}; +handle_event( + info, + ?agent_update_stream_states_match(Agent, StreamProgresses, VersionOld, VersionNew), + ?leader_replaying, + Data0 +) -> + Data1 = with_agent(Data0, Agent, fun() -> + update_agent_stream_states(Data0, Agent, StreamProgresses, VersionOld, VersionNew) + end), {keep_state, Data1}; %%-------------------------------------------------------------------- %% fallback @@ -172,9 +208,16 @@ terminate(_Reason, _State, #{topic := Topic, router_id := RouterId} = _Data) -> ok. %%-------------------------------------------------------------------- -%% Internal functions +%% Event handlers %%-------------------------------------------------------------------- +%%-------------------------------------------------------------------- +%% Renew streams + +%% * Find new streams in DS +%% * Revoke streams from agents having too many streams +%% * Assign streams to agents having too few streams + renew_streams(#{stream_progresses := Progresses, topic := Topic} = Data0) -> TopicFilter = emqx_topic:words(Topic), StartTime = now_ms(), @@ -198,25 +241,109 @@ renew_streams(#{stream_progresses := Progresses, topic := Topic} = Data0) -> Progresses, Streams ), - %% TODO https://emqx.atlassian.net/browse/EMQX-12572 - %% Initiate reassigment + Data1 = Data0#{stream_progresses => NewProgresses}, ?SLOG(info, #{ msg => leader_renew_streams, topic_filter => TopicFilter, streams => length(Streams) }), - Data0#{stream_progresses => NewProgresses}. + Data2 = revoke_streams(Data1), + Data3 = assign_streams(Data2), + Data3. -%% TODO https://emqx.atlassian.net/browse/EMQX-12572 -%% This just gives unassigned streams to the connecting agent, -%% we need to implement actual stream (re)assignment. -connect_agent( +%% We revoke streams from agents that have too many streams (> desired_streams_per_agent). +%% We revoke only from replaying agents. +%% After revoking, no unassigned streams appear. Streams will become unassigned +%% only after agents report them as acked and unsubscribed. +revoke_streams(Data0) -> + DesiredStreamsPerAgent = desired_streams_per_agent(Data0), + Agents = replaying_agents(Data0), + lists:foldl( + fun(Agent, DataAcc) -> + revoke_excess_streams_from_agent(DataAcc, Agent, DesiredStreamsPerAgent) + end, + Data0, + Agents + ). + +revoke_excess_streams_from_agent(Data0, Agent, DesiredCount) -> + #{streams := Streams0, revoked_streams := []} = AgentState0 = get_agent_state(Data0, Agent), + RevokeCount = length(Streams0) - DesiredCount, + AgentState1 = + case RevokeCount > 0 of + false -> + AgentState0; + true -> + revoke_streams_from_agent(Data0, Agent, AgentState0, RevokeCount) + end, + set_agent_state(Data0, Agent, AgentState1). + +revoke_streams_from_agent( + Data, + Agent, #{ - group := Group, - agent_stream_assignments := AgentStreamAssignments0, - stream_assignments := StreamAssignments0, - stream_progresses := StreamProgresses - } = Data0, + streams := Streams0, revoked_streams := [] + } = AgentState0, + RevokeCount +) -> + RevokedStreams = select_streams_for_revoke(Data, AgentState0, RevokeCount), + Streams = Streams0 -- RevokedStreams, + agent_transition_to_waiting_updating(Data, Agent, AgentState0, Streams, RevokedStreams). + +select_streams_for_revoke( + _Data, #{streams := Streams, revoked_streams := []} = _AgentState, RevokeCount +) -> + %% TODO + %% Some intellectual logic should be used regarding: + %% * shard ids (better spread shards across different streams); + %% * stream stats (how much data was replayed from stream, + %% heavy streams should be distributed across different agents); + %% * data locality (agents better preserve streams with data available on the agent's node) + lists:sublist(shuffle(Streams), RevokeCount). + +%% We assign streams to agents that have too few streams (< desired_streams_per_agent). +%% We assign only to replaying agents. +assign_streams(Data0) -> + DesiredStreamsPerAgent = desired_streams_per_agent(Data0), + Agents = replaying_agents(Data0), + lists:foldl( + fun(Agent, DataAcc) -> + assign_lacking_streams(DataAcc, Agent, DesiredStreamsPerAgent) + end, + Data0, + Agents + ). + +assign_lacking_streams(Data0, Agent, DesiredCount) -> + #{streams := Streams0, revoked_streams := []} = get_agent_state(Data0, Agent), + AssignCount = DesiredCount - length(Streams0), + case AssignCount > 0 of + false -> + Data0; + true -> + assign_streams_to_agent(Data0, Agent, AssignCount) + end. + +assign_streams_to_agent(Data0, Agent, AssignCount) -> + StreamsToAssign = select_streams_for_assign(Data0, Agent, AssignCount), + Data1 = set_stream_ownership_to_agent(Data0, Agent, StreamsToAssign), + #{agents := #{Agent := AgentState0}} = Data1, + #{streams := Streams0, revoked_streams := []} = AgentState0, + Streams1 = Streams0 ++ StreamsToAssign, + AgentState1 = agent_transition_to_waiting_updating(Data0, Agent, AgentState0, Streams1, []), + set_agent_state(Data1, Agent, AgentState1). + +select_streams_for_assign(Data0, _Agent, AssignCount) -> + %% TODO + %% Some intellectual logic should be used. See `select_streams_for_revoke/3`. + UnassignedStreams = unassigned_streams(Data0), + lists:sublist(shuffle(UnassignedStreams), AssignCount). + +%%-------------------------------------------------------------------- +%% Handle a newly connected agent + +connect_agent( + #{group := Group} = Data, Agent ) -> ?SLOG(info, #{ @@ -224,103 +351,382 @@ connect_agent( agent => Agent, group => Group }), - {AgentStreamAssignments, StreamAssignments} = - case AgentStreamAssignments0 of - #{Agent := _} -> - {AgentStreamAssignments0, StreamAssignments0}; - _ -> - UnassignedStreams = unassigned_streams(Data0), - Version = 0, - StreamAssignment = #{ - prev_version => undefined, - version => Version, - streams => UnassignedStreams - }, - AgentStreamAssignments1 = AgentStreamAssignments0#{Agent => StreamAssignment}, - StreamAssignments1 = lists:foldl( - fun(Stream, Acc) -> - Acc#{Stream => Agent} - end, - StreamAssignments0, - UnassignedStreams - ), - StreamLease = lists:map( - fun(Stream) -> - #{ - stream => Stream, - iterator => maps:get(Stream, StreamProgresses) - } - end, - UnassignedStreams - ), - ?SLOG(info, #{ - msg => leader_lease_streams, - agent => Agent, - group => Group, - streams => length(StreamLease), - version => Version - }), - ok = emqx_ds_shared_sub_proto:leader_lease_streams( - Agent, Group, StreamLease, Version - ), - {AgentStreamAssignments1, StreamAssignments1} - end, - Data0#{ - agent_stream_assignments => AgentStreamAssignments, stream_assignments => StreamAssignments - }. + DesiredCount = desired_streams_per_agent(Data), + assign_initial_streams_to_agent(Data, Agent, DesiredCount). -renew_leases(#{group := Group, agent_stream_assignments := AgentStreamAssignments} = Data) -> - ok = lists:foreach( - fun({Agent, #{version := Version}}) -> - ok = emqx_ds_shared_sub_proto:leader_renew_stream_lease(Agent, Group, Version) +assign_initial_streams_to_agent(Data, Agent, AssignCount) -> + InitialStreamsToAssign = select_streams_for_assign(Data, Agent, AssignCount), + Data1 = set_stream_ownership_to_agent(Data, Agent, InitialStreamsToAssign), + AgentState = agent_transition_to_initial_waiting_replaying( + Data1, Agent, InitialStreamsToAssign + ), + set_agent_state(Data1, Agent, AgentState). + +%%-------------------------------------------------------------------- +%% Drop agents that stopped reporting progress + +drop_timeout_agents(#{agents := Agents} = Data) -> + Now = now_ms(), + lists:foldl( + fun({Agent, #{update_deadline := Deadline} = _AgentState}, DataAcc) -> + case Deadline < Now of + true -> + ?SLOG(info, #{ + msg => leader_agent_timeout, + agent => Agent + }), + drop_invalidate_agent(DataAcc, Agent); + false -> + DataAcc + end end, - maps:to_list(AgentStreamAssignments) + Data, + maps:to_list(Agents) + ). + +%%-------------------------------------------------------------------- +%% Send lease confirmations to agents + +renew_leases(#{agents := AgentStates} = Data) -> + ok = lists:foreach( + fun({Agent, AgentState}) -> + renew_lease(Data, Agent, AgentState) + end, + maps:to_list(AgentStates) ), Data. -update_agent_stream_states( - #{ - agent_stream_assignments := AgentStreamAssignments, - stream_assignments := StreamAssignments, - stream_progresses := StreamProgresses0 - } = Data0, - Agent, - AgentStreamProgresses, - Version -) -> - AgentVersion = emqx_utils_maps:deep_get([Agent, version], AgentStreamAssignments, undefined), - AgentPrevVersion = emqx_utils_maps:deep_get( - [Agent, prev_version], AgentStreamAssignments, undefined +renew_lease(#{group := Group}, Agent, #{state := ?replaying, version := Version}) -> + ok = emqx_ds_shared_sub_proto:leader_renew_stream_lease(Agent, Group, Version); +renew_lease(#{group := Group}, Agent, #{state := ?waiting_replaying, version := Version}) -> + ok = emqx_ds_shared_sub_proto:leader_renew_stream_lease(Agent, Group, Version); +renew_lease(#{group := Group} = Data, Agent, #{ + streams := Streams, state := ?waiting_updating, version := Version, prev_version := PrevVersion +}) -> + StreamProgresses = stream_progresses(Data, Streams), + ok = emqx_ds_shared_sub_proto:leader_update_streams( + Agent, Group, PrevVersion, Version, StreamProgresses ), - case AgentVersion == Version orelse AgentPrevVersion == Version of - false -> - %% TODO https://emqx.atlassian.net/browse/EMQX-12572 - %% send invalidate to agent + ok = emqx_ds_shared_sub_proto:leader_renew_stream_lease(Agent, Group, Version, PrevVersion); +renew_lease(#{group := Group}, Agent, #{ + state := ?updating, version := Version, prev_version := PrevVersion +}) -> + ok = emqx_ds_shared_sub_proto:leader_renew_stream_lease(Agent, Group, Version, PrevVersion). + +%%-------------------------------------------------------------------- +%% Handle stream progress updates from agent in replaying state + +update_agent_stream_states(Data0, Agent, AgentStreamProgresses, Version) -> + #{state := State, version := AgentVersion, prev_version := AgentPrevVersion} = + AgentState0 = get_agent_state(Data0, Agent), + case {State, Version} of + {?waiting_updating, AgentPrevVersion} -> + %% Stale update, ignoring Data0; - true -> - StreamProgresses1 = lists:foldl( - fun(#{stream := Stream, iterator := It}, ProgressesAcc) -> - %% Assert Stream is assigned to Agent - Agent = maps:get(Stream, StreamAssignments), - ProgressesAcc#{Stream => It} - end, - StreamProgresses0, - AgentStreamProgresses - ), - Data0#{stream_progresses => StreamProgresses1} + {?waiting_replaying, AgentVersion} -> + %% Agent finished updating, now replaying + Data1 = update_stream_progresses(Data0, Agent, AgentStreamProgresses), + AgentState1 = update_agent_timeout(AgentState0), + AgentState2 = agent_transition_to_replaying(AgentState1), + set_agent_state(Data1, Agent, AgentState2); + {?replaying, AgentVersion} -> + %% Common case, agent is replaying + Data1 = update_stream_progresses(Data0, Agent, AgentStreamProgresses), + AgentState1 = update_agent_timeout(AgentState0), + set_agent_state(Data1, Agent, AgentState1); + {OtherState, OtherVersion} -> + ?tp(warning, unexpected_update, #{ + agent => Agent, + update_version => OtherVersion, + state => OtherState, + our_agent_version => AgentVersion, + our_agent_prev_version => AgentPrevVersion + }), + drop_invalidate_agent(Data0, Agent) end. +update_stream_progresses( + #{stream_progresses := StreamProgresses0, stream_owners := StreamOwners} = Data, + Agent, + ReceivedStreamProgresses +) -> + StreamProgresses1 = lists:foldl( + fun(#{stream := Stream, iterator := It}, ProgressesAcc) -> + case StreamOwners of + #{Stream := Agent} -> + ProgressesAcc#{Stream => It}; + _ -> + ProgressesAcc + end + end, + StreamProgresses0, + ReceivedStreamProgresses + ), + Data#{ + stream_progresses => StreamProgresses1 + }. + +clean_revoked_streams( + Data0, #{revoked_streams := RevokedStreams0} = AgentState0, ReceivedStreamProgresses +) -> + FinishedReportedStreams = maps:from_list( + lists:filtermap( + fun + ( + #{ + stream := Stream, + use_finished := true + } + ) -> + {true, {Stream, true}}; + (_) -> + false + end, + ReceivedStreamProgresses + ) + ), + {FinishedStreams, StillRevokingStreams} = lists:partition( + fun(Stream) -> + maps:is_key(Stream, FinishedReportedStreams) + end, + RevokedStreams0 + ), + Data1 = unassign_streams(Data0, FinishedStreams), + AgentState1 = AgentState0#{revoked_streams => StillRevokingStreams}, + {AgentState1, Data1}. + +unassign_streams(#{stream_owners := StreamOwners0} = Data, Streams) -> + StreamOwners1 = lists:foldl( + fun(Stream, StreamOwnersAcc) -> + maps:remove(Stream, StreamOwnersAcc) + end, + StreamOwners0, + Streams + ), + Data#{ + stream_owners => StreamOwners1 + }. + +%%-------------------------------------------------------------------- +%% Handle stream progress updates from agent in updating (VersionOld -> VersionNew) state + +update_agent_stream_states(Data0, Agent, AgentStreamProgresses, VersionOld, VersionNew) -> + #{state := State, version := AgentVersion, prev_version := AgentPrevVersion} = + AgentState0 = get_agent_state(Data0, Agent), + case {State, VersionOld, VersionNew} of + {?waiting_updating, AgentPrevVersion, AgentVersion} -> + %% Client started updating + Data1 = update_stream_progresses(Data0, Agent, AgentStreamProgresses), + AgentState1 = update_agent_timeout(AgentState0), + {AgentState2, Data2} = clean_revoked_streams(Data1, AgentState1, AgentStreamProgresses), + AgentState3 = + case AgentState2 of + #{revoke_streams := []} -> + agent_transition_to_waiting_replaying(AgentState2); + _ -> + agent_transition_to_updating(AgentState2) + end, + set_agent_state(Data2, Agent, AgentState3); + {?updating, AgentPrevVersion, AgentVersion} -> + Data1 = update_stream_progresses(Data0, Agent, AgentStreamProgresses), + AgentState1 = update_agent_timeout(AgentState0), + {AgentState2, Data2} = clean_revoked_streams(Data1, AgentState1, AgentStreamProgresses), + AgentState3 = + case AgentState2 of + #{revoke_streams := []} -> + agent_transition_to_waiting_replaying(AgentState2); + _ -> + AgentState2 + end, + set_agent_state(Data2, Agent, AgentState3); + {?waiting_replaying, _, AgentVersion} -> + Data1 = update_stream_progresses(Data0, Agent, AgentStreamProgresses), + AgentState1 = update_agent_timeout(AgentState0), + set_agent_state(Data1, Agent, AgentState1); + {?replaying, _, AgentVersion} -> + Data1 = update_stream_progresses(Data0, Agent, AgentStreamProgresses), + AgentState1 = update_agent_timeout(AgentState0), + set_agent_state(Data1, Agent, AgentState1); + {OtherState, OtherVersionOld, OtherVersionNew} -> + ?tp(warning, unexpected_update, #{ + agent => Agent, + update_version_old => OtherVersionOld, + update_version_new => OtherVersionNew, + state => OtherState, + our_agent_version => AgentVersion, + our_agent_prev_version => AgentPrevVersion + }), + drop_invalidate_agent(Data0, Agent) + end. + +%%-------------------------------------------------------------------- +%% Agent state transitions +%%-------------------------------------------------------------------- + +agent_transition_to_waiting_updating( + #{group := Group} = Data, + Agent, + #{version := Version, prev_version := undefined} = AgentState0, + Streams, + RevokedStreams +) -> + NewVersion = next_version(Version), + + AgentState1 = AgentState0#{ + state => ?waiting_updating, + streams => Streams, + revoked_streams => RevokedStreams, + prev_version => Version, + version => NewVersion + }, + StreamProgresses = stream_progresses(Data, Streams), + ok = emqx_ds_shared_sub_proto:leader_update_streams( + Agent, Group, Version, NewVersion, StreamProgresses + ), + AgentState1. + +agent_transition_to_waiting_replaying(AgentState0) -> + AgentState0#{ + state => ?waiting_replaying, + revoked_streams => [] + }. + +agent_transition_to_initial_waiting_replaying( + #{group := Group} = Data, Agent, InitialStreams +) -> + Version = 0, + StreamProgresses = stream_progresses(Data, InitialStreams), + Leader = this_leader(Data), + ok = emqx_ds_shared_sub_proto:leader_lease_streams( + Agent, Group, Leader, StreamProgresses, Version + ), + #{ + state => ?waiting_replaying, + version => Version, + prev_version => undefined, + streams => InitialStreams, + revoked_streams => [], + update_deadline => now_ms() + ?AGENT_TIMEOUT + }. + +agent_transition_to_replaying(#{state := ?waiting_replaying} = AgentState) -> + AgentState#{ + state => ?replaying, + prev_version => undefined + }. + +agent_transition_to_updating(#{state := ?waiting_updating} = AgentState) -> + AgentState#{state => ?updating}. + %%-------------------------------------------------------------------- %% Helper functions %%-------------------------------------------------------------------- -router_id() -> +gen_router_id() -> emqx_guid:to_hexstr(emqx_guid:gen()). now_ms() -> erlang:system_time(millisecond). -unassigned_streams(#{stream_progresses := StreamProgresses, stream_assignments := StreamAssignments}) -> +unassigned_streams(#{stream_progresses := StreamProgresses, stream_owners := StreamOwners}) -> Streams = maps:keys(StreamProgresses), - AssignedStreams = maps:keys(StreamAssignments), + AssignedStreams = maps:keys(StreamOwners), Streams -- AssignedStreams. + +%% Those who are not connecting or updating, i.e. not in a transient state. +replaying_agents(#{agents := AgentStates}) -> + lists:filtermap( + fun + ({Agent, #{state := ?replaying}}) -> + {true, Agent}; + (_) -> + false + end, + maps:to_list(AgentStates) + ). + +desired_streams_per_agent(#{agents := AgentStates, stream_progresses := StreamProgresses}) -> + AgentCount = maps:size(AgentStates), + case AgentCount of + 0 -> + 0; + _ -> + StreamCount = maps:size(StreamProgresses), + (StreamCount div AgentCount) + 1 + end. + +stream_progresses(#{stream_progresses := StreamProgresses} = _Data, Streams) -> + lists:map( + fun(Stream) -> + #{ + stream => Stream, + iterator => maps:get(Stream, StreamProgresses) + } + end, + Streams + ). + +next_version(Version) -> + Version + 1. + +shuffle(L0) -> + L1 = lists:map( + fun(A) -> + {rand:uniform(), A} + end, + L0 + ), + L2 = lists:sort(L1), + {_, L} = lists:unzip(L2), + L. + +set_stream_ownership_to_agent(#{stream_owners := StreamOwners0} = Data, Agent, Streams) -> + StreamOwners1 = lists:foldl( + fun(Stream, Acc) -> + Acc#{Stream => Agent} + end, + StreamOwners0, + Streams + ), + Data#{ + stream_owners => StreamOwners1 + }. + +set_agent_state(#{agents := Agents} = Data, Agent, AgentState) -> + Data#{ + agents => Agents#{Agent => AgentState} + }. + +update_agent_timeout(AgentState) -> + AgentState#{ + update_deadline => now_ms() + ?AGENT_TIMEOUT + }. + +get_agent_state(#{agents := Agents} = _Data, Agent) -> + maps:get(Agent, Agents). + +this_leader(_Data) -> + self(). + +drop_agent(#{agents := Agents} = Data0, Agent) -> + AgentState = get_agent_state(Data0, Agent), + #{streams := Streams, revoked_streams := RevokedStreams} = AgentState, + AllStreams = Streams ++ RevokedStreams, + Data1 = unassign_streams(Data0, AllStreams), + Data1#{agents => maps:remove(Agent, Agents)}. + +invalidate_agent(#{group := Group}, Agent) -> + ok = emqx_ds_shared_sub_proto:leader_invalidate(Agent, Group). + +drop_invalidate_agent(Data0, Agent) -> + Data1 = drop_agent(Data0, Agent), + ok = invalidate_agent(Data1, Agent), + Data1. + +with_agent(#{agents := Agents} = Data, Agent, Fun) -> + case Agents of + #{Agent := _} -> + Fun(); + _ -> + Data + end. diff --git a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_proto.erl b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_proto.erl index d9a0b994f..7d81de083 100644 --- a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_proto.erl +++ b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_proto.erl @@ -13,9 +13,13 @@ -export([ agent_connect_leader/3, agent_update_stream_states/4, + agent_update_stream_states/5, - leader_lease_streams/4, - leader_renew_stream_lease/3 + leader_lease_streams/5, + leader_renew_stream_lease/3, + leader_renew_stream_lease/4, + leader_update_streams/5, + leader_invalidate/2 ]). -type agent() :: pid(). @@ -29,6 +33,12 @@ iterator := emqx_ds:iterator() }. +-type agent_stream_progress() :: #{ + stream := emqx_ds:stream(), + iterator := emqx_ds:iterator(), + use_finished := boolean() +}. + -export_type([ agent/0, leader/0, @@ -44,20 +54,27 @@ agent_connect_leader(ToLeader, FromAgent, TopicFilter) -> _ = erlang:send(ToLeader, ?agent_connect_leader(FromAgent, TopicFilter)), ok. --spec agent_update_stream_states(leader(), agent(), list(stream_progress()), version()) -> ok. +-spec agent_update_stream_states(leader(), agent(), list(agent_stream_progress()), version()) -> ok. agent_update_stream_states(ToLeader, FromAgent, StreamProgresses, Version) -> _ = erlang:send(ToLeader, ?agent_update_stream_states(FromAgent, StreamProgresses, Version)), ok. -%% ... +-spec agent_update_stream_states( + leader(), agent(), list(agent_stream_progress()), version(), version() +) -> ok. +agent_update_stream_states(ToLeader, FromAgent, StreamProgresses, VersionOld, VersionNew) -> + _ = erlang:send( + ToLeader, ?agent_update_stream_states(FromAgent, StreamProgresses, VersionOld, VersionNew) + ), + ok. %% leader -> agent messages --spec leader_lease_streams(agent(), group(), list(stream_progress()), version()) -> ok. -leader_lease_streams(ToAgent, OfGroup, Streams, Version) -> +-spec leader_lease_streams(agent(), group(), leader(), list(stream_progress()), version()) -> ok. +leader_lease_streams(ToAgent, OfGroup, Leader, Streams, Version) -> _ = emqx_persistent_session_ds_shared_subs_agent:send( ToAgent, - ?leader_lease_streams(OfGroup, Streams, Version) + ?leader_lease_streams(OfGroup, Leader, Streams, Version) ), ok. @@ -69,4 +86,26 @@ leader_renew_stream_lease(ToAgent, OfGroup, Version) -> ), ok. -%% ... +-spec leader_renew_stream_lease(agent(), group(), version(), version()) -> ok. +leader_renew_stream_lease(ToAgent, OfGroup, VersionOld, VersionNew) -> + _ = emqx_persistent_session_ds_shared_subs_agent:send( + ToAgent, + ?leader_renew_stream_lease(OfGroup, VersionOld, VersionNew) + ), + ok. + +-spec leader_update_streams(agent(), group(), version(), version(), list(stream_progress())) -> ok. +leader_update_streams(ToAgent, OfGroup, VersionOld, VersionNew, StreamsNew) -> + _ = emqx_persistent_session_ds_shared_subs_agent:send( + ToAgent, + ?leader_update_streams(OfGroup, VersionOld, VersionNew, StreamsNew) + ), + ok. + +-spec leader_invalidate(agent(), group()) -> ok. +leader_invalidate(ToAgent, OfGroup) -> + _ = emqx_persistent_session_ds_shared_subs_agent:send( + ToAgent, + ?leader_invalidate(OfGroup) + ), + ok. diff --git a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_proto.hrl b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_proto.hrl index c780ab193..6689a0d3b 100644 --- a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_proto.hrl +++ b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_proto.hrl @@ -49,6 +49,22 @@ agent := Agent }). +-define(agent_update_stream_states(Agent, StreamStates, VersionOld, VersionNew), #{ + type => ?agent_update_stream_states_msg, + stream_states => StreamStates, + version_old => VersionOld, + version_new => VersionNew, + agent => Agent +}). + +-define(agent_update_stream_states_match(Agent, StreamStates, VersionOld, VersionNew), #{ + type := ?agent_update_stream_states_msg, + stream_states := StreamStates, + version_old := VersionOld, + version_new := VersionNew, + agent := Agent +}). + %% leader messages, sent from the leader to the agent %% Agent may have several shared subscriptions, so may talk to several leaders %% `group` field is used to identify the leader. @@ -56,17 +72,19 @@ -define(leader_lease_streams_msg, leader_lease_streams). -define(leader_renew_stream_lease_msg, leader_renew_stream_lease). --define(leader_lease_streams(Group, Streams, Version), #{ +-define(leader_lease_streams(Group, Leader, Streams, Version), #{ type => ?leader_lease_streams_msg, streams => Streams, version => Version, + leader => Leader, group => Group }). --define(leader_lease_streams_match(Group, Streams, Version), #{ +-define(leader_lease_streams_match(Group, Leader, Streams, Version), #{ type := ?leader_lease_streams_msg, streams := Streams, version := Version, + leader := Leader, group := Group }). @@ -82,4 +100,44 @@ group := Group }). +-define(leader_renew_stream_lease(Group, VersionOld, VersionNew), #{ + type => ?leader_renew_stream_lease_msg, + version_old => VersionOld, + version_new => VersionNew, + group => Group +}). + +-define(leader_renew_stream_lease_match(Group, VersionOld, VersionNew), #{ + type := ?leader_renew_stream_lease_msg, + version_old := VersionOld, + version_new := VersionNew, + group := Group +}). + +-define(leader_update_streams(Group, VersionOld, VersionNew, StreamsNew), #{ + type => leader_update_streams, + version_old => VersionOld, + version_new => VersionNew, + streams_new => StreamsNew, + group => Group +}). + +-define(leader_update_streams_match(Group, VersionOld, VersionNew, StreamsNew), #{ + type := leader_update_streams, + version_old := VersionOld, + version_new := VersionNew, + streams_new := StreamsNew, + group := Group +}). + +-define(leader_invalidate(Group), #{ + type => leader_invalidate, + group => Group +}). + +-define(leader_invalidate_match(Group), #{ + type := leader_invalidate, + group := Group +}). + -endif. From 03fea34962c8ef6568c7c5f3d7ee403d5270cba5 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Wed, 26 Jun 2024 17:51:08 +0300 Subject: [PATCH 081/131] feat(queue): document protocol between agent and leader Document leader's states --- apps/emqx_ds_shared_sub/README.md | 12 +++++++++++- .../images/groupsm_leader_communication.png | Bin 0 -> 327420 bytes 2 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 apps/emqx_ds_shared_sub/docs/images/groupsm_leader_communication.png diff --git a/apps/emqx_ds_shared_sub/README.md b/apps/emqx_ds_shared_sub/README.md index 9c4c15870..a5b08ee26 100644 --- a/apps/emqx_ds_shared_sub/README.md +++ b/apps/emqx_ds_shared_sub/README.md @@ -7,7 +7,17 @@ This application makes durable session capable to cooperatively replay messages ![General layout](docs/images/ds_shared_subs.png) * The nesting reflects nesting/ownership of entity states. -* The bold arrow represent the [most complex interaction](https://github.com/emqx/eip/blob/main/active/0028-durable-shared-subscriptions.md#shared-subscription-session-handler), between session-side group subscription state machine and the shared subscription leader. +* The bold arrow represent the [most complex interaction](https://github.com/emqx/eip/blob/main/active/0028-durable-shared-subscriptions.md#shared-subscription-session-handler), between session-side group subscription state machine (**GroupSM**) and the shared subscription leader (**Leader**). + +# GroupSM and Leader communication + +The target state of GroupSM and its representation in Leader is `replaying`. That is, when the GroupSM and the Leader agree on the leased streams, Leader sends lease confirmations to the GroupSM, the GroupSM sends iteration updates. + +Other states are used to gracefully reassign streams to the GroupSM. + +Below is the sequence diagram of the interaction. + +![GroupSM and Leader communication](docs/images/groupsm_leader_communication.png) # Contributing diff --git a/apps/emqx_ds_shared_sub/docs/images/groupsm_leader_communication.png b/apps/emqx_ds_shared_sub/docs/images/groupsm_leader_communication.png new file mode 100644 index 0000000000000000000000000000000000000000..48040ccea1e609f4228f405a6f581d3afa2b4579 GIT binary patch literal 327420 zcmeEv2_RML`nN=hQ&EO?_OjNy-t|7y@A*B?yY`W7>PoX2IT>kaXl8Fw zme-=8p(E1JFby!wfFnJ%^+WK_G-oZPO*E;G7q-*TL@2r_Y(?ovkSMVJhzC5siUKy1;KQelc}AvpuMFF9D?85Iam;O5-csq=ZFZ2hzSTu2#84F zg+;kV<;0M`gv13Ughlnp=bP@bv_}V2bMPeC+M05UD2fXT!B9MErsf2D7YAo6ZqZHf zd5gV^r7ipmhv9Dx9Q?H%{x2+KEG%v;$qPTqIXc=}>ROs@CBSSH#YH6rMJ3=c&sJr; znku)*X875bU}FjYC|R1@I3Sm7wsLZ?gCmN(GM4%c1c`{2E(_Q4J za?;W?ceQYrbkW*@mr~p!rQ^9pi~KZhmQKzD2YdA4#RSC##mT>L@o=;xAKm5Pu*(*1 zhz#j$ZVEq=RueuVN6Z|YEG(VSLogK)ZcznpAvyRB@}KBtbiv_+y(vtabm=x&Ljhs( z^#p9;cWiQV+rHUW*ICKcMpKR8scY#tc{FTHyFE%yrjAx>4i=>4wb(-*R!UrqJf?*Q z`GB~P6#2j|C+s%pU6BJgf+sp3VbZ)sgm$?SEG(VL*P>M;^YCtq9aP8lSZ{y3SAWU&#EAb9a(k}oa;cydGnE~ZYqAlQ&>`HSa~#Qyb_|DxJBS=ySq5ZorZl%GcnkVu9+|DT%N zU(tfUB-1bH4oMF{k3Opb@&VJGAE*UlsJeVp4TMR->hIKp?-~2ErfqWoEc4kriD3Q+ zStD2<@yULNY!B#F-{*R;n~@w4#UXxunxX)TeyqDf_4JpD6`~ksL8T#ref~zY5SBzk z$j{>^B+171OD#V|`O)R4AhB;OKW0Ne&+4P0`fICiX>TEipj)teb6ZnqXM*{}&^!@H zVa^tFush%~N^Jgmpr%A^(lvsR;9!yavzML_tVvOQo23&0kQ6BiK}Pt2z8lMhz7@eq zD*Sb~g}FM&T`2fWh)P>9kWUTmuw&mH=)Y$QpC>ZO!HZ!I9djv@UBzE^@PB?w%pNc^ z!n_z}ALO9<3&s>Q@%-^-g_#oBL`1|93;9zRL$Z%45SU_Se_~ZX(11|i{!Q#d3c=r_ zy+{i3uK`g~MlsAqU^a`G8iKukc>}^c0hI;qB-mRR+gh3;eADF20@89wv8Xe4Q(M9= zgx4|$BS2!qCM0<$0NEmk{=$x6VS#V~UrhZc#x(Nn{@(0=GQP&VFI9|K`-Su$qXaO4 z{{-1Tn(+M3NKO=!3l_z&sP^lL4qOM~J*Y&d1eV~4Q-gy%atQMc6f|-MRZ{=!o1O$} zSE&5N(2rE&!_J^aXFs9kiB1upmgO#jGl^yV$4O0WlG$N0`t`(ymMy5PP8E%E61GTG zvmm%}i%9H3e&m4~0Ev}xb}@Ca1Y;zh0h9>UN&i3-6~V$G=3^z$+>A=vlqCEox6-M6 z6IxIr*zfuei_Xab)QDXF6M(3NCC`5^(!mTB6D3ynpb{l^%FiuH5v&sMXGNvI!f-<} zN6FaW=N$-rBKW;|9woIST?XCL10TRFf3QJ1SvnIuP0dJM zMF{+IM9Su*Rvht7+~V?Z1Zif0bC5m_q$mdd!NJCIvxBVz(l%%BV2^N+(6(qh=@6RB z|E(=Y!lEdIMeSG=3#U}#r`Y<>ZOEdiKc-@)CgPd@4=_%G)f6!P^_q)?sk0TStBlfV zM1k}Fg&rg*p_2HzZ&qM_a$r80V-?3px?dicV-li5hCe?r7stW`>Nmu&at)Q=z+4Hn z+nAhA{RHtJ1ukD({vXf)0d3XaRj3sbT)S2jITiC2l#KpQl6caIe`5;-mN$Oefsp*~ zU&!pKR0gYNQmYJh>_0;QOm2b@!xA^l&;0l%2+>J>7vk6xW4HWeIR}xctzGi43F04D zi4Yf`nAB2&6_sjZIGb9vF%O9e`oHfDu^Ih9Z-{m5!bTO*iJnbRSV9tWb^jOdM(Hra zMx-FKwIX6)ce8Nj-{>p9yVaK*^M7?;Ny!ya5I|{)#Ps>c`^w4vE|eM|cFSMZS7M16 zmF2;%0gO!g2RcnL%-3KL5{tCIp40pSgcRRoZ|P3zlOU0S2odnli+f^N5{5Z|FJu8& zLFFfpdsC4LR$AZbK&rQ!qkIh<-tCI)n1eoJQ7O{@s1K2=^b?4u2~04hTu&aBd^&ck z2_<~;nSbT(+~UA7A^S8DrTgj)|Hig~Uz--g=xj`}F<(XHo4!bkVFrPPqyIjl!!QX| zT$$pDCh}xILEUfi?v^hBYD^^5j^!uZ-6A%%O!PlhUzl9*6T=cADpSG^QHv52?@we& z{}vbk(ERCtlREQ}oj%}bgfTf{DDMjp;ftOWyY45F{}*81WK8$ddr}IyL}h;1Qd65B zZ0{t8iJxfwr}oWZEfgfY^*`D{A%S^MDv4r;s3nR?`6m)63cCM}ikdDj+LR3vYHnIBqIfuYnFMr6J2fm-Q zC;7C$Ktz2v8;?YcZ@(ktM{nsCo4T!aa!v+?@6B!^|H$T?$=Qn(x*Aj-LKyS!Ke>xQ z_fh^GE@EO)ADWQAt3XF_ER+JBpoHYwzd~b^KW|5BU82)+oj%hPhmJ9D| zv2@3Je@LBq|F-^6zins=GRK%YYA^9?010d_2$h#W%nFCL<)bZD6S4VU=?0>dvM_Vs z0a$?2=JOd=kZ=M8R{#h=U(l)4@1u9mf+7VEql!(6rC&qKUr_D;2r&L`rH7S(-2Uql|E!V}6%i5o$$>Gp091jJ z>{Fp%R4@*Ln&NRe!zaEi( zWb{F4@oz>SvA;b-{T?I#Z0g^LK9U#`#b}maHu{K9EnNIB(QFv;POLV4hk7CcR>pK-w722r6#w5!8Y2jZ&dxedtAl7w@IEHJ-^3c zOsa-_H!Ap^vH#o|A-_SoD>0?TI!iHa{AIhNum;Nsu!Y>(;OuB>PinNmUhIqYXCUt- z5g@r z`&93X}7+d-;>lV5K4 zm2x3DF8z>B83{B|`)1t%iu3=-0wA^kR8b!5qWz!Vmmo~(BcQT)4BJs#Jmv*{qNqJJ z`6XKn73=fYa;6C_a^J`APq1yuhtK)dgx;7RY1nAk`KDq2GczAyA&fl3^zv^=eK25) z0$t$mCT$YWgE$3qHQ&T3f4KvZN@p;SOszB6vHuQ33n+V% z&i;6uB8ojChEt|&X8d(=%9r+_exnZpeRBm;c6Qoj#skZbe62Y44=>mOd_>+eA+Xcb zj$jLmhMcNwY3pW*d|wjP@5wJQMSd$R;t1>~8nVQ?|BH;9{8+%Jjk3z7wCBW*52rVVpIYQ$rXbGw}whpG~;-h;vzR|Tq>4iW5`k%BxkPjmi zGF3MO)++=8HMg`NucCmdg|(~m66MQyFv9u2Pd;E2IF$hYw`*FWU#MdLBF+{%29TzK38I|3q5oGQdm|4`s1g-{B~xK`>LJ)%fKlhh9i zk|nW=bFMkGm3jd{{f#&~(Y3N^yKbyF-N0{=B2h=zYD|k`Y z`LLYrCrZzz@@E)Z|Ml*2vZ_t0eoPf2IgbC8H>OW`Hz1{$fRs>3ZwUNcIzaaT|6g9P z{sTJD(Mpv()vnRMKZC&DJ_N-Y(k=(eZcS>%!}_NF(~3uW*^-k3@*aK#ZXrZ7KL3Zk zw#muV(Mrw1!V-#+ROgU>Ye6vGSy^j-zimlrNdfg$AjY$ ziM2Sa%)5E%vb-bvACLDI7B**%kDR~ygwX9ga4of_e^0})eCL5W8^aoBLmQhL`3D~b z!vBs2(9kk)`Azyu_8`5)^6=91wP!ZdOrv9#o%oj{EgjBw1r7lM{e z9uUT7RxO)3c_d`uE6Zes!r0sd11HS^`6w%-uw>HBNgwY8(iGu!YIAwMFn5|FR-cLU zC6B4-VD;R=V*o}wn7 zK|`vld27tUzTv92mz=h}f!t!UmmZU^J#+cjJ(TU3+0L$tFx zML*V#uyBgE!5eWo6@_j(Sr(&IAEs-xBUK$vwHxkh&wd#qtz#rS^NG;z7Ww?P=%B7M z#=g8)-R$bKvsOiVms{7QE|Pw8&Y>`4xWDtRZXy%fI(i%5>M;nxd!%BiU>dlYI?1bkn82Na7^PJsA zFr(a&TnnMNW}o->S7i*YN-`K98*MzC7{#orJ;A?XEtOuw2IPtr;G}>u}C4dq>7g z7r|hWB%PFq?#v2xLcV6v5pnI_yG!=2c4lmOdt=-_%eu=i+bsBgts#crU*=#L4IX8r;v$X zqg%aQ4{?3(?fGt>dhed`+GphZ(Xz8VE=YRk`&aj?W0d9&w$d}P*GjdXe;GiOZ1%25 z&3(g}|MAVaMw@0(^wrE-+hWRgq}>?vw6%_^zFh8+n%}do-m^0$$Z)(Z=2-Tc8=uOD z2fA&1hWmK6CDr5%GRzJb?k-_-yYt7cYc*`Y7mNh>8#r9{+V=EvMy!tz;oY&aMhiwvP>)w=bVb^uc9qaku73KNRZ+@3SN`zbEfOLF{nwl8qaxcUCzg#uBN0 z=GNSe4Qjddd&fsp6oWY}hi@MX_k49ise$2&zC%G(){dCVEnBaLDOxXbt<=nRe6cAY zk#TTC)8R>Y$1EsgAst5?Cdi+C9M57VpqI%N{=W0WzUs*saT~9{G99kYLw&hF-afB= zWsW+|yR40P(q==+jZ_u|##nsIzQm!d_<)!D{I+y%koolPR@^)BMhEczZZ~T5aGMYQ zwtH`?s!*IliMUFVN*G^BYJ8dM%*A`6MlKLJtmOuytW+Wcgz0sQ^6*DIOS#V&v{b)g zd~7HZUT*Kh*dsPu(@ygoo^_x*{ni}&SK-y+<;ubVi7GNd*Z-Id4d2o&V^Jl{jy1ub zVqYSLx;(j%4nv(lz#ZO2MC@n-U zRxl?E$0^(w-^8OjDdC!3N9q+b7;sI44IS>%kQBq@^`jfp;1VgqGtVH^}A%1%6 z4HcG}q!ZbT@FK?+-<@nTezFz;yxkAu7rhq?RC;m$;aO2NfA#2Sp|~xlR`zTUOD_ry zv8x~kKZ@MgOuuRC>D5Pqw=<^3u*l2tNbGvNutHfdY@oaToNrHc$~l>luB# z3t~_5gqOda-%XS7l)u(rh~co9f=1`2L<0!N87!Jd7G|b@1i8F1D)-dpwA9fN>VI)) z$({9VZiK$=@#+`4UY{`zxe#FzIG6RyYZ=PvGd?<4pWAwIfz1tep0^FTR~Mwj(&z6T zZeMk7oof=oMVncr{W(0D+OdjtI(&K~q!7XD|F&T8qHEjJq6^Z#-Q0^VpU|ok-llMA z#~DFG68wYs-2iZ+Yx{)#<6eigejlVKI zF2dUVu~cjAjq%a9X6_QTxq=&xmS^V7*H~ZHZE*(tQ2%MMjx9lvm)ja2uU$_HGy(GMCJ= z>P*SW2I!Zksan()RoY}muV44bzP&uo{zUK2M-jQ>HJL?M0u`m(_s}b?%O+G1GZOu{ zo9ENv03q#uQ3Y3arQ|d@`9~0|cM>bAyJFe1*QO@bwWhN2&gfusvvp;H z<__AXW0N8-e8g|(Dlftx&~|zi%)KvhVIq6gWb^}?qlGXW^X~MW-*3VvlX{+QiaT`L z>}SF!*|WJ=k6HEgyGNI-Cj%0$ZG>qMi{=yG(j8jjd#!#n&7rwa!mX>e&+xG5@~LFY z4rZ?KQrdNA#8%){24nuxqe+Zw^jfs>HfFnSglPvjr^xI5LcV zKb92u4m{gcA0plJd*b508soh?274M)d`E{7)vLF?`e<3z2F`s`Zko$2D`dG7PA@QY zek3m8^h$ZbT1S&&={~^x^?-#Jcy&LPewzuQZ*5M`&Uslo05=zmkB#WWsjwk9$o%a> z6uwJ9Iw$?|sOZVH%MVR)#x%6|XYE;{DxjbJaG=3iqu%;*imK@&iF(^p+un6tPRnLl zu6Wz19#XxoeRDS`t4kjW@qO+;Gui!ZQ>vnZf8D5|dc|`eAy-Jx>MIj*c0=!8NVv2;-9M;{+xcX~WqQp!lI z{5_a1f21cr#rTG=TaBUnYs--Yu_>wSt3tn;qRQd}OeqRM>^*`h2D#-2G>(*hc(T!< zG0)X`4ZVNxSh6NwvcYk4LjETZk$zQ@_U1(k_C5tGvW1C5{+z_!Z&aZ1w#;jO*!zG) z5Z%29Nr%EMXJYwo7fn%VKiOH+()J}96?i^_<*>TD$o)K|BPkCbUpi2>#H&lV-tvrL z*Ml`TTrLGypD%A|jTqL^c^-R0C#S(FNP6(o#}9VkQFF{gr0%G19Ch)6;Lrt#-?dYk z9@i1TXvKJ;`g(+nS2oi_v6R%PsGA`h$M$qSySgjGvv9cMvfF*`GuL?CMRz@RshABo zZ8xrsSf1x(C%e|3iWU9Go<`EDr}T*|iyu2#XbS`)0Jh>r+M`=fxHxTP-BUs>C;m%I70r*)+c zBsz^-HDvlj7kPDM^gLnA?HiMSeL`vR?$+7W$5mWZKk4DDFB2st>>Fw~jP_N8y02HC zoGXBZ>{*~npu+)cU@-S-k4sT#;Yq_ym$V}{_%x((tTD1xixA6zRP7-U#%l(pgWWhc z`g$i`WWBLHFQmOJ=1^54?k@cn#+-n8OH>K0IM9&Bt(=D9J0JdTQ135(Eo% zD0hkLDB!7gnP2c#yln3z!nK~pY*R)uoYl^=R~^u0%wQe5t>Mb#n>J`86R($LU1EQf z*i0|?c-E@(&)IdG!8@3x9a|(_EtuD~Y|zpppaLcTlkC=e zTaK)1U@b;4b(PeHa7ge@6}&5EDyfS;VaWL+F1pN^lbfj3#EC1)C{oSe(%5`0n?zwhLu%J(C@M?8rd1gVB>RFKz zwR5-~S1h*Jx@f-}2BkEGbkQaw(K|v>7aWnh@{XXSjUZ6_V7@T$WsP-^b*&CB_r!JA z=d7t_KVZg~i5D|dS)PHHU{fur+4bt)2?@vNH}M0~j}J_7i45U6MWNN!d2b6xY$JUJ zGQjA$yBK?8g2fCqmg;QFl+9&l(=Dr0uZY(;RzSQkeDJqFT6_g<#HueHTHH|`$*8DW zk~Ufi*@xt&%NAvaSJ7MW6-8+FE+@|GUOfcFRvyh(?hPyIfg5@Wh`}bOS{ryIyS}!v z^@3mU1S<5`X9Gjfh-8@3ZBhG>sLXdDNp?&AJA(gvq1T{1Sr^=!o-Ok1$!)EJkZjyj z{DQM2woFw{s2@=N_I$VQVuR8}G27H?j?5+cj`r);n*|93UWgHar>xKl*REctcZvVO zluRF@5$!aGeY`1WV>Oj5gTu>v+#2)U-7O-uZJ81vhS=l1wbt%Uf8G9oq^d$Kt8N;M>iVq2s&pT{(h57awku&6MGv9I3F z%#>KFzunMAm0nI)$jrLCE-Sk`yFPC)()qEJ{cY|0S4?jmtj)YdR6D2lC_=n~QlkM`FX>j1Xg?l1)NPX;*H)2gOsF5Q`Y)wbpKk)G`- z+How~pU4cJU$px+OON}hRT|lgWjt9s3BA&^cixu9U-*=%9_sdbq&+?|uxiVyYrr!# zdTQ|1g=ypu#M#`&d$&oL7Sn>6zfoa4Z$ECu3s}l8t_eR z=5<{okCJt6UE5cBehS^#-RBM-BqAV{T|loE0Dsq}l?RmQ>FEa`PUUYL?5N^*D;zTE z0Iua|0ffbE{Hs?V?XHYZ?+nGM?+!Y#1Q-dMW|dD#Ifco$-;*hq-(?s zw^_a@K2Uw#<3p(1di8@-O$Nln#>SRgQkr$}WI#fp&&VCm&FN#Oh#v_#*N4jR-1$eV z&zA3IJlZ#|M^^v@gpQN2Gd+IYN}i%8y+xIGf)}JXw?3Be8nAT_e;l9Z4K$YEE3LAT z4If|4zifP?a3nXnsIoP3Z%_4OMwR=#!R`&a4;gy4$ED6e2u>5dCC7gI!rJj6>O0N) z)k#HOBzg?gnje#@7&?L1HFOPY-19Q{TxPY)Mzz`aGUXVsuE?GzVC`oKOZ#?R^jPT@ zl5JO?r>VVMDJ1i(=&m}H;LNDW8mmkE?sic)jJyauGK@7(;9(G$szs z+xVVsJ{;))hcg)GdRg$jz>&o9C+Ccg4BRORPt0}}4v8+4F^m38e) z05x&9^_Y$vTP#yYV%&VSz00!1T;Bm>rFuV0%XMCr2jEHfPLoGxMOSgh+||;S{K9IK z&F2bN`5-e2CfE%SIvt6e0%lE_go;|)HbS9TywcQpIO##I5$`ZJY+`*`i$8t$OT~O} zYO3SLI%S->KuR3z2HvM51KBpl?J;-cTf($&**7?>QB z83^~1(hpFPHo|%V{2zBzUA6;vu5)##8j`v5d&e^n`Bd*NU{SsdRd78IDX9r=;kZ7n z=|0}7;bR-e(mBPgugzHK@#=&GB)lREu0mY~$&yY+tQ|6ol|jlNx0Se8xA#T@BCiCH;Xlel%s#B*P?>nsp^TT=? z^XmiI_`4#g(E9oRf0iSv+VZz@IYR6{(g0h-nYm zl6fceaRpIrfj)q}vv02MdXhb7(Yjg%4nqs`yCDLu=ZQkrd@q41uxU70U-^tskM}gCr(B?lDTovd2b8S{0T=on&}*60}0FadyYV&W_j} zT(&}S;zX!0yjEP8XjOS~6L(nf0?CMgRl$3-KGxhAv*eVDHBpKQW*ti4>GA+fU@njc zsgfO(yn1#8AMQY5|3%LZUXD5Rmbju&MFaYpa52J31QleLuAFWr>L_H=^?&R-|H3yu@t~fVVO+8}5$;Rxz?e7Idu;FR z$BBvl4%d4Ro^!fp`|OHYO}UF)a9sz)fC9*3`qt4gC-{Lh=%4PH2z!pF(E>IMm$0%p zQGGfIGQTcIwHFmU@$yB-=adg?aL$U0c8$t(!+D3t39b752l7=ChiYEa} zZ^Q%KlVineu|VaD-g#_r+X#vHL|Zii3r(sS&dbBR4AIZ73abDr%ycfET) zexbiD_+U$1q|HQb>koE@{5zq=%hirl)M@b{jDM#j@3kbL5Zd%2p|DA8cZGZ|wGk?5 z)llRg>8sF~UpW5il!hDb6XZngk2o{p`lDHohppCIrNWeDvnV;rmM@$y&8UbWW5|2} z2w#qB`fjG##gHd#Qw7dzj{gJ{xMu~kMEJZTifX*CG|P&c^=PgqB-~5FC${*;6YQYC z?~&`tY1ltL);u26ovLd@Q<LW*qrHx&8iJq#+}ZIs*xKSL)O1nKIX}+rofd2&Ns3DmFb z2w0Mxb+a|N8IETzu7JXjwk^wts9r|LK41FHdXap-a((i+kC}Jy;2Rt2A21PFk0Q)g z&su>EMgS|r2O(Np*YUn3p0jtl5oA1)h$KwO<*9r55W39f?en62YA?gLutqC87+|Jyza;W!Z3eaLDsf(64@N_?S&U0p2_xbU0kjYx#S;KuQf$MOqHx2`F$0LI8|Z}q*DFn--Spd3sh zA~-^A(h928G*>$WUJAJ&ZgX1SK&CKZjvS5B_$C_vz**2z@i5qZBFymMJq~WV-3f{{ zyQMz}80J;7;uGooL&lP6l3Y8hiM$SwLj>nqJzCYB$q_2qwC5Hyv2cv+c#CUe3w_=S zmem`R*0-46fbq`GCl|YmSA`)EKXPB=c#D!pdp{Hzb^ym?)?JGi>POHw&Fb33ZOh{` z53&Jj=L1)(Ew!;Jc!Jiy3X}bmr#XG*oJxwFbKx6M~ zu^eo5ZQBcSEIxK!ebTV0?aBHNs$JX%lo|8zXQ=Xv9#?@>UtsR@6l%p4gm~&)(}0+1J%n7J&Dw>&buHL(HMK!G%@*7A!jqIkn*))}z*p zio19ZYwgKOyyDiahkKrhbUSP|hN7!oTET!*Q1mLk2=#=-gGg75bT%N!b9wR^HR7cm z#Y}5lZd63Kf?M3I^|sZ7j+sXQFkae3jK{2;?*}W>cPdbkhd_t-*1U#jeV%dHdD`%S z(gWB1s_&JXz14FxWJ<#^%SOEi!wQpp^&9y)O2EqCp4-a^q|A| z)|!I8^4c3N@zXaM(`3e+s%AbCxk2lMjT-PrI?3B#WJ|5d^|MoWe5Pa2s<+v_$>Z)o z^WMEVYu6xI+2NCZiaPB#mh(Z#ab-F_vs1hMYVOuEg7yOZK22#x1$DvJr3c;>xml{9?LQ>uHA~4U<^0~UkIL& z#sIaYX9BF3qxUv$fIb_w7qg1+oav9w3l!l!ZKe2W8D1Tf)zmx@3>i=6yFKao zhTBBDlieXN%xh7a1CPa>asY*eNwK|Epz)`JCvagx(KQc_tCn5*A$y(W1$pBQ)oI3WiMIQB>$JDn z=+fs)SuPLPdPmGs3owk9(01rv9k=&mn)c<>NPE+{b?XQ6nskA~seNLqq{^+o(NI=e z6Uai19;W9+w^GJNnmLqW5Fam?`4-ux5R#e8veIYL46^AOyvQD#L)5bhJTR8EJ?KUu zv>g%4@Y+fE48&CfZ>Z6^$BN$}B{#H3oY=Q!u*SeSSifFArie8*aSre|*ZaGkiN3JA zC~SVP(dmIek4&#Ze|PhZ8(Dy+Dn}%2YfTVtg2ULmtZC27*)^&5jd{5?38R*3ueTC? znb=pS1M_v4-X>4i`V8@Ig4G2sIhVdXxdau>@+`%b}6vh`_;fZ4r!j%l}YKOpU+jnS=z9S)IuzRFSNL z>3%@8NjtQO@DDzQ()x{BJ>G~p9#D|Uw|hFD?SN_0=KCd2PNg&_@3?A~KL;x8IpLB; z_z)Re{7f?9dIBo7{wG9;W_Q^Kd25$w-#9fMum>xlJ9y#Nn+9Vkr)6bU3yYV{ajGkM|ax(1yC-lUohtef)UvhxwAWVDJx&?y^%q$sPRLBP4s1m>lc+!wdhl*6a%Xv58|o^mY^w+>x!HZF@e zk1hCd#5z~&)!|82qezZsP*JIjcz2Q+9uES4m%B$dVLthY)H0ZqN|4tQ&$dhFej{cw z3?JYyY6}^GgxU@|_1085sl`ZxC$E^NO3=LyMBsty>JSeEyyPbGo1&_ArY{tpsuRpOjob7eV!Z!xQz%#ew$E9Lydv+nL`6dbQ-*7M zx#cPNp^fk=G?0BxGhk{J=;nN0E^Nlk zj>h+sKZf!j?=exakyzw(Z)xfri0>z%dpQ^H1I?#xZZlPtgyItS03DN(Ppr-0$>8}az}jb8}b+$GU0RNQ()JvK&oZBwG&;v)18L zk6CaFB<)>r47;9OT$Eo(tmIv-bvQP9oi_vrog2PBPGbZu$8si?LW@rYm zg`z{x6HmM1{j=NPrJyx3o_*;^FWdpCL|DBKJPXS!uY*^)E7%Xgx)!2^EmRfb z?2VcWeeCWXUuw^2yseTwI5`if8|VmJ*R=cYBBUiz^3fWG^H8vQUVTpMth9UTBIk#~ z8C-&E{fWP&8sBK@?pw9s4|?ur!v(-E>1^Bfyeai&8dROi+Cwwx$RUz$88FpsiS=8| z$+XWs<7vUamE*G!K#PPSQZ8u~t$G^jq-7h1o==y2pNNzRI*>9!X>x$uyvmkV#!!6o z$-5l}(DQxvT)8fAe`a#0jPx_DPAu3G_mr>9tc`okdD~GD!rd&4T=EZubW&SBco3~R z%U08^R6lQ3mm+xf3fLghZ*t0miK-FL0(xVTezO?uB&KoyKD?xPO#Vja(@UF$qvh`o zSu5w0#8bX1ibb(VPu(W>^125q2emB^=T=-uJU^MEuK`owUZzUVkt=wpCEt$q1T2&&|4ojB>hK87~hu3^@HcYVY4DP zo3noOXiKU8IaxOBq0u!EA&sIS6xMrJ(1*iM=g9c$jKNB$j_?Z$2e$_M`N8ESB}s^!weT>ETb z=tKD@L$-+wW7*I0s^uB839GI|W$wxe)7b{K4P=-=jTv-&OK|)Qv5?9ydzVI>w=~v2KlWp zPFY@L=8Vs}<8J_Jvkd-}maICpZYi_yG{xNtT?fk-Qq4wd1tKI^-snR|DuC@DrvJSp z%-1}XYH4IwqRT7t@%C;3*b4QBzfjCI{IipHxf$y9=gMhj7omd<_DijFFHooJ+{?ID5+PtPTjtQ!) zueDqgm}ot)OLdQ}4ufQghiU(;Yn5ZF2MtVV1*?u6=P#8xm5mT6FJOkn`n!Px`jkdi zDAvdspa8OT^u_xP&i-~ye&Nbj%bo|C?26sO!1X*#mRa?qNWi_E0ne(l1lv0#U1lJP z3|!m26xN1ooDQtRT6^W@RH12SI^^$#?Hvu~T<#}3#x%rUIaJ;B>FuRVE)H~6oOaT5|wrbKSY@kXuIz06!w~ct5<{?+(Ikeq6=8^10S!kw2Uc%({kOwJG&e zr$goM)96mle5I>V zm%W)2bth*)Cu-|Eb~*{jCK2&w-M_nkI@xybZS;qA(+j(QoL}{3+&UJg*T#Ee?l91i zZ*}Hki0nFa#mjpwcB_-MaoGy=nd(T-w13@FwfhUuwUbPHPdvHAZ;px z>!yg4+sIE;-p!(X2))g^i>n_WL0yB#QbbIfKa1&lo0*69{TH_m=>zhJ>*&(o_EO)A zze;M4D6Qq|HPFnb&=$)6u6sJHQh6)Z=je|I_01Le!!gl^bZ7kF{!cQI0lZ z*yOB6T?b1Dz4`9Nt<4&*D-MWRrl+SLXTgPOwLW4PLT&PO>s5`*nEJt!Df1jx&_RI~e#Qm|fxAr_jCNl|Z+274=oBLXL=K_8GFcfDg(-3g7mBP6%CzPlVReHU|k zpO(gXQa!JQO4Ri-Ep8ct)CXRxsn3Cn8}+E%7xtfm9qeqWbB6||fqS^s zPa8t9Lu51yIPZ1@A`jKmCuTj58=X3jOj#yXBa{E? zSr0GD)I%d$TdENr9~!4;F>r+rceekHbr}k>0^eL!yG*4C_YV8fCPIifH-_xc_~TOR zG6wlgsIARACPKYWlItq%5n71551Xc^9+YK8NzAOCU2hz?R1T>dnUuxSp8lV3mI zws#RacwXMuGw$vt%8P%oMzV)c8^jYjt&9pI5w!s_aO|TGQQ%5Ou}p3JftS=mnD3A8$vmTa z%7+vLMST3tSs2hT#x_8Bv6kxboea39W`HN5raZEGloVbrK3pdj({LRH(kVO+Uxx*V8#F!{9g)5KtYy)A<9}3Ni z%pa?pEQV?CE@Rd~?We1vcsG^0yz-MJy4P(b-Y3iRY~jrehW+R)8AmcPUrM@S2kX6R zb+7wkWK;P1nljPmW{2QC=*Yt|HAZgrL`C(wXrKE0&yMuH23I6)qHGNrkTm1ugFgUi zL<*}*F5){r_uoGcmfe6#{?T-m%{a`xy}uhcpIY+QWm6FeZS`It_e$Rjix6aVp02@k zm#S)fe5JoZ6KKgwa*+P&Ld9<9TH^Md3&lI{UMnd6a5K5w4S=#1<5 zQvZY*TGJxQQU9rVvE_Ocue|TN6rcN9PG>okVecT6MUtbUxITn!dC5jrK+gO*@dMk* z>s9)4E=>fvGKGNe;sbcSo)doeRb>4Yt>+UUcQD?b+&r`xO z3W$<@Lk|X?s;Q+ZNS0lx&zWpX4OApvPnPI@-X?oE%1=>=5Q^Uht?$cYqb2Otl%;$i z84-)O!JoR3_r_cuOy>^eZvGAz(n7}&JGS>XQXcqV-?1-cKL{SU$24>WELgIf^mJd< z%;)2D-gO}Jb3G)p{F;93S@ZlQ=zD_;ISo}ce!iWq81%J4DE9D?N+7=uw z<<>O*E1G~|OioW{j$uYIlsw0CX1bGT)~qs|@nZI8ZPG-XzLY=yy*p~k&CB=9 zq&>-)wG$%9*@{Nmi{wI-8TvTJD?36lX$|`AN__p2 zO{9qv3-3BA(mf={s-YF1G2uHEKUqGX9P;B4(^?&OMxqhTdyFos8=I`35-pup`;BL0 zT^W@*%s+&Nkk>CyX`G|n^cY<&VU07oT02u$?dFa|)!W;%g#hJivei95OC#NFIdG6H zr-uV8Q{M!kIm{Xj>23;$Uq8-@msz7i*Yrvv%%x241PyHAZ$xjaUl3d; z4_gaK&X7+-w41Ver0F$WM^{DCYf)CII~oeS_XZ|zq>P6=?(3e}gCwKOt^bg)V*|$K zeVDJ>IC1#~vF8|FtUUVzDGdfnZ%L#<$5D*v+q2^4I<1e+SIORItWVxLLIZt}kA1hb zbg}SvbNMvd#83I?1jjvBh(Df*O>mD^;&RFakp~`YLLc~s%&CahD`?~u+<2*SHm2ra z3>OC8kE09tMG}cLMZaCNU6~xg3&_SDQ?`|$kc#F}uj_DIe5eEh`l5&Gi{j;+@(2Tz z?0TiB$2U^?_A=6XK1ETdgs(#amv6Npx94NEg0SVsE;H>-~OF? z=XlYzRF$zaKr7>P%S)8&Vol6&X=p7On3c=KX~~q*Rt+VtMX08<@42#e6-5L5K}9_rPo1)jCf!`^ zinQZe+DoYMZj#|!xF1_QF6K2+;<2o%^p;Rhv0ua+*i80yHE)~LIx8^3lgyQ(*RZgH z9WyjlvP#@1yQB2Wj(l{KSj#2?lN zNfUmwO|nJgBC2E_15YaqFeBhP3d;REcL*;p83MNVl7{39Dyp15cRkXwzP)WkFGg{8 zJuqy^&_q#P;N@-uuQq8LwC|AJuuYQtb|5*y_u)N5lE{ZG?fKyD`rYO6|z&4s@A<_anRc?4Y-70P5=X z(8|{h-G3of>Ck^R0G)^Vb{_rXd{BVPg3UcTdGjr!3&K)YtfDw9X5}CxbpRBS0suy! z=4EUd8#xyb#+08z*qX_XF@kpi^GLLxqv9jd{MW}rS6(LsEn0XzlzI7q% zIue|u)j&kHuS>(SUD~xsC2{CI@k)k$syGSgAVwo_k*eh*m;a`^SFV>s z^mb7ao6^KHu%j;2hdgPk&_F&59 zoDMBcgnf_?Sn=?ZGRpxd&0ia|ZcMbxen!+PI5M|VrMGt{#o$^I6wqpagoBMt==Mj? zuef&7oPtz!guQ`A+jps`|M*at4;ux9Xg5Ukq0S(u&p zpJz5%lz9Hy>?WOJ(jwK=hp64E;DeVEyf#UK4rT$9R8382INh>+L2xK1yebX4ugVKj zp(SCfq0@bCGLH>J6q!S)keP@?;aexVzxR25@B4j!Joi73 z_H|w7u!dtD>sSNZ9Icm!xzz5YMm6Z-X*6m_IA0w`$W<~s^@A_)rC2Xm2j8nh8~%w-28O07Zr)yLJ} zGZzJtFDYAgq%?3xAXh%hT8jO=VZHxkZcrtopSq-Up=BKc=zofpjLiR@HiW< z5_R$xLwiQa38?Nce%DoWVhi6d_Nq`O-Y`{<7FuSifT0!O4N7Q11xqSqalM@RPD3oyt{$j;fiCGV@Pb??)dBWEgs}vIU$Pmub3#1 zWKNmeq>=XDVsw3~YR>gko;Y ze($^4)oZK0E5(N8El}E!dL*|{m2Ld^O*n&0jA-}r7!?wGLxe@dye#-LXfUeoVVwa_ z=k}7RbiA`}`JR}hB)mBf#C+s@K(XpB6v(W+l*m9uXSA}K%9V%NS;l82y?O1vu`_dl73z?~pFbpc~`a(oVPV4VyZv+h7F zU<@|oTky3FgDo;IHqI!wVKOeHc*P>38E~zffXVjWapvnb$;W@pQOUpFH;DKTfj%Df z`->l6`k+ZJpQ-9aGKR{bK9(lEH89lP=_%6^r}> zeNP%6!Oe_ZebQxI@J~#qi;*-)3OYmdQ`_m+I%<2@@D%i;V-2f+S>w(|lYh`a-JWS(* z76D@rI0SC48?E2WuH&Ed>4zZbC_?RYE35FfK0Fx`hU3s|=JwJMYVt%H?dsL#$G{oo z$z-;4s1#tin@q26h}nX|v}aBZZgap#E?vkCM%1t{1~R{1sxvTwLOSn~B@gU@sDSnu z>)Xi9!yH@25_h)*r3KM@;fQ?IJNX{LkXY2XumBgKhMeUCo$2lM0>&4hZP8UVhPY3l z%hX8e3>$N&ixw$j4Vto=FP5W@LlD*b*>*nsy?;WSRS7fLu$VichS|J1;2PL6ZM_UI;lDI zWkO?Rg?TCTvwuW#FsLnR68a*%>l~*rI(==-3b`W%Aiq#{-+)K`j-Zqc+;dCU!(JTe z_l-jvU^}PP?Pv%~SW|^yj2`DToP-SIMdKRG)M=YO=U~TGe0lylY&%T-OY!~m_4*EE zbj?{cK|#%^0)z@WB`1t1fCcVerYdoAUr2qVVD;$nBi!F=w-sZp9=W`C&u@k^Wn~;xA&_U;A|jX`H%OV@^56}zT%f!a>?g}7Ou+BMDdu)}NE}6p< zv)i~SL62&)^4`u$aax6J(GIlFtn&1rLD8oW4pS$n+nj$X)$!Jo)q5DP9(Ha$`84E1 z&A=LG?z&Gq%>VcqLgx$UZn!@3ylH*1n7l?ok^tUtOnRUT9!Eo9p>V!5%JwswZ&Kk} z?|{VeZ=_&CokEvcsfJNRC>U>A85*~3@2jPh#hrYXrKs~&Y z$j}}g<*p1SkIN8R58KIXofLvVaeDuzDgjw^5^H)4M;K?`hfy^&h?s|#=^F3vVGLpO z%ZG2Xe^n=pRi|Tty!k-byyU~mnI}5&M_*3eF>&uP0^P#CQ6ZfjJM+>Rk6f(*?zG0y zLQ46}z(nXiDHDaW4XJ?vLHpL8Y?u$x74{%cSmu~Vf~|lqfTIVc-EAq0jzixg4y$}(u6hUr#?t%7&_QqUY>>sV4`_#E z52r*c#uLJ)pT5wUAp9jAx?-*UTcTesOmvl}tor?&9D9}Xkk?FMCEbwm#3=MSw$s_Y zQfguTz-FlQJ$qmFlgDd@jNi|^*a(wHs&snWpd|`!x#OQ-v#z+{qNkm@D#I~ImQzRN z1M9ZVc-+9YmkH}{? zWl1qNgl*<8N$&wdTQbXa%!D>gg;~qhm%kYW0sqEcf7y;C7rQL4xJ(j3-NmPz#lbXq z)wnn(EpTBM`3SYlGO5h$M^K`pLDTI`{J~(4_1Pa;pR{`J({0{v?fJubUmk$x{pb2$ zohM6*CHayQT)y?dxetS^lj-=vU!%KNL{~bQy zZkejaMqG(n+jOs>-V{)mIQqDf$k3iM=i&DE-{=1i1fWrJaE^8;ZqqSdT~K#S65UA4 zJ;zY6d&vL05OXx~>>xFapt9h6^tP&j&3 z7G=Fi`2a+eGDw8k|5!e0EC>{(AHU@|T)Fv_ejk0dh^URXN6jWH%L^WoAK_x@@MMUT zd=5T+7pWB^Eh`sD=cawwq$QzgrofZ;-)JB9S&^#_1sq8&FG z+Ks~8<^OIX3;Mkn86`QOHf*FR6W^LP0&O!MCJ!JFT>0@}?a1Qv1Fz0A zNcycDR6Sf?T-%D70hoZC9KBw$b71{odULssi24jcmpt+3gj>)g-A)K;`jI+0DET6p z(eB5ZsTK)vu_tdC^l8#=WVGsOQLQ1#P%LXOJW6UT0v**AK0=@abUWH7q^PtlB}zDa z?M!UV3((;3GHQ@v;>^Mb4BvpGqS!-|2Qtv*-x8e=BOB>F6UjT{zYhDwdTP1(NcBq-B^nEmd#%q z3;-U64P&XE=oc>jWV7;n@t)4ij%>(>u?InrL=G~LX%iII1DwG(91!{Mc5&rWfu729 zxy4UyMLmu}ivza+8FjFk z!4EHiY?(bM=y^KnRDt%G$iwvLeOwV~CR_kEY~U}Gi$@(9)iwfUklqaJvl727bfe>iW;Q_1*YAn!}U1k8BRnZ|v5;BpJw;##(`R{tU_TPFpE93SCHc zy9F-suuCA_@HWY9dI#eWXlTg$aP7SP1(O)aK2705HOl?~ZMn~uy=QN89Ql?X&Tc|^ts zeUH)c#|xDW*t5-#PdOQS6swPf@l9j(bq(={z8K6$Vz&jM{7AZ6qL={G>g;?4kM3LOQtJs?Im z*?FqnNHzC9>CzWKFx|R9cqpt7a!CpfL-%t%H z=8tN6lID%%LJCzTj%VB7&4X_9y74FHKpAC-{3FHpTf+4Io5~Q^K6q^Q$NqfBnZ?0Y zyl5e$PSW#Bo}Z!faF0_pmAXYSM7V{e(AGiM%P*QM%s+?}*d{cBQhructw_k<-dtCY zk_m%CS#)`Vtg~%d3O<&uE|kEM9hfi$XyQ)d4usakFnv%sZ=Vq!92>YYVdV|Vr>rsq z0OAdHo}&+rS7n9VGkIaGE%PHDI)Z^+V;zTOLfmb-=(jU-qDQ)tN{-qhrKLYnXa z8C|4<2ZIruDcSXdM4`M(YdZ2H$sXqWPoKJfcf0_i?Td5%C+mnLxj+&kTe8XaV zhRT%M@=Hns1Q7>!yvBA=<|f|*uyEn{opC>b4dXmvahv=S@zkJ z>Y9P;a*6a*Poac$P|ErAGuOOjm&%1su`m!>yZmGS)9~oy#fi7Rm2SG~R3E+U2kGFd z?LNh82fVQNj3%G7cU1Imu~HQ+9b#H!wOXk?9oQ{5|3Rg#^R9SDe}Uw+y$BOar4=mE zzm$H=!j|qKy=x~ib!bpNe?6g33j^ra|c}y0NNpI@TI@ynLMnSkYLX{ht6dmnV1+|wl zOw`E-WXbUrrIwWsuYRId&fha@aOEthgAG#tRWHfY9Y}cLm&DE(X z6MFx45BLXhS5>mS_584Mps(l|Xjc6!nY*-S=YXEq+FPS51dEI7;q5Pe5Ju1V@D+eI zp^iTg6~CA0(@Q57!K?AF1V-t(vRd$7#i=8=VNO>f$2L-pH$W0(hvR2oh^ukJVYHe*s z1E+ca9h<&#iN$m62jaKIZHP&{m1s+)K8#0!0ky{ zn<=bX3z%L?Zd&qhdfMFcJ9)+Yp2ofA%;s*8T2H-_drY?ZN@bVe;$<-#oPh!A2bQt2 z8G~k29Xi3coOBo#Vb+~Ek^8*roL7H68pPy{*+FSV#9f>2ZJS+U_}f!fWwPuN_7N%I zY4Atm;iOIh@G?Eqz(dMv!mS)G|?a#b{;BiEGk>wRw_vT)t!dZ2|xM`Pg0&UeU29Nq*qZi|+ zNFua~Td1`79@cY&?h`JA=3>87JBe>PZu5$pG22f+4f_hCHO6Z!&$m<> zsK(%)W{9oqf7UWrv;0PpubIKS=kZXvM(<&33Wo*X*C_p8>@hdWwK*Ggwur@Ce|k?K zo~t{%D)dy-PP$BJo-4l?x=oDqZ7j*O@_v_i5ULJ9Wb@=h-OYQexBBg|H?~l7MJQ7B zJevyIe&~|HNnBCNnF3kemXcDY$nAC8ZE(RnAANOOdFunBEuTb^Ch(Dp+FJfB`AT&t zGxu&G(ABiV$dGf8di66+G2pc+Hfo*@(&vhX#+LInN>f^WD&O;wp=F!3IP8JD7DgOv z0$(|{Ltu(N{^I=`lU+}&QeI!TpLl0}cB%8g$RHFRntW~-DRC)KY5g|8U$FY^IQ_mF?*`l~It}R<>d1Dm6`$Y)XwF4o)Py7-~Xr#43>9v?4B5xRui`=Rf z>YcO0`wxCZsFaTqRTZsgqy9?6K&D4WyTu3wF+MjkX%@T8 zWDh|NUK0PdfrWzP!^*22O05~iVs*3ug*h5CX-)+ax^6}7hIrC=)GxznjGE*K4>{UX zzJ4VpRYNExDeu6MwolgAcNvyG3qBY8$Fd{h98uO#D}X;ek+O=YZp(@_FHt zR3Q}FTBGl8=B(D1<9fNx_>$sV4>|{weAs?%{`iNpyAP)7zs}IIV7#`4xldp&?>TYtz-SVbD0khl;&TMw(!9lPcOPAeM_cfOHx&FVjr@hLXHXf?cfJTFk?f z0IPqR>8^>M^Pt&-H*LuDj~G*vSw(JgT5l-?!@=q=MhX$0)vLbk_XpfjL6n|+cL&8{ zsnutyZGc&g^y!oayW5ShA1-5 zpAK-Qshjz3Z{#c}P;6usuZ^w>>iF5ObSQjYxcD`@k^J((^H(T4&^b4x%MhS4arY1& zp~v~v7?tf@2BoV{6FOXxp8m`EmGc}nF+OAQ=fd6apLrFcw~N(3jus)e_CES_yWw!* zD|(?4@~GJGt5&Djp|Cs_Rbdt$GV}e8JPR3uzwEb{dzPlNTR1*w zKoyBtnQZD;n)n3}wUJ*e_Iyj?yWGQf5rSqfe@7>$zG+{-+*3=CT7Pjjpy4wM@Y5E@ zd|c7Tv#|{C(^M@iM0(@8p{l}*dZI){GY}Z;RS>FytK@dKGWtGx~e>LOr^3W zNMRVcRFPM_59fOdtzQP8l5_TDqbpV|=ZM@0P(UM>C{7L+6ttcwYKAt#fQ-IDCgX-) zW2=Bj)5x=BMZ{l@4nvSmCF4w=NC{my{q(xkCAU62$KyFj1drSg#OAd@jQ`c|f;}rW zy5@>pesERT7hl9>C4jp4Z+%98{d!5>Xbl@c2)*WPZ%F(5FC2d-ML>TZl)*$BX(d#; z^zqrb3F{9|H&t<5I>FmxUYO(S7WnGw@56+hLfgrVC|&pJH7I?OQ|6f8?@bnZ#wV6; zF%~_j|5Xd$)f&c4VlIsGl}nMasi8%xc@ylSTghD&^L-cE81%(2y*;v^WKg5jB3=eF zO~|VcQXhzQozR~*`PN$g`LtTFg;$5M?i`JpbWwDK?8jQs)88^fjumQ@{~@u4%Fh}y zN2TRZePKB=Qg8fV`~$0j%|jg>@vzI|mbM~a+T4^13YnfbC7k7pb`E_HE&pRU`;#un zVWFjXu?;!>dU>_GbJqj~LaLTJk3UsvDJJ=~T>*uO3~z00SN!+x0r!O9ZE@>v!$P+}P*2otS(Khp` zHJ^$u#^d)u(lZ|J0)?*er-=IjsvL4=;0!JLb3wg;bSNs>&{CMsS8qV=N!7CF92DU% zm_}j_kUjXOI{}rR3AGh!nRXg4 zOumKL#6dx(BnW8UC_0;vnb$n9eWVFLPPgSCF^86ww{LttBbw9@jxRvSp1!} zM$GBE#aL*(^5|eg9$uy*>bL4%j$|fYqmK+@C8VdB^6oW?i>=3e)l$R%Le!mhcO^Ay z%+?cpY*a4;*p8jQ!vV@0EfQ&|kGV5!eDL;+8+-eZo0f~ry^mn(`sfccXslP5otg!8 z#}1iGz*ib7dcM1yi~Nco7qIRS^gcsYP=s^?T<$#jWKbJBulglhr8I*X6O+28FSD#= zXpnaJ;UXRUK&9C~nL37ZXWooHCUr9<3EgJK|Mc(Mnzs#Je+w03mh-wJik`zwLYF_K zFLLex&g0uVEtiCp=Gmn~*P*t@T&Dzo7{ai1*qdJd&jo%y(SjI2Ri2t<$CVz?h?sb) z_dx#UskgI-nIbN?y9N@-+LY%%Y(nA$v$MH~LMNn641cF#M|jsChpd&-7BQAb@yjC$ zYYrynZa1BRm`Y&^T9NCLYQR9SqXr4k=Ueh8_7EQNkd)CsV$o1sB7?9#p}72$AzpYm zmOeIL_376wA3j&m0U4=3>jdNGsR)7&WHL81!J^Fb83dO|gv_<4U#8*sS_mdpX#EPo zm)M6uSzA&yqz>H?v<8CFTWqk(VhAivf^@jo&&=$S#u-L}LF-7*#&HX}yqD{lZ^aBl z?%%^84u|0e1jLQsAt)*{Cm8IjF^ZlqmD;pfsnm>yvlXHR^c!!cuLGxb50QezwooF4 zR3Bh2mHNp%M#BBCBVRY4a(YjFQQM)=_vgIA?00S+3m8p;K1pHqS33T({JA$f+mb4e zHJx}Jd**zC=0B`9SSA=Cvhfr~8LEWP_j?SVm8(h6Za&y}vTiWfF*oI@Q9c z7{`>8UirqS&eTrbiq19n>GLXT#R=V%r~QAI9h|KidY^u%0Oi1N7M2bMwjCP_c1_W4 zBQ^y@(*-H0+Sh=guS3%vKS(dJ--&Oy%~Xe07PdS4-WK1C?3vV7o{a;b$hYxN^D5?W zSAa@E&nAilEdl7ve&{hLyd01^XqNe13cH3ZeeXbX6 zKBGZn%VA^rqV}DCU+)WibqEGI)n7wjcK-WdAa&x(%nDdG)+T}-erPuu#}Tx($*j6ImO@+&hPFsSmNoL|A%MUaQqG0VB?50RlSH&yeD})PTPSB5vZcAF~VSK~WEihsM7r ztkxVW{T8rwe4x{_zHHavOxlQ%j;jl#UQPoY6@@2XROfzF9Ch2Y7h9;-M7a}Feva?H zj>bm71EZGHy;WDi8tPElfy|p%;2JOb54mIYC#_hm>a9&*^zbm%(ESG8 zQMhA8D!Zj??h?SfSHxIpn7CTyRuyA}4InZh(i0@K9?gT#x| zIf!LfsnHSnhUL`h!w+1`&UyBOVTw5^KM+Du?#@?%qOP2xqRwa@u7==vVx)Z6`pO3J?$tje7)k=-XPNa*!1*}E`98}diVHwj zx+Phf)Y#@WAjUX}R(ub}BxXZJOA3^zP*gbuQI#iMEuSK)g2mPRPISt1lTI;09h8RR z$)MxnWvl%mP3+v8b><`t7cd|9L#UdhLu<{jIo2l}Lkc{B*cu-VCE`^Rmn(pzU~Z)+&VS zph~m~I(+mW;SS@AZRm4CLshdFr3MXggUeL_iF?N;gzmGxZ?rpsvLqr2-*4_y>0lxZ=fn}_Z`8q8G=+>TF%@-E8h^;uBl z5^$RP;VDMy;f-7$EeaEiCZVJy9`JJi5NR3aVCDiRcRFn>wB0L3D6ixdGv69eSwF%7 zT{NmWRa(;y(6e2GR(3w<0hLJsH}2a{*q{>!ypY;w9n281K@5p%CrnDyebpXtvy)Kp z@ZNLQ2F=*)6{}+ht{v!3_MKB&0)ITVoXM?3%E|^0+yeoiEeP>Bj+c0f8EgM;tu?wN z+P-f=&&w%+dm{?N#=&)~tNd};ah_l6qR0aLNB19wL(T^;Ry@lsXvLjTdDUT5sa1bm zLGkeGR*k?N4h(hcw*Z_%iXz$CQB(WAM)v%kJwx$5=aXC}fq$TF9PeUM1?wTY2iUx? zPvYMc3|)ki>glCfBbbd_)Ze3q2VGcTWq!5l? zIvrw{DXZf%e&%WhRt<64X(_u4J;g37P^uk_uMMT(%Hpx)0LZD$3zJ1lyfJ@Hu&!I4 zuMpG349fpvl&rk6SOn9B_DL`m9I}#GylLS-e|%Dkz`5zoBk3{%LrfIhLoqjuF8h?8 z79+_Tnx@8C0&6wQr~q!`!1Pd<>W5ngT}O8#fw#u6=rc&0n6=O-JOp$1@bFv11Aj3? z^FSCCQUu1oq*`b#h8u!E61yo|l%%Aj7j+v%kdt$U7S(-)FkTQ@^egJzs zF0F;xcwY#`rZ-N6jD)@rG4=I9=633J?|FZ(qV{kJjlAkE8|>Vy-nLC!e4nxYPHY1| znbGjeK%z};JJc7>`3=Ncw(^C=ZpWPRRQTSN<^!oYG+5JTBu;CP>iRT7w~XHz z(`M;KhoN3l_-@9!t5;lLT*t%;T&;RP?K7PJ=QhdzCgb4gO=nd>Ya3oe643P&+>OOYi2peWQ0)^PmWbmc+HE__$o`D82#k}-0FFIG$bBdgpPuA_USiqi^AU& z(Mzyw1h>RC+Sxsa_hqiO?%RAdBWm%u?=({pqY#74^~c!MM#nyz0;T?4M2v6?0aWm+ zt~vLWEiiG=$G(W+vBAojM9;}1eaQu?DX6}oDU1covmrd;m+G=DOv^I)t~QJmw5T5} zt85vofmm3?5XUS}NcpD{$q%Bx(?*+VV=lD8?7+`# ze?U)E zg%+R@%fRtA9+41UY;%7zAF@6N6pf_zNlwd&1ywk{JW}757+QHoLTwR)7)w9#H)N z=JsW%9<3T}0zk*LE9lJislV2Hk2y(T_ck)V@*xAt38txG6u;%YPF$vFk?Xcf2m} z6x<)(1Vt;o#2XfN6O(rchm&MR z3_<%Tv*I_Mpz>ug-vQNg^idLR{u8BB|*uUd4qScF*?Hh%=NC zys2{74u(<{fm<(+Ae`8@?t-}(%^+0?sfiYOd@;AcGH~+aGjAC7*eRnKtUm=h>Sc`I zpYE{1I;XX7U*B@>hwKujw|JgK4yE<|92Y7*^aK(=4^x!|-UK(&Pm4Y3N+Oy5zavsh zWCuJ>?B0BY$woE=X4wgm-W$apvQu-M^sMhmoWBu$(zVy6cvpbzVY0FY?j}lmm~<4C zag85?so!bceDq0fGp9l~Kj_m&BA_R~R5R6@CQHB~{@ad5kXtTvpbSiQKL>!-^%&of zQA=?K5Kk_rNUogSNt<+59Q?|NJRPw;$4(_H$`TLf_A#z*I^88#bpB7vDyU%iAnmU$rx!XP%@T z+RG7>o4N55kI#9w?btX8Td32xzz+n)`+=>PfckUR`ixX zDcZ~b`#)uyxuK%kUO|bI^kQ}t(U{0c!sE2fIrIL$yj4_=BXhDoe3gO1YR8z z>Z_xpgH7;##DVXb(1yC$=;hCs|9(Fg<3<1k?)h6$^C~iqkl1CqRLs7?7o3Osva3lt zn4Z^iKT%I4jAByZd|45+n*+mwEJN0QDnH@u++43GTFb1_?@|_UWBRR@7oa9(_~suF zrQA5>5gU-85oEs3tACbIdf5vFUH9-B0HIz>_1N#PHmcZFQ|&I8_dVKBoZbrw`yYX*@W};q$2_HQ?tN7)1LJA@q+t zaE3P#&q1rm6PkBWLsLc`YeyGi2)X_U4ihP>Z-US!Z9L?)*4m#Dg(<|?Y1Z;{C91Ui zNR1VLaOoADcb`|6a7zX-kUxNbS3sH66ga$am8f%lw~jI;u(rK)^i6qNJOf4hD)YZq z1a1l!d5(RXe82wq>s5dOCL!Ff_Lh5&*$^^yz4mNHDeK&%+Uo74(Tl6Om{rXh zquELEW^&6M|95o|{GY$MP{L1a$YYU0Z z$o}uMxAGxb7x8IF&>@1mTmZ2>V|Ltc-d<)rEB({CyrWZ;hQBE>VzI zeGeO94eb%yg^GBC)#a}0GTu?*mJi3*pmv(oC``(S-|U6sVFROC=mU&kWk>He8J?}O z8ApA(uqxkHz8G^xvd9&Da&H{w*p;q%4@wjbp>=k#XZUBP*9%5}&J7Pu&)% zyXJXer>Ri8naR7$^ZTBghay$FE1v+JsTGP9xQG{Ec;9ywWg5H~3B zrMFJx>1mH$MYZ?$1T8d$hov@vs;Fz{^M=4>aB!{JS>S6!`&Z2^ae^ zv5UmLb^dV9BLo!mz>r^Op!+$Fen>mC*fEgCX7WO7A(SHR@vVeiarp&*&_g?u9yi<{ zm>`j~fJIn88jI-VO=%6n(dQ-~H*8T!UVGCu%PIkQS;*Mi8R*lZsQ`}3xg z6D7m`F6tUEYV;|`zO1xFehA};I9Tzw&Vx0egYFVQ#+VK)W(%0Q*n-gZllsf`BUeYO5aj6kYo{=FD!fr8g&OL18t2^ zXPF`??Sd^6B;Wh@$`9yY1(KpGU4*>#`F)|=w89|?5Obo| zJgw4mA@H@MN0A>O2z30z-{@~>7co>s3M(o5$*rPEl+s34_rHIHQLKiy?H$Om_TR8$ zbKEERH4{@N*h6{wyhKQU0Qo8EIAumko5}AWF%$w`@DmPmAJChZ`F7tw0WGF&&M@R! zcW(4`_2qx4ge>j)-^^d+X%mhtZ&F znH{ID>@k^bYxaC&ljocLzzu_Jm}qmx!f`7R_Ki3X88UHHRKF7{hzxFyy!Mz%PMY9+ zkO4kdT;W6Px&0a~1Mc%Yz`tlES-(w;cQkndeEv^vv3DXtiR^X>$eouB6n?^wXr9n~ zp}89~t`(XKNt7Gw+lNXoL+pI2Xvt|w@VXG zwYu}Fm1t&Z`cv$0S3oXWYHwRsl@ZwhBkev(Li^~q8zE`6y!ycwAA{LdV6+%%qFVLl zjcYa6d9}HegWqT#vG0y{f1p*d^fh-PQ=>C0peNKIcm%S8W5WSBQQ|%bGBPm+rKhR! zwvt`3rjD56KTkgV&iuWXmIN91vBC_!E)Fr98;E?HM@RH#Zk|Wd+NIAj+(??dF*a{h zR2I(xNG27kJX;FjSc<0c1HMFN1F}2xq>&wX<61vo(Wd zL3@BQlJl-!;O<4Wr&`h2V4)CeCws#7_jGE~QoQ8$DlSL7K_GD`yED>>k~$qk?U8)L z=n_LVvQ%O*|GTBSUvNCSqjkX928Yjprr@#IT!-#1K(Z6cCK^h4f*vtUE`QD&l^lNW z3-6}=iZ9M6;$zEg$AA-Yp8CFkpQU^wI+o%|AT;-?LFKez^{}K zipM4Pd+8z{qPgvP@&R03?oR>6cs_dJsiJO1@$oF}g+oAyo~~}t$;jx!0+>w-W7PU5 zG*5AKRMx7(E7x=Ok%bTl3rsMla~&>S-+9EYVpFPZIEQes|-P>xLI#_d+H1${Nk&`gnDCgL%8pR`MI0m8M zJ`UE~F4-}&0hOA13?wQwe`2!>O}#tDc$9I_s~9Q;45jMf#EGr}wHnF&PY9g(&8f3)cBVxcHm zW-fz`RC&o%h*-^PR|S{3Xl4UY z|AMvg9l>9tit`SH+yg55OP75SmIbm>A=iF`dq}ywlQ@LC^n`c+;>U9g0_OCkf!RX! z4jE-$ci*K69;ZuqH|%fvGqb`Fi{s84t$;->i*Kml%vLD5=Qs!m<=`8SqXN9ICGzIi zxX$=XiLrz{G7mNiV2FK>xfmDAnK{7u{x%d^{OIHxocP5aTWDy&lpqgHR&FwtLylbu zRI)L)i8Vhzp5PZ3{pVC+Mbl94;{Rb=GAbYC#IBwvVgg2iI@dN=_HNjyY{fJ@BLo}s zj!(_hJwua8xSAk-QFvQa2)(;8uy0ZZfEGs>($#04&0wf+xRTC0I3`o;AZ_&YlQy2} z`IIY}WW!{QCB6WfzKcP#O~O(>vmF0{bj<@RJGrm43SHFHwu=(0^4$(BJ}tWs+vzH~ zaJW}TDv@launs}%-gnH<>ewf)uY$*BuUryVi=t^9j^3%D8OwQxz-J$~-x%X9OKpRV zW8X1!<^w-R=7gg*E_t0R^3j%v34LsB@)m3nUks~|2iHG?YArquFl*!!r&^0$quIVA z7nsuliI5xlD59!O5piqw(jz7+?nyQpYcc4KFm9aCu6ljgVdZ_?j&=3`$~3nGX+x&P zevaGWqTpjvb%M<}NL2VJ3l5~&b5EsarZKddu0{);Een6kOfXH&;P@KM@GYTB4hCA8 zyF01rIGl+|E1469242?`+8qDp9pMBKF0Jjt0`HED-;O=!pmkQCjfpT;dOz61|Htl{ z+uWYalqgFoF+RFQ@L3(cN8;kKL#q^JwI+uPI&N1W%8L7(nEFFgrVT^H8|VaCw~JhD zXe1dOyk^hA2FJ0Bc~;A|1o zBP;0O)U}~8hRQ*oJH5?`vv^|oNpp9H&J*MaX zfN&Si!v0g}EDZAe|C|LJe)g2Po0?CY$MCZ=(6P2x9NR!g8X8cNBbI$|b%JO$`ucy{21+|@k+^EsImgMN7E`LMHLLO=A@Yy{CoBtS;8sk1>^C*qM7A1b1X;O z;$xpt7_bdvQB}$MaEBb2C-=wqhr^2K^^fbt+hl{QoY>wXT^idmwx13soJ24%bQP9q zXL=8mM(G#@?RzXThsO63h!KW0QSO=;%w<1=K*?{s*+#tYd7FW(XnQ^)+NyjUGcNi( z%Svg5spzHa4S)nfkEv}xr5sLi3=ePkDQhhJ|qjv>?Gcr_DRH z%dT+#cyFG+(p5G(o+;0`svjw?YY@+l@dgpShuiW3f}#zgBV)}-kgF`!3SVixz3M5( z_cFbnLyIU{{YN~;`s4`{&hbK-Bx2-4{0SQ0@PcGV0KxEg-lWQ1DGPaQ-dHf_iv#AC zqIw_bu;|{&h?{3uW-=m<8e-J7$JV{u(ORQNi+cGQ(s5L&M79QYGspDRJo^(`T5406 zS2#Yok?7S2RTWPoX7wM$1CAiccgUnAJenzLY#6mf-oJ+Ju`rbWn9rv$0ENtC?Y7It z`E&2CVl|THG%2zGVN!;l8-!4}^y0Y>u}HHNN8;7TKNtDQ8eth*#vtfa#W%;Md@Yf{^50ndC=vs#F*sOKTKn-}moNojIJq)CQgg$|Oa-{8jLE{x%j5hJGOPrbXGHFH()!YRN}`zoWd?kmzWB{irzq%)m7BR0(eSiU}c z@I5YiYAr@MkwZhixHus{bIuYL9Twi*ai>4bkv?x=>{HQOPJWEhtFvd7vJhB#o#oxV zBi0*w+)YH-u$nmG7Ct(1${O?8z3oqiPvPP%owcm^$yMXmztaoleC}Zxs*XI`wsw6& zknT^GLAqhwPFph-n+DbyAU)j1l6f5=9W&)G1!?#4E5;9KTTVn}&=KdfQZ9cD#eZsA z4@OlC{D_@|mezL2nL3&IGR0W#l+UwU02z51GsZT=+qU2Em5;E+u5UrBWV^628LQ=e z5X7{7I<*ACD6yvv%_jd~epX}%cQ}O*z-vpiH-Db}|Mc756P@qh#=QF@x6CCEnojN8 z=jVI#vgoJAV-KliFnlH2)_)q)F~rNnJB)}4|7gqSV+rLIo@f)X&Yl7Kv89bI%?OY# zUV0shx)VX&AgjSS&$kmsF5fum`-m98)|Pjr+csoe2C-kNdCvsTLBm@q=|a$*9f0!V z`^WLWroxKn*tG79_K#mB#aV}*FM_JV4n8JhB!FX-VJ5aH8sgqi_(#M(>V{UtC39ts z!NX=4Pgx011vK;LjRE6<7+)PVk1;F8tzP4l%(8RG_Wl-PO&T}&y7rEn?h3@ZqKAXW z%N4R!lJ;nNW0Wd}T=SSzXE5{z!4f70nt0U)iao~T$+Zx39 zBY)l!$E+acgEl^o*9y6(oaclF zGKq+5F{!o?K%=D5SNJWH*1oBBGW#i_CSB~=n+xrvp&=$+)5xrPezgmL<@w=<4$`w39-L#1fND?P84QDzz)?Zps~!h-G2P@{>F- z$q(HJgD1tsAKVIrl;9^6Doa9m)vPJIS9YeV%u&+W>+EHnjyUx@y!{FdE+*lPLpf2S zS-0E-AT!~D{tGEtsuNTjU?_lfwgT7Um9?d*DuTk-2sgYx2=OO8j~&$_8UQaJuU05A z4dqC;Ju=K|)X|(e_wNu@qx^TcJH$NaQ`KNmKko447{uRa(}U92SuF@-eX#}rAQ03j z^uFr)HF${bcDTAwLfdv!SySohQ@8LZIe*Ov@$TB;t)$IGtS@6fGRU#X8?|%U-eOSu zMJF)}aQd)=EXkYoD~PY&A5F7jy2+S&;)IDSn3wmHsnYMT3$2j{ge`48LcC?QY&WFV9Pb6BiLnn}ekPwvkZB zkcuKVm@du&nEBn_y>j${$ApZ!csB5vT(Tmno|C7KiDzFb)7mQu-~NRs_}IUI)QdHs z{~d;}mpzD(a!sBHp_4e}Y%-ajk1OcZ{-BJ|#EgaFr!6m9{zV`^2ZBXg|1?@Ce0-&g zsIT~=!sU?0@311gM*Efi?;jcoZH3pR7ce0o|NbU0uGfEa*Rce7cwzX>li5nD zDYw2g{^bJL$!)&N#OFuufBkxP;ud(5oAKG>7~K&rK-5(=YDZQDblQ1yt7i{56i{!T zt{~jzi(p+kTP>mXp#*;^8B+gy>-yR~S=?Gn^Y`3ju~DH_u*r5LAL9s8%tsX ziEM3pI8W-0EgV(fXYwycuN8m~bkJ}QY7022==iw~7@L_nEmH!O?%!Y&x1|-ws-NRo zU&*^I`stW|zk2`XSD%fF{RU-$Z@lfG6C7c{BC&ttA@poSiA@&emZ12}9TiK8mVnw+ zz3w&DZ!vP>8-LkwD5c=g-;WjCd>z3HT!`()#;fRQbUg(7_1ea+bN_Sr?q8}qSM0{7 z8y(5yzZ(o6%gA9Iu&A*9uDMlv@b~_nsPAf9V4an-~Er>uYKVjR8ni& zHf_-oh>$RdDp(a?CwZV7bxrZv=E^;LejAvSL434~9JLSY)izG4-OxHkZ9N^c*<22r z3k)OhJ{UN9V*A;+H-1sdbItef!fmA;`a|V03*$atKIaXBR9D+H@W2mv)EIS0I5y>M zq@(6~sVXVsnYPv6<4PO1We?0P=ZDA^F4eR|niMT>7;MhlvVpS;QX6LR^xzG@zZXPb8iN7P{blgb!+rTMma-H1HzyFcaCug{uM^^#ie{HlMa~tMtX;CZH>-VH zsIi;-o_!+$ckra$@`iP(wUcVyO?yLCQ5;tuHI}(b zLjp#8eOAPNg3cFe|Byt$zy^ft5F6aA!NjT?(KsP@dxkpc0Z!cRV-G#@&h1$S-fcR^ zO9${6SSsA_P4XbR05w2bLAy_Ebk5*l6uhsPH!4U+S-0%;)M?zGRobcd5FmlNhlD^V zDO-;H&bcw-lvhgr{Ucf0BM00-5h&^P0`5Nvui{gh3FIhCCMr^1%d7&{rdEG2Sr8oorEtuG9)9WnVeHG}sa(7FBNa;KG|7;08!O2WsSFu2D`S*G zD08L^aY99=sEEwlSTYMINl^-sGM3C!Wz76r_p|q=_x+yt_r4#09VdG~&wa0Z-D_Rz zTGv7}R3iwgMx5P%Q!cz)rAfPfKZO2fUl0Q%#@I0!$xTdr2IfMFM)S1AH7KJU81LOn zy7}1Yju5UXoc*(iSvhuEI13*(GTf{Qfs^$Nct0H=HGpPs;wx2Uk!<3FBUkL%yr;t( zVh`+}i48?qHloaHsNwlWMxc2}=ze_I%Dv80I1EF=j@$uY0ifI4LX_T`oMMs#t`XTN z37%XEuiyu0OzL0TS#6-TL_vcoLKo)NhsQ&$b|O|z4JsQN_EhJfu&n~^Fjd?~qNqvU z$5*a~=n!fm@4h_=>~^?^p+`i(&-aSf6a@M0P^-3Gmd>y)=}#cuVfWJ`7h_s-cRF?{9PO#S+FHcxI%M~4F(QFwgWnnA zP5Sgfgcwg91y`~znmqI3$3m6s$31Xemyd9}U)4R{lT3FQMm38qXXOb3hrsgj9tEGb z&GaNr860>^9>N0EAFa#SM@~ckrcn1Z3bHnHw9?7#%cCT(ly3k{ElV`kSN(w)zlgBm zKwFVr$MaC3K6bNZp%nGKf1H6+#53SKE()0Tt2(2J7bI?CDlw37m^k+RXj(F!te9&yycY!<1)TKCTd%yDEH#3IY@)Dpj zTVH6ES23H9z~E1ynROXX&3k?5UFj|w%YVCGV15}+@|<$)mBg*{8srtqVgQ57i`b)l zL_{*a( zo!w>Zb5l2fHu-&_^(^1$|oHKqB+ zzV2U~i2h|DGHO6U30zpy)4}>yC9oiOpgD}eITfVrrdQf)jxNlhIIr!qO+8-iFAl8R z%LH-9Ixq2$PoZ$NJKX673~R$;E=JLSVNTLQSaK2%l2~a*Q9eXcdS~ZuebV1Y%u^;6 zxr9i;Haj9uqKLxy;{NfV0El$L&kZI~anTiJ+?R66L`^24DdPJ+BT5ERkg}|g@*hH* zS_DKD7H19FEjPP*ymIT-pEHqxcvw{8N()N4z#~Nmghg^_GWk7GXn>BK9MG@K6$(2# z8RYV#yK73$pG#9s8Uj*uBe4`g}jnaaSH|R7TER_)X>^n@H*FhUY*gTX@!0v1}u=5^aaWiUIc5+758n|BqNbuU&l$7bU1| zR&%Q*fCTeP;KT!~Nv%$~IeF+x+>c{+q*#9gIgCU?c`*MYHcnm=;PBT_#+x5>kOvN< zumQ%och_YANeyvipUA%~{&I1TbZ^l9Y&c^@^|-8(*U@6j03M0Xi;QY95w zkgn5BO47SveT~juT{fQU2hQk|JL2tcWU^0X(TIOeOfdr6;y!#ga{byzKH$46M;{p6aJL8tME>f**iIzeR(>!Nwn=_RhxxEKAoq7mTiMksw=pyK6)TdsMwB z#Td3ErPF$W3K*<4>MeVrPudlFNJ>+EQ2cHlU-x~eWada1j}zsZ;eekd#LX8*^x71- zlXau6N&GfhZOipI`PVjcgNl)|uIbI>3iLm@ZGc5R!jM??vwEv=xcf=V$-cTGXkHU@ zKJy}MapcMin(q{AE{*OG@rt<^nIG{W637byy6WmQCo&`KdF%q^?xXYq4CeZA<+su< zBNt$%d=W^G@=bRyB1%~r*0qdz1UfRJ^L@q^3mrmq>gmHE@- zETE!OI{F!6OnVg3-vO2SWLauHo(X14|h-PmTaf-yG9452bvGIs9Q=^q@sljuM6W7EDXDm{$IyEBm3bM zG%Nv$8jb-gsCbWk$zwrZ$g~7$(zVxC1BM@AYdm>$7&q()!BtARf)NMZ6?lAg2&Iip z@gUmP1j*x`Ef+wl(G^mgzmOPiaHQtGmU_vI<6MdU`vKXa2)hVVcwZ2)b}3*bnvhjaqn4r{!XnS8S!CZNnU z21BIqpH~0YckbZOknbjs)#)H0hILomV|U@u#dlBOp*j#>e+$%=C!WE4l}4fO9~~ zers$og*r-@mTS`kI2WA2nJMKw^z|GhxHF%sd{!G@{C_Ow)qUqrqE`E4qr{bI2sf9~cypx|CDpemUJt0vJ;`v=YAhO$+l<7YSMAR~; zYGH`NTpwZ@K?nnYC=5E3P_(IVV2ZOO?Oy^r<`*>IBIr)-Br{7IfK1$#UK6^=yj0Yz z(<^N!hatx=1|TD%15=wYkhI^STEPKt+@!^z4Wu%8B?Ska5*`YM zQ-2p{%mahbt|>_Y#qd(?u-3VE_DVJ{8$(ga4)rRlF4sfmRBXJQ2p}jCLC~}(oWjh` zXM+9<$`|&LISYg6LW?R9$@;x^P5P}PS*N#s0Q=u|g<9Po`1kWtH)1WmuK&9wY}105 z{=VBYopBqbFt-=w1vu_HlbhdqK*h*Y(tj=W9_rMD`2!|4hgIX)Tw&YU|Je3e5{7dB zcA)r;p^Ov34o4x#;{GEf<94$KxxG(-Tt$r5WwVcg=9Cf+_Y;is~JQL!6s6l2dnZ%b9_?qDs!i;3?)Vx}dl)(CfC-`hp6geZqya?F@+*mHqH`Eh+4ux_A}3GZHV5S1L3>G1=nK zrT1b&t%!n?1AG9aBV`0vo-Iu15H3ANqxZGEil*@9ntp<>98&QC4}{;@pi##sq~3Ho zDcQpLycIRH7aRm@y7;X+5FsC5fX`_;HTdB}`vkIC;qtMAMh>r-RVNH`sl8^3ZCMj7 z)*kk2FwyIXeHkg=7X0q~RYogv_&j}ddI5BDy?|o?3RBh0#?@MFrk4B z|0%&z5EB3^qQ52#xvl&O6l8+@=Qt7zS~da&gmLK0$wwIx&|d9c_;0d5+cVUZJP!0N zb=E^5yZ}w>M9q5K?HPQ~)qIp-Q2kfV7u}BF>2vo&jGGUZIWjAp0#BFU zHv&qgcF=w9syxG9mL7 z?xExlJAJ|!#8*yjhJ_peRQVYqW-F*}N_83igh1~5H7yW&^+r4p@K;fg{tewB4wj{4 z5PRSIO9wUM5TN~^*w(7tj-|d3C8g#ZHF#;aFwly*9wBo_oT3*Z;aGQaazj=&jt{$F4s|HG#)Hg8$NH3TDoUC8B!S zv7Ayk|4Z9YWFVJ#3SAChBx7j)gb-5Ke0$ghI^a?q)#4p$cOgFZ7bHG{8bYKepyk9( zdgk9+^w%O%JuToy-qz+V=+K7g2+6TZ*tKHZ~g!X1t^;=21tc8qZ3CJDcKi}f7%V=*m{D_fxm=4wte%#as!3w+P zQ51MVGT>16E)U{-k3C3#4AJogtqgw2yuF^5xAIq){aOiY19>HO?LKAJnGtH$Ne&L3 zTy}mjgKU@SG{ju)K;)C?o8@Ck1^M_!eImu6u2m69mhY1P3Vr(F7IWo_|($UBCq&H$!GuQUgQ;tXP*PBEK&obR6COnJwVX2dzt9}VVTLr z?&t2}0M1G;jz#U;Btqf>JY(e$8vdxb3-r|ADXVGe1GY{TpYhKZ^W+Yl``RkJxGLS; zu3r$yIh+10U&;GkxD;kG!x{U_m;pVkJUVIJ6`ma+Xt#Cb8?n~2DL(SU3)zK-r==y( zD-oW#CLSHH=$P1go>a$RJO;xRWRr6+&!ul_N4*g2UT``<)nRE-@eb*a6p#hy{G&77ntc>{SJ7u_YTDUcmUx^i~r0}n%- zd?F>uhYyqX4au{dQ0oT@ab?V-f}?Hhxgum9##>~^(GlA~Rz=Y(XhWxS{WZD4Iy^Lt ztwFX0&Wt}b62>=;xPD0{-kgq5k~tv0_HG*}^06AtJvyCI*RNSus5#QF`D1ETehZLW z>||S8*bTsSCa;x?>ycw=yz`uvW$9g|(zaKRUYvZ6JM;s<5Bq?`Y zeMy7nXp#vOf{Fq5 zY4V@l;b{hF(&r+FeFTd~JH$v_`Se}NSP!QzKMK*%3;=}hAo&dh6+1=5$$FQezGjUR z%W&L*da>WAz$zR8VB#>&X%T*x1LOR-yQuzCRQw7na>I_ds-dI@P)MVDM_{JNDCkI3 zNOTlfpHJ|fz3&Y)Hk9DHjX@H5nUP2B3Ce^($4s+%{AW{7#O^n1p}HJ94{{WcP^j_j zN1-;mW&^T^_wmy3X&;RXqLR`^1YSI?S1kk?)Y6lXM_xv<;6|G>{%gq}Eq5$}bfJ|J z3IAYZ3?N;UrMiEq2kh=bhwZaFoR6Ye_!50M1#t52d=}H+WfBbKB0@7#ie_R0F zM-m|r`4s`uSS#Ln^(4ycvy~%xW+BXKhg!=eG+5&;H@D;r^l?6rZ$7b4jMKw7F9psR`hv7$f!^RXw%HR%$cQtuUG zTrTEywi4=rM})uMb|twOO?RTP^fwq8A__)gFXeA0RJbnyPw@ebG+d~t)Yj2C4>37S zeEJ7S->?0F6YV2-QT{N-P*wCLr*J4dgocb_qzE%!7IPYwBNPqVOE|ezs@A|~s}0kZ zb>Fp<$c#D=7b`}EoNytEduqejt{|O3N>+6!u2bS!0AvjKy@l%g2z4&SM@K+J*7ts@n(Gwoj9W_{xXhU0A?wxB+xNpeWG* z<*v)nP*zB@QtE z(?S0%RvAbB3q{SJAi8~X75FU|;=fmuHeZfio`}HJP$}qPzrBn61t0=F1}rl>+FVpOuzffT3(zuEB`iJf34Sd>H=ph~Q*N~dSgV86|WD<`d&Le_fu z4x?u~sUV06fg(;>^x;d^Lg?^^a1r=}`hAHhYaHGh62|{ZH-#R3N9IGh&q)`F;yPQ# zp{`eq5_AozVXkuM4A@GJ@Ap9iKT-L9dwJEoapWKIB{$Kuqa~cI525d*-Va6gU63mx z1VZZ?8UW=vHvs^<4;o{4{KDtxy#H$K)jM=ho(_EGlDA_(V)#^KXOxwbQwUTy7K&96 z-g*L-$oyZ+^r&tp7f4T_5(wBzG#is_)q-erudGfqV@fcsDkKOJQN=(?x*duVx+3+c zL-gYNIBOv12ur4Ax->S4kUj%PT*pkH^zKLdtt$>&nMtsdGY^lt=MyBuK+y{!d_sBW zZ*lxcu6k8)%V{lymFpx{E=n3y+7SySk(w&%WD6RFDDL(9CsPiZmms@1cBliWhUkiw zv8B;{w64=z_qaEy{C1Z-U<$P5TA$(;g79w93jUGSo zUciG+{(%SmbGSrZcRp64gA95NqkZM+ffT{SB1A<2Ud>nl)ZE(RKk zf=VZ3hvV!l>%3DAB2$Y)ohOqJ{&x}xU1qhiYoE<`j{xoATxYKm0-!#1c3Z7OFo+GY z+{&*%dDgAW8-D@(qD!}3vma2`@G#h>4C&xXjI9hfsfVry-_b-CmR|3>)CP2faJSJB z0%t#Rn^sBj~O z3KY;`4(Y3JpP>YfumOGGiNml$5z3}NnKb{)Q6wJ!&sS4QN#3M}h;fEA1*Gs}$3I%G zlK|A*zw`(*eGmX(k(!0_?(}x;b#VP3?**L}kd|vimF15gZ2KWN0al-M!wL+J`wS(! zF&Mx?v;1j(3~GVt>vfZ_;e@@Il=xP{gkC#^0pP~midn-wqUYqgW6eCr4?~F6FexkB zv(bCzNYCsaN;5&kY=yb)i*h+SNV@dko)zgeVGh#Uhwb3Ku7HN|$?n%Ij)`F_JDT@} zt|xSad=y-#l(aP_QIErDY0BMbd_9W9%wPV6_@V=@V><3YCLD*MJB1*;S&zg!(M^Pc zg-!(t_JECQcI0YW%ETDsSK+JW0D^&a$tYTPnZtTlP zRKO3=rBhFRu8jnKkr)L+!=n3LVJys9;vfOcNBn(=a9#AfH#3||M6yZLc< zoAWD5od(-M_|P0i21cPl<1)DiK^6ywAzT@9d->1EEC^2V?88vmyJ85kgi3}L$TlVO zzmBSh@G~xGx2)H-_Zb-|@)neyf)J#+mTsA3REbi=1bZZ|V;U)p%VkskW;hJu$dRh9 zxm~j>D|1yHs5}lqY^ng=cX&nI17?CG5{0` zf6xWymo2E><%27X9(R@R>dAMx9kOPHa&L%1-;>KKV3pcV;3tZ$fjP(M zLl-28&a-=psmCz);s>9V^S}o$f)v`43ir^i=>7~GAT*;7@vE~TJ+$N){Lc)G*Zt@# zxWeU_k_MnYSwT2{;&UD%bXkA86pO^gQ6-b4SKzNfNd2h5y!4<16Qn zcE_;jh%P(_Nehta3NoIpMFJh_i}#_0!S7GTXSKeK7Fq? z8h_n!=QO9y4I1$$iLCvVwEx?ceBUAyDEC10aN|ua*kcB))#qKCz$iZL5+(@OqHnJ! zUXsxc{#pb?aET3e_9BpdTmcP#a2-;0H>zov-xXt2TMQ|M;o$rD9JHh5GVmBhwK@kd zQ$oxB6Z@>mS-%(GO9r&DcLQD!6v&Y*N4?|>@F9?xRDM7P^}Tt-TShEJDWM*a!`w)c z!-PjVO1y6$@$88~xB1-*|6w|$M;%x61twGM1M z&g4YCKO`}klXJ7^tq>}**P@$6eXF(U5BTwo_y$oaYQ6}fr#L`%h(|4m>{USQcZtpK z$Tt4WD_+U-i0^Rj@mz4OT*Mci>FhAkOP_Ue)f#+0)*gKwXZH0u9I1ziL2Twet8!X0QSRi5x$YRjO!$+nv-~fXc&5LE?n4&zH{UdVI@YN{jk_Wdj413 zL<{JYRQ!QG+244t-)m0<_pJD`WgU!)Pkbguc6s%qb%UuFp|?>*_3w5mSpH@Y#+^+t}WgAju84>T$pd63oKpqpu=tApbkLIIMBEEQ~lgvBBXQA z0iggH8swio?YcaxpKfr_X3fF-5f_I0oqR301G&UUL=$;bE;2wm`I_>?g;9t*_iAhb zt{XQ*GpwqCvfG3{2q0RRhE{TDUZKRK&OkVF0+CxU=M?HIk2N12e0TplLQslfltOgh zgJ@TJ%u)p!dUmdLwzQrk+KtD+jY4Eq~ixD zK<8S2ObcGbS-u4NYYmz^mJ7{Y_JEKG_ot4|6jG1V22O~u{RZlhGXN5fh$Hh51~SN+ zv|6FoId{Jj$OtE3T3Va-ayP&;6L(K;caZU%bq0p9z-N~%{C?~iv+7uO36OR)T7meC z`2!cmnwYd>AE5azK=W8GOaSz}7??CQ`;;UFa)E9>ffU7{DdH-4#fy?RwexaKtbF*K zeU!~tA;6!w;pk1sV3CpFxgU!LZmIG8edN{ zo0CFl0)^Aydlz6?K1D&q58;QRsXtIak5oYSFSVdayl>qo^;)|zACT#&xS7OOWQx2Covn;P#w6{1c>#^GH6mOVjK5zzCH5; z&Cm?iYN9Tr(h12ngnVXsCzmz^vWN2y=o+0TY;yw}cHY7Mk@q9|otyI+EFS%7TBV>x zX$;fsZ3~QZB$8UIWPQLoI#F*0sGknEl)UN2s99T;6%H?;NVbc zxbnMzvifzu9_St1VG0A-qLBv1pm!kk*M-act9N7f!d#bKGT(usGpRh-%46tk(!aCW zHOFcg21TReyDUZ1$=Y@q6eN1M5GeV3%6OV zKFQ}%t8OsIZ9RRNHtN%uy40N{EtS>z4?dkx0Xv(V1&yPBZv?;#P$W(t z7?SR<0J5QrwwbcL1k5)$wA~=-1YLYmb(F^FXyfCPwJ`&YT5}U!T_77g?$f%}J?#LJ zr6>fzxvl9|foaAwW-4Wsk+{=MCEU|>_iPm`G$AWlb``XHt0Kbie)G1SU(qyYk;$Qc zGjdJ3C-A`(X}}0Sd+H~N3$XlypG>eqglZc@@ouQnEsQ|T!A&Lhowy>E8UQTn-mDb2 zbBfw=heM}};7RYXEFWYJHbHHRvQ`x$6yahzN|O~pb?$~LMYe`d2E}Uqt4A3h1hiEw z666~+qq4o;#U5B`15fCN=UP}0+$+dVmIr8eWo{BHvmZ+cg>u8|vH*V8gLErhckh~H z>X1!2CXyd6DtZsmgq;0}BiIcG7syU7U(x9PG$YT%f{Rz0OCRNlGUp0tDKsepF*m*( zm=;}-k}erEF_y1@cySqOK=#Syqm7>~gaBH~TlVT#IT5cw2IjQt;d2WbO_Cwa+BFS8 zeD2+U@kC0Pyjc7Q??a^R>-IO%V>p{AWrq_v!XWq zom}1*surCfI5{9Y-^VB-g>w{K-ZwI5b>U8J`p^-ot4QWQQf{YE$PQ`fyhSR;gJ&ti zH98incf9BFb-BV_GrRx@O&-hpmUPZyrl+Y%e6n9OV~*+i`}A9c*?!znM#qu5C}4gl z>QaLDr=HW%AVuzB7P4c$JZVA~Ad_dis~rj+4Fn`jj4-729pjKvcTzubKia}B9s(dk2}RG#vA6@Oz4z+jbdN!kz?jebdpf7w?-rP7E(>_5 z7=2Zt5B)24m<$W$KS}XheLkywXy!7FVqnbh>zg=Wx zBXf~(W_;1XP*&+rTd%iYWVxfQ^PZDRk>M&=->KW05lCkD%awlZrr%=kt-|l}(2IC4 zX_@1eYfhRdX&uEiS?yiU9S>rV)b&5jWhuZR?;qqflCZjT>5jcKjmc$@{B{c&z&Ef> zn{fX&$eoq{(FLl}-}h6WY+Cy075Vdi-CXtZTd{;B_V3g>HqSs7Y*A8(L(@O;zc1z_ za~f9wp9)Uq(%qy>)w3cBPx*eFzBaEfe{1r4O3d9uN(oAV{1rcK9)>##^N;WwCN-UA z$eGSJc}Pd^{W#&k`M*BAjx;SOd2R=@uj9Auj0{8)I1;YONybH~$#9G76rOKY zkIx~1UqKADdf~+xILTK*HM`Bkv5s4n236VT=HwJCj6X9#1HXt=1G63PHYY`wC}Y#o z`%IGDzf|T9MX}pnkfNFZy+}2M3;Z9n#OSX(Z@oWZ^v2}g){o8_BdgH6J^7t>8<*@z z9zQXn%{aWcl1j?>*sYQc>Xi00B~h)B*L5 zUnlz1oI4pPncN>oJX$hNP~M<(@KXPGEwP}Yn}Nov3$mf2saB7kbg6Z0b;O{u+^Wpi2H76}@bGlEvmAy&3)xxc&Typm-qfnDjMEir-ee)@0?K~MlIA(4jRITu|f zjhmA${HXsG$-k?dg00`5n(B+o9q`}%z7cO2gxc#mIw>a3xGkqCH$GkRqBIbf`SUcxL?5#O52m4Ynv}- zI|c^6dBS%!{%gj|ElIr&5<9;Fft&KNp%#1+D)$B`|uhDTVFRQy-s8qzgw={qbfng>u0HYH@owc z%0Lc702;Bkz}5Y2=OWPS=Z|W)Yw(0ChI6WH&t*`G)z+T9zPvEMz#x9&;H>RZ)S07Y zt-@57p2k9T8x5sfVzcAera&HAWS~r;sm!USP|S-H(Kj*bi;7g03`#QMWp+rhGhsLo zb=1U*dm%_m+}sAJ!Wu?%^&Dz}m+Oxe96oTBc~Dbq;)&jYhoXAcE6i<=tSWQW=yfa- z1LlCE8~6HgoOLFPV+cQ|@KRhuI_$nPtBDBL)aiq#GoRkUGPTyB$lIirJimHC3ZnaE z&k!!mVHW1Jq`H?UeL|$PoC8&O%yAvJti+@gtNk?Www69f;(e*`nw9y5zi4Vvy=6)g zWtix_Mh*XHd8TNIYF#nj=z#lI{f(UWw$JhiGkiJ`WsvJtct<^bb7q0_RCg&dxpPtnaI+N4LFh%wS;$Hh8m_#!AI9) z{Z}%@%vI>M>kU%(YMYN<+F4R`c;t~vhIUhtJeA#-06fHMT(fGRtgXb6d8#owEHx&> zjPIaExEjOm@Ak!S6=Jn1Qj#{-9f%&SVrtb(a49UCU`bv>puLbccv3toXWh`9qN71o zf^O)CVbHpUM27LX5O$wVr{uO{)~VxC8+Hn&;7Cxn+AHJt195IqmO0WVtT4#upM3nr z=EHxnGVqPQWLHy2_5hkdd6XuM?RwJ^`<+rp>xG)gERerHXb&MR_&MnL_#KozZRg}5 zxe6I1{{b6-L^X`|B`egGp1jURrMq$Swx4Fa^l2@^L6jkXH`n<$MQ)y1)-y2a*8l>F z4^4c1y3S&CA)LfclcuC`^Y+cWO)1R*c%V{vRO!}n)gzTc*-2IIIPqBZ*i4AI+8#~o z%qPtq=)2B98=UjNZ8+WP;?%G)IP!19JvYyXO4cfAoaM*Tbi-fg$4Fhpi9gW_i56lj zpyUSHP0@Nj<^Nfw=qf(CA_C(Pob)G-o_fe*0`J=Zk)qG6;{7Ja189bT7d7m0| zOf@e{k7Nx~{ho<;EVWcwNSOHU5Xc|zzjNRz=QEfT5r3gf*K*J0(c6bTvzqyLiILKH z=Hn3iytR8mSj>wFlN6xs!_^wd`Cz&_i}R_9)(#w4bfacca8mY(q$@MYLHF1>hb#?^ zysLxVLMFob(t-p`_Iu>8tMMmgK~Qvfp{0nM+hzOrkN3EPh07jM@KN#LzP&O$Hfk86 z_635$rCX)7yCW{DUbBV)p2vcPKj)0!Xy0B0V*$7o_jJMVY=QbqPzrKoI7zOfkUunc zm6e-2%>LD$Llvi+k;FE-L5*F7h51 ziu1_$!Nr9;nA=$$RqOFya=Dvp1+vsN+UDD8Z za7Y{#;*Lpbgo`{zO2zyS1cOeRKTJq!6U%L%+!&lgQOo{WUL=!mM|={?o%RULj5nFn zkuzL2*2W0750l%1~a4a}iX{Y4qBHEid?kg_6Q zeCy&A%1F6p1MGqC&oIvx?M|e>d%rQ+UkR&Li_d%lUT5 zyC(eu@h1>-ge{nH;w+C*)N`ZoT_L&cK<=#g%FG-pc2Qv?K4D0lTR~^o(8E1uDEc~0 zFgL?l0Ne!X3dMcqggHA`Pyc%Ktm|T%)x||>|`boF;1$1 zr_6AT394*PxLkL{!6UaVt9GG@YrgchS-5LRezHV*2ERf#|E2q!)T%;%jhrLzKm1VC z++P}cf*ZORFr=tKQf%{D6e~(ZMK?Xv_#&OpsTRF~JH{8(zpCpzo0e?cU3NZus`0ZE z2vJ z2G?t*$`E^@+6H&LOaaDR_xnt!G29ua)Fs_I8_2DRm4V-v2tD>88ic|?k~AF)R!~5I zaW@08Fi`w)&r1ZQ>ux8&;R3He?xZ4l-Bv{+NZEc-=sUXkH3JFXm~$cq@@L|wL8*#R z{Gh&ndmF}}@g`#a-m@mQB#wPz3Gul?q+buQV#TRACk*uO?IV4X96LYpqf(yTbj5y4 z_s|zx`lqaZA?yF_4#7B>Rs{)RX%n$t;k>1Ht;E}6L|oJvSC8{M;g$UAIS7S;{%gDU zlV~-G?WDp$N4Dj2^bMJJ3E!wU6tQhB;G-sOHb-Q*Ir`3w55$%DPW z+*v`exb(?y8%h5g{_e|k47ch%LBIFS6xqwC(+_+8a>p#5PlycjKdPhZ#sszPE5iN`d(L_8ZDq~4AGvw%r=Em&c-$bmW`Vgu#y@P9S zMBK1Qk!{Mn+lWQ2T?9*tbl;RiD$@A+I#KFaGNmDeWe$4pUd_IQDMc>;*LsI~EA639 z5Y{EEp|CIUyJpgM%>1~x&mjcligr~Zp^${$TP_xtEb4*=bOGy}cd{Ds%~2aUs#gZL zryuDhD`Vd|OrXuzm*na*cztSBDcf@!jwEKRekwe^qI$JkhJb!t4%!<$vtjx*C^Uo+I*0L|Fi6Yd}GTnZIx8L-Rp2>O9MvLYl&$ z;I)tdqmY%_0I^Gv{R&1C&my}Uvqd>p!s}R)V{eD5T7D^TDC_yu_(!ZkD10LAp}iw0 zV%UE2{0*g$63{GJ0DkC-XIqLg$b+b_)(fYjLo-7^4J3lE(%!=Pqko*@=lE6A)aoPd=?1E0zPGBUBAVr>;eJcQ@*n zT|dM07aFG}V$6R~1xc%11#VRw0?|x>61{W@rbK1JG*NX^)At*lO@CVvXw@!s*rvu% zp&}i!C?IHo;{_W|rR3ulZF7}nRye_bB`V^vKzfwMKshW?7a(NARct^bMq|nlHwpDp zuc~z{6jX@(j~3IMI`|>s2k8GzBtNnd0EXHWr{0atG@WRclH%>1-;j9Z9Ma~2{ypA` zRXh|X2k+e7jvdy-9R$zPyYw$-9Di`DPT3L&h;;&ZKOC9(;o*@yMT%&Klmw86l1JD-ECH{H##2|Z6;hNx__4$QXTPgDP))+KTq1n#NFV1^}cQ_ z2*`v9TNm+OIdx?r_1r zk==ms{sSYlE$c@S3a+}l0`UmFL_%!C&XpzofSNi3J1@*g##Lrt;`!txLV25uo#3~~ zV-nZyQ96_g!ZAb84YKmEbHFQgLp+$vO-X?tM51%}zRjx$nAJwdTN*;nZ`+U zKVSZj`*H5=hPu-D!tD4cDqN<_Jz*ce<||C8C`BM29L&v%rhtq+Vq)|8wVN&i9hqNb zWzg$!eNPV%S9+nu6_&rCe9d?JjUmvby7Mb#bzfzRe$8}5j{RAhzFy1(KyLxNrth_u zmaOsh_0%_*XWp)E0RiYSB&&nfB3;J~^_5gu(x_!()hzUkpKg7e)#!F(1#LmpBf_Pc zI>(nwVsOcM!=Q;!5ht2Q;B^Ok1&IQ1Mdk$}L zOzsjxR&R(&KSP-_W6%h$RKW2#Dy1{<%}VK5e}+rbdTH0c)Vk3NOAf|;2bF_;bKqs# zb}B&S`I$!}2r2dhlPBKa^UPscImTAKt49(~qUTZAuX$%wO;$Af%R^d$c2Z(-?jhj` zVP@Gib6m_M=>C)eBW$plOW$N);-M?YxiSUrEox#t8}S4i$oWW?vF=3jheaTWI7=R% zN)^@XHwk5>Vox}LPSrk8b3<->poK@HU;Fc(OM7Q>wriQpjE7plo7{~ekPeiW{RzPp z`ow<^F<4i3-p$IAnqW+Zkg|S?O*XnF{vQm9L(QH1sGPy1n9#&$SUh1fLU#4!{F}^F1g+yfYg|RYlawv<^V(UyPZyfhYEz}Cz|SsWKGdf0Q;_6W4``V>07mwc_HW&*Y53E<4u4Q~$x zjW*ODlrHXoo7%v7lfh+VD_K(6$lLguP;L6pK%g~F*NFG3xzdc)YW`285-zGYoLjrm zIkktQ)&O*pM>40SYkp61u&>^efb!Gbn%*cZy_)~cVn;gRPl)FVZf|4oZk_0Sp~=4a z9-Op|majg%;&{RY&u}$FWk0()?zbg_Up+=UQJFHRCFx0`5$WiTX$3Zy8??Uvrgw%W zY*wh)*aHL370nr}lL^`~6tREMpN>xwK29a{xv7b$yt=}jdWFP%4EZ;el&&RSS?rp` zpeIyj_KX|cdo4SUgWXrixu8)s+gYVbz;~x{ZGW*5224BAQ?>|@%e%B@b?kZ@`$Rv* zAauXxSrmirdFz+4&eJKC;$OK`6f{g9& zHactlY6g}OUE~HWlLN=2@$EqD9hy?J#C@BxrE1PW=)6#*s^eLb6Br=_bp9Z2nEgZRp0cQw`QdftU2J8pbZDhL22XGnWC?#wuwVVqRTxSO;oPmV40r@Aww1g6@K@COwZYE1{XDQw?$VNt zYMC@~4HY`5v1DLjWWyzmCX&!ASqIz*IL+ARyVoIDI|AA=346VJYl3$v;f1b41vtTNHw8)byj^vdG(zV_ZZAPEk{bH#75TEY5tHGL>`4RsB`l8uEe^p{H+ z-wU11Pm#TU1LG+XVuG*e`Vv$_YGkIj%Rp(T3vP>;-RjWZ*mdCb-tS2J4;nmNRx2XX zfMYkl;=~-R3;GS_rtkeAL?pzY8Eq6vNf8W*gSdmh&$~SL_a7s0SO~lIMdJvFhulFj zc!Q|Vv>2xNUZ{8h4IjUZR-hqf7>!$9+WSq!=hygrLf3QAW|WP~Rq)cS?B5^ym?6bk zC1Is#mW}<>*q!gREu@9^SS?z5HM%#mY&fm&$rifPPMBS&t8KVV%D5Bx)-QKNYMRJM z4s}G_i zb~8ez;aKX9s6RiRPlfvMTLHKxtbV?=pG;~84-PTCL{-_13Up2Ufhbq^p+RH!wza=U z^(h>ETIkWqQ*Kb>_?_bek>D*CCntP>X!KU05p)|pQDBecL#6P9V>o-POO_LUj zRMFO_FoGv}COU&os7&WEgp7x~n^=Mjl05)^Ia!A9d!{zTA3C@joe5|G#v?wDR$rM!it8^ty zMDXfZ*ugWiwVgz1+dwdr<;6KSbVk@-0Fi$jE?(9nUcaxdE&?xMuySSLGP3`<7M;`c zI*YO^zt6WpXKvCpA=*ZT7b@xKOEMq%%JVCz^cxY5W!o{UwW-Y}ghdA-smpVq7F-5t zP^WT=_7||o_@X%(FcW(?1ak>9AdiR`zwt2D#2#1}3cvIrZ1s=Uz#KV9Z(#|`t>pE@ zf|zNcNyUTF51`owNy@V+gVK5zXiRk>Fdr$nt$s=HSs4a;o}QT-C{V2oo?zLRxK%w0 zi=yP%RS3cD^M4C&70q!+3n~G_wNJhyKDl3Ox&`EpJE6g0RJgP+qGf2i%(Uha=zy4m z`d~RQpm{5L&Z4OKYKk)<8lxvh{bOi-utSeBd*nf(ik*tF-caC=4&|{32p~(LNon5B ztNlGoh@&P9f_{O-ZSBFxm46(m)_qaeMCjnDAtEk&*%Ap^J+hNG@&sFe#-Cv{;F0m6 z8BAl=(9(jKMj03!MoUaug>dqJhnoW4i|+JIa~EL|LZh}5Li7`aMTq-9i!gG7R`x1E z74Q4&FgG+%U3xervJ(W+e;m8qni(CINe!UA=`TBK69|{Zv=#_B8XqxiaV426gi2V5 z#&5C!>hx0}g9oyrDzVMV-uG+LVM^^=8aCOZ!xg!Otm{^VW<3z#f+o^!PJ|0~q*eF( z0ANrI0`0}1K0hCZqCF%|yJ`xRFzk{qUSQ6G%SbSPnn2v+#Igtx({(`;lsRV7nd63O z1a+lhITRwU6E=#G`1BtC^7KR`sF`108vA|FNo>`+C1mAT*Z<+KW1=xbJ0yt5`*PU{Fs&39u(bdhA2&ue{iD3OkzYnynPn6}kFS+klR`uzb!7GZkQ5-v(6x zkn^!rxNQXLZDxernaiWQ zm%v*3fe5*yc((oWrR)XR^m3#zhdM4k<=U5g+uW9AkXI)BdX5U+2B%`RQ$D`Ce~7U& z>4AFGhUGk;<;&g6>}t$=Iaskk9icxMgPVl@r1H>xY6tDC+*QQ+gr<(iwO6rJ8fuay z&>16iSgqnbkh3h2hmekkjxBYBb(JD*%;K&65aqi;Re{B%&mPE6Ep8#9HgF+~ z%pv@094L+}bI?OthIBp=QSa~qnk`1aC-&GCgadPFE|cCsA#9g2Mz~9vMjt%~VF_KD>GgR_=olF+1-z_v=bdUA zqhkAU6i=oHYrX>p%X=ZbQ_PlkyZ1G(LON3@9O3(c_>2=q9dc?)v`T>8w(aL%@6 zP7^o{egtc}CPnaii7gbp)H=J{fFRff?c2+A;EmlAR8{)nGC*9=2dndN7rKD7tOKkA zUm^cM9uRjG%~k3eK?gQ!_wfW-?gq10axZns8c$suigqrYa_<*tKuT;k2{Ght>fmwaYfS^Ffo13YFflxYg?jh6FZ{NHfi# zZWAobnOO!C1{~CxB6YSj6kUWKBlxr4>b+6AoiK3t+50kO)_YrqQu?e2c(McGLfrFu zO9nAb;p2iYIX2?*(6<{aLUwq#obp}LR$(8%ls$_XUT!+(dF<|eVl3) zz^mdvu1p@aedk;K%H>(3bjmEm$@^g&Lai5)_yFi8NCi^W&c32@9|C;%`=fS4K^Ljx z8pB0>*+tB9*&vs{XtH&5KF9%W+z8{U2- zZnov!-g72xAnkF)u3fB0aZBvXkDg<$p14oMgl~HIqX~2(d57Y`WP0oX3fZM}==#z% z@L3}1^0w^@#l0Z{pS&+QWkZ@3`gaCl+aw97@e$rjiNtP>#0?Zc;eoCm@32aX|0th_WVi%0&-+?W^N%%L_9cSDnLGrCr6lB%Iq zi-xNuUMIk~#n-R#>l0k!VS38T<1j*DLb&9se@i0^JQUZd3OJPZ05ah}Fz-w<)@ees zv&&%e=0X^k3ZyV~)=Jud!Y*BcOsxELd2+mlN>=hgfFD3xuH5E?-mmitC~Kbq%&G+X z!1=H6o`vv7r&_!D{WV_=l~7_m>?XlsIe7VESp<0O6AYA>?c!v)RfZd%LJDJ#9|2kP zGd7pIp*GOJL)dtTmi^W}M0H0)P#lDn<6+tb%7*H{$ud{qAb?SNhxHZ35*9Mgda(8y z79sF$CF{Sb{V2oM5+QTI&A;)XzY~D}>=`C=ByCs%2k{UybF}GMt+Plp2F-POYjN4T zimzIC6Q#~A77dl*WP?XigavH>F1tvaAE z5Ix$0k7P+nvV3!wBNxO7?V)XChK&|(ESh~sCC(3-bL_pnW$1)v3N7}6ZxjiU(vsUgMxKZk!zX;BhFlAP$YWO2RT|dtW!p;>4rYX*X!gfp8yOKd%xYj-$It zz+%@;2#lPakr3{Kom8?s+~+xVpco#?k~wWUFye6!uoosk8+UR(?SBlM@sQ7azS7_c zgllhd=V+kXEj$MueR~iq0DwdJ%C#aWIGG?Ao{O2oo4Mdn%NDU{5Sc@8@zvm!$Xzw2XZKhJ*J@3W8J zalF6d_`UD({daGD)>`-HzOVbb&g(qS3w)mjKszdWAK~0i#UN50b%C@aC#k1Fl40!u zEyyaQg?}5dgc8ln@AQX_LXs-Fw^0I!vE4bxU)A-W4XEza5UAQbKo+#6b`jZ|#oh9V zj%Y=67yQ)uq|>Q2hP!p1ASq5pH}l>NfxBO8%^-Umo5JtaTE{d*cYhYCSo%Qf)PMUF zs8JRXl|kJDhwd}abD`4kXOh3W_iqTXy-}{Qzg7oOsgrT;Law;stpSo67sppi521XU z`~X2=#zg_qFx$U;b42c|Qx2?tQ=KH^-j9=&#a0G>4&oQ~{sZ#0{@80+w1ex+Htbc{ zNc}E|wn=yN+0p63iKz+Xd&~-BZ`rx(LuPc?UxGm1d!Rnlqt8Lzs;s}-stctS^PRjC zNGRRGp7F9BFK+T*aJcIE{h}K#t@MmcPQ%J_grRv`|X*+YBpfrSaXzA)c*1zA1hV2Eua9c*cnSvaWf zp!e5MrY(JQxi{eNj@1EM9d)`zO>*p$d=Uyv?FjAK%* z*j*Yk0wRT!m`BFn3eN5ESjfEf_F;k)EIv;WvVXi#aP}ko^tNpuVD-4WU--z8K#_47 z+ltbTd&`3z2b4hohe^S9tYsl)&L z>Hn<2|A{M*DlvNpI>(D}zpp0?fMjzN%%%0{&zVsYX4v6CDN~BruT((@8wKMst*|$< zF)&BV3Cgxdh)1QobGv+Xz}~6Q&Kl?8+>0pi53wg`Rpi%_ARn_Y=JmOVj5a}BmPkV8 zq5bWI_$E8nfz4e)B9imYs!lZ#~zZ@NvH!#3fEJ|8g>-|SZoInpU z<8QA63rva)SFPj)gpGZ8Lgdb7`Q7a~V}(^OfD9_Go)oA@t|*{@wn?b}+9QeeDD+1Vy?)$TE32)5a{R|qK#Sx`c5Wx&?X8bE1p8X_du04eq`Yn=rsj^7Vop)ipjX+g+M zWiau`eEut_58Z)sDq&Y3wl2{HLfkpy(N^~SxbVP2g}c4{)jz=vUfHarq@(gV^n zJL{r%7wq);U6vj>Z|P8b)gyBs5EyUgInsuQiGC(d&n$90re8cttIx3V9|rcjc5dy? zrq6NWOehLs1}L*i3a~pg+nCn@dku2IH3%Y`Pq!(cGy^|?Cqqg9q4C}PAB`_)d?)_5 zX?%HJ6|J%To5mOYzhC3qV7egQ7E?`rhbk6}>I1oO?q{iq zwP>S`AS55hrtZ?56^4&?0CMgXKMWy8JB$YoA7p1!l)JtOLfGfTue-OxUeh^c1EnN0 ze+R_v>3%JCX6VOXJy~TLYQS=Lh{;5`=OW+)(3Q z;H7N{ik;aNgs5&XAGrC7Ar-YI*Lz&5rM2nF=}&hNFf~d(DZiuc4G!@)WW?Vb+prpu z=L^};UhfH-Y0h8xCo13yA8+7eiA_Kj&6!T}qsIY=Tsc%4`R+2FSX7Bv4<+mH`YzIkz442%M z@bctoPN`R5)o^LA=T_H!`bONx!v_Q1hO;O@>rwH4fy%c8tMdKR7=q>auTY0yr%A5f zPYo6!PRz0yI(R0taGf~;!BolcHW-i?UBqt*FR`!+SK?n9*)6VCks=aBVB!++Ban=t zGm!Mp1cFz$!jk`34ZLn# z!U^bGM&Ek)tE|v5&g633VfW6_%b5CwX~JOIcRuVRjBVK0MjenITbz^ac=Pb~RaWB2EjsI6EL%>tNJl0TLSSu*UZ##?8eFIdV5HtYO4U=LJ5HxeN59-wP&?ncMPgJ#)|K@0@(knWwyr5P6M3#z!EJp;Zv zTkBmid1{a^DACRx>-oh60MiAdtYE)@*SknB0p_N(sK0#$^o(s`)n}9xrMNYKQ!Bej zj^MvFP@`dt(s5vtQkHwwy$y=lGttWTs{k0+AkuAhAyx7_J1UX3MeZCmS$8o$e6tox z^q)Tt9SRJoblen1ndNhA+ye1c`B$ffK!-PNsO6TU?_r)-h{CNVG|Es)&6hqT)=z%) zlN}?430nXrCVT-8%v5V_j%Oy-8b3>}FRs6rkXk!yL{W>%J zGsL-?sAX#4A7_&E6w`1?zL$jY$zsna*k)QfR-z42p7g+N8#}d3R9jrBow(Ju3ZRkroX0<`Tu}+O<>q%F3ph`dE9jqg!=PsAsl$ z!O7=b_J{r}XU%Iafs)9(Kj!VVqDupJvq0#NbbyWdwRjmtJ5b7gJh;!KxJHM4(m0&i zh9{y}58l40qOOSvOMD?YEPgF;ID=xIF0mfep9>0VQcjQO2U=X@9*0%4H>sqV&l7v2 zkFlZ52l(Kz=tJ@h3dL6JQOX;S^+~H?;4qi5Va1zb8cy!EUKxJai2{NqTzViy-C72q zx%Wq2XdRm2y(?D<-^HH6NypYZWXleHOp01yUSJclEbqTi)JjIq@HXkS+x_H|X9I$o zK3>0vm~AEy=<2sfVY;tfWRhBDP3q9Yaz0{2%@7YJuqeBWQxd3A!L9D2GkGibtN+x4 zHV;Y@w#QTYE|j)wUp7w}{iYqx&;1AOu-A6J-?FT=P_!hKR1k|mIdZb(hPGDcrsfm| z@mdCpoHR_ArnRdKHnM82L2I{Z&%-68l<*Iizcb9SoQp!K)tRcQr&l1bc=lSh=?R6Q2Oux*VN*j+#nU3U5PzN>Al(Jcpq_MTv zP7IT@4@9t<${$C6+bRF_^!sJ>1*KI9Wge+||Ae9qSLov9CC~Yr@5p_T<15=+m$H~U z;RV<##s`;xUYUp|Vq7?5tsi#9Vq(crc~}uar(_TV-}0Q6QDY9uQw7nig%?qD6^Gf7Oh_A98~HfxG4oB@zkIH8kX4my1` zT#U1!e*l$Kg;fq%p=5U{k8GoBGrJ*2mt{4MaF+GYO)&XmxIe>-?S;J52r*-^1+2%L z+OR>y&-Sc;27|bcNTgAc&x_{EQoM3jPpU>ap`IWL4LXTx(rjPIuGpzk<5<>uwOc+rb*oJUT`j_x8y!I z+4fliDFwLz^^_-m@yOg~uuy02lb#5YB#(ds^0FIowDWkJb1Q4+ZIh8_BAwQ5@-ewc z0Xx16nc?==;YjlOIFtbyW18gbM6xVLp!f@x4(-cC3&5pa!mrZ0PuW>wtHW&c!Y`$7 zsJ?Kzl*7J2LsRc^s`9`hu<|IP&9m*a@u5v1aN9XH>51#^MfZpmyx5y;#ucT@ta7&o zYgp+eE1K>%r+Z~=rEg^rFLPP!i#w+e%9!StPNi;n^y>7Ge47)Rbbg0k5gly;7ZAb2 zJZ)xDmaB`SLM^VDv;0^ywGtQJ{SFt4Z}~*;rx4NSL4^_+wrP%x%J+~ZeBi~JT<~i2 ztShr8$#K16BhDtZcoC+AAs9egC9!0OPU~;8BryiyZOZ8Wf1xoJpB<4bJ2AT%w5)q; zGGcqT7&k4*yzA>nL3w%F(V$C~ZIm9s(y2n+Aqv9E$#y7Y&evg_j^MI;tR4>@>m$b$<$dz(&1P-| z7jG-*52A3T9@ig|5=l*XnQqHcd38c_^VlMQON-}x?!PA96apH@Pd?IvDI$!r(?tm8 zuM@~5IPeuJDEat%%uezQq04&kgfsY5LG`V!-MXU}ZTmYjy}(V`rqOb=%440miFKI~ zjhJsrg&%fevHPyE64R)q??YG6CqaoDJ1s@lkfgDaq^3_yK&BhvQ}|qxBN*mFtSu8+ ziQ^u}FNzqDyRVf-B3&2xMlpT4FQVf-MJe(5hI5kAsbzg~p_?S5oH6ZZ_o5<6DSc4M za*_p!TJnfZGSPjvrIU4NKQ;W6A^Ab&Yy+d9(xPj7gt3z8oC4C5W{`I56gW1k2>U)nE%irYo!&i-YR3bKx$Y5wrIECu(MHH zw*?(h7wxi_4@&DUSy|@nV&A4BRzMYcg9n^Qx;oUx8bn4LFQ1h5QKBx-q#80LCKb6# zn56mj*`%{jOiJpR_>*~$t{51&1=|8oQF)^rzF7z=j!pxn<>AB|E5w2UX1 z)Xpd16;?LJfA$JCTsK@-GqVHHi!G2*PZV%%8<{UZl3#+#x63;sgH@bA#+U=YDaHPC z$FjvnO#K(6(Q2)uHTr&!<(}oPd<09?6d-CQr(qPA-+hr?TU1??vYVb;vI?FgPBh2B z$g-8>`u+NN-ikg2iZo|iL*Wv)`$l2r?}wA3XgRcnwWBOkh{x+n-y7s*$X<|gQ;S2! zl6a#GfPWFM+;`qh&F7+2|BR0;_3?@tdEuwd#vOZ~?z}~TbujjwAc0ROtIwFmDiu-S?BEI9#}#9hsS-1fQ}?+*ion!p~HY9qe>5T=$6i=RQo8fcmZ0VyTcWo3bP zq-xh}crKz2#Q#z(j}hCRGVPz|n;WD=dV65(7)qH+UA!q&Da=Jh6M-*Y$(_0>@BZ8< z2Ybqd|AhbcJ`%sP6Bmb(kx+T;@fRKTh6Aqmr@+*4{_1dqYfWixc`BufK)%Ty?lQT; zk!M(qTgQJZKC}vIkEjo!&c?N$kGP2AeHNpOn$B2pNM;tCtFlW%*A$f+=@nhddxwaR z=@lE|Iq+u}pgwbeNk2PQo-{-%K(ua^5vzzVj!-dG3&X2Lo%=dcmM>~`9RS~*6zO*j z!w)cgUL0HerOeDXf3YG?wmNBgSL7$eQG-X*6`Tz`bra2*I!*iUBg41BQ>6TP&|dS0 z%%gOlkW5VSa*fWieXRbe_|+cVR*vHlTbpT)B8@tvzILW)hIp6|Fn#X1Gb@yBBkMM{ zj#xalR(#I^74X;$EFcnyf6a@q$kNfYDbb{Xq_ZVydCH)SVUAS4q-&;PNS?=^J|4AI zySv?S_R6z`&On2_{GNMml?BSn?ez>#t7oWPO><6$>cUV{7m+TV^wR)=w4Uy=iObF@ zl#j1T^gJC9|Sa@7K~yCW3;x~fBDt9)mB z%j=-zB+?)R_nb!*QV0mXJA|-7q;dez72#q6rQ&zyg4vgkRryEkE+8Bu(Ib;ZS#z6+ z*_jZ~e5$Xl%olZ30A^dfKwJ&3)&bR;BX z!3Tcz{5T7Z-?l+zh9Kv~T*Lzu-h4j70=v-vj(MOLL_S}3mO}2DH+X{D_cFeNo1w9O z2;u}j9^#h-6F*ZnkJo8(u03oFat?cq4t_TK3uw%oL)eH$2hcZBZ{+W5-WC8S^*cL~ zqfsk`AoHw(8;OmqBF%On^1JG~&ct=N!PpvTPRsb>k+KW2jepd|f_9?6ZqW~{SB3Y` za9J$-c?=~Y9@~mav`Pm$_tu*QAVDfWy37(Iqb3QD&gxma5Rt*1%3(tM+{p9e;a$*8 z7*3tM{Ko;~b+AUiiJ0Uhu-~3Vm{D0=CNemR0U4eEOMHOG%<&JIhf7|EO%S4)(uI&} z&)r46Mqs#|DY{}gjP+2VMeYC~IaE(1=^3q~tj-Z-z&;TeI&)k-r5B#~+|(6>K|k>F zkO^cm+;W#yHvqLY029-n9pV^*JAijEA}U}9xHo*xtMSb4&0pSJbqV?$)=tkbim;OQCT$C_ z*TD86ppUWIo&-nUauPpmM_82-5N7bwC2v%iPW^z3qJ+V4jWR1az$Wq$GIBkG@fio< z(KdH-U9X|X=p^Eyzt!QkX-!yqUpQGNWDO@OXmbqw-+y*geD=(G+zx5Mm$E~aR1fBd zp+8m~aL!~3@SNP$O$b1y=+f`3-R$U;HCmVxx(E>JH~o1SIr)Z)X|G8I>?s2qu)dQw z4KF?SFiAEKnqfbTd$!^(IXbARF9VPzMKTuL8yzy~t+S(@sS*Faj~cE| zOM@+1iBjW~<-g@R!@#FXP{lF3oseS4{9NR(vH*QLLoSg_7Anc4c7TkZQ|%%IErgTz zUE1G#ddRN;Oa`HkRtea`t0Rw6*5n5@QhFYqalixx5YVt+>W9|S)ZGXx#usvoY>KSJ zHZa^*K6ny<;O`Zcn#EFT`eeycrZLM(fo0{{ejTJdjJ`_Dd8{@HQffC~9zg7_@aIG9 zWh&0wAMiB8>B!FS-%LmUAqb#j?>Q4K0OQApa@aXN*JH*4=&eT60R$RQ=@f)WtGlXI z29OLNHIY;k7@kd6Zw_ExO%FIHMeCo0$Hr(h4ve8u2o|r(#U$2$5Et`TTmCzs7rBYmsHLvfKL6j8yux|4&2ka+~}H0u}9GOEr^O;2#^a zW!_?fJC&-$-w~lV&PfR^05=(#(sW^Lt9%`EJC$BFxJxa%8Jh=r@FPAYw{R3g%QYVE zlQI1Em8ahK`6Fb+@L#@lf9WdD^n^cdkOHjZ-W=0^L;AuMdbIsEn51A=UPB(hUQ8IT z=sZQ?E6Uf<3b!Z+4qjzgxlE)nLmf6;1xoycdspI`(^<$TC)Dq|7igxaH%~>#VF&_0 zCEpCk(|A$cn4EFh{%9OnGsa%Hp`0}bk-T5#y8qoKwhoPjr?RHK01(9|y+}$83yYq* z?Sajw=1yT9+MRA3Tg7r{cndt7jsx}5DIHJ%L`y_&iC>cOw*N&Dws;UvM1nD z;GY?GSOcci4bqFGUCGqaK@+;raVOwZWUZTf%vEzU=!(4PkRvG25e??rMr5&@(y9$f zqs?QCdKGF-9=DtI5gWwIRN$%h*nIH9*Cx)?3}RGdWl+#3VMIMdt(g!`q-$)rI%*YH ztLk51jSx?)k(9AWvc}d3r~Urd${f*8Zy+OhkDQ{N_fH4i;}x<7vPL3jvl$OY@u{xi@6A*@mukyfXKEo{0;DKZx5;JkIHTKJNmZN-YOM%?5|@Eo@gf^|yc8DSHy zxK^EjT<_L5WTnUCQ5t#KHzi(x32jerxh4$By`lW&UBkg+*Dl3Ul9kSpeN!jvioo7Z z@yGumN)zL;h=!NmlKRdo)sz~53e=hJR(~Z4o2x=>$7Sx3sbjj`T$4;Mzfc*_tlDc@R{zd)~U^k;VmFVZb>f5OVmIO54P>O!BkPp-Uu&$W~GjAz~BN!kxLdk zk;;IYWb|S%B|>aRymbUtuTGld=14=Vu-HFo_y23GNVH;u{egupEyP@R{mL!yGJ#eO=o))&pEFQ-=pR0Y6my_U?eG!3 z&x`xNMC7l(gpx7w5erRJuHQ?!$5`&{zV@00*H_ULy~!ATi+-BC?9@IR>(pOAP8#39 z^P|5JMqQ3gRTn{6^1Cv9hhRCQ@nO?_Iiq^-oj!J(O8xbHaVyeYkv)uCL7<7{)Op%m z2Q{AR5uD9aB!TLiBm`Y*7Ue{T%Tut6V~G3L#d(XHq^6;E-03hI7CO7a#|^?@7NUdj z&TuR1qYAKZT?l#WeS|QDPkH4ole2xe?VatfGOY5jnI8K@s}HHWUPVUFTn`hw-K)uo z6S21dF>!>Hz`REL!TXWpj#^;AhJUy75dJe?`DK6=WMgX=*a_^C7!9 z-AP|NHH{bc%~vB>eVgulY15weB|6%l)Vo2BWmSa9_@9C`}#QW-g11Y+zCidX%niaH!zr z;B$4F?E!K=tynRwprZxIeTitf3dHkX))D0C7FjyLW#A0L1n+rwC$`ACvCE(KB-y?; z3}UXDe51K_Px`Qt#|XI|BBHp-2D&nI1aVRcU(Y_ic8$%c#ML8#eNrNk)YOPIPmSOW zw~dUUnnsrwVQ?Z_!-GMjMtukaG#;NxnME|K4v+#O14UGNMXJ?U=Q*0i_I%l3H%t#{ zbwrx9I)A7reWAKVzjn?ac3Q*h+7rY>DhyM=vau7@7|dqB<5 z!42ZL7r|DjdMS0V^z%sV&VbO0nxe@p7~Z=)qIM35g;|HlG{_HtM$8YFIeztLvi}Zx ztbq?OLSzE5Ji4<1o@377CF=&0t&8VItIBqf8F|aKQ~I9Y+M?(d=)u8H{(yK^&DfB+ z`#zq-{@1O4@7_BLc4OBw(yX-gknBt#I(;sAM5mYsu%E(c0hkyGG2V!>0(Tt!3K%oE zDTp9+wj-svyBnbTbwxY~<{YBx#$FwfHACkH{Eg)xa@G!1v~rnXWBIz|Uqb)u%f6LKsWu9l`mi%2IrJ-$ z0YQi&4hFiE^jGykrMwQ7_T{ikHx_)@U^=xkBHu>@Q!w49?JvJiiXl|)Nq4HOr4i~6 zt+9R3f0X1uLxK%S>clHM^n|z_Ko)L@3KK!aV63`kf7L1^QdNj&s^gFsfYEVb&$Ip3 zVBFtYMbdQ+BOM5mgVlR-mWtGIgxsNhXU}Xz{fA8HJ*D+$?^!=2MEHq+5T=7;i zk~VS)z}&}ybd`kEQd&+JQF=@#=!yCU^cO+v#xVu>k2o@5Wp@Z662sjeJ}}DZUw!LD z=$_38ftZ*n`s7yUc-I47XWg>|+{Y_3)Nz)h!MmnCqz3aSS(BLd`6=G<6IhX~ZmnN;)XR@T(;4Tsuocf?s|^Xnt9hC&oi zze+mMo>CF}ukXsxbz|O5vR=prl7n9zfu>$4v|p9P6fT^>Zk*EH6R-FD1>A`7pnMQ# zb2VanVF|?RJkVl`%x0E$!+bR^bv~=k*C3M9b*K*@a)2{=6QPSEt)6(r*(_ByuQ{rt zdtQ4isDrqf65YiEI%&Jp1kc>@G}eC7+wkmNDCmW|hh8Geez9EK6~fF%(lk>XCT!NM zsj5ZLZuS6lX%jUK@}_Je@l@Oy1byycCToSp;uQJ$YkwdZXBp^qnfMB%tJH_}Ll5>K zHf+A^Z{_lv{bsL@ItoQhY0+omQu)HM| zoYZSK5M_Qskwp6U29C2nKVI05y}I>bwqP!zZfx+u3&he6mX0vxX0SK@JY?n~6shuI z&4SnrV3UGKLswQ8A@nrH)9oiyb%ZPpf<4etxCH8D(LW%Fj3eMQG8s&yRQ>=oe`Cwi8RrN5^{paJth5$cP+(03V{pO7v9@;d% zE6@v3CBS^ud_P!wUYERm9ytC|8_it=j?4kKLd5{>y zzL<;F6J<-n;1%o;BD^MhrC;LWdH-sEO6EI=EGky4)cgxb{g?C;N>+n#d=+tdHILvcP?!)KXD}GNd7MX< z-Hs4%BzMI_R@%@`?adKgFYO~HS(y8pi8g!T_`}fE`hSs0|5csK7rZ}j^lFgC=f_8+ zN#l^P+cb7d?445FwwH(?Wl!u0+@3dSxORBkKCX&K5ApYr_Ryb@m7pt7A|@uL!~dp6 zefw*p=2ey^_`eU73TCR;Pwsw?{dwbp$5$Jj>^>XD_6t)t?ZzXdzl&U`tlIkaK3nM$ zA2MuV2I{D2=#BK?Cj}Z629yPUyvYLjo+*yGOoXTNAv})a^>>{U!Bu`l8Ud^d_s*ae zIo|Gx?dd}O#K^y1fa!5570_;Khbph~L;?82JiGcF+_+|7-fmdv?DKG`SDB|`z$?aA zf=Eo3%%;+r?^>Dr#SCP;ZnD|2g>Bwq&rh73^`xc*JY?$*AF)1EvWj$%Pnm-8dWmUP z^cpy=#=rofoM;VGnpmk<1xM<{As}VqNw%<*O=fAnAy&-7P3k{s%BIzJLg>?XaeK)C zR(N36$NW<#5&7f+Uu}HzCGmM6gz^At=B;s+M#Q^(2KaayQ}qIuAA5d!@&M-9`#KZS z31v}#A8cXyuoc!ADz%VNUhr_u=d|9df`Nk#9p|2m?*0t9^MWR%{)v!)E^X6^Fh=`4 zM3hc<1?NR|hab74Cwoi)eWwpjiMi%jio_P{P5)xNjKqrJ=4a<}F|7*wKR53JZyJNF zf9Z9J$CnmQ)a@_lN#D+~8}MJ^wIsOUmD!=jtBI)m8V60yfxkmeQ&$a z4qc;xUIC|*%0gq82R>6m1v-ce2%dc^I{a7=lZt*`kW$%)&py z5MF(~=7%c({g31EEfx(hgi78Ll-=%E03}I24Zf%XFp()Y&L&Om=d06t)Zmzi1rPuA)2mS4d1DD8cq%gy1X#tNn;2nr4K_d?m;8%*9xS*%WLj&{SY(TA<0 z@NfymhHCxGC-?L~#3XnN53X4)F&Ro?;)HFtB6U!pkI8;-VC)-PVLVeIt%HIWRkCQX=(|5x(H$} zT?pU**(LYjN5f?EF%Ifq&2`xIc3?Vq5$iC=AFshSJ_D!T;%!BaPS>WIxXXY9Vf6l^y~uuB(dLN>nfZA z;m5y!_igccNDlL5v%p(9@llE%B>e5rvbs?^?&GB&#kLUX_R2aWVFwOwBW|c#@M$MW zdmKHX)`v-_Nrty9td<%LKmGnDnMDKS)Dij96PSt>2ktb&s6H;ArDHCC_#RD*k?L#l zBvBz|2C{kq9$kv5$wVR*?#~hzh?I2c$4!IIOp0QsYm?TV*gJZO3%G(DHb@uG1$we0 z6Vu4RxOUHyhOhM`Di`64Uw?N+kjC-d!OnxYbe+Ls9H&zcNn{+XSGNil>m61z@wOj0 zR_(tRW1I#q{dOuk15cTry|eNRPVuXX*{p1v)(qhFv>pc!1#-m%sEUG>^nWhGyot!i zDIN@C_2F*=Au^(1D_*0{CH{6S<{YnA_woIGE&BUvXrHYY`SHgg_GHegdEN_P{Ym@D z&@7m*?c2w{dL~y#5Bo$PHO{PhVd{e>K&Ct6sIoImXJ@kN@_|LK}|q-1#=pA23^5Ppp7Jrp_G) zN7C~pz3R_+z&Mp!T7PH7-xZ|p3e@bTS&Xi6$9Z8!Q`7-%z|VrmX_oU%RP!gAgxT4E-eR+Vyu@&Ftrl=KD@p z>Rrl-_nglPA;7+(Jc8fvLC3T8w0D0xeTQy#H1z+*+o1Is_$f$`$8jl*mX1auF%h@< zDcf}Yq}^a%qJUeIUIJ$SOc99x_BEH+#O+{QB_A@obaWG$xXtwR7*B+tprD`FEUrxp ziy>}isNih3TSiH;n|gMiSFS7FFs164$k}2Z%^HsZU*woNJt*EPJGG5vP+O{GB8(;b zgDb!b7$E0$>_yt;MKHC6PcUa@;ObW5=tVl^dtyi6Teco>mgD-EPt*4#C`^O#glNx8 z3GDuOjQ_Xc1sqL!G`p68LKDlJ;N7hVmKuA%{IRFT*^BVYQMj7BGN$6Y71HjSa!h6I zCcREz&94iDC|y(t4Jg(Jszh_uc`{6lFW6&`J$c9CHboo=9ZUu_G4v)-|wLe z!Z0L?{D>QZ0B|{iQ4zzSAhIs5c7$Bd67Yl~(XAPkyEe!r;{pvA46yn+oD>5xOB0hd z@GeRnpzkB)Jn0}NcB)?0om79lP@hfLTcnvYrcsB0W2{kf#|8IqH^0Ca%-jBmO6ZNZ z!JJu>rrV(Pj6r%z$e3YEU{V%}Zb4hR{mSPAUaNbmHKZX!?c}$y z;CR<|XC+-!s}a}(BeCG`5QA`=kAoSElYw`qG7Sxl1YOvHA+_^QQ*g5eP9AO#4^!NI zbzbCbis-hMrAks@-`6-$2qKk|uwQJo39<6VnL=`G+1k zsM8r|`o+4_PUS4TK&y`C@Kq|kS5B-*Jozn0@cQUa@q{2T+Z{a&e%2OiV_uF$aMogh z=`@%CmQ~}HS`~fEYpD*ne9mKA;=h3#u^)yH?bbO85e@!~HzrUuwqXo#hvJ@Ky zP6ZiHD)VYzm`d$tUw#`nY(`y9yUlD#=BR%}C7rMt1Ch&Fm-FoPg+r*jNSL zXG^{AqYFnT`AgC5*BdcNsrlD>pXM*{^}c!7J#V#jR6?AmOk%=WWiAz0wL|E zZLr60|LU_Y*?dv*Wx$7NO(^16lb>Z>32U$8G=G zV;q0`L5C>}I>`k^VY*s7{Ud5(0qvGdG981iE8v51B1DPmCixN*&6z3jL@+)kox{N) zoW}hXEn<)&<+tqVOMNfYRA&8Pl~HMi5G30nm9o2#4vxvq^-fap7%YzP;GC+0G(L`g z-hF)nrK9utG`SQ|0tJVX?1=t)zQo@~?X(zm#4%C;9BA7R=7L8K*k7?1R{|uLS3crH{yP6RJ5nS786E+b*I+-U5(L%lY}q;L zF0YW$(WiM#{bryAF}iu_-T5Nazn1x$4GuBEZLxbhy?67T(P+(Ds|a7NO%$+Yc6%l6 z{wrSf{(cnUjsxl^p9`Q_vCxVba-zNY`e11!`g!Z69>ojMVAVRZz=W7E-`NKGCXb<^ zp|GhQ@B?G&YXmrpe}muJcFBh&s`iF@_|XJNycBNzv6E{5>aEhz?d9n z+GL?6uz2U*-dwfKOn?Vk--|Q24V7HCiqi;=q)5pL4w?bFUk_XP+lLWq1g1YCQIMdM zmu9V!8uB%-K$|Cx5K6BOzev1whOWm#2?xn&NHzg%dPnFO7t*zzZMF9!}m zHPHqhhc<|fNi;-r3|JVVo-)1<)NMR-;|jF~ErO`s;0%O$NZb*}Cjs82$D{TZ$q;VG zu2+DuFYexys3P3LO}aH;I1bvg5Ucw@n*_aq0unE zY|jy1Pr-IZ->zhtGjgmt7`+Fmw~SF55mn+9CCR=sEh!iAJ_m~Z!Xo}H&qx%*h3G)$ z0>si`qO;)EWeL6r9G%nQCsyGa;zq!?0wb_3X%VzuKHm}ovh~6Fmv2w5spwtwJueb) z2Rzy=!Qi@ZNIj3Eu$kctT=#s$8VFGhu)qBFMD-q$pKH*xDpk}cUn;fu!!?E-mSECG zkbn=9QSt!8)_d^?&*1)2Gf4j#6y6;^h#v)mkJ^xY-|{g>X>$W+`7+*}Zyz8;&UcQ1 zePPib_m6#X!el<33Ay_-x&8jeJL4TwAFxA7?_USUHG6mug6x~9nxAPLW*`L!x-}8V z+=gb=Xanp!cG&G*zZ#q<5KL$gDcyK}D{Tl`ZEdi@t&j|hJq_{zYmuMduEq+uy*W&Q zIZS@-wPGxPJ4{7lcl!Xz@n+%-_a(SUE24uWPD;+nKyW@+lo#4V*_OCp?;bn-=LY7a zzWWd|{SYp}qtkU|*$M`wL6IQF-zD{xrAkWG6O7ihrVqXuusAw#30VfgM)f-wxR zvINj((WwpYO*6znlpV(ris8tFFOjXUT?Xag2WVcE|LzKmvRKidA5TTr!7!8_CB>N= z$lVeV(3&C${^;#si}_rz02|~7)o|1r+8DK}Dl6wpg{g2S5YKzQeq{qw>H$^&FcEz5 zgByaJVL_y*V>khso;`#y7*Gntw?TL}SrIpId!T2Jm^%y@sQ33miRAeF z&YG^iofl-?cB{*&S40881=t`XLJ1m-UX1#Qkr?9ZWLI|wYIiK23VflAwT^Os$1qmh zq58u@*kxaW z;C7?z{i<)Du=wmajI-JJp0I0|2eW|m-$?_HR=X(wE9^+Vqp7w(SM%@k0SZ{gvBEx7 zG?Nw^1DV)B4%jqBxM&t{fJ@8;@Klc!03L1SYO0gD`7z+v08n zFsNGn0@~C}S8&RHM|sCdvoUB^t{NnTmf$K{JTVKvVrdiN=ICxm!mNA|yhmS-=AE-n zF9<54J(j-)XMX5MAL@977P$2OAb1D#>yg5)5*suNg$2??5+rEL6$WvRu&8#5fdvw;PyaHxV*w>n7R_a1MXY(#B z>lr4fO2j~wSW<5)NGe)9FK`a7Ws6U`RxkW3ISzR|UcV_x=l$vS~9cngR6>7yVXL< zXkoym1)d(`E&NcLKA%R5n)|cTOPYR>zoNvy9O>h4fH3lJ!OW z5sK@)+)SZy>BG$jxhkKYAJG~u!+rF*|H*~-R>oA99O8z+M`2-{k=@nj1J}DeJlEbI zTBL)0)`B1cWs{hNGnLuQY*EkB)65r`0OaP9Wyh8Xq{K&fuNjTX-DzCUHhE9?#o2kn z(L8s-1a-li=-it>6!@9bP=C}HZ}KcJ4u?qxHE@w~ex!Rt>h|<~A3yId>&@DqlU(hv z3q9E1qH+o%I$VVh-VD(21Z%mrpz_CMMrGnNx=NyuP&<_SE5Xc&Cm zM4RMd@Np)?<*De7Cwr_x`1wu&J)M%6`uLO%@DMk;wE@bxB4U;G2*1e1a~^m4;)sof zBNm`Lzxb>n#S>N1i*$G8YxQjNkH@fwcil;1=M9v+| z*-~35tKH5e8)MV^8kC?ws*qy4AcvO?z@{yKIM+g?6oA#yJHs*n<%pEhug?4Ahw4&v z=R}G2?_Yrg(uz`{1om&Bjr0N9Y);CG6`vSyK0JQfbT{NxeNdq{==aQwjZ6a>+jh6d zcZU`vHVe<6o~-$%;80QQW2`uHrG>y^!78>J+9bNq1ZGuqmGW4(tN@lS+tuU`c&?gO z$=><_s(T*`g!`}+^KP~rzFe+>dXFtwAs61&X^eVzxvSP~Az^#|#u}dmMpjJr$TaJ< z);jraRGmTEv&Jm0#~@bXZZ+>3k9d~K#F`?}K&(J4Hj$E+o@lL)!S&-ZU5QBv{T%y4 z7Dw)f+?9$Sj6C2htZ?RmsqjR=IlF`~LZ2g;iJb;69~0vqfz}$eBT>(fd-kfyXzg@N zLdaBQ4O^&3n=Pk{ZnxweZ4n7>h*Ep|=u{cQ)o6nt-S2s8puHR73oDj_7e4SPkBfDt z-=Fs}dLd;a0W^B;NrF(1EO0ZPHawCb8RD58xBxEIc88yt4hVK%?u7w_!SZPlg(pgk z`sxk3HT4fOzCw}5_4aMv9HHe)P}a0o{tzj$>I4&SqQ`%Jxl}4@(Kkc8Ji-)fXNnGH z$y@`eT(2YtPYE!BMh2(?SuJMdHnqmX<)&WRO~=ZQ&VP3aUr{nA9$~ybTvgHW{KPM#6Iq8TCa(l*-5SlO^a&r3EHX0 ztYjkn{HVLOQM{g-52TI5P#QmXh4SlKHbs9Tg1-pX&D!fWp zHt4tCQHz?^1vT)zHoM$GeDd1oA*D~Pw_t?8YEYVzt zRvN1N*Rn3pBTjlkv?YiHXQZFgN%!_lYHoArpxW94Jyjn2 zwwcEGbyrUb_ZnSgP*hiI7oJqmwFR4lJ7q7CHe?YH0oEvMxze(V?xR~6KPG99rb~tt z&i58?XWaI>L7ldU(KBUh*;Ku|xtynn${mMk1nK%rQH62E&sY78ZPoRa`-N@^Ley5~^BcUq3714asZ4FC^oFkp0uaV-_RV4mpE>UO z#TzAh7ff5DDsn$iw70oDd)hvGN80TN3O}OcTK>#>dP?RQo#FFEL6;VsdW`1~cqQH> zL%6&odH3L>djxJ|eiXl&cT(s6ty4?9ei@FL+G^`@CnfzDv`I2k1w)Ja=+7L+7L>)p z4^#QoG|k#p$DK$r3q`rcAI4*hg=ix6QPeTiiC)cmJ)Cxnr$3hEqM3)oG8D;L==jzf zj0>fVr?YIW8YU;7BJN|k-s(L^x#eP^9g)}F4k*1zv6h0wd$Ow`=h9$!4DkiK!Iuti zyU0a&wECYE-pGnI)|bV(%5YWbIC>@}2gx1XgxWhq!ivMJ)W0{Nd0+g*`FL9olP#*5 zT^ej2Qz(=rC0}G`8!5TlU>T7ciRwkGC2`%z zdXzlT*J`7t9l>j0kV|J#D0@4zV!;AobF}|qqxC#P#e#a~Sha0yqB{L@F#m9K|9mE= zG_8!HTvGY`nSp#Sy(rVnNbN<+D7EByy1-+#GuHORa;<5^BrRe(3W`mA#2R7rVS+y2 zi`s4M2V6p`@+0_2J%ne=Vav~K-W-U)XqHAdava(0NDwksv>sL9-PBJ<%~ncG(40^KQ&+a25i;N98Rb(O zC?V~Sx|(X0w0h_Bd7q(;#Rt-00q$dM6o%fC(35E zNL7+7VN7Jwcy=EI>#>Yh|Gb?^8Q1V<*hUBC{qmTxOUGL8^7kpV=60sQh?*Ci!lh zUkI-SdPp@}t~0|SWTcT&27JpRxh6G9-jPLwT#znnS8HNBCtH#icX_2Fs@Y{CH4>xF z^lbLrMmvQD*kcx+ruMN-$H-TTphr_GbhN{FU8`*(;Y-=JstbVic%`s-bNso$?0L2_ z!!C=3hUl|SQCGBFq<+{I^v}hdjk>c6CV^x9P2HRBL%hMyoBN+&mKlqU9odx?>8ZmT zww_c;uj}bnYS^LSiT)?wGAYUm8$sd3A3Kw*C%h9rUu@q=CZWOr4?(v! z3&Lc@x%a+>*~KU3fIM7$xOtRhgS4#h4k{c)4_fjad7?!<)2zzSO3q<9o(R`FveH?3 z;V~21^d3%XYtg(BC}uJp^l?@9=r7v4)?3L5QmGY@xNiC!Zox2vqIU7;v{5Hzufd6z zw@2g|YY~klO#f_pq1~P{d+gG=G}81%Lp}Z-gFu76m-8 z{TT(lo1BL!xGZ|l(bQA^V!zhv;lH>;8e8SJAx=7L#>yzI#vpz-wDI1M5b{?^DxFr$ zstT7>+~5mYnH;k>S>|Fnh+0ai6v_KmZlu#6R`W8OunO+2ck$U8scji~d7LA`yoJS+ zN=-BPw{kC9U@|yEaecQWzHVxKi7d22zxd?%Opzh2H+`#4&(OB{Y%MRQ_=Id2qIoK6 z_+%)J;eA*<+T}>B&G{)454nvxhvOh;v>zmn9zHou(W~?9cCKR!6C6?fSh(=GZ@aAx zgOmBf=F(UET1;(rw03oAu|tWEoA7$d6|x+igDV+S8nag=6!ow66>qDL7%+CPd=}Ml zrm40(k4fuRIW{XKz0 z#zLEC9Z4^$JYug#KmG;Zlra3|HbR#2PPrXQ=WWZbY5u_K#VdE@w0kr*RxCsUG2NvX zbWX{u9DM~O6EDA)K%*J4dpc|XsHXTPa|J_hEA{d*&$-o$rEJ?DPu=AnHe$^D2Eu>M zp!%x0|A(;m3~MqAyGE6P5l}~pprBYF^d?A)TLk~?UfrO^? zj-wO_(g{eDk=_}p(!c!#nCpAbIq!A;LY}twz1zCiTK7V$Qz2zl*t@*f{`x}Ms-LB~ zxgXpg&{261Oduld-%3mX)+yxl$eufYM@iVUT%iOWpWQ+ju}TaRKo$ zpn_vV*$q^KjcVjcN;I6#33|&n&c9&Nde=-(SYtu`GyUJuuZ#&Zm*)#u7(V9!D$Xd? zF~mI}{Xb7cjxg*VXRLbT?DNnk`spva0L9w5IkaIP23qwoD3vFexq!pvmh2#7zQ0w| zYv&L{h23Vt207Ne8}82uG8$NPr|4vf@zW6wZ*vBgUOtW~n6$4mX=rRv-peikKe!+> z+s5c}IG5E0av@hV6&(jep26bq^;edOCOVxpvl-#v03>98q1zE}kado$^)4~D$497v z3s;=osI!L6VbXYg)vHS(PV;wJ*a%{KwLDG`gl`P(sJK4yDuOWymMxr(v{t65AML** zq;H#q5nd=gk6Z#~1GgadLrC1?g`20ObhbW&(qN0_>ZtT8t6j7K1c=M1K0_*Zq}s)u z^SnU&?@U?z9j{B?IxC)s@iu+GBEhC=-Zf}9InGm$th&35aNq|RfIH?UnqLR3^CT^t zJ~rvMq#ptGODh4yj7*=4;WxQYAC~-4mWkG%tc|8TX>sBISBtN+Tj;QPUiXT-nRTdH zRG?lM__qEVp0X-I?7iI< zMk#(?6IYcn>%OhT!fA~C{O@wB3!1{u0aN+gl6w83LIp!x=Z;>3%UL5fp=|5XJ98`J z{8jIlY!70lMmCjMG6!Of*^S2w*^M)Ds&MwE@3KPgIbQWEPN|$ShO--AFI0(1R+TZ= zG5tBh-0RmHbCU?em!{%(aQ;%N6fJStlWDfOl8J!(*iM676Ny@PZPim3Iy3>buuAo? zsgC&T4o-xHl(L%&pg9@!}v(yyQMM( z)%L^adbb+ESwHjPrP3vBt!G(&ys0)})Y*9ku2<0hUHxUCFzEZ?Y#13#J9Z9X4mUqD zA6_>ZhJgj!U>pPYuTot@z02xee<9jrDPy?7Ke*NzbkGZkI{CxdBQf7cF0gaP)=iWf zs_Ez6@-4u^xy?HTKNU=fKH~XZs5lac808Ka^d{@B<2daJ;On7$Q4}>!juQneQU?vn zgt2@4&*wb>j=bU{Q?Y~)OE7EJqwCmsELtPGKB?5o-KyW94%7IQE07TOw&vLe=7LzH zLRprBOskH^8o(uTozlJb!`=vwr_6fD4VdWC`lGfDTn9XjpXm)7nO`sLNIWPQvYju{ zdjcN8J19!%&$9?$_D99z;^X`O2HZ4FuIah=jj~zWGK*wX^M8LSXFw^hX`!Ts{wIevv10vG*@_8 z%*ME(FQflOm$Z`xb zT;Ya8i=vQ9x_@lpy7KBZf;R{0(L4h(TB01{>9Si{|SZF3%4{)-YIF4wbVF4FJNFtGt8#I_)aB$M*SEQnlIgJdee%8l)*0fMc zjIF7v&{R5bi7V3L@S3EDjXT+cEQpn$A%BXLzli%sg@YWDq4;th-6DVB`kIH)k+?gR1xF5l+x_jSgC+$p=#??-M{j65a5rV5H40QKw|1b-wUE9j`Y;uI z0L@nk=y`aQkio+92>>5Quf%oE9+9UJWbZ>UARFi?MBIVDug9H^W&G)}B?4HCrnMuk zI}%CWQ)!Pf0MRc@2Wl1#K5T~U9W=kep5-|uAU=`5;e2!hXYV?Ix$-Do!=kl+g8|($ zd51HH%eKP=eANw5jR#~0kxoKHH8mVtadz0o*{9v4_6s`-f@EV=8&$Ep;no3UE=#iy z@7TO5I}K@z?~t15r4>(9w8!Pvc%DwdGpxWE>_#-lQ4=>C^HSn&$KJ*=2Aep(dN!TU z0QK|KO(=Gh!*I4QVx?}H;_ozEehu=KPvwWm73syma#LoJVG~|; zzvF}8ACDT55Bs&Hi!^+Y(sS`OEAwUagq2qwojlzTLEA13>M#Q%_*_%&;U6YvSwgsy z8>l1!$NQ-~CAnhT4eFZ~FOZavT%K6<9scgy`GtD&`tW&)jMp3k^pjttv2eaA_HNb% zS_dNcj;x|A^5ROuDYlnT`%?tdC`ERD%>UKTYuh27&$pcAVfKYV^kMQt$>|s_s1Zsv zpfC&L$Bt9;6k|>5_$8r&X!6^Xu?Brg?a0A#c4@r-mJKJ{YI^ZNtGk;s)N{uJ3U3c0 zMfa^(+?(WNJ<^})gNs8kDo$-Qu6+o#b_q(7;xxGn_OhGRp=Kqw>`TlhL1sk>;@jVFJicI|CTal+B+6*{G$k#@-_k6u{O{jYa;v2_Ehfw` z2P$~>Yx_>esX=cz840~zNFrg_y!A2khuCQO9A&M3HE9O${l=YMRYekDCMx%he-ai7 zi5cVtj^|plk+5@sSMyZ9ipoo)&nwKK|Ch!MHH|sBfv;|JeT$c7s!ap|FSx;%h?;ynzzzB8moUy?2Mjphv43|!bxC;HB>G)B?Bdv|# z)#Pu?kQdX4`6_mGJf_w6#mtRIqQKFev!w8GZ}V%CbRKi`H9Q;q+HO!suPVGt&EU9B z>vLUs=$8tE$aOJ?i^CsQD^Gdc;Q&k{l)AUK_n@3TDJD(@YahR0YV99Y;?Nt*Q)cly ztVZz{fEFk0!bN0B-3RV#!hW>Knbd{fJ}MW;OsA&VCt9nk{g)kZ6pzMOO-m+yBRl`) zQWD_xo*C4oz(~KOhQ7h(xR)l!PBlWVjyKrI3SMI(UM9|?(2bftNIZfD`4Dis%6+4* zN(yuRMSfHQx(Rl?top79J3y}y!WymAcai54{%>)5K}Gymkdd!gyD(Y%_ z()cycFO;Mxym1Yh7JeJa0HTDdqGJm@L^;3kJy9Eh*A zG$KtmNUA_UpCVn^wDRZCb<)8q8Q7&izWyfD1DzK4ODT5N!$jJ_MlsG#pNax>+9~pb z2{s@idv*POcg(5l?~ECa8+V=Z#a_LMri$d>pW^e@`|mM5!COe9^?4k$NgQ?%d7-DQ zc&xYCC_OL@lI$x!M5xj;Uul;j)0-Da~x4MyIc zVfZh^Hj(_*NMBPY3S6&6?teOU4?K^8w2a-vQhSsL!=ZLl=AbD~PN6PH zi={_Ef&hpS7SaHcV95}2?!xNfHU)?p!@z7&)~9-6)Z~N<07E{Qle`Hk{=QBEX{K#p zcLR~$tS-pIr|$n0CZ`ltINC6YxqpGwb6~8%*q$b8G)I;YG(R~Hu*EDlcrG^f(P{EM z0Ah8MAFZy!!i~r~_W>`vFbPzQq(%I4bH7OLp?q|ksrh)DTTuHG)K3W+bojF4rz!jJ zEMh>o$|k8S!V#Ld13dgl5J5#%lf7l&Ht-e{0&IvD$=anB2u_;@K8m7e`93^N0;T92 zEQ*+fHb4rUSt!Z1ew!y-YEL_+@eov$O$1{3K8jK!l`c9+O(6s+9sNh~jJE%6X zgQPX?Wi)jrc--_fXfBC{gN>qg@KR2|0LCD*AjmQQ<4Y2)00^MJPk2C_6($+KQb^|l zk#!aqhFB{Boq(Ch2{6?spv)HU1>WC)1!XcQn==5D5g&28`(y+d1U>|e`AL_7?J2<^ zdk8ljp0PaK8F+bf4442;15K{@(97u|KmoA^wRRu0z`Cakxw+RskPd$_G_hyrC30w| z*g;h|N`Pscsj(Mez+*@ZD+uL~Sq6nHii~5*)z;Cj&Y5acSUFBDEr&@4E&gKiMlZQUnMWiziM|PR2vy(c$;0)M2r^VI_NneBJ>1ow*wa*%g_-d#;`ZTZ@!bv{LD=5cQfl3)XNlw*1 zb>mY*%iNL@E46Ee0?!jQ_Kg^UBtQhvkSP27pLRH%_qa)#1}UGOTFE;N7rtpic9W4` z$8jDAy&mzaM?qOzTfkV90F0_A$>qw4MNPuAaldG;kRBz-Wm6%*z&1; z5J)RNt$@ilO3l^Es?Jshm9Jj}`m)WHW0j{{g{QXU)J0DqBpFJVeD4K_Ma)!eVwSQ6 zz~X*mPALQVs<@A6wQ?7;s>B*T&$zlfncSODs|nNCcy^TVW-pTVo&0(fuQ z=E@M3Gni{Tmmd=t*5TSkBRG!cQISPm^~TM=Vcl8ht~gsGdKqO=6L!go|<2bU`VQ8MK zI}4=wQHt##R}%Me1PCDGcFcw{atp!&W8jE7?q36Ua;wtqW!WzSULOy|a-26Lyci6s z*|{%uMksFY#{`_x3+&5ytp5CcYIH8Ub>nDf0f(lNUMIS6wtp^(s;ZUM3yi)t)l?;a2 zlk{HX?<3ey*Q&7Rco?+C>@p)E-AAuh`E25CD~->ZzjpZT_X^)YWuyD6^~IBlD;q}{ z-V6wpH6jc0g;2%ZnOf~*g)@aE0R0}qAbBu_zDUH)Ehr-z=O$W&4z|i-UH4PG?|m08 zX6ASGRPYGcEh@v$Uw`LWW6N|@?)_pkm=`EgL|)-~mJ$53u($38|`_eTS3XG(zF4ij{+mX6g)myNs5h8-{TiFz)70Cw4|(=RS` zZJXvt994V`g9hq&hxshqqu~u?=c|4TGWW(<%iNLcT4jknh=o$3rWTVrkA+1Mca54( z1|lX=FFp<<{Gv2KhoZwg6%UdNF*Kn??SMuD2Qw(h=DWQ1gs9xw<;rsVsw1}wp&6%IqhyHpDsat*B zw3wXG&zJWpgcu%8xnCt;RfRSz#pH^+pQ9U0-_nxy0}@9%_6n!G3UK%^&YD0N;h*mKJE)~^V9|+7SXZhA1vC3?1z3Pt(dG=n3j?ffmpV&{ zaA6IPIj~ZE98?Bc;Gnx|z~Z*LAUOLQn~ZU%E2xE03H*_@oa)$vG_B}_;zR@0ZzP=N z_=T-D4vhJL&T_~}L@)OCD}^dEq;4hup2PXm88{4C%)(f~BD;FL$db-qOG~Y!H8%Fu z5wOx$q957aUkfxO28tOzNS~7A^8c8bPNxX|ChkoPH58dS%jba>_h2!Mx9}FI$$odr zTjsp>@K>|7id=Oi9AoSl^0|7`=PP9pMOPez*c_ec5|-fT1I=B{LJk?{oGrvowlukxhM8LMA|W09b52VW=1Fc!mLK3R$vLkZ$kVh&$OG<>LwGJ4qu3FSKX!QP-9P@c?AXYsqt3u<2hP6FAF ztJA(hj*v+0U0_MQM85*7_y0SSa~bz%{1Dik8@96QGmiav#p!;Kg}F7MX*TWmC5b>e z7nG*Dl6uq}_hUMnGb~=ZsM-ViO^~?ip+OMfp2HOEt+A2Q)ZaAwaepQ>XHO4o0P((Y z@YO$p{YfW6zSFYuGi!gMkfc4aB1kJIb&d0nOA#-*QOD0XPWLb(!nB1U97$Z4iVHR1 z512HgX<<1-%0r4X5}}tl@2;-}9qx{8hPoBbaR>OBq+R1zJaB3+w-RpzhwFTAD2@m{OQ}Pnw_B%fGlt5O%HsT8+NXVJKd{T zjvW>AS^Q4uEcMyvcg1UFf1WHeDfyMR+z+yvycAvL>7?B!x?5}Zp1wHNYW#^b3fNQ+}5#Bf;X85Up*7N2LK`E zZ|X-u7*xT^1_5`vF2_Om!^pM3YTAj|HkMDMM-)UEX59U(Y-Ix!2W1o`29|`;yn@a2 z$vtRaY|DYWRa1U>b77#P)y3AZ45p&#Za*s3PN_U_H7jn7FDxO&Y(yQz8e?=za%y}vfYhPApmb%Sye4S^IW zDzG0&*y4OndQIk&*Ss_jqTX_lBP5!z;#7u!Pf0ka#DhaP(gZelhHQ}j2tgSfdqmn& z$-5olqRCqnOTt*B&=w z^7Ql{PXXwC@x?J<^0l3C3tE>iCY^x$;_T^2&lZO2jxULq?|PD_B(8RnM66)jI%KAL zmE~gam%pUHr92)F9ko({QG4MNqa$Q`X>up8@On09+*<}> zE0cQQ5@2?eyt?1j`vVZC>n2k@Z0D#zzFE?Cq|$wK{0=CIyGTzCKpAlbr^wDzlF|Sb^b$|$5tKOX zt{KdKXv9&*V11d>kA(1^{$b>6**+`K#V9LQzJwxAX(Qo%!2d$&lu*+dJ6ZXPx#Qk0it;grA^$GrxD6y%oHL*sNAsS*`yZ<~7v^*lXs006 zp1mISI_X{z3U>4={-ZGwe?lM0eVn`~P+qtsn&X2sL4vM7fjL$$%dZVDyTnqDbK zVKBjWE8%>$T$$FJXVdPv?&nC@g8aL^! zqduVddj9yto;>_EnDjKzd{CODnHDI71y}a9Y>qp{@VF%mmv3|&)wsHM6yK(>Wd^Sr;TeI|FoC}8Z*ym5n#t0 z0-f}nYJpdOIpn@$ylg#gM;{D8f@VmwNIJOzUm)w`!kz<{^e4-bYeSAjNx;i+6c}FQ z?q7ocqy|=W65x=Y2K)t_7c1?XIBC$i-A*AiMV8-23c4WAtl!~rN2nWU*RQeA@M4Ha z^aBV&J8+@4p8(L3b355tzUbe!yU+p$i7z`XK@o)Z{E`k#Wf7l_Kgd)^*t990<+p*P zB7B18Umc;VZ0GxMwg6B0s1wcOPy=j#^Aj<^IH?^-0OG=L!9qE8(wqjy!H62Qh6BgP ztSW%+EC%3r;|%@DZoXy?iYHtxdJ#*FslkrD*(u7)%j;_34G&771;!vyPrXY#?$l>b zwg7S<4xQCyJb@9Q7$h5bW>G9R1r}pb$ORB8CcSit66mmi^kTpa81aZNJOD<(yb=ES z40=Br`+(xc5i&~URlj8BrjX2FFdFOa4vtDg)(~*Q`_#UPL)TUh;b=>^Rz!~gVXLIV zu-k95e%xhI#{-$l1$b4F*-=PO0ez1I)VRhF`&{LUGx9_CqwC9%a7OlU&ofy_4AlC5 z#K)+vhjM}HLreqhO!35PI)vX6aAB9YaOdBrEL)JL2tZ{9K7RE4*-sT~-iM^;xm!z! z;PEK8TgZziCCi}j{Q3;)G=s}R*@9(YflC5P%14V&?#@&Smbh1}Lb5Xz(+%VR$9%Y3 z3E)3<0CeYLOX{S2>zO8LApcQH`vGr`z06gDDj&4X8%(!I2SPckHnK=9>sT8*pM?p~ zW)IiL;3DmP)_n6@fDaQ!%oztqRGq~UhpT!^KsGA@n9OwnzF?gd1~pTR(A=SSI9vT* zV=kZ=1NMHk#A}t9<(=W8&G9p7AF(dH5L%R5mSGLf^u=Kjt3{?daQpgLv1~~T$VdfL zX^ow~(`8&vPPpNKxoJ50cz;Nb4slV`)plAEN|lm^lPzgDIWr)3KIe~jJ6v)4pzwy@ z^Fy5agSt0mm`A+;ka7mK zIB1hUPE`2Lqz>#&a>@yX^A)RAq+5cGwc&mHdeLj`b%lAqVWaVufxaaGNJs`}mBA5! zactg`xG?VsitXYqviPv@ilct0EmvZH$DPmSk6b>R^+e0`cj>`&il?@S*R1ZYb>suE ztZl9UP2=NsKCokDw-u%!NsAXl!t_P`I) zrx0J!u3il6@(bd;@%4Rg9OUfJ8;n`I1Om*`PV^O?A$zx_z+Jr-Qu{ov`~>uO2;-oG z7X$PNDH286(P#pgGsrj;3I*OfN2bgurcQZD=!Z$uH(G?cZMr9E)Ps1pdkFzGhO~0J z;v&C@E#PbT?~|lT+v$EX4_>Lf$CY-U1j5*7OHpiLJMV(#+&fR0EZUFrhWHTeCjG2y^WW?2lQO+zz{*dWDF=NzXK{5W8Ay>ln>;dt~obh+V|`D#HO@ zMK+%OoN78vDXMi!h#@%t^>wb~w1AZt zK25{;M31{>Ksu4%hmO!2WkONnwsjWuU&fZ7#c`)W2LxL??o1^`<>4GNK&^iwm1q(3 zy31F}vdqmZZFiNQN*tW6zJFs1J&jt(dCk#+DRUQu(c0g_kOB`Vh zeCLEpE$96q+K2?_A_yAm<-gNl(GHgBhZJEp&$6Cjgddd#bc#`(4XE-S|9lAnmGg+x z8k)}HyU<{N_H8*Gt^ml#e6A!x@|IAV*30Iq!YY=t`%(sUtdKiW93`DG+FU(3hMu6I z_rR`!`>tA&q64rXGs;fWhZHd{?0EY@62Nk>I#(7}3KiAhT||1ecIwA`*WC;+-{Ll; zV%q0^>jg}0hlKjjx5saU>d6FJ`Xl9jbuqw<(&_Zkfra+F!}L0L9vsf~zp&&bq|32n zx`QQe(9nWA5B^M5YxQ5?YFr%j=vkZj7t2m`8gsg1eXXapX7x4Z_7;#*5tI%Lg?-#e zPGiJv%$?0PvC{C&l+^F_cRc%U%>S?IL$B-j?KQR}cF9>x>F#}K6E*A~sRmG7OM&~Dv_T!W|$7-4f!9gKA7^#*Q7xT5YzTxuCWSq&|G#_xvIf^~e$WoG-c3SGBg&pu$QCjzHIPrh{|yJ|7y@3`J|>EC!^YeFkJ4l z7P>10xPHuDH{4NRdcGt3JK_*PO}0$myiLkw2Eod#fl*S0u7u`wk-OV&Wj>!58q~gSL&=Y{138M;@5`{+-< zOv@6{x6$(Yc}6Sw@wFbHHgg$8adp*Fgx<67EMpOg9>$S+p6JPgpDBw}r3(QZyK3X+ zLh7|c8adbqtf&d#$_Zn8-f3Px+8qHONst$1P!MiCLn!6Qrr$5+{h?w-5F9tl`|Vp- zAiW%3Kpn+v4ISJoX4lDOh{|}E14CZ0dfDY##M1K~_>Pa0Yg`q|&KXnx-01A5h)MsO ze(TPo6ydn_(kc^*adxxypeAz#pVVG8_;%Ub5i#UN^9+~^Vly5L8LZ;^5aT6-TjniF zq1=(C6jS@tjj_`x5JO9wK&*}C(~rkk6cI~Vv|R)+LHJ>? zX^NNG=f?}Rq89j|rOZnlKbt3Ur6>#7KxQrm*@KtSo_Lq&YhgrnGGOID1zK#s{~^)* zlu2wixENIEU~mOQXV$jD*N^#yvKOxs8*nZrC_0q@YYks#pu`ELDHDw5B}W-j z=WE0-8hWK708zASme&+JAB?j|Fp2%jaJ}G?m?e!ggt~hH{6Ip!-O z+o2zJ)i9w(Nm}?pEXN$*+gL-ynEhr#f934)i5{H1ZiDpV>$0wIX9&W4IX+Db;kfzn z-tlQU+M@WxupQRSqZsJ-l)V? zM}6FMOnI4s1Qi|XyITKmilJn4)k3$jtl$O~t>LS{BV?06KeOi!ko7=?h1_a|A!ZIk z2g0e|+NuKQFvP~YSnml5)aB5-(jXmo(_#1BWP0op*@PgwK69(K)x4{%M9< zt48n}-A#gto_P3}*=f}Nw@_}myqq2NK&g%*`-`gZ?a%OG9I~>yx(ZO8hD*ouIZ*^I zi!u|jXM4n|3mzW!>Dnc0<{q_I)~Bpdu$R;txJ z@FFM~9u>8Stf_uo-$2wh{bOUNF_hlfnX7C*XLZ}rNDMR5%fNo0xQ9XPJnGrd)q%?Z zZw$jbI}6_*?gFp-dwSUEw3qktepmh0#Lmff@9WFt1PVPe>aqy%QG{3Vhv+fIx-Tn#^{mgqg>h(c8|vSVWdGMDUJOh>?J*#VMdKfzeu-v$TfijDZbQgirft23^r zEOu%fLytH}@J9)94=eM*d|+J2FRfxhZ84SzyH8I~+k)hM;nf*A{!SSoP_DpA&8x~i z*RCe*T0*Jh#~ZTfM|ARqxYsp}&oIRgU^#s$eE4|V$nq$trXM$(R>tQ7CKQEz8wM6* z`Fj`zpS#+g&f8q*bUl<#owjT~ztV3!a}X?h!H=Dj`>s}V+{74at{8{?Lh;;s>D|x<6Dxux8=)E@r#(IIkBYI^ z&P$IiGx4a-v1@{Lb!;Wwp$nxi%Hy4S)1VhZ5j32>%c3K+cBk-EGdT%M z1i+jbLN%2I2fOKcn}BVA*Pve8j@(2*lt` ztmQ7@iOu}WXw5=dQ20CNoQ5B6TUhhZ@Z0#(z3^~pqoUAsB~6SKUW!*|AodgDssLl8 zR+2wq*0a6cS%!Apb*;~r_^f9mjv0Y zlOzV<$bJ=~__0ViEi~L2$f_1TvX>-1d1)=J`7U)uBsDUBlX#`cR8s?&qhh7eGHbT~ z17xwTJxmii8P8pOAV$eQs(=F!Me5eXFjJAN%|iIcbQbn&J9K=lBu>a3m*KYu%j7Xi zA@z@W!<$?QWZhMC3V{f0OrgR43mGb^0Gu~P`o3T#4|9Rw&T?Djg3h2EAyLnn2W4&W zkbFC&z&k2!ahm$Mri$HFI*LI0haiYOTZ+yCZQ?BMQJ`VM*c;(q7{f@+ASja^sp)rH za+R*v z#I&J3L#AVX@TLs-k}dhC&-r`FsOk#-;Ns4Ap;E=^oV86ybfYa3-Y$vjQy~Aof8}CJa@06Wi5Nryt(E8 z(S2vU5F<|~CDn3CmAS+w0Z7rVGI2I`I6vgFxAjCS6C}s{u?*4@ZVf z@}3}u`N{(OR0}q)<`jv(>AvJeTsgi<=BZ8lrrYo~!+qCsbvJ{D;bOyQS_U}Zecq!B zBP#FyjAuE$wSJeX?dO+QMh^BU39`ai4v;a3WYAuwN{IZfp$q7iwCp4bG6_pPN5WDy zKOKI)W&7=NGDR<_0zybP7TnYYX>_GJrse`hZP#g&_t)~yn?fUXaAJ-2e+aECkb7|9!#xs-#WCe5<4)wi3c ziQ-VL^u@2GaS@<_s62?BL!ctA(waf&NzDf_Q~zTw4t0mo*Rq* z*hg<*{1CygkG0-|K1vZrgHL1zc4OT^>Q^YbV$c(J{<}yKx+sht>km)u!155vO!F0$=T>VuKV8Lk!4PMqc6x+&@+yBOZ=o~2cYzJWx zFSO)^HHN#^9~R_ub?d3T$wC$|l2kSyR{qL>G9sifWPtKNNLwpBv)CZAV5B+);u62x zC!c2!cx0@{tQ;G$s(P7p*=CVSA#@3tGoH#Dli*+$dW7tb-yl+|1Eu#?i}=G*8A5&` z47bMI225I_kh|v-!K;~%BA37pLKvKHKR~t})+yanZ;D1O|gQsw4?fn~xyPg_Fr@ki_WsxQWH*1Uumu1F3!+RRbT7Z@|HGbZH!r z8=fZtvoy@LE(m#o>?la)k3;06^)Gz4!0>Q+Q~fJ@a>eCSTHs6Dz0q-|?b zYk+hZ1Jy*-3rZmm7N&WPs5EnLZVgj}?}OAh{@kN8LZt5otrkrIg+5P$6(OCm7+3t; z;NNt|+d^W%4T~8C5StI5sco@Iwhn$YkP8$j5|StbZK)l1_IzIBAZ8Kd)8?}e_*sdN z@$3z(NgA^h*X3YXNLw$Z5J_!gpHI6(8o*E}!b$zJ+=Se?NNmG;DEIAK%|G}t)SBUA zyQYE6w9Qn7EV8uJ4mV(>u``@A|tc`Oni*rUL^hp(NESw(&Om|h5dae0P~}#Q0`;9 z`%NHHl0yTo?gm*DPXkk~+`;s6cKp2^;Jh}r*QAS2Vr-1l*%3kRj zX+(+kQwE%b6?Aq1Gis?7-p>KN#h#{s2Aq3$4rKg_S%9ryTRzb1*#cYQ;z4bYF-d@G z(k?oJ)OgujN@Y=AX8;*&18nD$Arj>^)?cLnXCH?1ZfpSZHU`Sv)t7^`O)>y)FAEVLQ@JU1 zP)*A4;;+?{va{2B{SO|WOlFgb$BE3<9E=lw=&YOuybctyo`x_FP6)`f1!=J&uN!}R z0gH*@f+w2%WwC>hU819&7#5;jdqOazHNGTcy#ypezPp@(tPX0N_w(4X&5>6glHkia zh^m{J&p#k4;LP2(DeKHO0&;mZ#8g&mo|YB(xq>UftSIH1*aJd-=NGaAlr$EKyIeM2DW6?gI`DttlYahU_rALDZ&qDsmFW+*t26$ZMVvmL6duh zz@xBE-3_@(K95qi7afYv;rH|Xo;GkUmDGCTC5*RE(rc$Fe}nA+?ZVB{3$gE;hGz%l zW%R_rHu6dJZi$|yvR|D=Bo1WK#?*MKI8J9T`44e${$zVQS)rmcwF_pn#eNxeL7AUA z#`-%+c??7dBg!wWa|6uwgyZi~^oDX@s;Adg zj@28Lb=&Ja{2ru2e9k-UwN`IZ&cuH0WRe&ntvJQxzfZ9Qgongx834aGX8W^%?^0(( zA5R_UTHMMJx8$+W7+ai}<~NV4w$!~6inH&DI+gxdeiYv=5b&3gNtuiM*iPO57 zuH%(s1!jsfM#hFkU%+e5lZZwl=5VufCz}_2QkT&nw0`uUI{Slu5(KsvxQaOa z2;6n93F$0TzGU79+1nn7Xu!91wtD;J7?h0uUWIJcZ12z62Y}jl`d`4&uz)9ZxX7}h zS_HQgSj}jKZ>8w`iKFH6__->2<46S>DW>=zmItBAki}$~+lsXMR2Xj8d;=saA%%3M zoM|Q8Hb(_eCX;)8q#0maQ&E3{Aybr;>{5->w{`$!%)Fao^BCBA`tbk~#E=&@0>?kbEGHZ@eR%;f_UXksHNKCmBEX8+sD z-8yD7TwK}Dc!&VrV!)le9iA#hbm<)iv_u|LIhJa`OZZ}|hg$-Mahn^t5jdzefd-$9 zg6nM8`_b7TNOJ+E&Jxo-a1>0P-Rap76^QI~g=P~-GP3`+a7wQ1f>7M}=bSw1;9^Dr z9qX{WkfbuXnfd*NlUVqCEHWDMTB+f~gSEffD=jQnH5A8tO)6Tn1o_10m$(80VtoE>HNe2*oTK01L@4tn0a&TK|NpzedqUgfTSi>fV2>i z_ALjyhy}KEc^;`II#W-8r*Jz2K)lhT^^x)I3}A|lt_H@71(O(1L;Tx^2Vbh&MXm5! zXXQS*TmgG83#5^e(a*GugiYj2lEimuLow)NntE{;N*N5i4|oTverS)3D}Tj?(IE(K z5OIlDXDUWif3q9pm|YG%wJhrs0Be3%he5pouI!@Sn{ufrRkfc-HOQn(&rv-D>LlLJROK_7%}!d zv~CPr&4NX*A^@M}ph$F(($-mp>RKiNh?Mu0nitNkS~%P500iCel2;v8&*17TY1RSqT>DRbQ=hc&8Eh7h_<@t1i3nAD|MIGx-o+)e(NAK8`dxrFybVo(dj1 z=Ad*0N_z<>^F$>QmX0N$C5|0BrV!FdtUkXJ7&s<;$B@DKgt`!3`2<3F* zbhD)(uZE1hX( zx`sz|FFfqgqnZ+&#<3AL7e%&IC`r%&`&8JQxlM5g*kUY@f z`*6ju^y9r$2L1jA*f`VDcF+=Ko+%l-Tlw^`ch$xi~bq`mm3c#)s~#R%*1h2!Uy zjppcXB=eZ3ZTMxn`TFGFt9N#NMo5xsIn3gZ`T2N;=xF2=O?Zz86&V5eHsY1d1v0 zi^4qJqMrI^j;XwVXohK?bfqrc$%H2%@*qp+odxYieZVe8Hel!S)l9v1%Jw^1K|+r#DUq z@6pSad;jW^Jo&WjV=ESofN*-Vc=8PD1Xtjmjm^tzTVEcnX>pr(-fr_)G&@VS#BtJ1G~qCiUO%`;LE)b9H~FnJhmP z(zZbSzeXjNn9YgMYMbAE8}|n&k8yAv&SW0Mh#B9_qpssHV@x@j3{R0CRjeDoq+m~FV_bPhhX$aKy-v3nRgXqq z{(tkqjc5|d7#PF&ZMj(pP)%mL`KJ8?pjBw;yz3n_Z9fu`ZFW4;3yB({CzFEB@wcIT zTigr!s4l*D^PN|y=(;(G_sJ|+7$vJ*#AO4P zG6I|-DCKCKJ)mX>ZaGKGvea?^de1LURBO?4IsRP5-YDJv$4|L$cXP=R+OCbHJ71$7 zO^7~XooiOwP*I&sD-p+}@CX-if!?dZx@Qw4RB?5xF1KEV>?Wly;HBjE~B z&+>D}wWSA1w~Zj&2Q-kakvw6FU-;3O0r)VFs24%0VPc`m>R9N1;+>Yf8oW&BLLb^gyaxJ+K-WNFJQDX&0?@x&t(Q` zM=iTOwn2D5CL8kKCe6j=zS~_gCleM#uY;;TX!t*`U}gpr4*S<_b*gBsxLC8m&ZAW} z(C8XTUPhV;R*swH$4XB`%K#Uz#zqDx7WX*JqIr#OXX-1K=YaZX{`h6YMSC2LH@diw zXI8``%c#ODIc`-?FcwiS+jYs<<;4wzIAuv+-DDOQ!FaWl4Hk>}B6e zn6VTpOO{Zw7L`3)D!ZbDWbJ$2W9a#PzQ6DD`PXaAyqEhv_c_XS7n(P-9z6OZ#cI;Ted2A(yL?cKHKS6>Sz4a~^Yjig&&*aZs|8 zoW%N~#{n$t#(R&1=jqXC-~~r3$$IS>&XtX)9YZ(P3=zB}khhXEncXaDnU+Qzvy+H< z%EX(PbNH0HXQGTzRPn+sQ`-IOeezATZq(;xyxe*Bh%mMdXL=7etM%^Tm!zimZ!~$W znCAeFZ+rBlj;n6PtoV3F)?i^!Vqvo^MkgVAeIGMXGhg;;b>pciR61EV!Fm?d0#k|l zs(k6M{q~GianLjA8GkHayMowDbMF<`7g7;5Niu%PI)$5kiiy58m6q1mWqg$T7F|FM zYr`Hr(o4KX#SgQ1%GGmJ^2@EGF`Ma@so4_tQ~BnbZ&)wm_u_JpUs{|rNXpXg_0_TB zqw0&GS7J)dT5l+=u!$Kogq_*TriKt|;ID3El(VOU zq_fAXC1m>E1vH(8_BDr`Gl`DSiAr*fe#h*$UUl6SqptF%cfRq}!!EKF61#6s!5Hj# z`k@fAn-Re=h>WG&&bh0FC5ZJAWs6IL+6gImG0Up>5+&|me0P62Q8#x(=IyMkyQy*j9ttI>ifZCXlqD*7#9M)qcp@o55%se# zxEl7blkm~8nQY3fGq|PI#u`4R9h>2Ghsd8Dwz>>bmd68S{)2m?>oSV#0G_Ed@6iBO z?mln63%u*kfT@9hZNWd6=H-+I*=xFgO1#?o8kH#Bliia?vze~OfcLF~@FB9)o1+oq*;tY<4k0(&krAJE zLEhB^`SS$J1xhDsDiGPF91@rQ+&UX4O7_rKk&jv1?58F7Z=s$Du-Pd`3K0g>u(6Z8 zjV#JM&hAnExC40@mlc;|p0?d*NVR{)Oa0%UhU+VIQ-C#P?-+}ctJ%aZm5;M8-Mw%C zUP6HBp}R*CvF^Kt)26x-esZFJ=A5{5!1FQnF|CFkJ+}UsUaxhyPU|8n=$-rJ9{WCQ z|C&%e{N&6CKWxs|^fz+#*iqygN6e@X8SQTqud@E41u2UW3x4UiICHn4)6Xwoq{`*t*elDUW>QhV9&sJw^Nn~`(bjK*Cf%LfPZ2wY(l;=AQ$2Y3BL`wMs&*&*BTAwi~ zmXEUHMT&f%3~NNMVl+F3mUcz~Bbykg`ULly?>(cxOI#IKHO^HyFx#}Uya_x>DZ4O0 zi1IR_3n+%}xEt?=_U`6?-6+HmcVa=VZ4dHRf3gT^VoQ5)0FOvP*R*)$$gW-DF}KQ_ z_OI`oK#KCJBQ>6=C8l>zNnSmode}%QmeS1Qy+T{4c%P^-?4++&8YnbbH)=}Rxk=C2 z-vq8F+*uryjb|w?*pi`@ol34Ah1Do7d@C&ps!50T7W*ics*MY_&mQN%9h4>Y%?ExVP@AaWqks*bWP9lmH8 zOxFjEG&|@FjKHXBS;SsiuW1<|W*KM;CpmULI{kK`MBWOz;&xyal@%l}8T5meQ23mQ z{${+yHgWc=eiu>et7x2({0y_tt-Y0&8 zuTT@{#)z?}tmzB7EPZ0yP0SDJXn`a}@|PP6BcSxw=5cvoD0=_cK%sMQXZXtConta0 zj~=U`W-bGW+zcux+fFP0l@G)ppd#N6UG_Y2#H{BEbnzyYX7@YnN8}3QpwPB+xqo#r z+p+s8iTl3I6wC+XZhStDQl_Omv4Pn~Hg<@q86`)h)PLE)0d>5BZQ0T5@kpm@hmU?_ zcnD4xkuQ#b7OALhN4eQq(&;RwX+)!coA3COW|>}sw^mT))vBv2l(F9+_jUp8;Pr;O ziACp*oS2ADhcEd+6I6HfZAvppusAPgBVIEw2*B3z7+C8x4Xpl>MGSrt5CTGG`PJyojRaK=xG}-q}c3?C10ltE~bq7bg{9#6MN|zWJ8fs}$^c3b7{Z=?v0@!pj zIJ>#{%bfpkG+hhb<^AiMV_i766UCATI3dwB*P_}myp#~B*G04|UJDMJAy1(^e)Tgz z46u70js9hyBN zneE*EV2zR#&aK!}iIM@;>K>5*b7DJKxDiK(>+%N)@3gZck)hbRZRwra_(KJ7gGta? zlQjG#RinJ*`2?5K0fV?Dk2qdMVEL3g4?{eV>DZ?W>=F=K!hx; z!dif=pc#t#WNvEy>*dYhZ0o%IQNlUg5>P-Ua?3&C+pn^RX!;fT6}$Mv^r>7^g3-B~ zd9LC4`ZQ~ahi4vEa|0n|fjp3AO08!#wMo`0=$e|1@m~4iwCL+9dzYB^<(-V8jYWm` z0B_|Ux-;2t$8AP#A~JWO>5k_rw0La13E~bzDTW>Al~&IT4*tak(n$} z-FnQ|#afY(mIb)xA|C(c774?s{n33H$9Z>dzM6mQLCxR35w9O4cbn&vkEQOry7wo& z2*z@__dz~EoYXQ222>|ivoAc$h*A0A6elIE&3lN?H~R^~QgGNyp8$cpHh9~f&qE<- zOoY}!t^HcJwDYuLvxMeg0H3WuKs(BOHgV{mJhb*FhL=^aju3K9;o$qPEXpjWiNU`n zTGDS%TJaA9U8wk2SCP4f$*W7e^UM<_5tfN}Y65B!_|Kp!%)L7SOPkz}Os$-ONY(o! zEBiwce*i&ItHly}RwC#xIZj=9#j0f^=?%qzJ@&>yXWGQQ!`LJN@vYN*mI6m=!h#A+ zSCvWAokZk_=WvQ(tmu3rincvj-u*mVj7~%pU7Je~aaFl}ezA0_BF&p{t?$mxsMJWo9L6IBTjV6iNzg1?>UHn3FIx|Ds=O#*+-?$Dy~bF2{VC&&ZA_Zp^&vvGHP%S5 z!2VqxURK*k)#ZeI<4uIsra7v0y(Ibw>79H~>I0pF+H%3E-_7KituMYufsmMYJ8FQ1fFbNUtskBc!+jJqc)HyC~34P+|&0>j-K@88n&Zf63 zHS&<2DI@tnk*_q_S3JhTNlQ8*dMYzoUR#)6I!LaIu%zR5;_3T4Wi4~kjF^63p!IpS z%CUzbqq92q!RV}~!*@CZ@&>0K)U`c2ExA;2Be^DIXb^@ZCIq#*MWUBh$M^d?V;4)x zOs15Nr+o!2M((S*8?5?ABmIcoaYw*jN{xn`2dV zb#7a>&~A*6kObC9G1$~C@+DX0ZE=RliNQiRJZ;L7 zX*GUyNpCzn@-i{_1Q}(5qKP`{HToOC#RKEj${>$Wt>S|GlM(tKK1Cz0;;ifFIGBw>J+}4C;O?=sPW)IDmK}eU81HYye%*`bk@2Au z)itC8tpJroYT==*xM2FAn8)MwDl_K@`T&|xgr-Fn#}ESHJc zedeH6mAtJLiT2i|G0b*fe8>K@>-cd{0~9&;7QZF+%}iu|E!~p*WBR+X*UC|hBtn4( zsn#Fim=;&A@}=%qyN@MjUm6aNXd?EP6@@Y4%hFp9c0{-4sB}oZ6D|J0|DzZl4z{#1 zjUncG&X?({rVsBK5`xTJ?F{mL5l@4gjqKAI+~1{S zf)Ze(3|`9i$hkT6eQ$h7a@xaUAemM_uuGYS72H#1fc1;IY?uPHK9{(hQ;*^!Pene) zv4)sq3Rn2D9ZFh1FMDfYnF-<-Ff2j2v~L*|znuvoZx2ETiN=+UE*{a;9XHojy2vb0 z-#CXBPZv_`Fo`6%w|JF;JS)7Fwad&^*{95_3qKUiAB5VT52l*?BBJwSgvksQm96GQ zk`JQdf^*#WZILScy*w&VPCltV`qSGRwv9B{xFkS%zqTp@5a;*J9>2GgJ>MOE1WByR zb)OI~8Jk3D4~(+l5$dy^!5f#UR~5ZRde)k0>Y_p`2F(2Gw{~U=kj-0MgC3b!`E^YF z7?EX{Fj*nPGa1!>u5~Tz7<<$&r<0{uixltKBx-0q$u^4I$->5R7U-p?Q^b+<)XPML zey=o15W5XRsJZpIjiuQg8b?(zpKXn$8Eg8gUHdRcx=6_HD9 zQh@F9aeIA!MS7W$JRF%1T7B^D_V7ysJ$wzlKhytffiT&wTgNDb&kO|)}aERXmw zL!iz*rP=rDo@aiKLRisP$P$>``Qd^3Z3Vv^ml&_zgs4YG#IxegP!S}gFtIZWOn-UC zDQWki;fVCJ-ZM|Eu{U1}7Pq6U*#x8e&BeVjn&S$QXF)KLr>x6tva&Rv?;c!LU|kBL zio+HyZaH^`W!W7sDUNCzrogHF%l6qpx5?L|Z|9wkI8F+cz0Q}9`F>B6k6dkOB~3EV zEif=A(B@~75uK5U3TKU6W%OCi z^EJx&iN2{fZ)jv5yzJIu$o0@Ba_V*>k8Nh)QPM49O4vSGu~E_Kt|CR%oA0IbZ<+2D zm%jL=M%Us9y~Lh~;YCItzjx+0~Ynt;IpvtQw~L>m#JJ`L1Em%cS%T6b&{(X>Z|td zi@F0kx?0z>?tBZ(M$&hD5|$gy{R76&G-jx1I|%)3tRU~6&)GG6f_Yrp%4?xE+p`&p z^ViEzmw9@n;YG6$(HyD-@R`bpLPPvSDs4*Iwdm^63r2JATgZiHk4`qNFP55VCDir4Pxuc^N%Ye^5 zx46sQuR}YEZdcpWOE%FS9=e57a|(bW&_R^iDHx8716YIS$DtL&P%;WTarO3bo~f9r zp|s>a_84;J{c?x9=GGIy+AWd}OxCzCY6%nEHCt=_E_fX?@%{vE;kZLAI{D}n?b?!Z zuSwGr{Nss8gLoW!HOc|!mxAcO!!Lrv)Gynhy_ike=pIcY*}T%}uRa*8V`lNU{+}oh zf?);Vu!8@@ePh|Ohcl-C!QSyIguHF}b4SA0+miNg3l&>~LZD7{qGW`-*8C8o%(4TZ_~WIFf)1Cw9vt5bbk;# zyMjqgFxVa4SbS^DRKZrpRqhpqpT0QPPuAPFPo9Ht!))slcJ_l0GPEA5)u(^e`XHpH z=UI*_Ha9(+cV+C}L&jLI2|O{y>Wrlu_Nf8plZ9Rm?i6jxLwNv%W=_&N`U&;uFkkM@ zcAD}HQm2=zS~Zrhi^(^L!x*^XvP7+W+)k0 z-kkD$J6~Kp_M=cbXT;DeWJdc1`^c>eL*_5J-PE{gg74zJ&t(_Az`mSdxFe!0sT+I- z-St|fEB~w@`k|047i~4mr+MNj-2OF>edK6urSi*jK5tP^tZ4`Tq&P{vhYy||>L9w6 zFG+hG&fjxF)iWhp^Ag}kd)b-q6l5D#4GnVICo{+AoPIlP4zP{PF2Z;uHYwwHDick| z42vsrWA&6TlwAay<@LNX<0eAGGM2U$ z7Tqm4gJkT6Ayg#Q)V!c+gYs~h+E}}MPa0|bsPcADKSem=a&Y%L-l|Q@bi=W!f1L|J zabDEE!*qy}(x~pGA~o^`fQ=>f#?)f3W#p@YirHc4G{e_wR334<9ZMw zr0~j9e}3qZlqPaL6u7sD$|UIc)h=O){LNTv4fo5f7Hqt`o;zqkrioBsV(#9KIE(ya z7P$OSnw6sY79;C)7u74&F6vK}bF*tFZ(Tg)`Nd)W9>l@ap&P+|-g3JsX`33W7Nv!$ zeuS3sZvk|gx)5Ah-@~3T;{IR6iEsaavD6k?(1!I&As;vbe_tt5)xyaot|u(=(J?U^)^|xN;9#&x00v z<{KRsQ-0MN!rdoJReuWdGirOMw|j^xcQzr7C20F(OwQ{9==Pkw{(p2wq6J^@yzltg zAVHCdb!hWCSJ}793{3o+?|`CqX|*r2a3U)ufa?$yQvdza)_Z+9M@LXug@ao(yC{GB zacu2TANh*RZ&`afY!G9CoHV@smNn1bx0H|*@P%#-9U8xfC z-%?D0>g|Y1&>Y4fAdZe4+58W@0BEH+%z@`&y>Q&Mz1@7F8EPz~A_?_n~zk2Qd*(LK>2zm`(?% zBp7G@l+m;iO`6o!uz zIDG01GV&17B}aQ4`V#M|y=BN|{pIoAWm*eB!hE|0ULmW<@D1%3CTWXuuOi1QNFmk@ z+N7B;=1#dqP4ztf=RaN<Qp~*AvO6An7g-s-%=NxHT)b;R5m}G{ zH>hiy{>olXgajI3dJtY{_xXb6CIENr(n)O6G5MwEjN@_uLH_j~86^{XK)Hv!dEzV1 zbpMQ~tuIAIyY`ABDY|4eUBO{ToBhb5A;zojPb0eyRN8@0jvbhm%YrLgs9y7%j(h&A zR|*gXUmOS$MB1KTai0OT(+BJE;I!6K;Y3Sp$hk^V=pFa<$azf_M}5{AN@1Z0fkJf` z@=~>gJim4}SwE?JlJm8*+Rk_e`Wcx{^EBJq%vNwFA%Y?hp$ z;#a0@;+dH!?R2uWKM)qsuDelxe@l(}TQ`(KDH3y4QPx16c8AQ0fGyi^smc5NI&Vh1 z0v^5M>bDt-Mjt6l;n)rH1IT4_(bWhNIgVJK-w`h& zOq}W~m2VNoO+Z6HhFj6AFkeSEySN>E#)ybx=wO}cYrh~TcoGr+dJ;&70Un!z{yTx;XOgZMz)PR%mNV0B&nSW}L<6TD(Z>4eHSOYcSw zsr$$LD^HhPIBR(lJ1A20{z~4C8tH)?GrFC7Hv*sy^&6gxQsRPYH+>Bs5pdky?$z{% z;-RRxdGFoxD>J%ZJ9E`}cz(T*8`0cz#6k_xXV{ORyxpA z>oEQJp3nK43BMkX(NP~_dCb5pNjIID; zHJRrl>}?suQSU1vek6~cddP30O`|gM(yr!=t)00O%&-U_Axh|%*SFd=#fy_o@-5jb zc3f6$l+U+10%0uTx_J$p(#wGJBekCHZv~I`D6k%^g2mqKv4Yhw) zqXzXn)!5D2zxL2>I>hYaH#3!#U|M%IXT4odF{kRrL0{Hb`k&jH*$sT zn*!uC^ad*Zc5vX`N1DVT0}V~Suzx1Z_r1qDGjkKIFm-CC@u!!Ak;r$oYEm1qz20_} z>tC)xVax!*YhI~k%_roCPt3qWrLF59P)!AJ6uo+FWcL`&fxV1tKFn}ni(VIUo)L91 zzQD2Z)Q)bWaK)ze*Pf#7A*Q0NgrjG#gJZJax03tb6X>>FGgT z+BPB2i$(WEqtLDvdbDg`Rt@2L=DY7583X(4){7t2G|G~Msw(S5C%@JT}E1=v9w8&P+M?N?yNM+lJA*N;dd zmGYg_ay!Wp;TH(W2Hdh~2V!X$D;^S^d&)kEU`Cjz9d$4P27C)#SVTnkz~}g90mR|W zUpS~;~3)uu@2X-*o>7BWzXJ$J>gK0~{gs9d`$#Ea`oFKH7 zx{o%vo{)$5!~3uJgQCVC?2CUIn2^PcN>EkdBxMOn@8M(zATux#%#QmtRoHt?i-@)( zO$p}`Wt>gV?+F-@P8#at#lfEaBVT_?Ub2gaH705N5(pd1N13f;XNk- ztL`jst7aHi0!-5CIS-s1JMe}O*|F%>^z85!(!T)l^~0cgH(wkQQSP$Q7`@4I3&}q3 za@I+=_T@ft_U*QL8>fOOHHj$tfs)*gycGte#AH*PD-h1w^)%PCNFvwlnG7iDo(;@{7TXta@ev3kMFTom7%J9$ zzHFR))0)xKcmK>TX(qj-BGh9FIIUC{4T8tBv1h zBta9JT1-y}LEbXL_|cRm-}C*cT>wQz_nUdZ;X}EkUju=q4l#6C{f0n&qNat9tpJ>P zX9x78FyEIjXXgZVDO5$wb4<>ks;f_F$$|#24_z@MEJAx z9bAp$IdMQZaV|^8tq|gbCvL?DCkm~s(~ArrQjeTmmHsB+{$+X7bcm&Y;lP7=Nw{8b zlF}77a%^Y+r2%KgW6Ku`z{fNZl4m8!3`80W=UWn3a@QfQ3=#S4HH0CpjXfN|MXg(j zv!CruA)oVPxu7`iIIAsk#*(2$P9uyqTB9BIp379lyrmg_0@{d@2t?1lh)AuYr{+Nl zoD?f+)>Bvoz`q>>IZmSLoBLqAYxE%L$a+NMHC$)q;|zGZkVEIkANuy3(4VdsAZMrx zb@L^Ec1?Ai zsm!2snW9%y<+364S!27(;X0duv2pfDZrUh#w}&RkT}&GXfVTqk491ZsK{?YS7sNK4 zqoIq@j;dVG768KlelHk$#NNMvU>K-i)w`i^%B6gh+s?Q@K07QrE8xV@csWe-^{qRa z9-S^YUuM}`>^wOp_=vE{4-#NIAU+f+&RSeu7)H2c%uRV$_gsAY>@aKU-;l(_y0$#n z+g{b#!glmv(JBc2`Ta6ZBWywkxLjG^Ld4NCqDva3Ygq$wya=sV(&&jLcU=dA%b|YJsNhSuX8xYF3*od>02c z%wxqlOn=%v*Y?5m4e~7#Bq`597;PXh2*+Vg_$CSAWzi7R_P&et+{pMt#$(s635+%; z*_%Fdv2Z)q&8?vfiIUdF9{9hY6T#BJeS*&A~Xp?#HK>0j#GA9U_pE zI6?7J9>TZOs}JW+PkAKH$ya=N`b)zSdE=DlHPoSQX;I}C1lJJ?+9Z*K^1WgZuCD70 z^A<*c`y*+fJQqFQnQQiOdKMwX0CQpB<$>{=OA zr(B0aA^W&wmQDm~$B1uJaeCt63w7?JB@7!`pU@F%-1=xA3L^XfS{C>ke>{AR7()&e z=_9p!tfWM83vq9Q-o}f+@|b9Eto#L*NFPk`M$h1y=HkcKDhXse5h#>jAZFG?)x#W@ zfDiOisB|Gq%a3L(x3j3b@peiBaEe@^|BXY^_jtq$eHpr8$dm_(b9JHO2wAnCI-Ng_t(o}P@d>i;H zD2^Zlh7be=EcHFl2M-|h-M-6d$>{?oeKVq6N@MGiG#H|rMBUi)c+Ec$tB2qVBLqZ! zZm)r^{{ev#Ltt^0eYa0e8-cJVff!D-O!**0ZBg$^@aIWDOx_C-ZX+b)2&yC`@?LJ# ztTX7Z-gLu?!Fc3$=oGxoFZ8_W!S->5l7lSL2o53-Y_}%JO%nFJ1k=Ma2*siR9OPoR z8k^6EzWFo`;tFaFv_{v3FwrTu`q20(8;Op4$uALoyZ0Ts$lHavw+^bdGYTbp;p$l%kMAB!;h%I zww=rmUaPg5!RPpJ(|T%f;1p?&8>~J?U}|k)esh9k=3z%aD0zRG?2=qhp?G&uw=35@ zj}bb`z7YIDj=rdMF_5d{P9p60u#$a8&!~Pn5BlHjX71SWtB{I3l>hbNoWT7n|6R^$ zi3IR+7PaXso<>32PHO6Fp+ru{m3ltn$x~;yCe_J81%r(D8^LrJ*c(W~z^!e9C!fp< zgfDO?%B+E|ltNdVs%(jz)sWzzYv+X<-u1DapUieU>*CXK=w(O%*St~j_1hH1BoNal zA)MQ95NR<@LVwUur~W+NwpLeBikVH+KoGF~j*ePtC%qhJ%)`s2BlMqUPAe^++fa^PxE=0j`E9yjNW-v*ouu9xclGH3K}JMjfyUtq$lAaA81L# zjNDace4_Zhh}iLZC)o0yFKp5Dx>O16$lgUfqzcl9?)yGX`2hAYO#m&Kx<-a^Nef^3 z0&i^Scq?_{4IG$S)LV$TwmpJR6^b3+^qjf3wa2F9G@5}8>jF=tEb`DiaGn$4s>?I3 z{x?>sTEx_FD4kI9TqjP9?Pz{%66c`YAd@sqz>tJNszj~O^@_Ck>jI8CGO_y zYi@Lq1gR-MRwclE9}T_0wNUJbUD*#luYvQ@g@lBSs4q?hvc->im^0F7BOmSI&TgKD zjrdk>aA&!I6(11f9qq!TJB5a#4eJ3%H{9}>cO$*Kdx4rNeBKSCs8r?QvR)sLQJF!& zq}gxXZ2-#`r|uI6+I<`3aM$0`y92(lOOi4CZ|_^1n(9ZcMoXyBX=tSKabl)_2CBB{ z=>xdWh0#HF<;OXMR!u4KAtg=RV|kfxrp)Vy;+H^yrl)Vj_AOWd%=%x+;YFsqfw+BO zzj@KB{3}+?QC8W8u>aC^r$AMlMl%FPBBT5elce9GM*(o_lK`B^IxH&$dDxqwvA@wv zgCaHnkma{(3p=f_-X>1tam5#N|Ined+m~S+H5mxxURecp%@SZOgh0k%1MR_+-KZRA0s%*V^;%-o5fi5@8#3=i3_-3k>u?MAi_RQ+`c z;OxwbG~2W|l@qeH*p7cbvi}$-9%gp(QGC-x~*CP&4)ya@3%PRm`$s z3pZ^VYE49E_B@$p#3+)KUxMJD`f{IXB>_ErjyHRJ=J?1Qs6IaWQVYw7L4c?K>m4o7 zX{kCMQkJ%1b0VFjgKwkC5cV-7WV8bm`WpA!d<%TXgM+XC7RtPzsnNNb2X`xuk)|Qb zqiC;{J!vCTWq13Ml-Mv5V z`Za@1oVDIstN|!EuV>JbA_VRnjk8J~P2W?|k zidRqLy}av9Q(up+z;Lyko2rqRX0Q&|yK_Lfi3$-498Vlt(3Ci?@OLLLQxRomWnGUg z0lYK~LY>=aBQ4zN#cLTTcpW9y$53Uv!t=`)hq(@ZvLWPs8BUamFp~D2Nic=i8P2Pw z`^4ezoefg~)Yr=X=B0J8Ly6yRNu+2&kl?S?wqIXc&^o1lF7Zlof|ekBPWX6=(@6oK z6iR=T-q6y+^mvMeD)>75ksoA`*5th(iVMESwYACV8(A_X622CzskQy@*B+!sqeS1r z1VTH6dm~m-RUzCeX);{WY9o>`g|Dq`hWSSq2n-OgahKZ?+`8eKUf+uS*LyOiMl9nM zI?3Nz@VT^uSbwJ9;Ec{Dg7grm#n4(yTzHyi?`01@7#G=)<;F_(hi^NzW#coOXiC?v z4gHZDr@e_Y=5_De3=u65^Dn}#MmLLnI5E{PmF*JG?YkY5t2UFcr@eev`Z$seE{KaWa;I>3`p z`>b_@DKJqrvbO!hnCzQ%E&x=;6idZ|S`M>#I1^7u%%R@NM$)%#G`5)nsUaeG>}*=n zYi3nV<8Wn@TdJp!J}LCr8hnBBeD=0YtfjXjq8f>ZW@rE5qITSacRSqA{8wnMOomr? zcqbiU4}jLqh^_}{(GzJe_CwNR2M0146T*Q>x$lRAgvTcJU;9INU9Jc|5*lDMUS+&vv#`vK(cfx zQSQ?w5fad`Fa=o`wxr8fBWT|EVAE*Ed=ODioKwe}) z_IG{7wEQPB_!!}X+Cf9)$T$+${$Ke=ofM;hmgXcd**I>earJdM4!ug|gIW|SJZO{4 zBH-#^U6|zd1DDTO`XVl+SD?ymYD$oFF1v3ET?!;~ZUVrS7!KHT4XvE?G9zTH5_gN9Y`+ascns!!y|n-+o#{nVgHRC= zJDa75?F~whjN3uEUwjP6>s`7aR#JO=c>e5vyp1QFw?0E?O_`N4X`LlPsN$g;qP!N` zXoh%^6a&2o!<>C2>#-a(DP;lrvIl11tn2B@FPjiX+O&x)&9nHsKeTzJ5y>gwf%Z)Ylg{IQDu@0@qc`UHbCu{h0B*1DMJ!8$xv+roR}X{w zR{Q*B_sJ&G?gzv*VXD#2R=3tKwIO=XT>6_}_~pLFJlpWz>YF?CfQ4|r;y#Sj{!nhQG=#*R3W3- zYes7!(Qzjg7cdKu?#Iq3w@K*fh|i3^qwqWs*I?A|!DLs|7BcHb&@kkq0(xq@tysj& z9>p_68oB60n9+X}`mAo`?+Wo|+* zCdyNA)a>n*7TJwGADFoI>Vp@vbJA8BnDGnzP!kYAd5Uy@y?zBh%D#0gp20s3aKt?j zHujk%olwgJq19&tM!BD$Fp@^bl|l{YF%l?O(|&wAw+&^?2NIY1GzV24}{o-jVyJjj&nr8X;@hab3z$$O*p@v#w;$0gaTwI%Z zk5qzMY;aV)nAGa!i3S^rw0qSo+XPOhN@YDpJ0NX`u)P=N%YjnMiGU*U;3eA5Ny5#d z(%WZ?df-RxAWocG@&pw^xwYe8jcY-l+ncRF(FEOW#ul88sO?fy2U0f|R5OU&s7x=r zgyjLTZVV->UsU4$4I>>-U+vL5T<&Nwc7Qgl?U-eYI*SG}XYw>mSKU&70pNiTJvgkr z4*-4GV5Jss+%8^|YduvCrQp->rcGKfL-Wo7X8}j^nXVdy;C+eco{IpB-#$DUp|H4N zbm{T=kbm^-nx$q}4TT><65pM&0KFNeoQH0?@*vr(e)l)ugvJaF$8iuv0*y;#;hV_Q zF~px$?JQu~b3>s%7OkHVMW*KLH;@{5nYmB$&w+-mXAup~;)VIooVwm7!x21^@4w=d zvb)>m9vtb<5Igf`$Zx;hK}8YNPhkAn#lLjL6NU5)m{DBnfFn9vA^KiGLMCrcb-UQ@ECv(YRm6O_ zvV#E}yQ;FYzaiPVGUoD!K52{S$V!uT9WO%2&~=-L#E`@9tHg5Dd++Pq9U?-t{_G0y zt|TLlD11)~5t9J5F6Qmv+)Eg9$@5qi?5TB`$Bz11UjDqbT(q1gAky#NzCRFB&S5m( zmlP?Nqtvr8w>zqWO}0zZD>XhvkWVlTSKves^W-~!k-i6zR10Dh{4oUTK@s=BBV>ZB zRs`qqNsy~HYebYTpJ1v zt3K`cz5N$!m`_@7h2Nr=?;`v#`>L*It?Oi{mqyGj>U?U&o0cBgW}{XrDP%0p?nBXwdPhcu^xmSkvdD?3hsmzx5Tq=9ZgJoGb<@gqQSbfRJ#@pssnQX z%V|Tr*m~jRPKM(b7m*=_ZIK#A!I6nP3H%AET0Hce81}JOrMQx(Nh%GXo6?b~)@Ig= z6p0k%@tcjPxWx~;SGCzh+z9I46tFsUXG3un(HzAHiL6V|8}+nZGj#7+fs<@4sk{H= zjHi>GN(1w(zu#FjW=pt{L{Ytlp(>KU;oHMUcsKt9blN`6o-KeZbkr}NqTKIFm&K(bBb!Jm7TKPT=H_`omx`HfiC=QWv(c}$TeMjW%Wem! z;lQC-dA2s%GAbG85t&CS8sd+gO&k%G7Ysse9wo|ZpHcypc(E%cz;T&qov)?Xp^qE- z0G%hhQ4*Z|_ig!~fIu1lP z_Mj`yp|wxyHi&-nKjPXHF4cCi%m0F7t4rPvGU5Ve$1;DcYsSq(O!eP6F|xue;`HOO z00Jucg|9=Nz`um%AVZ0)S^?s zq9%zqr#RDQC1DO6Sgu5WIpip~@ZezSgITdE?K&;XvC*qJmWE+y|9WY^hIjg`<>Ta} zZ@0CiF-p!QpKoCX4s=|K{DI3E0>|;wnLI&qjXeG*N-6qQLFNpLZgV`A)Veae`BdZy zwdVYHLj!k^3H1ypX$z~t2EmJLVd)A6-GlSB(Qn(7e|hmdCdc7WNp%dzltYjbpB&=55*y>6d-r?UCHQ6tE$f@_D)8?bL(Ml+u#B- zW$lC`bJKj`%JGEag_9!}mj;YsVdtPT+B3hAwK(G#Ne%4uyAEAp!|>7%%4!P*2?uSn zGS>=4CM_2mJ|PyB6g_YkG(e%*VD zF|~hk_FyAvTI37X=mJrNN4pl%)b`K7^RP$ouO!P%2hl3D?R$)#JpJ&WZ|!)4qPdcq zH`1gyEK#_qvAG*V&kbt*xmKo?Ss6?knjc14A6Dx6uzYb%(weq7YcA~F)Nzm@IFL)h zBdg*VF}UuH$ZX0_Dh8n8_7>?kunTemGfm;R*k3j{0;e!+N?7yK3cPne ztqct!VNV^!VO!a%qcccJ1cyj6T4E~PvwXZXW!589CDeY;mCkA5vK5$QehQtgl?Vsw zyI`UDm&%Xg-HMXgieM;wesYufuEYOd%_~VthJE;?LdwIAAg>?>hBW+Jo*NPC z*krs<$4FMJLPAj)Mq#y_W4VTX(B_|argU*9=qVG)N&o0r&9=?;IaU5JVvD;GPrUB@ z_~kawV0DpTU0=55UvX8-q>te~*nNbI*qY2J=T&7lPd_$4lEpS-g4Nd=j2HSQp{Vq_ zDH>*t$GZ9TAEH;dUp107y@=}!vzQCT~dVkmo>X^|C&8zxF%X7VZwJoG%mjN zYw)Aix8B|86mLBVgJsl^x%>CsEdD)>N6NE#ItFl2p;2_RcsapW`)ADGgQ&UI{0|U^ zidyW0ues?t`bOd+vyL~%MY05o&56%iYfT4ST)x-hbB(rSDhg*%6wCg^q+!XB@JuLt zK~wtV;$KK529Q>U)cb482&>|;D+o=`7grYpV}wT~wYMB%ck2kn1-M`^rB_k81lB|4 z*_TUO^Or$@_WgOAknJ072GUga(NdL1PW6hQPi58AT_~r|*%mu7GvpMUhdVxxOUn3X zLPZ(Q!(y-4s;HE@NgR6!JDx>{S@V3X*&tuxXI4%Cv(D*1@Fe)}q$eukehFgdgKl1b3~B zbm88N21F--1)6?9a)Vj2DF6%o?X89eYw5k<0hvpBR=SS20l!Y{;0#|<&F2|m z!3T#L73z5vhD>OK%-I<_G+y)4L5xMIYA+yZ%j1-=(c%^#4!){mgJ!pjF+Q0sS*gBT zxn1+s4Q9N8(HD%h*2>C!)t}~s1El`x=#S*+C)W6Mlrf=7TRNjqBxQi@NxH=XDyJP~ zgZd|!9lsjx%fjwbf9y|UBnS30eiy&Cd_U{tugrr+q(H_hl?Z<2XkzA-XuPiD+a6H^ zo|+Xl4a~Fb7o|EzN+a>-LR&lA`}FM-Ge?phqALiI1RB-(mvgx)MdXo^g=T-fuyk8@Qmzc1!+PbA1hjrVeS58wI!+Faxq^KtBi%1F(5p_21;5 z0yb}!-AG!*4jJ4#f9V$f%}0!A`RbjTXj(#EmQc#~u244`trnih-H|5xj6>K6e0yN) zLzLQbAlgLL{O8bv+Xr&~d422IQq@ zMKa@Y8FxDg_f`BtTh$OM)1+P7G{F_?`M5orKj)k~MLm0(Sm7x9!n}OoV`WyW50~K5;*m;I`7pu@aZw90XY`^|LO*1U)vYG zzU$(I)heJ6?dAIhUAB{T4%?-IcKE5bGCJjH7Y^lYX$N4#=N^dgGD6Z+%#DbcQlTEq zBvn}(wbB#ESGD{zq#`ziYq~Z%B)(PtDk`8Z(*Yf~8eP%W6EA))^u~=u{#^6+$Rj(Y ztT3UaLgE!cWpp(7JaalyRCEb`sL^LO7sII~ z6X-rl*YlZ#p}EuhrwU)CBwD@F>1y?a2;9Gi^dhLVcb6f~Ap2I?_u%MA0{(T94hgRj zPBhA1DF2Cxc%9jE3VV!zC-g&Vo{~^7b=%9JU2d6wDeu#>sfbS3_xc_#^D13u4JWLA zRqPTVAJk8d&;AM{;7m?mjP0*Bw<<5vKA404+1wvx{#Ea5{b>5R*>B^|dHGI{mFcxj zJ1!}M3n$r8RMKP})ks4!vrp6m(t#(v+UF~XRk=3{ibv<~^}3rH`ROv)YDuW6Z+%-2 zjdJa{w!E`R=bvhm!RTf{M@(BqOUmYiu6D6jSqa%g6_ZesD2mm=4@D(f;+sV}Zv9LT zoJ~AZPB?0ormoi&y=H{WUVV4e%nG)q4l zOx~A-GJ;A~52}3@y%xhfs~`n!G0e46AGAuD7Z|X78z$^K4;XBMlZe~f{v9iZE6pj2 zU6oTrNsK7@Y5Gs8Td+iW$!29sIX**O{nNwNd0Dcud>xKBZqcsx%u_P$)q&_gd;R*$ zFMap!zg1dk)9u-1(e?iBx3Ash##n_ipK)Tv&56K&4pSvnr$;sb{M1v3&fRS-mlgdL z5Hz8LnHMq^Gm(n+-$h!{#x7uLibCRpwU;7{6;61%v&|t4ScO;7f$yEV>zzi0wP{IS z3p_78MYT~Rkp;c_tjf30K_=F#06%u=sYWPmLSou|0=_}oL_lXi1c&V{vHu=>=t27R ziM;n-=up9O)ID<>f|pf~vHqV0MURD>O`gkut2QIfN2Oz{Lm`V1$+(=fv@6R8pQK;R zVk+T@NM90z1aM5YQHB0^M>p=c2RpL~>#>d;H_u8!Mjt7lY{ugCnLH1@s(alac61Gi zf2dtqYHHgZ>T%>m=-vqJe2>Aq&s$3h&!tbD%aie6UDgdd+Kd?2J#fk7ao$778n_?Q zO3Pjmhpa(nL$Qhc2QO_y(8u~E9Flj50a z5lPcN*P_g^um2oIZW!Bv$9@4t^6C6BzwAMwA>Tcv!^9ixZZ2%~*akF)fcp-}$DOn{ zggNH9@0=Ykq8g0x%2K3DWj+MDQ^EK5sD+&S+lHH^bdhv;CwA{_InpsPC7yMyl-01* zCi>W*xphui5bYhypT~=$8Jr`xMxHJF$z^Ny5&08^_-7!AAI_vZtcNg0ipgLGo9~qqJp^Z(Di&l zb7CmlaJY8E*Lg*;Lr-)h^_k7ppY3CBLm7x7c^A-9-sSw_Jfm+@ww^ev?F*uZ?zsx( zy0=GPl1Kq6Mp5kA#y=1-=mW)*Fzm&E?DQ2r{LS<7Pt63dpXW9)KB4Th0IEqhyL;&% zKWDWi>C3Ax2E`h&X z>^Zf*lFg-1L0EN~%@-+{;<}IbRd6jzuKl3H@}NTL&}b#4`z)B;li4RF+o^Jox?w@*B?N z?2q$0cv8=ayRwIrGxvDS6&mW$S8DJ8Pm!Ma96>r3FnoIeMo&6i&u z`MfRKxC*ovDBtG-sAdvSe5HrxTRNKgMx4WAr2ehh?7iM?fe;`SmIo>NpgU$8EmJAq z29>RH_=>AQzfNBG(OEADI68OFVKxr4aN(6=SPnuE~F5AHo5Zi1!%4>RdXk ze=+P4-gf!YuU)gzxnK6~1TTX4h6j?R+L`aTVo% zmVyUbD_F#tvlS}c`sW@Ny=F%uRD_7?J@H9;_bWnGO~|JE^A3V24}@D@Jl?C;R5X!Ev_j&6Sx%#s-xEDb03Kd&#nhm zvHzE!5GRteRoK8iT{18g7LKj+NJXf0WpjO`Zls)xw+#fM9aADM3M zZm*RyE<(9Zc)_>Q)H1w!#CBIwm%X^Otz1j$ECEZk@8DSk4A)GN*C?G9t=+S&2#pz! z-A>jV1TTc}QNI}9J@6m0BZoOYX_KDfDP5d`8ib(A(=b)F?d_lb<(Q|o6k4HM16DEW zI8y6ezj~_FP@OyYjU_;faIlz4s0_5;5`krdqxJwKwBq|Z8|Gowc1nVfT^uypJ=B05 z(8ifxKylo#nUjD85W{A*i7XO#{=@CzTmSm{7ApLZN1xw5XP@sq^WR#$y2aBtxTu&D2KcnRh3j} z(bT6@#}Blx++*WyA_+a4%j(K&nsyez*lF>u>HLCIX$?Q9!7-x`ZMEcnaAs_UCa3w& zL??MQ4l2zDHVBy;WNb~7h0<5Bp0jR3zn!@B>6P732%ZPgCG6Ad*6Wm4|2bT1zie8` zd>x=qvo|LGq13eYBYi-w!Af22z*Yx(YQ2u|_!Y%9;;IBbKdgLbocP zN(g&u84j%hzxO(BFCD-^55=N?*@E-O#@!ecL#KG3Y0~KqyD=}I0uHVh05rrc?!Lqn zC<_<9Q@1l*lTpZh`AnxDslP8kXH3mCN_1{`Zs$9vaN!)%HWE%&4=n6KKgb=>vIR-y ztL=%pBR)$JNRVIntwSJxGtZ56CO)5vF$X$6vsY$34M{_{bV5fEAj*>t->Z(bD?j>B zDV$M7Ij&N_v9Z)o^P-Q`AU4*UkMrkZpQI0$0|em7N4^h2z*NxqFQ&pxBsutoBnLK2 z5kO&5m7%YRy~*?@{kWh8-MU3H)z_mNubJMQubc=y^Mqb{*ulDp)oApU$fs*wl~P>Q zjGtwHl5#H2%?c*Fb}G}HUxEIt_4gP?3rd^si4C6st(dZY%~>JvA*=&)lkuD{?$9wv zV2TuF{N>6&Ph&^nV) zPntx+u{-bR|FTo@t%@9HHR3m9Q1d;sHM$}}+|5=t+O&Ur>ZER9!SR&~btpfSw5&R# z6HxDHUSznqnzO_|7y#jSOY0cBP;94|v0-ImLvmINlD2xT!#;jaq4_EHQCtd>$jzQ4 ziV=fC;{>E@w~`SoNUzLY2JjZCuHxQAJ5-ZbQ4bsniT>$*eOt>Z>8vk?GOs-GI$c^+ zTGy@T;|)#qk7to`ay7{6!L_$xLEqGneCor`1h{FdkF^9bOiQ(;NDitj{a2zEE4a{9r^^l$AhNfO>BG$$LhAa;G$rN*Hxn7vDErUTTm(HM+CY7^ zm?JLHif8iW-$V3xhq5W7m?}D(l_QIiEd(IA<&mYN2|5YP`+QDhx-bVak2GANPi31Q?}S#U4AWe4%{Os2Ol)(rO#dNdy00QlZ#gHN^#p| zQKVFGLkxOvsPI=EkqV*vZE$iBRJHm^7~9O|l*YB7c=?Gx+8|W8+e`nXi?VJuiON0Y za4}5&%4jE@n{;wsr=$~y@9euLJUV*GDd+2fNa7%TMq5YSg3&N09%-idV#xXQTUS<0 zD{mP^-{_wy@G?Vn7v3;up7RQ&K)bg3X|lT?WtpgXUVlr4ruZO+@m$YxIOjw+Ra&1* zv68rgk;-Yvh-wEF@?xEbOs4=|ocBTPukWW!8U@lM?Cx4UY#qNod%3_WOp5&qOE8Kd zbBpC$7zY)d0XjH!Ao~biPe8}vGfkAf(@yGvVN^~_pc`PE0EmawFQB@nW>e0c?L9M9 z#F6~L_pUR2_lm53AS!l@iG$(ISl^QNY<%Yk^2s`b2(NH{+e%%!= zF>80dmb$o&ScXdzXx$$|%x{^gdQ2s06bd1^v2@GH?oNe~EE@jf%NKi&Docrn_*B^s zDHbo&hE@nhukqFjkm-7;@xZ`tX!)ZwL*+X+wxQ}jEAI>~#W|x+B|9CCT-%MPpBd?; zjjly@bw8XQ=&gBzR`QF-?Asm0oF47Kl|hAkS+dUaA1P6YsLxEv>Z39=ji!0X8}|4p zZCr?4^G(yx_|UNphfCNwmP!%ZZuU*4LcWMW$2v8hiApZ3h5D?ciw?93-yJn$R6TLNzwHUx=f`Hmf~UmB zTG-ZP)uTC4vbGBk>!oTqNLZ_b4ALD{An10;QH;#WmYQD2C?nNEZEW15q%b;SrW^od zSn3sja8wyRsjJX7{qUyyk{zk?btS93nRi&mWJ)1G);tr>UGKPpGN4emt&7)TMw>Hi zK^D-~y{j$BtkAu#&ab|bQ=*`fe#a#QW6}4xw{#(vQ67-QS8uV0QpOzYiDwZ==343c zzI@|ahjmf=Q*o^m{*ts7OrF@|o8lVkGclE+E7d}n=_=4YPhGuwiY2Qqwj^Ol&rZ4Z zW!F08A*RZ-oNmce9r3R@A8rYMrVW4OrNwceMCUJ zReMg|S--=Rr&BY-cue=F7|%r46Y8GL2l>R*?@yc3LpQSdcPC=YfP)H?=OF-|ed|cB z?^sudrr_~C!d)tcqjPM8G65qnJ~_U5W%NsNMAq@{{u%jM^Ann(c{(mLa&_^|WnJ-7 z1yutgMMp|_^U;lySI8|ui}Jozc?U(L6tpnpo`4Apx1F@||8or5N7|;#5p#wk)sUb_F{(-l9~IcM8?!ktasiYYs& zn3bt&^SZf&sQNrC)H4j7Zvie)nBtvmp&p-p^9q$D=9U@m?bW&K zV|CT2s--B0vb>A)Sp5y(-ifP=@3h0(0l#Y3tJ-2#NZC}uuyTmqtovI0Ces2c*H(k# zyG3E2o4@IDPr&{IcYL`D9f^&~OttNHP9qJxYv$Vi#*d4XYdA74N;r4Y=kvbV_AQ_7~hIU=eb!iJ$;Beha(1PzIJoxz3I!$;9eFsi1-TXO9xSs(YwwsY zhHyyxs?ZD44qP*(PbP3);H#$^EjV>N46K0Rnmo3ss>P`wblbw<+5 zHHW=oJSzXD!D=7?24;&t&*h8OWMp&;D)#twIRyGeRGJ1f82vnFD_gyxKXyDwrTLNmdKK$!zy7~Q^X4a!r)UaJD(O$d(Y&eQ&L0%@xoW>2 zSC)d4Sj=IKHG8yT?EOwWHSldRRlCtUSydA6gE{Iaqoux9Q?l+FQ^-(gV&}$5p3J!! zb{o3oqe*g0*$i`4q7jRr7enlfsvM4>vHm@_lX5h@N@nBYw3Jr$4rOIF`x%AlKO87{ z*a}Bzn$*yusaU4_Z+E2Z`Lh~9Y68dPbPB)bB28`5)+l; zGO|KBu62*%buRay{V;-tA%gQ;Q|=a|#vh>HB?s;Z1`lO`YDbq?XEfrZB z3HLj-u~na>Hox2s{ZV?^+_)mf^n|U#vGqeWD_)csxrc|33#$~P^!_06n!?!7N}0g3 z3vxw_)0~Xv1@SswSuqb#C!rXa_fC-`3>UU0d~ZQup-di0%u(!13MrAgj{?K$u>V0~ zieF6flr~x2m+CQ5TabKP`hO3@6q!{IP}YIN>UXs+SBy7`7uz%52f0s3hO2=>iL#z8 z*_p7!dOCDT!uFk+)7z#U55*0&!?5EylYL1f8-Ztla{hgzh&nNA3v?N@Zyjz3@|&@H ze7Wa^t~V4gW}AkC+BIkAoVKs^@T7+Z=xmgo)b%Qkdfx0H+i_p}B0!L}!(;#J@I)AF zMANzR$5p_2f^GX$_wxA*hWnRn!@@}t%`aOtkEE+HDlWN8tSBE@0hZZLs#iU^3m@oJ zcP(gFr6h)m=W7auoC~4sxuc_-6&!P8Z}>>9&!ZFwH-B`+pu2du@xL>`o3=4=}%yKnfEZ1<19g7>*nzeJCz}A!AIZI zZS^?J)a(o%D5!VnjOA|T43V+O1NuCxm1WCV!NIUDD|?aZrBtavhANYO8=pe7vJR(? zO0@#^^{X)c6HE$m7>*JRn>1UfPu29p>JunvFmdyCO-VUa>!IYwxzS3M_q4!vq^?%- z$^$)x_T<~KWG>H+{8L~Sv@z7DvGzK%v`l=9p0ImZ!4+f zA3Y~tdhDg?IGWtGU{v zzHHLDCRrtrysJ(gVkRxMORV_As9SxcN{>r=_R4k@WfA;Xq}!UZd06@Ppw70frpQ?!VkrTkg?!W(Fk#76V^H9XlsosFejDrT9 z%w48SbA9raw6X3=U0W|dVCPg;P5rl55b-Xz$x(L;7in~o)^cRNGOKd>+wYZ$jM=V| z_4~>ryJ~gcc+4wKi}WPx$UieTQ7l${owxHOqmz;&YbOcq6o__FP+RACIO_uPtpzTU zPggOCsyt(P@j@w2u0WniKLC9*)os>j_Wc=ErksikB)!$=M1}s-!`M<5llpx!_yiOG zOiYpaZ|$lf#P{;`L|(JB&$Jww(qUPOP`v|>BGI&=-C75k)AlRaGNc^|?UZ9@)F)HX z7C&;DJp0kLaAuu|Wtjl-(13t|M`vO;{rw*A8nJnM{7g&rKo7A1zIO5j0?ARl$o9bJJ2)l)ul}*zQW^5{F z9Xl)fc~>dHEHqzxZtl;Ob8hJ35$bXUJ135p>SZ;8#}ST#^sW5P#fsw4r`OoeSF*_# zm(%yw6ASUCcs4vU1SPk>R1U;(x93q&_TlDXTNR;aKFziqs8mhtQz)Ye+ ztT_dVSgx;{?+pX@W&kG7P@DTWrQYWU6`5S$c)bnX-yVF?M?Ob>NBjJjaK zQTbU-PKY*MKJ`&dL)4?xS!Yb0ozC*6Opw_$^b|a~G}h+4wM_62+IMgf`fvO^IVdGe0GU{9pLODhlim?C z)d6E03eA;%<-#A1H%zFuG)kCN6O2)};u1?XOE2k{CCJHcTp9$Dr?k(`J^NmEEjZ1kVD5tI@26L4Y?5jJ3kH@9 zEd_qAoPlVaj~I<>ulmnP)F>CkGRc$HmY1Yv+n2R!yCoRguLRDU4AqAZUO1q!@*2&T zm8cmJhrLPlH147;agJ>qX6I^LXNEQzlEj%A8a^%N&gFUuB_n}+Es0LY;S5ETS<8mr zlm_qZBSIY+Z|r7f9sQP`sqt$qMV2$M#XkJkkE4X%6j>OC+-o_)+PE&mzpvi2xV%zv zQjf%t(F|HzLPY6`T>xzP`sOfw5-m*#S#7K~jSzM`&SMbD;z`%Y)52f;qHe1B5H;~z zoZGhalHQb|q`oZF=TPSVLFGB^>Uj4GJa-b8FRFZ>^=;g9g5gI^Q39h889dLzltInX#;>Q~M*W z&L$x*jscVbCU**bJIeENnmC@^>pIKpOHex6cV7cXfTak-Xs8=UlHyX>38tvWc{a-i zQrs=w&(-b9`s+bI6-?O5p?Rb8EzzA-#RGHVvFdqdO**sI^-l7=j4r9F+LJP|J-A3N^!TiEumqGIX>~@f zKRjCVXu!FJd&oAwaPlTpu+%bAgN7>HyW;H^$wg+3Zx{vE}q}=EWJ1{*$495s6)cX$d@4X0rRmC(l?%3YB!f;&M zrI{nQ0&FS%6yza}sBI7opOZM`*)aIxo9I6cNu@pUC6(U~_B14KtG!OldqK^+gvg-c z`Nqt9T3V=Ul0WdIk{85RPYU&pMM$3b@{dB_y|+%bjPXc;R{TF|^C5EDN0qX>$zCV9mz58ldxu`%yGy!CwCA`OMyJN^bj&Q@Z z;}w%Q^YN1*kJ4wwKdRZ~DQ>+ZkNdc%8P&XcCes7=5_j8;kXN@x{hNcVy;aqj|9Bg> zdl-cG+hsHw*SugOqWyQiiR0fY)EqC-9VLeUM+uq?$Mz~x0W#LO{&_H_`P%S7WQx*6 z4;F-+vHMMZE9almbMu}+ckDdYx&|3!6(!u5I}+#q)8|fO*n2Bko;`i;X8fZJJQpbb zLh}3f#NPU^5j4naVxNc+UI&Kmb8Ee;hlj`IT(^1xhEO?vIN%z~V`qWDjzq%T0qH-`0O;auiOe&w7#cawlof-c;@VtEQWWya~_Rw|@$RcY#n{k5}bpaKSGh zfmE5Ze{n3$dm|THTSE*k%>gn_M{sMJyIswa`R^z0X>xN8>f@!<(NO_H+LqkI6dKAw zWcV^A$$E zFEN$m@)ZK>ffUP2GP(o)-w0TM@`xzBP;=TT{35ho{GVI*9HlOz(`4fGo@)CDqB~d) zy5JT4aCYaMP}Z>NPm%8iaiyP;Vw0yuEE~zn0Yp)a$eXSF92o(p%U$Qu{i46+Ki-(J z=zQ@44^Bd-;-`7>lNKZHFhk}?7d9mP7Q)rr96jzojetNrfNCt9d;n1;-fT9k8)NAP zsM#Ex^#%x8s*2ZVl+*$E>%f^#aX}jbFLct zdP6{@7=<1@PrYEVwm0xizCr}`VLGmCa^s+(i<>T=o+-SBvXF*34R5p z@#1H9er{_?jIhiB15_q@c=28uT5G^(scrjXoT;<22Mj=UtbuV8H<08UU|R1&e>|=l zI{ZT*i#TehtgHcwk-i!kvtU!{0@T#yP3%H9{CS_nBLT;hD>e2up1 zeyyhB)WpH=W(<0tAB{&B-EUGJ|2+TM5-%V!9fm)uH^q1Nwl6ZdiE!J(Lj&J|=I*Y0 zb8w=#8`c{%4YqWoxMyfVu@un*dIoSNJEm=D7=8t#<(SYYlH|pGZM^vP@DYW7?#m>J zzj=*J8SG*b#vF0UwtE2!%QWBqAisrG$A48%O07JBRVot)Gk=I&7wGGEHKn&py+NWO zd1u&UZg$ogl=Tf~T^<~s?vCDr@W>yzTK(y2*wdzeTqqPA!0-V8D6@6|VV|8sZzm9A z>iP<0_pv(zox-p-9)(5V=Scy-Hte0mpZ{AMdB_N8seucSn*+pd4mRS9ImU0_H{zDQ z7CHXRC*^NDjVR{hPAL{hm}b-B2LN|51VPdGmAfF>S`9mB50^1bmv9rVX#*c~siQfZK4G&k_U;Bo%x6UE} zKbZ+)+*g+`o{Zl|_{)#s_#Fs@XE%T+x&hnW14JU6ZnBAKe7Gd@A-zEu_W@C8{OrWE zldk^rya~n}3&kJ+Iz)zZ0a-vyis94Kci?c({39j`A9}y}B>rrlV7s`V5WGau;9nOy zunUN2M<3|RiFhtrl{ZkRTDfOz&c+{}t~>G@KenZp@v}YOdUTbb%=20p{{VozO#|b6 zUPE$4ulyrpSOv&3_xes;Fn1#5dL97&aQ$Z@?*LEN0}5l_voGWRAC`cY8lp8A0a{ww zE+T*{+MTQ`r9J-H=43Wrv@@6j_dUEX@wX;2mH2?vljxERTkWGScwxq|y(=wA>@|J> zktt6*fq!klEB*k-h2*%Klc0{?s#6yi{DKd+Dk$&cPkP2yzD7dWBqN4gW$Zbjg4Xu} zcsl#9bKy3@mAy^y?1lV6!lic&7o=$h9G5yNE%uUYZe^*~aOJfT`3a|ZsISO^B@ zxx*fk|DD@Vb9+1pN7B$*-I^pUp7^--%(z|d%I_%d`IEOS>C(ovENF43wvr0|XO*W8 zZT<^z8uT_jXYj)39eRGA>D=LK`siZ6K8MiM!Ez$KeQ zgY%}q4b3C`JAsyP6;up+=Y{#*!@LdYq)^Uu+MxjxZuTjPsAF2?WEC~k;FtS6a&4!O4#!+-5ajZc{U&rOKv~|H~`Ah-s{*@yydZS z09jTcw0eZoG*h|Y20?(p8o-h3U(g< z*Q`m@KmO&qALDI>4=;M9`M_U&1%U4lR`ta1U>q0|^lp8*OS3jghZf2Rn~frM(Hq@u z7lykDU^p!j{OEGOXSRwWtXN9aJ{nzh4OP11=-sU;=^?}V4gLBgUJb-wzRP4C^pj5A zT8ews#55fDpN73_g#-Vb!fq>;(}V@>452+-E8?J*@)gjg3s;^4G;c#(7YRLf>7;xr zyaA;<1To3{E;ok|#K+|TK)uN?doQ5<;Duz|8U1<}e`@od{GX>BJk}U0-b3IN{}VxK zBKJ%#i#H%(N}eGXKHOhOfd5Tz*#{0^CkSneC7|qz2n;`6HeIWtKmIx4`krRIj2hqJ zy^9bT0WZj(k;j9`Z?df#m<-xGgmJ(6#RM1;->)FtJ&k~;%K@Xjo(km$69bCg-KnItU9LzP-Nh$IO;;FGB*LoLl}+ zK=Wx}TDafp*-NzO2Fv;tYxC`K#%gMw5 z!a`Mov=3fhg-C_b7Q`X%B3ix094kP(LtM1mpUgtY;9mPxz(BMLR z$^h5h_Q1o_8&WK-4+Jk8WGg4(>TzEfaixjA-aD+ugdyt02O}FT<~w7;U#>qKIKqsn zTPbv0x0ph7Y)x(^SbqiCqi;p^J6@?q+3xW274UhN+QhzXVn4-qmdI|~bU!GN)64cd zgA+1@;sY*&E&S2If~&0{v&;_ylrUAWBhTn?<(4eh+?b&vW@GZBFHinvaHrehj( z?NT2~_89ILti~kwcmCLK>}(5aVs&%RTZ?x@t@O(-r{H4rFAVsvXyjs5A$&y%=!nfe zkMA7K)l6NW^Ig`iH+7uJe!9v8e!2UXK{lsCf!e5{KK*A_jD|Ce*VN1S!$SQ%aBSb& z1z_dsU_%qO6XDR!zSIcRP-=)pA7agF&*G7oB2?(c$*x&Y?dWAj7O z^t1hpam84f8%HQdrDE8~?0!!+#aR+U3N3Ej=jvs?L#%Hdpji2v-VEY+B(g4|I)8ky z9vDMNoW1}7lq*tqMiMRoj-YGZ5ElgYuq-WKDTJN1x@K#Jl}9R>X7_GMWsZcL=k;djpL!DgRSWukY+HOi-7{GB{}+q3VA1`fDhFw(c+1C*g36Yjp02|9>%Pn{u+ExWfl_GXIkG>(JGSz2X(>o>dn-D5QCE=3_aO-e@FwXBF1iUp8fPWu?D##?Y>0Mb|IRap| z{boSVT>z;2s^biD&jGC@5vk6gQhy(Vf_>WHkEjBy@ct6S>+e1_KI*c+!TFqnu%gN1 zL*TdGWlX|`UVQ&)y!7|@hiV`eh_>f&JH9(m>e5lbxozfuMyLd9oO104F4SD@?O#}M zI4M|BoG99MkYLm#LjpVU9JQ`^O-Tl`#By(~qI%08M3}c63J5uOn`u2D3$x^rHCSQA z29eQ>EnkT+)=yB^7KbgZqJfW5SW;8S2(okck3mLBC8K6@#JCFb|FufE$a-s$+XwJY zZcrX6Fdr+2MccQd>0;mlr}RCmCT!@3kV0e&ckkW50{^q@%^4i8;ziUo@Z|9Hcjj?Q zgC{rjwXaHw*{y<@%>Nqg!8;(><^{MX-Mlu31wLIG4@*H{d+zxKC7~wJCXwq!!}C*+^M_wvs_0#PRjsi2k~J_Z9}!H%O~iH3xe zPb?CwQ;n55i1#sIEJ$nUF5#Vr{1nPsmxX6h2 zPf2qZm6mP9k_{>H;yfuDodd*q@%Gx8dtY@LxZ; zMs}U_?<;gQ90E;_k(Y5>6|v zBlsnVXlcMjV`Q)S2w9EXpww!=^6r4E$sd(UmkWVCC-k!~y@xVd;9liXPF?svga(`p z2C_BC(SdU_3Y)3s6(1l18TBKy6%%HYXI z_}w0sxssdL0e`kp7J?)~o#6r?i$kbdiReROfGFp@$5F`~Rgm5?)8kO%)= znN$Z@pUI@j?NzPPF)HkY8|XWz)gs?~=gUPAklT;bSl=tS84s|vH{y#=%ESkadYJn7QukFolxf^MgnloldlOy| z5$Dyv%a=V0W&qv@scVVJ7x*tj;KY}>frcFoxpzF#eFKtwCalgrjZ@?wqwq5&kRXJ} zUu05J!F|OF97>{8lE=++epdgK0!r!}E~^>GeZ`maL{24G!BTKjXnzvL5^zpj`)u!7 z{G25|`+V?ktI8I+aw&D;gs-oLB-BeX0qm=aeTShQxCT_$O7K<67Tzy+<&sP;BzL6p z0R>p1WU?TOIQ+o0Y1|Z6O+gXTYNb;X9 zT@skRvp2KPj4l@we&On#f-;xsOhY^keiiXc9CbA%+G|!{JVe7tn8ibT-9Hj9C;kFw z3Wj`a(o}@@Elqq~ZdD){A2}i!jQ(Ep8*3=yL=_|hvE>=Tz#dYU+0#m)_zrN6s6Ghn z@4bg5%qa9a(J#VM?n-DfAa|%I#Ek22O7OR6>E3T_3Ou4CEUD+Pq!7zNquPy-YORAG zv7FsA*DunL(u?*2C+Lu(h+Bj^e4oN`H+pV7hFRN7;Zb^ z)88F3Ryo2E@IO%+ht7Xix#1?SFRNjE=Z?P!x{DISiKx zx`SzH_^Hy9<<$r}bYBBm1u@I{o_Fw1PAZc}tn`pRk__G6zoauf^8s-Jg8{i8^W_aW zZsgqW+B)4Pe4xsndt~euKm6lLI``kAmE=3Z)9?l_)JC3+%I{h+So3K~46!&&WxFL^akj#5xC?QY~!A`x! zi0|NkQnSh;>{3cX2Y*1n=^mssLCATlbC}YAw^ttj?gjmgxBtF7&c6(+V=)Ukqz8cS zpW|>*5MY?**hRmPOS3%4s+sZw3AxJwQBy&`^Xu_B+AH~o9oX_`fyL$yMAAB3-la%u z-#we&FSQ>JVj#{rSWS5UefK~4$O9%rgv*QK`KpXj>^;N+dhOZBR^vRRl?$LKU}tBG zaA94Ley<4>=!O8{)n#DrIS+d1ywyk*xDEb~17MWe=AN@g#;p1%Ro7>+$(9EoXsjWy z{lZ4xnlRbapJ8zVRU8EQs9_vO2U6e7o{$g<#Max zB@w}>dF0B~*;=ib&ItPoq2v6AP#29gbhadY=K`-yY4h0uc+a7nXAZM)w-;p%Eq~!% z&Ogo^3aUDo?hS|T4kQn8|KdLFg(hGDZXD+Qk|@;lqJDS#=TYe#1O1%b4<6TIH`Ppa z0FBfVI8T^JF!1D@Jd1y=6R<_&pWbVEmd%aoTqvFhNOD0$Es!3mH`I}9KKFxu0m@aN zO9~9qBfC%owW|a9t4o&}xdX2|K31)BpxX(A1E5Ng-X#`%o4W9(h>Vn?vjs; z0QT54eaB{cr7QQO$}h$*{toT7Tk z<*5t#Iec@c&9}x*J!`M6SUMlJz5*zT%{ws_Ep1~BOmT<+UNLv}#2skvx^ic-5Z?+( zL6QU2qK&w+crU1E+o>L6$wCay>XU>@0rS^b2nrxOsm5bopyAvjMT<|L3J?J|U-zt8 zgBi_CMFV|Q!tZ+DQ0G(|-q1hZCSf8{G_ME;l{lSD5+MTYox|H5f*}i(-wjjKNl$^?>%8L0V>R>La z!GfZ7cMGRaLdMla2{$oV)V1=0C{>N&G|iG>QwcQzxT@hZ+fZM)FY*#LOb)&O81 zw@f>Qip6LX{hkD420%f;?3oGv-d^53Z*?`>ZQigj)qO$0+Uf@PFnAxX1(v3Lj__lyzLyK0y__-0L9x4;x)$<5QXKv7-I4=Oam zt@5@LorVz?tY-!7C{!p^JHn$k?5XNY?#ac)%185C%aK~B7zvB_A}4>&9hUiwz;Ce= z%x{)KNml4w=GC1AkNSMy9VAl=Qz9lJ&VSy(IdH!{wc#pxY{w_&NU$3n{?dOr;y+vWYt^=%j zHPU!~VpM-cuB)c+Q+Fw)mPM&i47=VOcRuGwPI-OJP$-%wrl5)ZuX7I*rIG{@I&Ly= zAo)p5V!+M7U{QQ0$LCX5n~wO(qFqV)X_(basez39v;ATfvhSP_QfA;j&{=ZHS1b;obf~*ASG&HHcB>jGgQiEt z$d&QUo?^d=kzXsoi}R$RIv79O`6ydj+3z@e3rE9tS+Q@MGP@odx#wIySq}Q4XSTt-)dO!DpgFM z^0!tww4SDxk09Bl9;!P2=Mt2oZB;W6l?3w^C2p@{D?brc(s>{=`gvxzo9HR}ZVryP zBj8@$)hG)CgUA7-dYiyzfBe#~fKTl@y4&dwkBNUI)j7B6J+hgi81Nb359h(vN7x(hTl)GcHHa9V62F}CwDWAzfbT3)d=9mB)U4wqs+2C4Z}H} zz?{~uQ17?Bb^4gppKl2zM>3}_mnco?7ExKGGqU-`BbLjXJbkR>2B`p%S8S6?Oj*c4zm$HcCy5iTb5L*)MUELoN5 z*Ql8f#YC-~l%z6V%$+c`ZEn?a4~sJiBs)P&?)&@bc&~fMkBq!{luzerTHhvvSC6b> z>XzX|kah>#I%oUSZMG!J(@B6nsFQA@pQlgEdhjTLpBH^1OC+Go*p6cv(SGuF%E~Dx z3d*+LD38p9laYf$0)m;Wks_lAj29nEKV?Jss;)Frs{MJnW$r;5iDgG;y-@Gfce}c( z<*RVT?yfod&H=R1%6Ev)H-~p)$PF_5Tk@Bc$`egKu*+{Ou#m8A~YE6k2mWSvL zCuSkJ#_*j&mPGKYDWym8HI>i(Ihv;*; z)_06~PTL!rxy3#Ww?(;9D5P`fUf|m9=GV4cj*jO03UHk2tynjJzyIea^OARhR7s_xezk$bgkd=-x*U5Z&f$X?qx@7 z@euEi!6XtYw-GDQoz&*rF}HQXLOHBQ9)u+1W7c z`WaopYaT4HzX`iRpAFZC)74Y=$zyz2>{xva?VZdVaWE)PmD0P2ocp?@vz7BI=Uj*$)M3 zbbNMb{fzAE6w1xtI)tG+x25)eoHhibxIOLj_>4hB+#F^#y1_EDpvd6#+$EJo1-nP~ zU>bfcltaKC0<53EgR1I=o1Zpr>5w&>p1jO`60tiu6GWAjR3yp0qDrB8ou?X1 z$=gKyq`b_>v&5a4a6cHn=db1pr?w2k!L`bUA{O~h(fnn zuM73|c33C|fCF#)xTffg^1mu%UQzu z5K9P~>5|vWtYNjQBMsIu>cYRJd%s4JMeqoq$w^2l5{pI-5cxijh+A`ToE_5E)Pj(4iF6z`7qOh{hn!s61A&LAnD47VT>3a zi_eWGk+tOsDcreTr|>eb!8q_kPhtp0aLUjXlUFjHacG-1nbO*DrZBjJT7=G8ilg@? zjjk~(b|N9sj`LA5XfX(d(Q)Dk;(FuEHp0UL9!Y%^-d-dYY7z}*gM_f{-N;x2O6)$MR>gU2(JMh zYB_h*x`TQTQ{SFTCTmjq#qO+=U{UTJ85*CMG>V}Tjmbh-3}9f&S&aua>QtD1`XKx znt60QFfLN_ zz>Q?SwajQ(NFCRe5xHE(er!lqubP?=_MM=^*9zktm8dfv8DkwH?*euIfW%+-u;8f% zG_z`yxN&oCwn<0MSSK_Yi*~hP2ipf30+A`quQKC5{Kz7)o0oscH~%eH@8P0IeGa-_{{_*bfCzQ zpzY6uL&O#abrHXK8Cs%Pe|;eam7zi!mbp zn#zF|Oi^^4Af3{Bdx7$+VNhC8kD_Dw9l_!*a$Tn<#dB(mn%v4ae9UfDfmY+UI}-zK zc7uRQi0z;(NkmKbss!o26rHukSY16U*3<6G%9~?D_rbWvrX-V>P)NH2RWTv45nSpi z!Wq3BCvH8$uB)PIyL?ERE5epF$CV_bIQh7vU(CUdr`Ky&?jZeyel9P8Bz~u57w&2j z#~NMXCz07?Zp)uWRe-x$vob@T zBxpRQJRN|zOQ(g-M$b~EWf6$4p$_>VrD2blgrx8Sr{yy1<({a{ zt>c!1fNiqvdksP%3scspdvb4n-h3dGRJ}p8)fTtY^Jz)cLZx0p?uBni@5PZ!-}hoW zkYCQ#{f=<_GvsN6*kMp{&m->wRd%qvQ$M28( zasOK{uh(^*=W!nAah}I>2;65qYz!!rrQ=PvdX*%}jhYM{=+`E2aVMTeaN={8G+X5> zYi8Do)J}}W`~M7QG|}7)AWu53XRWpzH5u2A*a|b|RcOSF*SnEBO6*jbv3%MR)!3}m zCiZaeHk)xc7GxPd=a1ZB{Eln?$q3JsA_gUx}) zQ};=*T3p6PLpLMoP|XW>n7ph3O*hApo>JW$dbZ7_6Uq^%D0&ycP|-BE*U@ERY(LJr zNK=i;CPnm_CA)#I*9`9rV^jQRW=;`R?yri9+MO3ooXns5xxsbWg4@~32yy?iYF5Ab zB*Sau$E8ERNi3-3hs~>8k#;eyk^*k!T(vfKFdiPW^Gth|*-oeIZ>Eztnul%S?PjrYAlUujFJ}0@W^j(nevpk17d)K{Oq|99j7%>y;BN?X@ zlPI&x-m|Jmt;=Dg6#J=qY9u90`OyK+mKNAeQc|A$gd65BfJ*I{mU*zZd5-}l_4&!e zULKN_jq&@XqEp?>leu!>R`JrLehsViE|=BWTzpw^?`UGmE_56`qZ`4$-Ef<+F6X1V z0kt3ADsL3I%{V_bcQ*&+HTygWaWFQUC(kMv%xk7d&oQo$)ut%u`%UFt-SLZYJduOO zO8GO*d0W=tiEGB^#)Krc_!jUaW#n(uKc5j80jSXnoOe=g-ak;@iH4F@R_41x;5k`E zMr$?QG^NwFzE9g68ianxaowRUEu>=~nOVH^HWLQvxnJg>)Di#>9WN@Qi<7z^jh=$) zff;O^qn|SQ167QBuWnwO2?o*rUS25!(y$l+f6y~*;$B#-lU|Dm7TgK4bY4Nt%EJ7m zo!Qy9#>f5gkC)OXdD|S>yW;6v2`E@GXh(TFn)9%swC_nD8=-7fp{y!y{Ox1^d`VuS zIn<1A4GjOKwTs$xY`unc7-V-ymQE@}@@%lPvZQM1y|n&zto2sL8yLTWZe%1{Ml$F5 zu%53}xacBzS>d70-2C^?zECa;VrJ-F`LH1IVe3%-)#x9k?%qrY3Yn^`xi|6YUOUe{ zKxcN$jsBsfHUgywJo}$D&3px};*|<}ROTA?A1N zEyaL0H*Fd)V)sK5Gl9gTp-=jMg4q2b)A=pYKgD2OSC$e>N=j%?oZ56@LF5rIz8kP% z5G^B+B45o00KbJkx_YCk7cc?peR?^b%_?L#oCHvgt;X&xXaWWZu)x%neTPZn`>`C@ z*do?Xj&_dEZq%_ABydhq&jOWJa86`IAZ;2)70N!VC6oS$3a!rQEfK;As z7Q=A_ltX$3d(l^=;ol9p2dCy~G=`6U6A2m}5?YcA|>qw@OSvZx(ui`t=X zCo(JRqA-k!yE8tf0{xv8O{B(N3OQnDQkQ64nTokEWiKRpemsYX3O!II4xQ;6s6Q^b z!q3vcy4KOFb~G9m8BjB$8MP+3OM+J!<4L%J*p0;N?#dF!Xkf7e1litsp8Nn7BJc~; z1W1AU;w1te6tfFko^?VbP|T_|(%5t|l^FAN@g>+hYys~wh`t)Jh=4Qe7f8mTo z%!L9)-C}H-;nL)?t>*nuL0n6Iju_LjXI_8jTmw0NUi)Wn2SlC@Ogh}mYBrR|m~`F* zlddw-%4Z`Hy6!nEfVevA8lH1#Mu*{75aZaoK}K=~j727LF||smvtaLrvae~W9Z=C#E<=gvm};|xPJWmJ{A7^mC3dGTO=$% z!1OIBh|nub;eTP7qw(@0>3c^Ve4n@FPlaMgK$>dp+juUb?_y-YLaD$*C6Jz!p@D9U1Sj;;==L1OD3*$b_*bP1 zUH4ki*Dxbey;(N56P_{pYTxW5W(>WF(THv_Y+8?700X|lo{fqvah$ zAkFIp*{Sidhi8x=U~S&l7VpQ^7z>=rizHc6SkA7c9D)xc%W)V}0bFd}prFo2ouT1hP zcOfGA2xam4A>L@L0a;ueYs6w}1#NOi-(zDy`ul3OL7Wd3zuB5%MPR59h*TG zSj}A7mGM*M;yz3aA+keg;5g}vRs9h~vLQ8ZgN*5v`d4Kph^JT+vcQcg=jxLC8r(4r znRNTbT`Q!SIjn?5z~&z%W5>zSx)+VzKbIgZ=}V(_n>s;NXmJq&OVnRBt=`S!83t!;P{g^uuyeFTPl3;9iMI7@moTOtN0KCG*L44^SUR143z z9?3*k`717>@3~(67BKazK^Dt9rZm@|@TxN+-55j5p2*4~t}Cqc|cGe|!a)s)i+V`lK{# z{@okM*u>~K=R9t*wR_;lj{n5Qfu2|rdk7%m4w2n*Rz8ln;vf5)Vgxn^cPR`%T7^AQ z-AP{Rl{D~(c%&09l-AB0Rl`IzI~6!e*_RofV#vWIY8ialLu}f5<5^{*we5`+z2dGl zzC#bi4lkr0Vm_XD6npd}KKU^cj&b#EV$ff1i3y+&@K^;ApYl`z=O~7T|NB0_Q$;%a zz|Y&F>o&pMyb_&E-hA#(&>0K@82R1y@yeliD{%z7bHqW+Q*b8mV@Z)%` zw)x_F4}<%nC)dclN=;9%TYE;hGeW=-$I}XYovWB4CC8NQL<#ulQ@9l1+2>S0J~U;( zzgaSg`!b2u6R?^nvR`^igzl6QqpN0~Jn1w&rnv{^W+JppLE7ki|#X zZKV0PIkj6_dSk$%YT)k5qbiJeHQH7qc9pc=!U&x)Ii2rhk$c=3zDw&LUSQ4Ut+4uz z3vd1JAN+(Uo!3nof?8KysQah{N*COIH2i7DRQsGm zJ2VE5OW<``fzc1&DYXBfEmQFTku!uk*bmo-tqi9B&bgo|ueb$e;^V| zaNd)BaC%4r3-;PiK2}j_`21=GJId@Mz3>n)M9|hS?AB{l;j@WsF5$HgSfqHmoptLo z+Suw^J6q9MYK(95@(LEWNN0dppi*MBK&I@yM<=D}GCwni^x1vqur_1ICqOg&O~ncc zZXSE5yO0F+TIz6ByJ4Pq6=7yAMFL@=(n#36-W?K ztG2ke6J$N}V^3B#^PRH?@fWVN^yooJiDqA3jP*&0eH+(=6Ni${ipTn-h-M?+SZ#^t zoyZG@HJj&xmK}lPRs9ezo`gop->e*hHHin$%Np;AGd?S98~`ldG~eKX!J=-^kR4z;0!lZ}tCM4r1 zf6c!&H>ghTQM94k7={OE$L9*BYorf$5t*i|%%`^LbbtcQBpCjf-u8nW&(P}1_GZp7 zXeY}Fd^{y$+T2zi((Ne%?YMifC&H#x0#4P4&z4tf=&HfLPBR zJ`|%#_73bpMd|T);H+TK-$@GTAWSjSGfZZxSzcC{K2}+X)fPbK6u0nK6_evxzw-F3 zM3j+*T4=8E{+vwqtrN-?7DoI{|3l!2KB3W z&nG?m-Fbmg`+(!nE&*s>7DKlp-+A>A-vAO1KtPKnBXIC3MgVJ@6V{{KZNN>cSl>^u zYVq?`5u3C7ti(;$c7l>ZFVI`ZBw{MgADFzg<;XRp2~q+VyS>xnEsOMk6H-Ge2}J-1 z>p{rSrK>Tn^Qp8#4o^Up;7t!$-%J8T&GYTus|d;d9|wHZhbpF%?mW`Z!N%5W&vBqE zYk#by=u`+^+)s7#2`IT$qYj|!0-U?wFKBc(LJN9nZWI#$14F*A!@~d;3$0~!1GmWD z7w&6abdgv=Hz2E3#tSrF!KL?+o}ugy!?8EDy*DuBaxpVTWnQHJL)9vfo)sSgNDroNE*g#Za8=RA~D> ztbUMEkFC!!pDD?{*Y5oFdj5{teq$gbnh8mjn$3Vqjl-M`bd3Z#L+ijcZsL6TcUi7B zhS;FFnc!^aN@StDUfizJ&?0Xz)<|Hehq+?oMTCGS%>*79>|RJ!}sHW_kRi$n1x;0stuH z&!BDT{>n0$tjXIYV1WRmJ%ID0z5c_`9O=WAe>Ow3C}lLLVd z6Mz37)nU!bLYtC?Lv{Ol-If=5>o?V(uS|W{R>Ay42q0TXGp`3Y`JX{x#!K#55>_WM zl{vg$DNcW&M4fs;FHKmM&6K;%&$;=19DlaAcGw~?iG3G_x!lH&^*QL5Zymk4cKbIl ziesFDS<9$z0Nt56n&Sg;T#cIc355W=V*d5$tMk<)4I!|^yk1Rzw{RWRg%?PL4iU=j zJ1*YH%w~*FGTZDT>^%GJ&NJON$7fnqdObGncJ`&IiRLWmX`Td_qeZ;$v9!tfxEk;T z$XI}Ym;fP|p741v^c9yG7~uk@wA`VQg7U@8ue#i0dna>|qR-WvatK!KTNd^eIK{&H z5;F;^tDFFp_G4fImgl$b)Dgd`pCEKr3>f$9FD3npSqI~**cuRtwf_qsm+$)P%D+<7 z`w5vQ^Jx!gvkKAfPk%txv~=~6UC2X1!3#9pu^=Ds5vp^nOE%^x(vT~J=4$`x1I#h2{NnUYJ zcy|YI3F_WFd#`uDo^RVLVGtGiE!V?$-EKRmHGMkJA{icu7$YSE-s~owrw1}@vVZVi zdU5l>!q{x6wzZdT11zbIxp#SF3VDxP%jW7Wk7WqXfWKZzw#%gH#cyK~ZH}!%4{gm_ zHFv2<`(@8775BA87`hsJ^~3tsFZXSD4G2-gSod2%!~Hs;}RhXrHeF8dn~2-Hf@)VrSp%|GXb4}GGTcZ z1MU78vb#g-EGEu?JS{;AM3zh%j~54BHMUE{M*3=a2A_G9w04Kk6Q+bEFpTUNddQ;} zWK03pai9~Zv`{cq3pnZx1+TZCoTU+v?uHou@)KC|NhKRPzd^I}SiMKnZ(r}do9?6- z0PcFtIl0%(8S}SgXjnsx9$$0$7<$Ph-+#W5CudCSgX_232Mb}Tj$Z7Nx*#ppRsh0J zukVLSh6|Dl0R7WaHe7pV@fTB)X86I0^UjQm3~mwpUKNnWvzfs7q#>P5<(WOG>D=g~ zh{#FH^&e4-XR=}*&uKXEsa1ICtK0G?#^@~eO`JMsFTYI_WGs_CQj~B5gif3ccCj7+ z2RUSK+MYe4X%tvfoYe-?;a%P~$S%Ctp!o1TxE31i*cx#@Lf7OWvw}TnPwJURA~xwe z)=i%f14EjtM)=oYSr_vmTml8VWMP8+HzLaZK&;5x-J7EXUQt}9j#(%l#qJdp1E%@QVJBh-jA;3~!HZGenqj=aF`VB}Mw2bMM5|E8+1z3sAn7%9Y<;qQKe%cCAWzH0E5-zlmi^+3!?wuaRbcG&FkKAf}2uSgC*MXF;iyB2%bd zyW{yAvmfs+?vw5UT$}pnT|X{X>Jd+#`wT^PON^315_V8nEPz$qovnmb90u1uRGHO@km*!^Bl`zA6SmtVx~a6x zK0e#U*aikW;dfFPlR_>!882(F>#@~L+39$iYJ^{&sneJ#VlWPSt@I#ckX2T(`RUSi zv1Y+mGm$UpE7Eq@F=iWrxQsT;FR=U<&>xl>dQ|YzBun_q}i~$ zuZOZM<|KH&)a|K@wcakOH!~}UI%q@VwWJ6F*A_sH*Rq4a^(wu^7E!I^5 z+Y$wnT+ZCDji!^bi-rPGqueV*Wc_K%pK~Brr(XB|er@BJWBMNcCV>fQCxlAQ^rdIJ z|L*vpk0zD!sX_Y+Rhl1HOjMpR^EL_I#d8)0KAc%IFh!2ChA!`0 zrEZRd#?cx6TusJV$boGpLq-Q=y2L}75NG|IY#zgq*iWf_+#ug_Ua;+x*D+48elx6R zxsg;RL0G?ex$`^<;^)2g;YGOp{VNBhHKqAdAs%=h$E|w>iW^@v7Z0|bQI^!nl}<|S zQ!SM;4mLl`9c=4rbncUO>Yb(!FK=I^@#d*yIL&{h<}ICUeBDscuB6X7ilegoQ|^^# zZF~AaNUc3MU!~=Hjb!Eg>nMQGAXZhuHVv@}E!ktAB275G92c#H4a->J|Bh zgtiSvE;V0e`S>G$w?B&s{H?rEFRdwb(8!#@<#Cqoxr9O1L1}6=lgmYls&f2JEn#ID zce(@~Z?slbH>TF&(`TtjG-Kw}RaTLFqb$og{HZC$AumTWNG)+|sG^5PI7i?`lgo~s z>RRT*yoFuC%o8fqXXnNCNARSm-h+@AxSkfSvOIj!FDy`eQ>VaV-Nlm?%3PVRj?u?U zcAh5R)09+@C(R(eJ>_7{3q@JEZh+Ya(g&VsqN+b;nyal)cf_uY)^J-Mj{u;(6Mh`t zcV17q^9oJD-Z6#g9+;AmQU?9pqzIm61`|+j1E_aQ>d3)F+r)y2Uz9J>-DP8o6v z^~q#tnA29M#%uD|Y6o2M>YirQvWy3`KWAv#@}Vv%Ib&+IXIwMBeh0mo&xQt^njee5 zn3S><$iu6u%Tj%q@*-5ji>5hJd0BCmv-fHBZBK4GXjm&=?tE4xG*+oM>WL4xUsy9w zW=fEXd9RorF5ikE9xv5{tX*+>6JE^G-}v+5IAu|Y zvXn@VUeu6pDj;uv<^yzyq~a@w&-$6M)G%5F>rnPdQs?{}h&pH$Dx|hWVwU!4`{)S~ z`h^J)hMAEK;xA`3XrnRAlz7va1!_LdhKt4r?l4Uy`MjEcTQhbco?J-tEBp2q_57z! zx)O4EJhJa`8GX2&x{rloaUy6j%_KzV?5oWZm1A6PYl;l>J`&TsN8XTjrk%lE_3cTv zB!=}9ymx850Q|)wy^&JpYo;-ECshBA<1J<_BsT2EQCn$yZu zVs`o%m7y<`csgbGicE&Z`wfx}Z$IAYL=M=sbE!TVaHM>G{zz?BGfxUwbH3k0w?j6C zq9DhG;UNS49v~}e?)oCBmsS=~yuC{rjgpjPTGqpe0nDb23A1S%NZX^bIwz;j)?(&t z-NASb)sK@lms1me7J-zd(lOk}qW zf=^ox;nUW=Ii7!nW;OZM#V?AcKh?{J%7>Q2@%N!NK#kih5AbFCY( zTc^@u9UV%vqGZK%3mM6bYBo+YC8=_q&2wzc-O5bC<#)gsB#eYh#N}nJRIf980q$L> zIdtKP2dfjuCOR+Xwi!Ta%@pN?Ond6srRP$dtBV$U#AYaoZaOT29wahacAc}VHce)I zea6mxdibRC^U^YE2gR)=SIgr`TW46PBwjABYZd-zx{tNQUv; z$?f6pV#pAyQlZ9nogzUv9L_yqr!sb>zO_)IELP?;H5LjC9j~sqrK=M@vty^nu#o z5-B1|>C=wJkM&Mg3~JMp&T1v1Iey6k+==>k%@Pz}v(8sujA1`4rW@F%dR|g@zrQtJTI;NRZ*sf_h7b9H;p>ssohFNNj zS58WGi@g)OW_20c?JyZ0kvNUxOk!B z1}47*J!#z>-Tf}GLTPIm^i*5epQkE_8NGjUS$L2^^PZqrSo0W2pl~G*rlzJmn^%4K z{(+QIJrlp7HyL%3n`zOd3+>5n9UY%Ht$VljabKSSRcKt9gRERis_Qui*@&e~8Zpq| z@>TK80+_bZFH=_; zSEnD$$>@5*(kb$>M>PneUN|Y9tFGk0uGI06E0NJMnox>vQtxgRSnZ$(|FN&@#HD8oY?uGTG@p)Pm$sjb>{LUqejxJZy)O|0@pc#nVNV%+bH!pauOCG zwP53�yOs2WKckuFWte<(eN?m@jdVwJ<8un69_JJQtVWbhQ?&)i%8WYqjp4)?5z7 zX7LRE?WG#$db~N30^j;89%hXaEY(qv{rGBMi*rjPeH%mCv}ECY!FS_>l>5$yC_6+x zZg>f|c{&fdn|uz=FbcAv&c^o`9yrVK=9h&- z0jm^8&RqC%H)~(N$zTm}_ee$8_HPGKWM5#KVPWDqHg&m%W$GN|7d)O7f)2)IIz%ero_2cDQcEsueW{vfrOKWJ)nkAW;J$Mj~k7ce}Jye;MiEod!eQy?eK0lGZ7*(HUrCaaKu}Es4 z0V)hB4+y$}y410!w-cH#YU-Vljk(!pkjyAv`%1~(oKs?pXH<)DMA=dHlfxjgVT%qe z??l7hmgZW7<=7TeXG_AQ$c(%mm5s6ANYgEM-L?>_y-T*n^9p%d=L?2;hA4(R48K)F zWoHs!eZ4=wsJcp9@0`R(yDYw}l=sL+(m6kJVy(m$m*h1}ug_^dItUwn;k#?ifeM`U zTGw8#GmU>36{y-A2+U%!5tQt4!E|=4t-Y}R#ze|ISGNdZ3+8I8YX`USdT#bJdzQK)IM$pL4vK z)n~vQH?T{Q-j>tYG*Mcp?J9F#PPwPK+P9N6fgFY=($N|wwO?AktF~R)yl(u5&oW#` zT}QT@*fC!SHdU!4Z|~A zS?-{_j4bG!c7&*ebOMGuTrGv^D{s$v-bd$bNu6I&Ppkn0P(0N!8YT(=F3A7oK-P1` za_ai|{Mf$QLf{+E#?3nxaOS^%HNI|j?cQz>1mO{Rjyb*jPk^C;or;rB6jv>SM2F8a z+3^CUUfvp(Ql)1aO*6T-b5X-M@~87-9Hazi{m%p-%0BD*AtZg%$c;CGP9A%De8rL} z5pLrUAX>(adN+Iq5U2!d>Nj=uM$)O2g^lQ=wWdGBKoluEi3U_en#T$~? zKk6iQ_;^y3Q(=|v|J$}gyV?8vHjJ_9v{zv8BL_lG@HzFHd!#y_WebWzB$`&t5y~-$ z21@Z9p~W_uVbDrC(o?wOF~;^PxOy-8V^so-=dGaoid3%AoxxkHY_GTYWMOgsfft{I z@IMN=I2odLXwRL5JzI&X;=w=luzvGTQg3?Ing-Mvy~pah8$%0-j^KUeE5h!_(VD{U zSAz*jHntm%yZ2%Xjlcjzg;z*s_|2^TeE*I{wc-pf=+A=aE3Q>n;K1v* zi&Ud2QlY^OVg_A4jUyUW2z(G5Ma*b}Mor2}URguDt_S-AZZvc$$BDcWQ|7U9Ci!nJ zd|2Cw=g+knlKO{Vtl`=!&9xUdFmX4)XMrh_@-s>4Y!zA58LW%_n+aWpu5$nXY8toX zU;682XxxeAW&hHh6&K@nQQp}0)Ai-WCvkD>x$Two%#`%kay*T#N>P&ge4*+6jq;aS z+&^!5@AX|*a-peuc}B3mzjL{#wEMcf{gvonj-?Ln*RM82jh-6)+~=|ob(JoFp9oJ= zA`q(S@R6bZI}n11Kw#A%oWP=SaB}^Rzns7{i9pD+UkL=&e+>k|WgfeU*3`yvP&xh^ zyc-DMT^k6#ng*t<6&PqVz(Ca+-plj)x>W)UbkCR&CvMsnAiSns7~G8_%l=xk)tiYm zyJswe1CIo{NU{u<253!3{>5DgSWmp~HX;(L3A;PNzawD_ff1lOLgFqg|0m9x23u%= z-fbMa^$~|?8xc6rsw?ZRl45klwe2eo2y3*)7q*tLd*R9|DIylFbqV3x=0va{-u>m; zWNXP#>PXvoQ}LtwE;bBEKycbf&L<7 zW}^-643({i!I#)ic-{z^IUG#mc??DpYXyMRSiHRStNaV_i|s)|P-gXOFcj@ckba11 z&M&KD79rpbNGlm^@O;~jz}=+$e^Ycb0a~Kq5Wjo)uPU=MObgWhXKj<~MtoM8b-3f|Z>1AWdyAfvZ0f?%@+ zH1w{cb}E?eCVc`G+cVc0Y8+^3=$J{XZI%0fNyiMVCI~iDFx(3_Nfzmu`DyoN!1pB> zgMold)<@)B?e-bo@RLQ+Fcp$OFWGPZ{_*4PKp3iR0S`%)!A(&aVyJ>RaDz^|cW9^( zZUyukKYdUnMmIr;hOmH<0*NtJWA3K+NVG z8fql!;J$CSlyd85!2Wzeb`Hq#1^MayZhx*k3GfVCgb;BXAM_Wo2jQ~<=wp9%erju| zG_Fl4Z<)YIZFzWb5VN|?-oUsYOaz9-No5XQ;}s%p)!z@O zLu`vDqIAYeymk|+x-YaflmLIs<#i!k8KEdXbr)p2>FCMiC2l`4Mt2>7iH?p)eo<(K zk(T^>qzQ&Y^Am!X61CeXG-VE8J&$Af0NTpDJXk?%o=U@DSHY{z>?r8+MVG+Z z+zO`BR7H%p}j`s&^o;`av(l{wxZ?N0XL7!G=|DS*+BT_2ao_(RH*u4}i z+8j569B{zyc1g!N$Y zd`eOPnuf(Y4<${!s(eYxfTDDS4=-E(u=rW*UlsjC1NBUh`x7WomL9A{GVpLK%o(#y zZJhN6Sip&LNW44qTe_+)gH+eATBAYWGIO5aqQ-0`e0D-)zeO1nOg=CZyYhwJuU%p+ zH4fe6bbCmuu&dT^6FHBYlc^Y6Dem9V5_6fKwgRQ;YmXj1vViqzS!XWI)0aj+SM}c|fUzKg}I9c2+D>Mw)AMga*<`?{i9}Udj!|*hx z<)|aIJytj=Nklx!aa~@Ikfn6AFup3#E6!SgLHe}j8Qk^f1ZN-7+ecCmUqv=RB&q=o z*=hW$ffAl5srlMrsjMhX*6!t0ItRs(}4q##QmF#j@bfd)s> zKYo{vRf`*e{A+_WDG>$zSCQ|ruIfeD(5}PIc20*bZ_yIEIBB1vWHuDA^}Sy&g$F0L zUHzA;E|Z)ed}ia3-s@S+zj(|SMSzJHmG@bHK9|_0*{?CPGLlJ{SJu>!Wf7Cmkf^eY(cfg<-QN^V*ilKk%(xN2TV6xbB_KxE zjeJk_O4yBk1@c#olUT8zqoO^AOWjYk>||?D`slTjJ3(UE5j3Tlqd$j#SL1cB)~xqUz?HF zuK5K}0N0RP0kVgV%Kyw+@;S!>L|~DItqs)KK`Ko@(6ue{CDd{F=xG7z!gnU0;UE0~ z+JesUC&8p0D5znIbm)OhUr++Z8kjk(eV^mis}=wzo=Q+x- zZ;71m3ZWRR;=eNgrx=_hN|dgr-wT=04x>-(dsE#%!CHeJEICCK=8+Q@gia)8Bnl{Q zjaU)eF+~BVX`|~^)K?1uRmqW$inFKNBK-a=M_OVz9^()?f-Zty(4bO~EJ(Z+p@_4} z=1p`}CNc~9tT9;D5~v8DF#ojxyH^i@-^U6TeSw&xc=VLtjfV!$t$adPy8bssONei_ z&lJu=Tg;1{@Xht$5IX{D%#YUSNT>Dx9uX*eKLUly&NCz7ZJ1S>=VV7U8hNW24Tfdc0CwR-J(+cV`a*V7h4 z(DZw9S6NvG`U&`Px2))-{kM$~6=C3pi+o@fCY_LbI`fgsiEG9XyA5?v+{F5!_mucrailov40h{!w<&k?^gLi@DNj#h<}W=Z$I`AvBPAVj3%lACQvQQ`!B-=`9opi zetI5TUt!hRzvN;#iF~E0es~alS%Cf;*Z=`07v2ps5y2#2H6Y$ma{X)4!lwPX|7`aJ zt>L&55QQs#j7hu0QDnfa3{dOHEoAP{7~8!i096Za%4|ep;7I-oZ0Vm>*l*FKv@~0r z@Mj(a$cZ~8MnbrxnimBK#*ol;vGf0dMEBQj*lzoA-hB!x`Y(63(T<>i#Nq^Oljz&^ z#-ytvM;*_s+^P_E{xK&Wb(b z$t_wD*XPJXU8ldyMSn2pA1#l1P1DR_sB2chSOc{> z-(R)*f8C8E)YLU9uZkiuYBlhZds;~aUUG2qI9S)R_{v3|=c1{Nmjh|@soq1R#cY0g zK9p9JrD5;cofC`J&abz;?+W2P;~5@e&QIu!nFlA7&W?zGw*TO}SRt$!hgS&AeDb5m z?w&ie7Gq!maOBsIcOoV5vK`0+@Jo=&xhaM5Po2+_Bad6fMFKuvzl8mtBtgyADCh4r zWwVZ)na`fJWR@?@=k3${!7cqp>&J`ZJ7EjLqdkET(2?nkGr*|F-=$*r$4P1^fw}cJ zgT;yS!b$h|AxgsXC-xl-I;mOa2h9Fty|3p z$Ijh+J5`u0cu$kcuAOV~Nuqo$7Ka;vrO3g<19Y{_3X>&k`E}84(N1^T*^4pc8fe5v zX_=%Z_5f9<7cNtJ1GPH*9JI(c;x@2oINYg-U*^1MV2l6qktv^40GPkP9FgG6L^%}$ zbMOU~4Pq#-k{MkHS5#RjkzF!^s<|k4RktPy*~1K?S(dT_qL+^8O;0*te>W`OQGrh)Z&vY&&B&cyjEI|)jNprj+v2Tz`|P= zzASzo9qO+Gy3ZC_Fl#XBx&To~Fmcj#kB;qwZs0KK4Tk=+MGK4H{17zPS;EnT8YLIJeZ!0AR{D_O*zKM9RMT%DZR3ws{8__jUA9c z(z`*|UN0W1#ecB5)*FWEjDXba4&>TY1OoWIthdut(Iruj414B~?)>Lx?|SyxyxrA} z4zlmp%l|pg#BL)7oR;c9C~B}>@`Y0M3NpxqQ}}m;X{Lr?1qH*Pj38>)cHxkRz*tAgAGCj z(G6_vBk40~$|LRN&QyAYY9V`e@+2MIOC;3v(VvkRt~jg>SUr$cERSoc|>CMy5C} zFaGL1cXDzF-PVEMV0RJRZ*sSzf<%9t-(QQ`hZqH)E#6u9DDb)Sc}HZI&DX*OU)_#b z0OVXptQE5YF5jsts(pX$a(jtlX73&f9V~f@+E)Zg79;duMk=>{RpHCq8{XJ2>{NOB z_)jHZO_&x9J3rWsnCR|&JEhW7G~mM3xjwaganjNv#^tBiHZ{#bUry1#M(w}d2Q@R9 zAQNzaQG-E?Ya@cqXu+PPU*c(eUm*X**k4B}u!BQOc$LKH3hY=-&GF4&iHXpuacS4oA8FV=*eN#EYsPu+)p#DJz52S)Wn~cX&SZDM%6( z{PO?_;E9cD6`An?gA3^Qzl%f6+rUA!2>kBy4wV&+Mt9Flc6VT>oTTt8-uvw!aD*LB z{J|h>A}&=n9XWmkHBn!%;cZCuQGP2@iP3pGUy*VNepixtgzhre{H{!3cG#mS=oQWR^{Bu>+LB%P`kf*83rRGNhS0Y@?7 zs7j-ADhlsh^Ak<^AL1WGu~9+KuSlf*seC-AaT01b!@KWoK`9P>CV-@iRY2%)B@jAX z(|_x5A+#wf>oYQO#Wc80#PZFRzZKJz2~Ceg18yFS%j`&KC%m%>eNRfOMuZ$fTi1$W zn##W_rVanD1EQ>{GEumx8~ZGWkW4cJVKzY!;xhYNi0i*x>tOFY^0#Cf zCn3b8Sx6G%TJp!?MSlqe8DZ+w8b z7R?Ym`D=3=x*Texz=K&{e)~C+Ufc^lw8GNvhee#QOI!Ym+gu{j#U79JaGQrv|DuCtdrIy=Xw3zo!b1HB6XqXr?Df&;MnG)Gpw zA%);7cV)E3Qq%O(_CCl-m62wfP@ZDK%}|GlHtq;%{jYMz+qic3Q(1WOfAbelV=Me7 zC;s;4^;hCZ*L`&7u(wyy+bm5v3flDO0pXne>M=#8FjGIzIBlQTS){k29N^V`mR8Oe%5rgtXVyYDn) zRY@ObIE@msjJTWx_xXpaC+s1SOTTr?!j%X%5jqu(d;>hBKN!NKe}W6cybopP4qH~- z+qI;~ptil=aU&EMN6%szBfDsm{0q__R7j#M)2Q$-fO3&F>?J za1C6qOyIsvAot40u?tDntP0g>d-bmG0ub?TEP-tMEyaDnm$O;~5Ba&6m*N}oQtzf{ zI+ovxC{-#8)H_uJ)#i7bw6>M!lG0?ad z|8n2@%?*DFA`Gy_R7nA=0K-ex5OMNvEqRy|!;5kf$)btHsxU1rS_hxPv`FKbJ#yN4 zz=K{5^pc@1JSY%6q1lV-Dft+Hg|QF9q? zNCojFQ!j6{hP3ucU9mLxVO8V0e&oXCFO|0bh4p4^r@*fEZBANk+Dhak_YmSd$IZcO zs3HwEXt_$EZ0iecA-;zQ;SFs$9hkLN>3PiE7Wv%VA;xkPjn6tn$p# z{;b@94fsg>IC|C#uLNK*|0Eg?lNvX4vNYwD$1g>H54}ZO30L**cZlTdpm%LWY-)rk z7K2K+d2qVH;ePB`&Np5LwOR|B*YH4T_?&Cmt+|GB1s-@Tqq<<}c7pOmF92#hn>C+n zMEs~t3e0GZF^WH;7V_ngKIAeCzZ6RS_wN61dmgQhSWWbska7b@8-5s10vbiz|8Jh< z^hpSv(a_g?R@v}tG`HTy3rslVDrEH7Q&MC%?j!1}9vYd-&c`SnDqEf03lpP=&*w#^ zQ!CGpZ*Gu$Y-bW?pA##YKMj+f4LpR)ETuYD*pc41no#TsF1PDtj57(8+3+K%JW65B ztJ{Fhz*Fr@3jK|wFgGA-%G$93bA$Jnv3M+gbinmaf{6#mD6EY({Ess2mmT-7?2RgcfmoyMnlca)4a$zIkyim2KIwOW1eybdXstg%8GYG5LcP>^|V zSNr|BwNAt`UuXl0wy4;{O{2+ZQ*wcC%sFo%1xFFV|3&qSWx{chN8ejFN}00n_^xy0Zu zlAS0)_YTM$5H2E19gfIwQ4-l`o-}8qY*a+9f;j&R3;oMqfaxG`9sKwBijVwuM1YxI zXaJ1D&5dvYTv!OSomXZqhYv%4Aljzo+{w&t>pD&?>rQ+QvjF4IS!93mr0)&kieBA# zGlL=fFps)+-!y}zQN7R$^RyW z0UyZ*HheAeM%J(Lv4RVzq7t~hrqwmv6zvGuBn+9OtMuw|TV_N;^URO$LFI*`XL=U- zow9=ka{&Zjxb4{c7pK0xSnF%wGWonQ`=SHz5_(>3;o7IwpLk*(ZfARsb(;;Y(<(X! z4&v8gT&OMzgQ;0Byuj7#&C<+8AfA(yI2dn3oV4Hk$4MKCrlw#1M@BvI<=?J5s}aJe z)^mvU%MwkI7c_4kX&F3_Kv1RUjpP00Zijnq1xMj7@_LtY30!khxJG+~bpZmUM`USg zhnu{i+;*^u2E0S}Rbi7GaXI;|q-M^n+cFU1?Y3~{1}uZUYWbu_qb6P$Sbn;){S+xI z0bj1ra4WOtA}ya$0H&=2=7^&WpkE>>HX22PFPls&nrjFWjc4F7 zD4x`=C`0zm33tfbHiB;H!H6Ucb;?S1otmoH-HX&B3@db(zVBIfVi7`06ooKB{;nr| zb2tD|qd5EB0dCw&V9`F;0z*!acS=JoWEc5;UNq(f3t)Bo3GVqP0{N3{{Y(_b?~j6D z=N6Xdu?ovbJ;ZdP$Tm;&3zBG6=M$}Ad* zIF~aH9k6K)hOTyo2e#l0OJ(xaXdz~^*r&Zu)wJw84PE++LwFRuyJ2w;*=93T#wm-9 zN>gkHB*7!=TGpjpp1oZiU$)i+0z@B;y-sH`p)K^i8 zQ=i5P&uf|UCv$p7sN^-=Re#IXCfj`c3tXPQ($)q|njU0~`&HuU(G33RYsgz83m11B&oH-1? z+|JWwiS!aMU>%qJfv{k{DP-OrYPsql@8J;?k>BiLeyoyN1oD8+W@FilEbM1=kIM4b zY^ez`nzC_OjeI$K;R00MzD;^sXlZuP0)lNiP`-o3qG_rhw1diy_-Co5dDVcQ%6XpM zsnLnqH0NtuSvyqUYUr^>a4S1!44kJ@sLlywQBIOGIxWXqC)ok3G_p1)IQKr)t@9xd zy$#DjMau9EOGt;#b)AoP`?=bKBfj9CoBE7?Rlc<6ZGI}Ie!AzS7krbwqv81AR<4_A z^)Tky01SC88*Q!4_>JKjSSMsgJBU3e&qf%&Wd*O(UgG?jmAar=wbW0Zf#}f!;`2p0 zfw_GQjzNwr+)SrING0Ib^E7Ord(UPH=2=@CgAuPiB=5+B&-zGyo0N)rLG3ha>YW>M zuZU+(wrM`kOq9m8&|!@Yg@Xs<+uppNTMT)pjOj%1^$F&V z^VS@!^q{25^fc+Iae8P@Ak~FStNUZBesz>DwPk{n<`-a6`G+cAV$O6fkv>HZF8r5w7=lZlkUVqPXPuT2ZHeP6XFvwwH#-Qtj( zr>zA%CEnd=`+s+_^NhEKhM>tip&ve1`;2Ztc66~1iI~{2-{FJ%#Q+%(3;u;BAzJmkNx+2TJpB?N*4a7HeS+(oQ4|`Gip66t{^~sCbV0|>)9=rOnm!bA$T|08t z>=1_dJ3_AbEAFiS0oSd;`Q_47#{1LS4ky_Q`@t_v|RDnZp&xj1-K|~>6!oA z_+)4V4{uO!+@w1F8SypZy3Vf)?X>wySU@l!hObIlHz$~rzB7Sq@$7y()ys9^^X%tU zq{Obm?KyJMd_;25%r%s%45rOIIRns1C-BfrUUMhYaNbhqtnLd&W!1WIWIpsJIGtOK zBZ(pMR#=ncL^1$#%KTxu=Fp{xX?FpQ6?}x3TiY{Tes;cDUO>uU{lXgB1!uFz@0XQ* z+v6|7mL#UGTz$&q1GN)ku(m3-Ottnp>IiKdnoAGBbU-1ZyZla5l0*kSc9 zd#vTN_t%mt`Z-qm1SZzh$tS)Q@o(u2-x$+I=5LYgh7EO)5g6}}#FdO{Tbm=)9?Ge% z*Gg^o3G*JDzbr?t?xd-^(@sdr}tnZ1M_KT%|)TI6vUhB(aCm zpk8uHmH9YmZCMsWy6c78)q*;~+Mx%r52jBH+&kO@tw42$(x9;BkW@ug4IU}wltP3I zUb_DzBL5iMv&08U+;>0kXvdZXC@X(g0aT0lS)R2oLN#0DFs$VdS%1*D~sGEiDj zN~OQYIX3R+e(vXae!qYBI(NpApX2j+$NEhNzmH&b9dvFP`Gxhveit5?)awf@jCobX zD&M22XbmiKe!hd{YAqwdjE(na6jmtRP`E~o7a-8gVxfwrilY>wJHCyj$MCW(8V^fTCl;J`zQz%}l5h=sl$ANp0+@r> zx`GQsj+W0PuTGk)Gz}>JGFVDZXgz-mTZSxW)+7?*|H-umm9t`zrmq^Ic>V2}0AQZk zv*0_zP>^4rfI)w`y`~NyS3}g>MLvf@*meu%iRNr6|bo%JwZTDe5a%n{>PUSfuWlYFgbx+x<|vbfgmY~livF$FKPHW6Jk@(hE)sP6ro(9~^ilX%O=8Kzr{P{F zX?x~xV}XjwKv?)QwWfXm*Z=0|pkf*Wt2rDd)&HFKoxn@Yz*TCMZF4+qRE^jXw%IM! zmWiQ`bYT_Z$4#VLwhhJ}AQ0_qGs+(J&qeYV`N$gU>+lJ=O!PJc2JTvS9&7EMkyL7Z zuIpIbe=m(06=}8$pf=v1R{IL|H`=bvB0e57BRG_EXp#5OTf;7sAAh|YP<;l%i;os0Th0QF*=ii0Umygjf{e5Dg163PN4n6vSlKLkCC8E2h9b)7~^WSY3GSlu-0O765MIh|uRW z@yqYO7r&G@@wHRZ_GV~MrhWm{ho1aXgi~Gu!5_gUHu~;T7btM$Ox#Tv=X;DfFmWoA zou8g(T_A<0Ys3ZjX4L@;8a@BYARL$D`5fJ0QQYGM@8o?kaIUk`jwRuG;bMY`lo<2$ zR+tO3ftOkNvp{@W&Ud)%!*~k9Aa{z zQ>1MdSlez*9J4|;y;ut_vkMRC5lH~jAVe*7omN3P9i1Lp9VJw(le4A1FsFon_D+*% zK+nP=Xu`&Lj+^p*nito(!&vv1OOZ#m!H)MnOC~zzm^-p%S>(LWw9hH!ixj@dCjflw{3j= zaU2@XvM9-visSG{nxkE(L)9ckfQO=`+o`2Ya7t?M14x=buA~}>${er2R0JY9F0){!d~Luogp;G%XS@{J=Q`>2i_#jg62bW7YxQ{%A9hBs}_G}54|A9)NYp;s%(7k;f*UgI#P6O z@(7?@lB{G7-!^|`uH&XP%5(SxhnZWN&4-aw885-`6r3_DD9*c%l!d%Afk@a|HnM#2 z`P<`iD0MMO1|59<@}VdOi|o(!;?zV#4cYOs{#Q{-nghQ6#TP?mB#J|I9~BO+;XH9` zhOkF32kWnt$Uq~ntOZ1?#;?>O_Q|duAnBJ-ePVCU7AW_k=HmTxHDS*99@eer5`^^R zUg4*5UUD+)S`5mT57+u%TGaWjJF&>JX{TJ-sID*%;r|FrQybF%`>DxEl&_n)Kv7AA zi0!(-BSeZm99In4c)}x!IScOoXyeU3;w>V^JHPd9gqmsO+tz* zdvq#H@1P^?nHF#vtx(8y_LdF z&>U)Lrx|=rtU3@_T^!i-I~&Zf-%+|wmN^m~zwIoEX1f4(bys?^Dpy|1@p=DSCdI6_ zoUrAd_irn1uF+5V6t9GRb>Q#rnATh{LcEU!#g-F3|MLHV>W*z=Vx-8klp0fpdHA`9 z)tqQo%(3ka6mf?VmY&sI&b}2Jz!E0%WRxj^Ya?Z@#m=t>LTqzofA%VGj;FmokMPLG z-%7|cK9#j`L{ZR0nt!9oWcsla6GB+FPT1y)x|#iE!nfy&{KxIL<~}|Nm;@Gd}>(X(nrg z*?%shUU#0Zq6`_&oc;ZT^Q=DIXg81Eap~_DvTT`|jW?DHYD7Uo$h}fJPjs3=y-DXo zA1lgsr$Dro%D)xemngjb8&br@iiUF7%G+;dhxt5tIMMb?zt8%s=Qas5?Ts!%*a^g` zx-wmMeiZlG-hN$+=cYPv3Ga9n<&mJ=t$YAkkm*Xb#!768=yw|7(V^t^;zttUvTA)|h93 zH6}oX7=5wbv&Iw^p;%+GT9U0XXU&{59Gh~dnDa}rtv-wlV)L;uM=bk!=cV2ctGs49 z?+Wu_hPHiAld1$k6Vnr7>Dq0+nO=oW-^#JXUMKP;6rpTsw;W;z3pCMB@9)V zSZp^}X<`mP%SgLBBpr2$UDW zp~bw^==&C?+pFH?C+kF5Dzb0Q!;@xAqvq8dePj{kdwJ|wxV&A~Rkn)`KPgW0(D zLwpbkoAL29?!XgDm~UEo7zHE( z%a}Y1Gz?tYKY4+P-bQ=79oP7pRB%8xBegHu<0BkF9>^v~T49JNjOPDi# zm$J`Ldip2{on^&{0!hG35EE?Q4+QA03=)pfM1KK@gqp|kX}JNT3^$M!zNch8o%f5e zB`vvTj!-xT^jF7$5dHk|E1wU5+Qs&J$R~DO0hIrKy_Z$h-J`r)O#MriXRox#oR&NS zlsCu0p4>fSCY1gp+2M5d?D1tG9Hi#kFO9YSoX1-H*Fw>2vlYF0fxf`&UU z?iDK+QjN?YgG3SzKaIs@3m>uD`0wo3M?)#L%H7B23*zOGoS{x`Ww-qs0rbWrT=_> zR2%U^zqucZjWGR^klYH70mz_jOlB_=D-dAeAV{jK4O;j){|5D)cyqp{khHHSQ8kMo zl3j$3mA`5*hugSg$F6`AZS7h97eM86>mLmo1P2Wh;xXrQAOTut0|+#*BhM$g8r9u2 zFkIyjC%8QfsrAqvoV0}5%Rt%IClP|k-ehl@KIhs

1k^Mt>LLd2xGWir0L3aXTZw$I3f4v8EXWv8nI9E%O3*y>;){rWK|B@JS{pB92u~F z3q%m^G{M9KLQMz=b;Qizd0fsmCtiGuCgWxHWhiIG^nd287*K8Tn4@|qulJysHfQ@5 z6mq7c0OLdv*9{d^0XaUl%qsh1{xXaXp^wl?5ioUn5dx@#Pe{4@U%kNh;nkQAnBwCg zIrLp_^ZUh0KpOI~&tA|ALvY^!w-R_7b-}>U!pAnBA1nN_LtHLbW%SbC#E081_`nz44Ufh!C-sXG zOjdV-U)X=2$aXqMBNfU|lYW8q@1v(ae>M7LTxDcMFa8(|gXw}&Fq=BVyFhhsVAr&$ zpXURb)2ui>nSst+V=O);uY=dz^1IiEn}>IQgs_;jeEN$B#>5+v@ZKk1t6XJ+^0gnI zKE3}m(|10>A6Jdz!?mq$43q{;UYzGSsQn({W&7@Tus{hpg@>KDM)oQhYhtu?fDL%k{Nq&FjqHlVPVsi3 zoi#TJ+?#tI5f!Zn50y#|xTEUu#kQgz$y!kAO$wRETf_wSK$Un=|pYW0EiG!2jQG+m0$)gM5gc8$#zTY}9(e6|1Tsy3g@0rSD6z~=T9@08W;e#BmMx$e=fH9#(0{T|HtN=Z|IdF8hiu+& zgXLG{MqOZ8<6`o;n{#pe^oqvzH|Hp&Pj%*zZ(>J>-#58qMCj` zH<4ua#KX1yEQ;cwmiM%PGe0p6zswAHfRsXc3c$L;-ABP`1(YWe0HVOo@l3pfWJpB8 z1DRBFZhKxIEX$K*eZzjp=3+B0K9-Y?3fZfoh+JT8_@i?3?|yDvyjE^eYIn{XrDauy zd{QBzUOP6`M&$#;AFylwp&Hs)J|}#YdC&lLyLh9BL;UXZV2a1Y1Y-!OkntnU|Rn9A>GCW zqG#ci6@+|Jx2|eJYS}x*shAzpVW|@`KOaozOw3Mj(=*R;CNQF!Ot<=zg4J3unEp^n z=Y)6}*sbCS30sR0+)^mR4_i8f#{V9+$}$WzJX80cE0>_Z;#fDO^!uH~5>ecUwIQ74Ze{mA!-Zn9nX7 zc;kf}&$4U2K4WTqbo9h0pXsv=O`=ZQeGSBf(kd8A!*s($7?(42%yj4BOhi2kfOfTw zP5X}?9s>!X(Zio!9{|3D2e`E6OsMjpOs+gIpcuFb2272bOqDizCoi~$!_3v# zu8x%Pa_Nup+(*+PO-mFD`EWVJ;5SmyavKG#SBR3fcV_Lq9T<*}P#NM+c}t05t%iNy zSdwon+2lI#~9iBu#Jf&}1`y<3>zthceJZ(y$VDIai@cRMeaubVy5_`>l?giLSUF!WWb)#LS#;=m@aMm3I&Y0%r>KQI9cm(kgV0 z8dvP*Gl5Ns9*-RSyKoQKq|XL!Er~eQ(GG#jL)>|-$0#f}j2iV77PvNgomR=Is)ngh>l=v}P1XP9--uYP43 zB#mNIVLr2}qeDnNBokq&V3wmx%mI_fl*ypGt_pwBzbbH|sG~^V3U2bczUpbs270Ow z7`08;in5^K-$d*Sa90gR+W1;wFY*@f*(#1Wm1zTyjyY4)KWHX%j7W_t5+oa^;f`7> z#0;SFC07!Ip8do5zD*!?J-rnf(c@|oemVU2(|?0YC55;L{)aVA&oGylLcF-gN7?j0lw4;I zFL^m&Xu6(%g}@vRtAps7n|?)_g+IT&O>?Pf#TWS5^(k!4X$8MJsGCISO_ZP0$9{d(V4iC?<&HDAcA2UfoN#{3JWV0N-bjO%ob5_d zu@elRjfW$kTBvhNRaHlmZuA9@FA(-uXE*ojes;}591$=>s{}o4&KMz^S@@^< z;zC&}a2a^m+s9g3GY20kG!p%kVbQm(`*+5-kC0tllNFe0S7=lR?)5qnOtG3dKZ9#$ z7#9e(nunIKat78KZkJMl^cgFwNc1hUS(!+bKK^=q2>;CNsd2|~cZt>uoq>_YT3_WL zOjg5N!)x27qX(aWR7ar5KQ#bM%Bp;QqzSz=R*Z*i^WnsbL3_Hi)Cx~~!SuXHi@vybJZ2zB>2Wr| z93kA{rRO$gnd3)P%`_<8{9$Jp-OOsEB_W2rYDh3*qE!-Pv0b)nH!LW{iWPL}6-YBB z7}q3L$iUO!&oR1mI>P;q1joD)4c9POD+9C337yG4iA|8(e|J{c?QB?>k#9SJ59i^6 zUVQo_^G1ObgFpP4!vlM|Y?BuN31q~QGKUDsS;EM}JvV;``IPw?TfP6Zbv&DDDKhAkM3zgRd5-bnTaaJ(0$0qc`=rH(r@ z4ZQpVI``BS1{=}wS`rAXbt^~$h#o3tSAj4h>_&wfz_}H`|hZK&{cUEa#dQo&P>0L^**z9>ELRAU5vl} z{qGQQ5;{p#0rF$RPYkWSqqISc(#V#~wExDPQg#`f5RSI>KiVGS)$gdbru+xUmeTe!pjWRC;DTZkr% zd2H~{8_9~94{xsn#gS=;0`ubEYAO0P7E5tQJWCCJ`oDf~FvL-%i3Hmi)BW41d{&Nd z!N)@`zSgp>4RvsB!tn|J3Aj5lgb)qCb1dpsT$vGYHl)d_HXzuC zCzn}+ToGSjQJDQupWOB;D&N-xV+#K)?310`E)mLwI4=8-u0w1H*KWroXKZ*iGR z{k}yNbPgSG&5{mrgwS|Chpcb^h)U->mkT!2*m}IG;>)ObsXCu^k0gB?LySqkuL3idX1)=fBXnA(W1+!GgZr ziY9{|n;wA;mw)(izHZ`G;}!R}os)wYRQ=tR`enTIqT5N#_1rOYf7u*!+9VKC;p;0dQ7r6B<7;-;+N@hgD`9$@Z6}c@Av4_8d7IX@3W#B;7JhCku(Uu zqZm0(y2XFuogtp^QZoe3d`QW_F9m0D;YYj+N$&3zcW$}bQnJ2fWbnPMKl8-^psdVu z&=zGtV%pg7N0CJ2ZEiCf?W;5B?B&zZnTD4j@{%oYf(2(9!Jn~__!ZUu8f&D|!i=cK!S;G^AVV&90`s{n{sey2G_b@B+cqB#L+x>KOR{Qf@4 zY5mN5q3k_@caQ)TBD08DjO3sghnEK3j>{1-_{#W>^bTJyKDtO51kcGIXB^1N34I(F z0vp$YBls_qxG4VV5ykfmqIyveVY6R&I62K_lP+Xv^RV;QqUv{7o#!S zZ2t&qMsBZavk_`QN6m{=0GNPRL)bn0_=>gWdz4pgAse?7ig&3fZ+-Gl;NGZYx^ozM32{};st8UaC5OJQo~=Y4?ZPFd6+YqKRi-( zp!yQ{5fF+%`K-p`;xM|cm)01y)&e8-tm}L04ax+2P3{7qvdIVoiWw1L2In8mpI`|v zEX@Pef;Hp9bW+DTI@!~UP@W`4?qV?!ff{Ix#N484!x7L)ghAoJYcRihKr(623A4k*4oSP|Z6sf2}q+8B4 z=#BKGY)OXIYjH^jDRF)bnRyQ)tHYqzvAt`a--c-(y&(cMfTf`O7zS~<6ZRK2#QoiM+fXd$BSo*m4apOmrG;kAt zAzKSL3v(3V>3gaLyp*T?r)t4#&LESrT;xvHb9>y9{4K$(e-KJ7l6f%G1ninM@A zit}Ijm!}LxoT3gIDGv@(A#SN9ys?Ld?a*rm?tK+u6SBwy(ayC;82VN(Mmr7oo$+8z#()2Yc|7>M_P0q-Wjx+y|CVgeB)IU;`dnzbUy5kV^lfEkF{IDMOt_g|uKg?i7 zQtX0euMN37+PWXw7J!*sPxFlzj5C#m=KH1S9Ln;{Ns=4@Vgg;vDkRXu98b)GjAkfE1Wm%OtWrjfrA?Qc4e0+>C)uatHx6Q!x z?EB>7Jm??-Fh!*+iB*&h+D>+FTO8Q%Y-R>OzTF$Q89Ren-8|WkPn^qiWMi&dAPwJR zqOOXR==B+by!p1OG|oaViKKsCVNg0EL_z_WF6^v;x7?fMCk03d5)cRNkou92%8)kY z^22sgTyx(2p*!We2M_kvN7zR?=%-7F2j2_k4v_X3!>ohsdjbfgXOUx?SJKefvU_k~ zSa0px-VXc8>9gJb!2G#Ox$r!6VZkGCq37!`b_Y^y|8ENZ4~p;GPCo$DuHvTl3Fsqr zPaG`Eoy#`ZxK45ya1MCf{TJnn3~7MJ#8j6liGXd$D&q>~Dv>cJVh%{)ZkPrt&JUGl zosqj=)%U3!XkL&;^1{WS^OP9!fr1g#c)))|wEL9=^k#qLsU|Dt5Xzn&9-Min(*;OI z6{DZaKkplyaK1*Zw{;!_Wn7c+BqCJZ(b;jVYkv!tgvJ0cu3@zsy+f^h;-Cg~OO(U=@d}d4 z$*&H03GLpEA1H8;cQBbE8!OQ11O`rN0z4;?gGAh}#Kt}8qI_fvNg@MiM$GeS-#z(U z3-smcSRiTM{a-pb1xRoq;VFasp3cn#V;J53ZhK~L$iXkSrzmjVNK=qv8vw$eLp?oT zRxcO(laWUM`$^;!A1B+up1epfAC1KQKX+}=VN;Ju0RWSh#N&OVsAlpO<5 zP#*a`^9;`nk@Nycozbz^8JAx&#s%#^+>1P07usVED!i5iL)(em8Irfe^e8RdO-sIv zSuk7ycgs{#yOE9ry#)$ z2Yv;vl@Vj(E4|SAQ5qtp89}+Y{PA7@Q@YVdE9DQ4^DzAj>k8_$clR zt9~GFG~bK;xF3T*59?_@&LitFIB>Y0r7XeA zB(*|lNR*MIO-W$kmA8TwIRl!1GyV^X><3w6K zrBZOgl$}uF-^7c9e2;)7<%DpQmf#-3%Ra+9f{rrbR>@o8tYv;gHz!yCj(%KCf8s=1 zr>6Uoeo{6MJFdO&8F+I@b?9WllLt2OBS5d;{7>bMsZJl*)vo9a6dvY#hl|?lntrV- ze^cg_!Y%TO)FEW_cd8L-Pg5uDY0JJ}Bb!LE)X)bHJ~PVjup*ylt;}>i4h5OesH*~b z%8d?zvRnsjv#T*#2dLdW?ggcPx-G~Pvd%H4@XUDyE_dt8>38Y&7Ju>m4C8;(GSoeP zkTzxLNZ4C9t-Zlt!Y}#{W#)F1ytzQUh1H1Y5(aBQ+wwU7FRQ&lnzBUcl>e4*5V-c> zy8!MGHF8gEr~MTrGCDvNNj^JAmT+`EQV517$KfOBy^l#+ry#Tb{d99X6KrR^7v4TO z;SE-Ui|>yd07)xSh%eqp8EXIjQJM_@3OM3|6;%i1ct868Huqj24s9?}(4-?eT{E$e zjs&D{5WNPnwBf>GBoS`_Kyulb&Hpht4G9<@m-(qs2^6ybG>i_Do?7AV;_6K4rZV!O z-q!Ef#f{)2@$FHVn7Nr50UI?$l?;G4xQy86djf?@TgBY=kG7B^cO}rA{u2}k2p|wW zzun@bXVjm(IBhoo%uAPE>qkL2V&myy^AimKia)kc`Du~k{?BWW6zeX~GANmW)W*HR zziT1=`rYq9{n!hL*Lwnt{X+oc8X5y=!-0O!G^t@y5P8x4kvpZ4o|E5oIN${51}*hz zm`wmh?Bn-UAXxh+k?V$*CaCY3x&Q=*x2)eU4@^>>@0x?1i=FoNQgomPLAPjA&f^@?^adOI zq3TIV!;bG|w{Pl@;l*hZioO1)ctEbf(*pV%U`Gf2t`1^aJ^b5K(^^QW{+Z<-5B;s6!1w z1)#l=kcA3zuoxe#IgKBT>?$Tg6d@O2fKa&hGm+g$!>d0YhT5ul3Cb`-oiZyOyzlg) zlfeMV8iS;#rvSSIV=M#&Sn76tWQB;u^FTtwYo+6&&iHSVt5I5Ou4`hNVB}bgZzH;H zMQLRN(9qYu78AV^l3~pdK8mCz0^=E%?+gX<=fsJn5!Fgyh`Vmh6T@P5_n*`aQO}r) z3B(2A>TeiG=YGab=aQwz*}4+kQpj~x0O)&s{Onw!xtH;8AQVjGb!;Pb{oA-yMoQOT z(Ai5bu-!ZJ2NJZQAWk=(6(G%g$W~x0sUOk+QMfVD7+U{#6i;Nmg5zkcg8*b*uvpo! z?aZnVKM(-9O8^OwK1hTeQhRp2Ja+d{sckOK>3E&oy`P^k!Jd)e^Tr`*%X-x)oA;17 z1<;Q;2PxVzcj@&^K|n^xMPLyUiSJEn#tQ2*Tn0#=>#X$d03x}*!~Cd3uVesomV{3| z8h<;>zoI*`@ioA|EyI%D=-!HcjJ zh`a0x225l84IT^g3DnT9-+8a@H8&2v@>f{_P+!2Stn()WA`(Kc|r1`=Wh^IJ!* zAx_$73MO*iT1~B51fx_g6RHTdUpg|Ke)kQHd6oU*9ETH_^W(V4ieAWrU=g59g>3jk zUoO@y>iqavSDTSLZwcf>0;I~O|15K`3vLPZO^bwE@-u}EqSFw?6cL?RtuvGG83 zuP;h4$pX4Qhe1-x=oCNu1y&@0 zzTBo6VhOFk=a?2v{e(w1Momx56HKBibFX2nu6lLVZJOh)#k%Nqp!MuJ>QY-QuUV!z zo#X&Xkp8$lqDVjEmTPo#S~q_d=t~P6!9(>}&a8O)T+Q5Wo&bQ>oO~0gQ8EU~DJ}qC zwj|+|Q=Is{;OY2KsEr=UvKiFpmrF`C zOD448p?Ke^CmYO`2>I45H!bcZpqnoG^lV1%Cj1|CNe1X<3&{{%5gZVb3RD|` z$;9GmW9FA752bE2vqn5EW&2COit3R2b#u+#75wg?(|Go6+-+Pw?)C=W{R-=%hmn!2 zf$YWYpT&1(>*Kn6{lASArK2WBu^Ox^VD)^@+?2PY?ts?>tc0*@<1gV1?O?x4D`Y!u z6j|B^6nSqN0uk~tF-WfIEQ;nU#9=tJX`xHIVRQFp(%-3jKq!HtNQze2jZPPMEJ!uo%CD6R-9eVy@4sI$vYJjNrayvigTDg+(P>H$ow?PDdh7FjAHkR&s zXCXXaf(G2z`Oz`S=Yd0ys>84JGoMjaR)@Z1C!|_Mr)l80wfgRF8~v{3BO+S+|c|Sla{)gv!!cc9l;84>hF!$^^n$)%8Fv zrxpjW&%dtlTi9Ff?!S!j?t)1Lvss{cd#hU9t{llprBZ?~@ zO-l5e{as=w6+0@`AzLti%{Vlb*r=|r@UZFqbS~D~WCwSIgN8-;9V&pU@==Ot7SD?@ z{VV2u(GknGuK^j+YBNzZmnX->)bxBv{QTR((t3x{TyY?4?4?oLsM4OtBhKTc4M4(v zX4&kh`lqp{y4~dDmk|pdv9{|&`gIJ!@7UcV>K-d6=jF5kIM!{Z1Y7|Tl3N?^H$0xB zB{^(U@@vxyE)Q3aHDeHguX`d_tDa_nRrprPWX9x}k)9O+v!3mUl!GxKR=j_GA70?D zW}3`f7qZWj!t>_CO}b`svM;^!C=oR2tpQ=vNZ{3L1lL92OR$}hD1Y_ZWS0*Jaop+s zv>#7Jo7M~Ty2KyCz(#;M%794!T24t^@CJM;&D(gp;@8<~ae z^@AhBzJ>8<1T5ty^29L_zlOsm)AV=X4Fb1ggA>DYWO%H%o_L1}sRVyxZKXo|#VqTj z@vORq-ck;>tgOMnjOS*{o8Z{Wu$%wzB3Dn`%P34mO{Fsw$Wu!v<*H31NB(H!*6;54 zX|x!C=Vnq+vNmN{hU9{o96M!1djkaZO5W`B|NY- zS`aBz7enY0jm7&~^#fDVx!*9EPc7faQ+1vhKVx;?hfpARHa}-FPOo)iJRR*Z4(jtx z(Vlupf~kh&O1PVBsLzg@oJs?1Lqnxq*<9J035C#BIaD*59`>|fzkTbzO?^Q>Moz^j zLV1)ac85t|+PtL5_!)j&4oB4Xn7Z$?E=%pa5$2`5 zg7eHt7u#4-deV`zH;t(@ns{&ivQ0>Qu}-g4!W!n)yUfe167MlT*g5r9Z`nbGC_q5c zi-^#aa~t#2p;s@6&N6!AE$rQzRoQ2QJ~FH5!syq*=HnRul7IJ$d{i<<*|q0#iN*Hl z8n2~EwY8aYWX_|0qlWuH;7jsF#kb(O*rOiRjUHz4$(WSZ>%mGvY+ui_~5gjhw2KokP?qiW15qR5jDp*eyK zMC1WcSej)ydv~b8B-~xYrk$f_;_W+I*+pEpR8=wE2{prhQ7H=w$qv7T$#DKJCg5y?1bK z_3D!JE35C5THGgik(w}!`g-ivL^vMkH+D|-FKY-lfwofW(L!z%7w`ody}6%QpWN%% z=I8~C94i@97TFPIDn`}g0_Ob|=}Yw9m(Q6q>oDtjv1NJFnuEPz4wDxCO% z>|SgG=QP_S?dlo0u&bcjA124}oMsNc?_z08w6~+8;mivyk;gY1$O6Jc@XNt+wO>}+(vW&x2Jh|rWK)g9iV*FKJfg8&s8#1 zhlg{|@;Fs|u%a6+y)QP&DOjUvT_|s|Goz+gJ^lk67c<-e@Rs%Lxp6v&M`7&;4F#-_ z@6-c;^O9$E-nS?vbeGWC(r@1}CGe}N)kEc`&$7M>6N2XQ7tIqCYc~d#>%>46zWilr&Bes9_odo8b8~DJM10CD+ta1oojgLr5~*N6w0W)C z-uvV|@v97OZAhe!X^dOmsA~M`Yp$o^d&3BK=jkSs_0KX`nVcr%OeUF3!DDe5yJqzg-ooRT zM__s?49h@H$)VEt+Kcs)^+_*%bYA*s!t+$WymWW+mT{5w3d{bM@rSgdD^YgzGXi|L z#y2YGtJ$@89@SYGDL>}G7{m{$o>)nE3KIse^{ia_^Gr|f^yx65-1oumc;Ilhn<@<1 z_&z0%5zyuWUV@h&ZQ+s+st)ICM zctnVo-iso68hTXE3|p>S0ke~eGL1jKz;U?6Hq~V|^IPYP z!683pjf%vp4I@K)O=l=APbyrJO48p!UWxhC=H4Fg(yKj*0kw17mO;$f*;$GvS0E7> zx(q*Ml&5#+wL*>oLfnnqdAtDrC;v%`U*G|ZDfTx;`YKecfI|4MfA9n2N^-_bc$<)}0yy6jHFu>dQFg-nQsm(s*y z+QE@C6kn)LuZwxoriY!hvO51yPL3UHDUQ9*gPb>;&o`$=19dN2nF1XDv&Sb%A2HqTV|cTX^3bswsN25s z=;0<9RBEkx;E)IOrVb75fRS^o5X$Oa<;ygiATKROR?B32)@zka?a>=OYvT+h`d}TZ zaqa!oWa_5W-no@A$@7T`EFVKm#=OQ8CA`Kt<74P@Azk(+lZBY?jFp$d@mE%q+F9R4 zo=IxQe{;K3y)?M!YeG9cl{}T^^%xyn&>Q8kUb{TYJ}HEX$AowEoq7d09a3!H)Y$PN zt$GNtT@%1SuF=cO5NtbF0eqxKtIsmqPe!dIna_Fa^;nu%30z~V{HCT-En^&3=I(aI z#T>z&M0}r8w01dEO7Scu*<=$3AnvcLZZFW;4sN6oN5_Q`f{71|^(4cN_AHlg<=ATY ze)j4)?Q7n?Q2xD(2D63rz`&64C79g@LhH}4jwWG71?sA^I*yKo+xVQreyz;R2&&dg zMq67_3^HK~h#b2om)7J7#TZ!##M=q7m3@o)F8q?eHCky@epCgh>kEkap!M?NOQ!ih z54xU~=Dk1U5*(oS5vEcu6*OZJ_dsulO9czlS6`EYJqxs_RR669tQh}1KNna7<^X_E>qbKbbae;!BDRZmsKu+)0 zN`8}YfQXPvA}lF!>L2J>F|9d+*=y1Ns?dH3N?HWH=on#bD_X1caryGiFDK)Bd9b?P z73A|`ter@#JH`mr7`uFNy+?uYNkvU6H13;3Azq`Mwn@4606eDztqyu2#?ftt39e}z6%1BcT;$pz-WwbCz3)a1?mHSTRGeNN&D9g&y_ zgaJv>SbMe7IAug#x(pQ_+8935dANU?v7gHb?g#D>qrxGD#0Y(Ek3jny>>La_>pck9 zuJ&FrmV~FsQ5rBINP!pPlxD zL6S$113xY^aL1?CJwY^o)tkGZY)ewF>PJZ891N~3<=k8Gk|P(`BJ@X-Z_tp^lt4+z z+Ic|b8nm?78ps!q1Zrqc(!sx>b3ZM%mRpKDfSk6Mkt{0=o2tYsu7hLk2PODmz8DET zkm-4xsbjWpCq6}jN~(mxo*cS1AamJ|D9aV=L!_2EM>R~o{Fa|rpC9G538c%JnNddw z4lbq5@=3~5k@AmMAy@gmo8xRVd-fyV)1+z}vxep@XJvUbZ$PAXZqle#PHr1-GbsKj zz^L{b6TdX=NqGRLE(zPiY^o`R)@$Fq>b>Y>Vq#L&cRKk%kKQqvdza2q9aG?R%mL{P z=LazL{S37_3>xDECouy}-B?apthaCME&8(q_pDXJxw@ZHgOWno;3w7exblCK7vsXg zS-_`wj5Q{d=fp(ln}ZGC>xnEa1l{7Hu_28bL>KcM1@=SMGp7g)Q#&~#}PRyd4&Wpyxc3_ImV&2Hu}^fN&Nhh*2TwLpZHyuO)1m9_`USOPC{ z3y*ZNJM7PkVbZ*~))X&-+d^Hu-E|2#sXia#yXL!9$(EBl@p3{PU~SCF|McjTG}(td zoF`TCABMx8q*Ko>eT4Jmq5)vaW$e@1odv?s;9zr!@6rV~CV(ZnP5z>cd{&afG6~rb zM0&6%Bk0%-`K~dijzL9CLBgR5Xhfx(+`Bf zoo$%F2fk|6qXQ`2YQN}E29`QSkl{zEfhVe>!16267$Xy`{ zvgzc{y=xg!4!QvkSLOUSK6uq+b>2M_*Yb)@HC7H(>RrhV?mB(&9i@Z{jBH5r9NrZF z{|I~YcqrHSZ@i?UQ&}RV?E8qaheR@#u``HBAz7j<5rtF^M#$KAhGAxG6|yCD4#}2m z$(A;Ibh2jc_qiGB^?RQ0_xYchx$o<`KG$b?FGu3+bsp+@vRtc$l{`rOTl18T*{v;Y zw>~5$A2e>^C51#_hWCY8z&gGX*w`YnQ_b01@Q+R)(t&dhqYcGyCu^JGoPXGtCHPni zx{Xcmn0TdR$7ZN}wGK&;iAgZBz49i-(Y`t1(u_d`q2JiCLD@u8om>E?Cgsw&8!|QueCMCiB?7`RoOi|Za11)vLUwkxot(=!W=t1LuipQ+r z#JF!Pk9E%0zcP~6%_q7Z*-#QP_6X$eL2In9C^Lu2nBoBhuB@v;jR}?X+QFN{3c3w5 zo|_V^76x!&-Y*|ugM*wDvJp-D)Qn>b5r`PFnvClR-!5qz%DcNbwc)_Q)bgW%EW%QE zN={O~=a!ae#`;dlIbb|)U_{>@8nI`|zqStEvmiBK%KqpiI&Yq?&W#U_v1#1rjJ-Si zCRH&xE-sExcCgJ*#(yN!7fyf&^-G#{9Tc{_(z@j809TWWPQy2p#rP$^p`$~}oq`r?W zMCa!FFa{0ZHbXa87Fv^|n4?H=$v8zl#XiM>SzC|sB4GV!?mM1?3XQlH{u#Er8&8+mLFF1az*kx>Ir;35V z%6_#Q@O}2wqnnuc+H)&es@(`W;H4BdPHrp3Mc_Q*c}SlPJ|9B1(l2UcknG<4ri~?q z8?F{f5nVcsjEJV8&CyQQ{z`Ndizwm+c!xdIFKUss9%I-HZ=cpf4{I11kqJseW+-M# zwrBYziz=gbq8_Fs$$R7^O4$CbbeDFllvlSJelOpFjdeTXW%t?|xXUnYCKY;cH^|bcbZ@(MooPu;fU6!8tcR zw25y%WBd9r9Vd`O$j0(Wm^hHr)Ua?$?eE7ZmrftZUI4$~=imHueiht3h`K`0%GY^YE&DpEYVntM+fl-mPC^aE59}!#KOk^_V*W(KT z0%}Zqs6}J^9MLznKZlNjA-U~`!1tOOlho6aOE%qn0+-fPd?2I08wdB3&UFlnBu) z&^a9Are>hFacz4xzn?t{x#ncz6Hp~PtRK@yq+q7zulQKIA0vZ-nl_v!+ z3u)YdyAj2cGk)syK5w6|*y*!>XJFDqWMKI}(cfp3L7a!=0dLS!;@IKNVF^TE9jG7^ z!4fc0#@VzgVgbxkgF>-Tro7UlNzFhk^^__P6D5r1aYg%1VlQ@=>*Tx`5Sy> z>O~7dHTw=UIPNFlNe!yxuYZ*sClmPNm5?_3cTRRh*gdl}_j{U-kk_M%K_R#Q&zc8L%|2aTZP@%igPcMJC3ovt>#1 zZd(vRaHig>fh}AcZcbkGu|B1TWX##YDj}_j01)bXa9HZq15TyhZ@_QesrU@|KU`yd(9{X!$A*FGwZD}`yYNj zV7}w39^<^YNt8SFWH@t-@Y&&E3B7=IX-W0|kQlb>&G!suAQqG~&W?5HE=&!C$4y;> ztvn3A%8x2?VV0dsDZf8@&1B!#b(Yb33h|J61Nny%kh3jq;XdQLUjH$Y1mJkQf)2HE zNz8XZ2FV~2u2rtpVDKQzj%hP$UCT($&o3ui(Vp zdQM6UNfAd*T)ZPcmK_a3+=iW91!W&BWbVNlR7PQ8Q&AH+qxE>Bszdy!o{v6o?t|H1 zluqDcVKLk>q(zV$YHwGLrW3Y;Y;S4lF%py-_KtR-1z{XD`cX&AV>Bx-nus#>8Pl>pyw0go=UVPsV_XB^y7|B zmkJRP+@dEkhOA$3jbC@1eb$$?7y(sep%ob_W1aNVen~~evjAP!WDWEUfs1zd(5i?j zBAS+TouU!o+|GpBq$6yjk)a~loNh-GANvw=+fSlO=xJJX;q~~yBD8?}%OwOYcTT0W zx#)bE>*ENqkqm=Xv0UAwn?9zn(KR>FV8kD9_I3)=nsC2G*ncNrn}Z6L0}SdsNY6ip` zb`ET`?CC0WhFMl~IXa0pAOm;E18h6GSS>=<-j&|7JW8!^w$t%bX*0rEd^*b923*KFCWMJQ#s5$Oy-Sc6d`i~W_~p?Bd(e4z#mTLoqDf^ZVMf73 zO=ziR6ASRvDJ0G5DHBndk#ronctEC63V>r}6@Z>=ei681qW9DP_EGPg7_lu1PiFt3 zxrE#afFqZN`$+}>mXe??ZKUh2QAa#TsB!E2$8D=H;BG;#d1wV`>5SynjcaksGf4DuP+fm^$Qme>Gs0o!z+Ou_; zt8h@U%nJQC!~M0Gv*niP+WqP73&D-EmtFcbj_ke}y_)0&-aY17I+Dlp2gAUKN)#uq z2cQW4dq-};7#*b~oNh8u#h02;0^z1)i@dE) zh3>OY7V7%k+ANj%$(Casxsh#La2Y(bqR|RV0!ZS!6vzh-I-EbBJT_2Oc39p$+XqzR zn(p2P2qTh%R3bk+#WUcGW%PO>vL1+3_%`9sH%DYulCuc8omXj1cI@TUy|1#5j$h0P z#Ekb#flEbS`(xYsB*dd_FDQ&QlHK<-v^HhdJ^uXye5etBaT&+1M04!Btpa=l@57RW zq?2xh!%zhV-2(l(_aS6NR`3*T_siP>N88i_Q)iQR@^1U}Dr?*d8-Bl0Y9|Ef!_Pd9 zJZhn%>^dr)v^S8J+NGe^W2J1Sp4_(7m;cIj$=93hjLCPVk4uasBo`Y7*CAPC+T8;> zp3ThK#|+aW)ZCzUk_|<*2 zw9EAAUr3hIQYi_B=~BM44(ESexHWkh28#j30A@-?gyrqKk6|TjP$%8Rx4dWXQQz=B z-x=An#qfry4alU-TeI6~5`pK0RMuJP{$yf;!f3|5X1IB~r+iZtZald=4k*`x9)K(A z2Jh#%``|Ad6rH{J7WL@>_4w#s=achgJNKb`luvV+=Ra#2Lsbg&PUO#+XiLn2pF=XT zI~?kt17NSMIY}mO;H~Fad)?l{lC>C$iK#g;=ik2*C?Xe7Tdl=V>S+?J%sf&pN=H%W zI@6nOLK9>cb05lzytdZ2gkbVuwi2j{mHqcsBB-*(`|;N!?6(1xE1_eNGb?G z;v@$a6Q$+z-C8f}exLY6>L(`3?F$a(QMo&N0wH2I3NQ*OpeS#A!;-E&Uwm~6==`$s zO91cWttt@kXS;AS(;Q?f8|x&yf6Pr?>*C1xoysy&W9%N!Vzbs{2d-w)`pC`n4duQhuFQ%K*DtD9_}44-~LP=}WK`P9-3S!On0ROZ9_#zMy zn0{O&>bRsUH((b~#H&>bZ4^@(btt9(bK7E(9$sgv#0umf@TwUQ$b9cHThqv)=+*po zQq`;jh!8ttIb)PTuSV+Pi*3b#;Lq89xy$A4@=MTbb_Nlg=dUX}N05x`8xRg;w8X7$ z>?UP!2RdbCje`~!v`B%TAlp4;X5lE{0Rbf$G2zO8Hr=>$x!{(u_8QWz28GThianhT z1$8@RtKIZBwH79~D*x*52fxz!lmV~4mSd1`vkr;_G*JSS)*g>6KxK5SpEoQyGY~CKYpC%u^uV>H?m4SB4(AzvnmM4;{SuWuGFO zz%$htu#`_SyWmr|4sdEzy9{#O653*=F+D&A=xrEBR8IjYFE#I{`kgJt^FRpR(CM*S zWS@Ca5+__Y_jc`jBA<-~&Ve0*=oQV9?Z5tSf8m{=rT>tm9n8Qwp66@kfgV{w($ucO zphuKf?psU1NaBNO;P#(9&sDnJ`bR1~`_}|Ta_n1@E`#E>cwZcc*ZY^A-#;knYVLVD z1Zlfu?HKHSPA4!5(8p^%u*w#{(5i6DMfL;JLk~au2tUq0$9ycAv8pkyEGzizEG1`nn`5}c&DdJ7AorwBf zp(swz!`%r`?a~94H;O?iOwLUgq zipa5cBd3*IR&}4dx<6rf>6_6hVIfFdIReI}MK0R@5h6+@ zDBRiU#^ZQNUv6voa*vvKJD$>prDv(TO!pPti%nPml_lnw!ON57bqyRX?SZzV^HtVB zF=o_slI;lV4+9%-v7K%{C7b~(ZhYeSeN%_D1m?8+!V!*x>|o@yo`~br4u(?~FF)wb z&#o#AiJaaiq7nIi+=4mCVt6)ra7FNmeo@U@WjJOLv3s1lv3k7belV}uiX`NgO=e$+ z^A6u`Sk39TyFg$rvz-RCda_XZ?RL&Qg5?>CGf--dCq@eFUrCg@V)69zN8BrOm-cgt zOe|MU3-kN{y{yYD6@0FRAw|PK1(UDZ2&CiU-c-nE1xzH3pI=)sn#x7|cN#6ct&csf zWwr+bQkbRJ!7}AS7YJt0IOM_~58rW4_xgoD*HN9Ba?%jZ(B#w&ZkP7+?`;`wy@qfR zogAOW$7VvKz`_hO2PQP=6FF@&k=*swYox8L!cV_Kc zF-XtzJ#Kp8m>=huUlz9QR_TW$c=61oWX~${=7bfQyTYQYoPtGP6&4UH8tuw6t1{J= zm3JG`__Zh4YzD z;&-D!(jRv<`o1Q3kX)UO;5|Vgob`PEN~7L1IUZA?iRkn8k|I>mk_Ml@0GUMsNEJUU zgB?#vizAw=12&cngP$j9C-9r?al>C61x7uGlo^t)O94`!@O_{EecXOenzP+r zldp?Zv?$0GBv3-@WW2W6{=TLrXoSNNBTue8c~fAGmIP^V8An7UY* zzFL9WPF{X&381jKqG!_>#cMLxkXA)Sxt2TT?>tRkOhP3=cR#-dR(iPQiRdMn1&t7o ziNml0HH_(by(xs0>>=m;$A25**UAm^h8~=UhFu3|#P71#5z(qXA?diJYF?1wZe(}mtU60(e;9~_v9tt5Yt&4?$9ilJU4QopJ|eQ8VWy8z)QcCSaL54b`ZE3 zXNmv$_RS@6c66wtNvDv~sHlYoI!YDKgS%^l4duK(myt&p6kSrq4?zW8%p|V&P57lt z=4E#`R|WLU=oxjip}ucMo4b^i2Ez?z9lc{B6ip`W-U6*|GrOoaZSIn#FGEGVt-7*? zba4OSV-9e+8AZkPfD6G@4(r=Lg2MHdXFK#KogpJ93=HU?nNSvcawfEHYMpX=<{81; zARU2PtV&yql3_m}up{2Upd{+&;^kNq)5oiEA3L4(Zad^te9lGxXIyf3{9%8`oB+tl zTD7~2O|(xJ8qK_%Je}(`akNFdMSsYm{h4Q!9X>zl?BbjVZwp#4B3D^zG1Bg^`pY~A9|q##Bbojg^D*U zW4A!0uqWWx7p`BA7M`EBo=qbx?DTt6{wqPtR}HHFFzjB#q5an$ zNGFFUeJ4ip0FdwT?OtV^^huuV*BpLiZ!|-%Z@K5^f7p!8EZu^Xj*au2XTabR4B8|) z%Wmfx=1iu81`J17sD*^}oneBySQPn;vSI*pKvKoh91VOOXSx<+#^=Ic@S*F5vNgWW46;V$xuFth#Q zG_@LugqpzhqS~S#ADr;zeJ|0W!=xRw#1rs z_IrF`>$0wi0DRrJ=I5nCF^uaJKY59d=izY~tMq9BC*n-lOz#2B^PbqwpZ9-Snv|q{ z_*cCABMqMxk)UZ+U8eP{YN#FYx2^yM^%mfkI*LOfn52^ntqQGw$(w_w?R2zAl-q_x zxja_m@i2n5Wx8cNYYRUUk=HzvsLQN_=Wn!VgicLdoQ3R z;BPkmvc4(woh4AvJjQxOAg0lUNRmP)IiyH?S?GRH+t0_%oT71zh`_OvAd*=K zQbsk0ggRHV+?yrp6TncWR{ccf?Ev3jZMQ-rOiCP#l6fpRH9Ys*DNEF1L#sf}Y(~M5fv-Uqn?E}JXDH=M zh=}qp-OQs#IQ6LhhqKq4c8FH+%*@C+_LpA8G`#^(XxHoDSP(y^=zjhbr-{iK8euW& z6Ubx%nOlLoc)LP}MPi(n*3r;BLiVg;S@z?HV3ML|_#8ENeh4}-*MeJG&4#tpFz+kx zcTZWy&sy42hVCsY%D(D})%PcU(?;PP(2D^kw%bdW~xwKCsqwUMh&?i1BU)JTt zHWycOSVh1?bYetW6*l#upr4jEy5IK4^qaMf^aAkbZ#K(n=upCt2X$f(a%W(47PS(y@ab&l6~1s@Ep<-j6<#4Stcm;Xk4_hA zU^B)G;yh6?4t=Ry8afr(lWwkz;T>Qge{kgO-FjgoE$t`*bFKDI;ns)DPpXW7kx;2_ zDC=)%AlkV+KPPqUPMe49st=q0N4%sM3jXA_?dSC6qLo|7>PU#(5+=tI!H zZ>6I{wo0t9?oRJL4o1I#|Sg-Y~56T-#)hx>Atr3t$nE4?7w;bH6V0QP-!w=KTj zH8eG@%}L(nu~B$}QFW}PHPdap=afHFK#Q4JRAXf0Y+s@93G_V^(P-^(v*PC;93UKj zVf^f2ve?h!mUp=qQ&Pk$f_j-RKR)ul1K)x4B~rlfdKlbeG;fc?Ea7u_j_Ou##$Y$H z9p84$20a0<8^E4|(ADew47MM*sf#$CuRAdlMaY=?` zAUG`kU1C?uW@YGNhK-t5T+|kV)OygdB76L;yNFwtWP_?rqki^O;Q_!mOlX9#EaU z(PST*fcic_G0e&$_#Xc3F$WaUYNWkZUG!_dtfr^+bahP?(bqLJ{eH!HPQRYZ@rjvd zgX82*NJQ*!8$8tVk#XzAKJ-Oi?AaJN=g$+SqZO-imKEdBp=4EtyZ)XJrny8ez(h^x^I#Ig8#V4GUcTdR_>bq%qE>Q3 z`x7-W)(*dHE7n!Q<#1e@o8nl7ggAfBii4DZY`4-!HtPqkn2i1PPyv=Ex%jMQ`u5T= zBJVJ2ED|3m%B(LC(39s}qkO!?IVVmCBmF8?4nmiE;rEf}?S**EsJH}c*~w2j5#;vT zBpzrG#q*ptEB%tq`?Fs>1*d|gonX>9ddWkW;xLxqWJ?OrE+0?H)`pSlfc^OI0mB?} zd{va-K4QJ(4#Eb#aj5T6yeyW6Yl7(L(U<1H3>sT7yj#zvf}plrth2v9!qG+18RE|= z-C3#=I5E%l^jgiTSMn)Sbkq2B{7iZ3_gUYg5<^YfmGY4yYO0e zJQsgIFCSVW_8^OY)ht{s>L%&rnamn9-kg=0+7TFDraE^^TA$bt4ent@_wM$=ImH@> z3&I)+gzfEuIWf*UHm&JG{(W=*{d~Cf$!a%BR1=&L}vswUFd56eJj>x)ptQuNkX8^Id32Fzj7K znr}La<)wLpQvcNv2rAgl_xG19O*j?MDvt66prRfbFT4nTK0=9*wigK+i(p}Fbn}m~ z(RP55?0KQr$A$q6f5~{JLDi6>d1BluB(>VDmLmK)EPp@epywI?E>`zN1`3?~w9melItwdCV1U0K-Rq_3v-v4neqCBw*9E&u!B zeX&Isv=>)2kA~8PXKH^I`@t(RPUx~+BdX54jJIs8Drpk0I_Jt9+{l)7^irY2`J3@a z3Jm-d-VKHuem17TqFtg#-DfOaysKm@oogo}mCXpWd3nc`x7YZm*POS>l;;3$+J(y3 z%GKWGDHnz=SmRBb=RO!ag%(Vi1(azFG%?en>dwV@e3xx{+``VmDswth*-Rk*hJF<> z$2Xk=CIW5xVnOe&oa>Za$R0KwT@&nb9si9O$V66zQ6E+$pduyd*qI@(s+nIGG<7Om zF-e;K<+XJRW{cPQnI)=+g}B&u`;pDE?-h<%=RwL!PnSC4PCqs%uKaMVP{2)(vwXL6 zss4y?QN;9aM7#q5L{zjHN{1Fseph3QeP7jo5%yiwDnP25{8c>U`9N%IA$R zg74CMNa*aBzpL66&p5vZB!wN4eyT%8ji>RbdFc+ z-tUG@T@%NDt{C)R?zumKuUTVO4v&6C!UkAM8)d#eR%*Bqn;lt|o<;bwr)A=#O=Eh| z^M218uPUCu(us<6(*|h-mox=4{#P;edg!Ker71EK#2$#qujE|P^Geh10iL_S$7o9y zss%7eCKs=E%m*1Uyv05Ik(Jfy8M;sDzH?$Yj_b@<+Gw>P&iAXY1vv7CtQE|7Uw2u`kslfEMw13zzwD5yPYjg&Op&#fgagz1;8L8|E~6|-Z?50?lY zK#N+fNXq~9vRhJD*-3l1<8Enz{jFEt=IV1DsUwbZ2{~H+b5$s(#Hun{dg^wWoq=sy zP1YsyZTq3?aVo~vR(9C63ic6@>dqN(Z&RH)vlz}_IqmJ&S?=JIK3ylAYwTZZ{&-P9 zp6D~>-c~*z8di>xUGk}CNhOB-qbfjAc0Fv9Ih0%|ZvAn@`rRoqq8f?J z;{u1*i}R^L0U!ELg z#@6Je{t5OS;X!ZAYE!(_IulqRnU(UH{@UNfYy|0(n_a-lsf+$=d5rz5&L_#TcCR0* z*}2M3#vj9~B@7pai60wTu|FvrhU8&!vm+F*5Rx31xP5v zXl185;qY^V;V`mIbYawT9`QO(N2E+srStQp?0&E8#hd6JL<9GmU}5nag-Q3lu~uhx zq&%#-m(KfsR;Wgp{@Ju$m$RG#J&)MG9!*|nvrL|ghaR>HfXPOMrEO*((~_@wWW*kx z92POkGsmujtHI%?8SvuaD-}u7fBkXeqWx-%7M+VeUZ$7+7nn9sTQuB|@9>6MoD`y9 zB%fV)fo3ikjNE=JYiD|Yr^g{fLuOA-w>gcfSPGPVraV8PkV$K(0 z+h^apkP0l0ovUK8lE2kO?5R*LTT1{N1(l!nnvA` z9{Zn`;oTj-evx2{9+mwA@&F)gvfx2uz%>-NjAh?Gy<^v2VTGARAgJ%4r3f*yDiK#M zHRk2qJ_)_8r`nMO=7(khbF*C^WtHB8qujB3_M&PUZCt{Gn^=HYIYOrLm6#@dZLLzQ z{^AJq>zxpPy5!uqt@>scj(_IjsI2*M{#DfGXbJ(!GaBn2%V|t>x*aGUe6U_M<7A*w z)AGmea@vpx%I$Bees3HWzr~teaMY6@imC7#@GMXUZ;<8?Ccb8vt6uO4#-tpk)W}S1 zW8LK0ZVTK{T-CmF{B7XWN%3eA8am4GVOK&Hg{z4%T?&(20noEVnB!F2-79@I2b6Uw zUS`y7|2XL62D|Y7u0Lk?7uUNc8eAJO@m*4xr^dqrHFK8NW%qq(OeC_`zEkqEK$9_4^_-4*P z0ACkt@K-B|^KXtM+WY>CDLZzb)B{t$_4X}7TlRpdLr?85Lv)o1;;A|>uv!h{z|fAr z+6HTn=mu@Q;Ees$Ati_FDe*)r8Z~{Q_qWgp%H(?(IWZd|N>3Gl+rmuIR=~t2!R^bd z9ug&5|Fry%H^Kk&FTL*=&yS2>S-1+0+6P z(`J}WUX1`BRy&2UF6RFQa8W1j1HPS!=Ee4I0XH4d1 zkC|_8GZbLzn!N)*3P!;#S~<{Q=mao?blF7yL<&BG%jav9S4H~$>Nio(^edr=ZFk85 z^eGjjXuz-mkDzdRUx;$93?W0CTRc9c3=(xiokhPkh+}I%`E16O4PY85Dp^1z)&bTj zw_W6g&#Oa(tagcgsv$|>McKas?f9qkZ%sDAHG`60wavO}HHe^Eu1sYKnL74s! zGQhqPKwzxQj)lesl_cF}+%*`ZreV^0VvJoigdMFJepLR+cuoyf#>S-U0HMZ2=3(^j z5j}3kOWmY>6*osYrGXx|s8!FfW?%edS-hIhW(iU#Xu*Boa*d#mYb4wTIYv0?Lj2HgSI-s+U$_$)ny1t+B!qbMS7H;r6WK zOYF(JmRscK#WVkL&F=wRA%l!K!0}pk>^MxcU3nS8ypN0BOuoNz7 zQbgBc7o+vvoj@@_edbIZNFreF&Z@Q<;&4vR&U^}#6%p2N4;~WKn?hOZ6U6{0uv z#kgb?(ewdrwA%n?deL8)TS~+Dq^5Ly$K2-AYzt(hK2C^Y!2@SlSKOUSs9jxjVSex^ z{Iro>jA-wG9)m5jiuhq)9*H>!zwhZpl}>8eEe;L-THC99aC(5h2dL5wU5H>_ z9|`zX-&&ZGokrC-u;oK%pm)s|jI~QE9$B8ziItF;-rr&|3#_o%*4UDi!Uh=?v9GH(SPe{Ev`EQEPaH3b%-xe_js|Dyj@+RpNW`T%xr;6f+u*E|;^9mSp z+l~V;RSu6S)P!n0J+49PrzZa*v9s_tq6uhKnY&va%_5A!)Ob`7p!`;jkBws!ubyr9 zxM4PFq88h@>&VrM@2ms-YOxj3gx5b>jv>^q>xAqK3f$nnHBsBlN@7GUEb@g7L&*p9 zPQDbB>0VC7dHtvOIfTziRY(m={p){}`WxK2xNX)MI|hBw;$}KJTI`PW9{`nf(EJ{A zEc8U?D~8;W{&)h?wLEdDLfim4d;wjMF0&u`05`0&_p(P&2>JaehV#A7UEC-M|L?%V z-S>Z{7}Dk*&^5EV5i9@SiybYXL+23Bq^;vvksEhV)E&c=(U_IHAQ3wEfA8fW;_KsX zBPPN*{f-v)xth5Pg8_p8kZKbD1Q)uN|g;iccaV1g+G?pkm2OVJGB!3OIiY#{XrZw$K=J74OQK(r$~h`bbWmap0?ts z@!7xkilA~V3x&1FFmCiNbl5I%th>;c$eY9R5)5fNb708hS6H|TL%!iJk;+eN%=C`M z|7uh^fs%U%sv0gJY84$5uJ%P1X!K+`M}!K9{qm;R{XC~?r)IOZZ1@qmH&d1tn$Xt5 z{Ou1o@+7!zn}(Ay|BvhKAhR*ZCnA@J_czJvvkXeoWh?C_a{%n=6Gp`XFtqvk1<-WS zwq&_XTm05FFcw9(Ye;tO-V$bSAlAfwidTpR&J=f$e|$1z1uqmLJ~ z8LMe@dG&W#+@dq=D88a!C9VHG7;^{YZ?rx}vdf^#YL7d*^1#qM`H!J_=e>x!y#5bek^V!LApU8dgo~CP_CugQRT!kiaFg&zBZd1bOCQbZ5~}3V^F5 z#K+RAmC5*$ptS`pP$MtBQYry>r?jo8)%L^3we@b$XtfZRSg>r7b|T!PX4X#CGgt402B7Qbvuk)JJOzL2mefIll+R18|GKQO2tGbmfox8W-^yBN6CQoR8(pW|I9bZbS^BOY#kP9#J zL|Ipb^}YKK-Rb5)uEVMh8lQQF{wT8;GxQML7_xv?x#pdpmt|ITK^CV1U@iWfm@2f5 z4zo^_8SaHU+9$h2F&+PSgiXLW{@6_Xe-*YN344^k+f(VKfSGN3rls%;BJHPo*-5j# zbMjfjt?-cVK96>iqglWA@~Z}zrJb%3u8(PIh?lN0fFC~a#}8LUpi$gurN?8SxifmC zAZ3E}fN>#b!%p;BKNus*9$o@;=_T(jV?7dy@TDX=owFH1%=hy!Tj35GWXHU*2Jg|{ z)WU9QIsW)*4-K@HM)$+npODSCbjA=Erzd7x&+5cVN#T{A9z-~x7_SFO1y2ni;BpyB z&eW{U6``5Z0OK$WnLvuDwY5?I**Q7#ywPvmKZy;&8m@~nRJ*>cC_18{L(DDhDg|X6 zjvD?qdNw{>ZH8J8)MmPwmORvb7E4ynRZKy|y*ixgeW7?b$+@@RsF#!p7=4)_&dM5@ z$-S|^%Q|){j8*2@OBA^`R0J*SQvT`hr#P^*dml`(7slBe)W+lP|LuNdNXX;b-~O@| zT#c+)H4T|wy6&24;FQSTHgHu$->Gcj9PgTeQLoux`5#;wJT(oQj=*Gy3+1d(R#In} z%hjy%ZbGBI8PRVo+!sQw+^^G3# z>Kf2d;r}};d&>9HvWLHbk~#kYKEZwG;Vy@)gtYm;8D(%M5&w6WT-JeHjx!S`Krg!B z_oZ^Hc;u#=?3hir3J}BycM$FA%Dqa>r1-w4P@wM8uDduuoWPKbnB=o5&AxkIL{ z0|4ko1)(=D0lJ^dvCh2RsI1yIq5G@iL_bDNV4JoX62*1bsftr{+2)NcIbh9((-vdP^IYh6MHBUX`0U3eaMhg;_X;?qeJFFrNa>A9u4L8JP6^JGA6B+fa8@{-LI) z|H!d+o#B=aG-#S@oMfYKZxVRdUk_x{TertJw)Wo!yB(frcIwk9y3Qz-$Du*bR@Ude2k!zUb69DYMAtK0?bhGF zMuz%pWV{(iRH5Ul4m{v{JCHXv!*MUc>A3_^06#ya-a|peKTJLa7c9hwAEzz{Z(SrMX=~M@1zerL`$%X z(K`ZEP_4c`=T@b~LaOz~Czm$y#$2MncMK=RDH_dsiO@qq?0bF}>P2@I0d(<9i0~T6reW^%O zLJq&@_nPmat~pD8h;-%;I!2Dz2noJ3z6!5+^hBY>0yjZZ{?lHkr}EvF9^vv};oc0e z@s2_tys^boAtpd6TuDUAG1!guM}0>#hrusb8tFoSxqPhKWOzpz$rd7c`%|h&0n-1` zJh1VMu{nzPr`buMIy1(>$()_*eburFz7k29Qf__@9ftf^4k!wj&Im^f>B~a1DlXQ* z+JBKa_BQk0I-L>HF+`#N$T~d`I-RCFWz+)loVbO4U(+#EPKZdwVMldedU(7Kj~lKX zZdz_vG==vGdd-*PhXx>-&4G?3`jY2lZ`W-Ew+#d3;rR1}gsCWF!NR(>p6~s~J!dfA1iI)@2o) zIyXe@1<>KQOoZ@T0WH&Sk}9j!Nzhd6Dv8`O13;ugxYy16O8Ub~_*h|EW9m-uzoVly zu@TNI6uwV7ENK^VSjH*FJYaRM#WqsOK7JS-8+t$}2@xQFj>HEzStksja5_%E`2)F0wQ_8CmuruxWa3t!xQ^blJm-$7TkRZNo*lFOW$+IZum<^K zo&xy3kBrcc0Ydv8k>s|oV>Oe8${~FbYN{3lOs98x(JomV<30`zN(p8Yp_J=)ea6YT z-zpjvOW)gsne2pe_`}DKM7KmLuQ{+V$++}N&%pNSCEm7Z@6OU- zD2D4dc|V83*KN05A|HDGG8OsjKfF zd5KN*l@(cyBgq#x>7V8ew&k|t7QORy%JK{IncVUiWJ3N zO_Lz<(=t`+N`wzATnj?#o?WmZmX>xM+Z+o(B-Oxdwe6)r_X)e_xk~OO*B?6-_GPIY zq40bpWFeYNJPH(cN{?dvD7iP>ht!$~!RNzKzu7jg?&_15muAOupwZ*DUu+@)_^o+{ zM>n}JU_7_ZZ$+w})PJg?1?zYr03XviiWS^WYm)fF<|yZ9kKqW#sS`VP?|;yi^o#Ru z0_rce-M^3ZhUi8Jx%QeUDqjY9^|}d59PrQX9z6FGdZ{h@t69eZXIb1JnEc*>WK;F6 zw4{VTX||sU<`Q4kgOh#-!By%(TN((ktk8ii7)$@)-vfMiGANvv=?sg3&V#h`6T`$S zUmpcmiY~cI96Llc+^`#JD{^b@qpny|U)0}faEx$1Q3q-GHH%#NA*dl_Wu%opa1!p4 zrg3)5p&rPQFCUQKlzxEL>s@NjUss^XHw~exE(LNBLN4k^BHTzXb}&dIP7W2oZE1d+ zc~;cfLG=nWN>p`;>|47A7-Z{Pl2Ta4&DJV(0*G`~d7O$95^|6ho-z4A7sMz7l#NF{ z0G-nTxz6=Yg6X+SLDTGsAKRcoaM6RjXN$Aj8BT2w^i9?WallJX_$ypz+Zontv+zKO z0f@T$GQvnV2SbGQdBi&lXR+e1ctb+|fm7iKlbj{3K$R@!-*PCEszBfFUA@}rFVHr{NZ18{QBA|$TdPJ#HCYU* z9+A8yL&%a;oK5_8vd!3o;TE0597vodAXJfj`7#)$J&GwhX7k4;##0{wU!6;8;ZyNs z>EHpAXY8-k?L0sS9zGA+l7jOkJ2z8~SJae)eIm^zLtv4z6BW1G%)39ph$Qi1CVo4tGQ zp%X7;M*ur4fw(}(1533Tyxg&-2TE6pwByAurydyjn7w>tB>s|}ML8OJ04XN{DE7S)T#8=#Js=2)?7S9mRBB4v{jJgLp!ohx zC9-Waq0}TWbkMZLny1YW7IgN}JnfUG&yG7vtcSsm_ZWHu$AJtn_O2lLu(JQN#8W=* z=J(`o{&%qh`rl99gnC?WUk&%rjra}138xa_Wj?2#&y_vEVIuUO694EVO4g--^TgK#ivN<)mR}+xiE2$t`8n@6bs#jtY3c{jdyM~WGkyEn z5Hjf2CITQOc{k_H7vIXiM4)&34BY6JK{yuqwxWq*aLWHjjfVPpSXhmpmh3c~R9Oy0 zx$5sB>|cg_QQnYR4JGtL$8ud81X|>&V?D)J&X_iyokOy}?tut@t$PJ%V3zXz!7P=g zCOnSi2PXP=LHc-Vn6uR?ax@gkE)ldNQz2_7Kf?gpy;J`J0*59pv zmd&iGoN1zs7Cz)msfni&Gu*zLp{#b8zBWs!>*Vql?>WIb@LDLC8P?Vi?x9N$T5zx;twzqyCPmW zN_aBd4Bfnom7Zfp|1VlMU$2fQHhM4rz#Ww!+IDdg{-6x>K7$cQ5TXpC0MntTh@e-K zqWCo8&<6CK&X2zxA6tv-68YmC?xVI^WZv#e9>fx|*l+&lpg`-S#$A9MX(=QhfrDs) zL!7#Ra3}SpK2i?yYw@seA79?nt&iYnLfQ&i3vzE{cAOY2FoZcQd-YtE z@fO_@q6BCQYI8fDAud7sav<+;rdkUq)F8?h3Flne&y4>IgCwrypXIy%9>3N-MfF$T!Y;jxDq+`PUF10cR_6WPDX@pxY!?|~>E{U1cC z^-0Wa@5)i()lM!mC}(G0T=$`qmIs4wAn+v}ma>1JW{T&A9Cc&?I1hU$Hc(==;h(|5 z>sjZrbDi!-idL`s)3K1?eYgz=0uk1coi2gl=P-zel<_;ZC(ENmlqhpmv}pw6j(V z=@U%ru@(w<2aSxRQ4ygWe(#iRsQYpNl7MMV3&__sBg4%7Al2O$1M21p$UU@2giV+D z%xL$~>vs@ZRWsD9=Fx|lkmG;ykK_M>dU?M{ju<7~c~pVOOdbrx(8eAxu6*dO%i1Kz zs@hrZ3{z53d+2vQT?HeZfNSax4qbw=oRLSTp~g_cW~nv|FIWZ{I)#~}+=FsjPx0jM zZJV9Ck;3e`9?#6dx=k`3s()r4gNTfE%Mq*M_xR_~+5^cF83cUd^Ceua8J5VtPmt#Z zSQYdsvsvGEE3C$mDj{ocLFA8NI1@vsSc8W){;Rh+KTS1@!{zvyf^iJW>a ztjR2Q6{ucy{lv``UGVXqy;z8tL==R|j6xGv2B9J*LKAz0G6Q31xtisH@6Es|bPOK2$D=$_3fPCHp-fVWJz(j{M>-$%g@4yLM z*G{TRM~z_Hp#dE2D)qkQT1S4!nA=z$y-`)+ia3tdP;5GVHOe)EsDDZrjT zy<7z)dL6>>b7+WVHV~C#6!4>kekNe>r>;CtFk;vvHue{f(;8s&6JZAT;VyPR8ThAN zdql+gl4YFZ7ALHKIqjGK$_O?2!uxKM!ir&bWO^*HUs(-qDd|6MDFt{O);rV*gxHkK zV3$ar&7f)Q;4l;2mqg`=NYUVU7LV%KOP8|%r}rNFD- z+4`&Hn!jyr8E8+>Csa*UHhqJ!y(ujqeX?FvGnjO4)1!z4s9V`>9J={q8OHzr?)UnPr-vi{-6)Clfjj4OxEq-3 zfoGZvAO&psO`-MPeDfi1guBn}Sv+kFZxNDBCtF%HWvBU$i`;q#a0J&krXjiH^F!Ok4-mz5qc&51zjP{qK8BSty zlC9eXvjKKs8=oo$A_n#^cd|YBc0t}|9A^5G2%Xa0cZ|rFblUXm%e;) z#}#QI1lC2U7p2%idR1Dl$z1@m$j9T63YywKX5S9xnH^95QqOguh0Us7YkO(y+-2zg z+mF7@v=5hgccQ3zp54%xHB#mMfiHBF6#>7wgEC83eMU2bucncSaTbm0<2N8zvHv(e zFqm}X0q%E5T<WOnupd%~r~3+zJ$cRRHIfT-{lTa8tfkrv%FJ?z#DOA{2aG3smb!LZWhD`6 zt8pqm&_!_X(%g$km>@=Wvgt9*28fuNsijCg4(Gl`LU_+9Z98(}Dm8n_1)uZRNC>Lr z=`WNo$ru~XOkZ_b62z7gFK0QUwHQ;2QjX(orh37T)0 z75m}QK&rchuygCR6;Nj?ad9)`YEZf`n2q`%2Lii>Pcqk#=&x>sU5J~;bX5UFt$O+O z-0!t#hFfZ=yhDU6H=k$nSXPK8?I;#t9$JQYKD6daQQL~g1$ zSj%*Dh2#!Nai8z%XS_;$Wd(mG%gZchG9u(5znC3z=>KEy&BLi&-@oA!k+!4^wNyxj zB}0Wmh7>Xn%TP;(W23&4ELN%iB-jLV} z9c8i*N!V4?CtRw1b2jDpmXCkfzCF%s4f|B2ln@&SgoNia6B<&3(<0`3#F)eP+`|TF z`}g|^6fVy!FC`fmK4aHg2ebEnr{1o_^D8*FQHyLI6(Ycas^A}0Icc%6Sb4Rm+4kNs z)6yPEdOGN1zWZ-4Cf?X1=II{x&ODH8jMV<0U2@YjF;~)_z-AtY0eqLBx>*#1Ak!qr z+`Dz`>8ftd7M~@?hUz!pU$jEbz z=4fy{I>N}e#{eHRqkDPX=k@BshG|jVfB5Ox-utq3Sekr3r8sAD2ZT||nBfLXL-&iO zTfn4~^Z#De6@zrt>iLrk8Ygp>eu|~7Ow1$P<Huf;1zKS(-s`X1&Uf7#Mror6oG> zZ5dbsb0_dTf!N?rRknti&ATN!w~cFNe>RCqOtW25IZ5B*W;4!Obw9GIyx0n{p6~9H ze=CUlJ?ZL}zqY$t14tgZXx!N=W1)er5SeV{e+7n`G-Z{;+^Ac(Qjh&6S_gkM*?`eJ zcbqiK&8r*#?A}wTg?3i)4nIrlwFkdq1CP{9-KaIqZD+yNSa0LzTe+L$o$g1t=}KIC zC>b1P>K@#d${d(izfXEJ0_NpOz`VT1Z`xXO`_-#AOV(T-hoVng?{zHr0nPhCG>HN4ICpX>&n$=a+rIbP&gO{l>>+350#Y zB6`~vlDMOBaSNZke6shnvj0WZzY2ypK#P>|XTaZhO*ISP+A?h-M6&z8Y4RvwBPApE z&Q*rvBlj9NgvpSiF6+)^{Z5V7+=Rylj~h=$@5J#sd=EYswM5O&|LLZ!!+#u;(SO>C zrA9Msc3dYy3?G?sobRJB-^^E;)>Nn7zFO?e1wWuMxvJ>ZV5lSL=gNg zhx79${ptU8e%|{i8Plrsg1bpKi$rN;*Do?A6GyM&&l0!kLh``nTJX-wT}2HSKI)6< zEuMD^=D@i>sh%KmM!mJ2xzlnyOkzOyJ=uQloLR2#K`}o16m)2}pS6yP9B7KH`JH;o z=uBdHybDEerDp3VRvd6xw+4gINW=9B?~5OEQ9p1 z{(vcHxi*U)XRrg`TE2v)u@5YVmnyH0$tD{Y@rY`S>pGcR3QYKK9W5)O|f1o33=T1fO}Jk$S`|b)V}If;e^)*MY4K85LVB zXVoqn3@m+dE*>3L^~oQZZ<_ceT=2zoO3}mRTDe7|M1o;=1=aqHzuYagjIIEc~ zA6Y5$YA`*

1k^Mt>LLd2xGWir0L3aXTZw$I3f4v8EXWv8nI9E%O3*y>;){rWK|B@JS{pB92u~F z3q%m^G{M9KLQMz=b;Qizd0fsmCtiGuCgWxHWhiIG^nd287*K8Tn4@|qulJysHfQ@5 z6mq7c0OLdv*9{d^0XaUl%qsh1{xXaXp^wl?5ioUn5dx@#Pe{4@U%kNh;nkQAnBwCg zIrLp_^ZUh0KpOI~&tA|ALvY^!w-R_7b-}>U!pAnBA1nN_LtHLbW%SbC#E081_`nz44Ufh!C-sXG zOjdV-U)X=2$aXqMBNfU|lYW8q@1v(ae>M7LTxDcMFa8(|gXw}&Fq=BVyFhhsVAr&$ zpXURb)2ui>nSst+V=O);uY=dz^1IiEn}>IQgs_;jeEN$B#>5+v@ZKk1t6XJ+^0gnI zKE3}m(|10>A6Jdz!?mq$43q{;UYzGSsQn({W&7@Tus{hpg@>KDM)oQhYhtu?fDL%k{Nq&FjqHlVPVsi3 zoi#TJ+?#tI5f!Zn50y#|xTEUu#kQgz$y!kAO$wRETf_wSK$Un=|pYW0EiG!2jQG+m0$)gM5gc8$#zTY}9(e6|1Tsy3g@0rSD6z~=T9@08W;e#BmMx$e=fH9#(0{T|HtN=Z|IdF8hiu+& zgXLG{MqOZ8<6`o;n{#pe^oqvzH|Hp&Pj%*zZ(>J>-#58qMCj` zH<4ua#KX1yEQ;cwmiM%PGe0p6zswAHfRsXc3c$L;-ABP`1(YWe0HVOo@l3pfWJpB8 z1DRBFZhKxIEX$K*eZzjp=3+B0K9-Y?3fZfoh+JT8_@i?3?|yDvyjE^eYIn{XrDauy zd{QBzUOP6`M&$#;AFylwp&Hs)J|}#YdC&lLyLh9BL;UXZV2a1Y1Y-!OkntnU|Rn9A>GCW zqG#ci6@+|Jx2|eJYS}x*shAzpVW|@`KOaozOw3Mj(=*R;CNQF!Ot<=zg4J3unEp^n z=Y)6}*sbCS30sR0+)^mR4_i8f#{V9+$}$WzJX80cE0>_Z;#fDO^!uH~5>ecUwIQ74Ze{mA!-Zn9nX7 zc;kf}&$4U2K4WTqbo9h0pXsv=O`=ZQeGSBf(kd8A!*s($7?(42%yj4BOhi2kfOfTw zP5X}?9s>!X(Zio!9{|3D2e`E6OsMjpOs+gIpcuFb2272bOqDizCoi~$!_3v# zu8x%Pa_Nup+(*+PO-mFD`EWVJ;5SmyavKG#SBR3fcV_Lq9T<*}P#NM+c}t05t%iNy zSdwon+2lI#~9iBu#Jf&}1`y<3>zthceJZ(y$VDIai@cRMeaubVy5_`>l?giLSUF!WWb)#LS#;=m@aMm3I&Y0%r>KQI9cm(kgV0 z8dvP*Gl5Ns9*-RSyKoQKq|XL!Er~eQ(GG#jL)>|-$0#f}j2iV77PvNgomR=Is)ngh>l=v}P1XP9--uYP43 zB#mNIVLr2}qeDnNBokq&V3wmx%mI_fl*ypGt_pwBzbbH|sG~^V3U2bczUpbs270Ow z7`08;in5^K-$d*Sa90gR+W1;wFY*@f*(#1Wm1zTyjyY4)KWHX%j7W_t5+oa^;f`7> z#0;SFC07!Ip8do5zD*!?J-rnf(c@|oemVU2(|?0YC55;L{)aVA&oGylLcF-gN7?j0lw4;I zFL^m&Xu6(%g}@vRtAps7n|?)_g+IT&O>?Pf#TWS5^(k!4X$8MJsGCISO_ZP0$9{d(V4iC?<&HDAcA2UfoN#{3JWV0N-bjO%ob5_d zu@elRjfW$kTBvhNRaHlmZuA9@FA(-uXE*ojes;}591$=>s{}o4&KMz^S@@^< z;zC&}a2a^m+s9g3GY20kG!p%kVbQm(`*+5-kC0tllNFe0S7=lR?)5qnOtG3dKZ9#$ z7#9e(nunIKat78KZkJMl^cgFwNc1hUS(!+bKK^=q2>;CNsd2|~cZt>uoq>_YT3_WL zOjg5N!)x27qX(aWR7ar5KQ#bM%Bp;QqzSz=R*Z*i^WnsbL3_Hi)Cx~~!SuXHi@vybJZ2zB>2Wr| z93kA{rRO$gnd3)P%`_<8{9$Jp-OOsEB_W2rYDh3*qE!-Pv0b)nH!LW{iWPL}6-YBB z7}q3L$iUO!&oR1mI>P;q1joD)4c9POD+9C337yG4iA|8(e|J{c?QB?>k#9SJ59i^6 zUVQo_^G1ObgFpP4!vlM|Y?BuN31q~QGKUDsS;EM}JvV;``IPw?TfP6Zbv&DDDKhAkM3zgRd5-bnTaaJ(0$0qc`=rH(r@ z4ZQpVI``BS1{=}wS`rAXbt^~$h#o3tSAj4h>_&wfz_}H`|hZK&{cUEa#dQo&P>0L^**z9>ELRAU5vl} z{qGQQ5;{p#0rF$RPYkWSqqISc(#V#~wExDPQg#`f5RSI>KiVGS)$gdbru+xUmeTe!pjWRC;DTZkr% zd2H~{8_9~94{xsn#gS=;0`ubEYAO0P7E5tQJWCCJ`oDf~FvL-%i3Hmi)BW41d{&Nd z!N)@`zSgp>4RvsB!tn|J3Aj5lgb)qCb1dpsT$vGYHl)d_HXzuC zCzn}+ToGSjQJDQupWOB;D&N-xV+#K)?310`E)mLwI4=8-u0w1H*KWroXKZ*iGR z{k}yNbPgSG&5{mrgwS|Chpcb^h)U->mkT!2*m}IG;>)ObsXCu^k0gB?LySqkuL3idX1)=fBXnA(W1+!GgZr ziY9{|n;wA;mw)(izHZ`G;}!R}os)wYRQ=tR`enTIqT5N#_1rOYf7u*!+9VKC;p;0dQ7r6B<7;-;+N@hgD`9$@Z6}c@Av4_8d7IX@3W#B;7JhCku(Uu zqZm0(y2XFuogtp^QZoe3d`QW_F9m0D;YYj+N$&3zcW$}bQnJ2fWbnPMKl8-^psdVu z&=zGtV%pg7N0CJ2ZEiCf?W;5B?B&zZnTD4j@{%oYf(2(9!Jn~__!ZUu8f&D|!i=cK!S;G^AVV&90`s{n{sey2G_b@B+cqB#L+x>KOR{Qf@4 zY5mN5q3k_@caQ)TBD08DjO3sghnEK3j>{1-_{#W>^bTJyKDtO51kcGIXB^1N34I(F z0vp$YBls_qxG4VV5ykfmqIyveVY6R&I62K_lP+Xv^RV;QqUv{7o#!S zZ2t&qMsBZavk_`QN6m{=0GNPRL)bn0_=>gWdz4pgAse?7ig&3fZ+-Gl;NGZYx^ozM32{};st8UaC5OJQo~=Y4?ZPFd6+YqKRi-( zp!yQ{5fF+%`K-p`;xM|cm)01y)&e8-tm}L04ax+2P3{7qvdIVoiWw1L2In8mpI`|v zEX@Pef;Hp9bW+DTI@!~UP@W`4?qV?!ff{Ix#N484!x7L)ghAoJYcRihKr(623A4k*4oSP|Z6sf2}q+8B4 z=#BKGY)OXIYjH^jDRF)bnRyQ)tHYqzvAt`a--c-(y&(cMfTf`O7zS~<6ZRK2#QoiM+fXd$BSo*m4apOmrG;kAt zAzKSL3v(3V>3gaLyp*T?r)t4#&LESrT;xvHb9>y9{4K$(e-KJ7l6f%G1ninM@A zit}Ijm!}LxoT3gIDGv@(A#SN9ys?Ld?a*rm?tK+u6SBwy(ayC;82VN(Mmr7oo$+8z#()2Yc|7>M_P0q-Wjx+y|CVgeB)IU;`dnzbUy5kV^lfEkF{IDMOt_g|uKg?i7 zQtX0euMN37+PWXw7J!*sPxFlzj5C#m=KH1S9Ln;{Ns=4@Vgg;vDkRXu98b)GjAkfE1Wm%OtWrjfrA?Qc4e0+>C)uatHx6Q!x z?EB>7Jm??-Fh!*+iB*&h+D>+FTO8Q%Y-R>OzTF$Q89Ren-8|WkPn^qiWMi&dAPwJR zqOOXR==B+by!p1OG|oaViKKsCVNg0EL_z_WF6^v;x7?fMCk03d5)cRNkou92%8)kY z^22sgTyx(2p*!We2M_kvN7zR?=%-7F2j2_k4v_X3!>ohsdjbfgXOUx?SJKefvU_k~ zSa0px-VXc8>9gJb!2G#Ox$r!6VZkGCq37!`b_Y^y|8ENZ4~p;GPCo$DuHvTl3Fsqr zPaG`Eoy#`ZxK45ya1MCf{TJnn3~7MJ#8j6liGXd$D&q>~Dv>cJVh%{)ZkPrt&JUGl zosqj=)%U3!XkL&;^1{WS^OP9!fr1g#c)))|wEL9=^k#qLsU|Dt5Xzn&9-Min(*;OI z6{DZaKkplyaK1*Zw{;!_Wn7c+BqCJZ(b;jVYkv!tgvJ0cu3@zsy+f^h;-Cg~OO(U=@d}d4 z$*&H03GLpEA1H8;cQBbE8!OQ11O`rN0z4;?gGAh}#Kt}8qI_fvNg@MiM$GeS-#z(U z3-smcSRiTM{a-pb1xRoq;VFasp3cn#V;J53ZhK~L$iXkSrzmjVNK=qv8vw$eLp?oT zRxcO(laWUM`$^;!A1B+up1epfAC1KQKX+}=VN;Ju0RWSh#N&OVsAlpO<5 zP#*a`^9;`nk@Nycozbz^8JAx&#s%#^+>1P07usVED!i5iL)(em8Irfe^e8RdO-sIv zSuk7ycgs{#yOE9ry#)$ z2Yv;vl@Vj(E4|SAQ5qtp89}+Y{PA7@Q@YVdE9DQ4^DzAj>k8_$clR zt9~GFG~bK;xF3T*59?_@&LitFIB>Y0r7XeA zB(*|lNR*MIO-W$kmA8TwIRl!1GyV^X><3w6K zrBZOgl$}uF-^7c9e2;)7<%DpQmf#-3%Ra+9f{rrbR>@o8tYv;gHz!yCj(%KCf8s=1 zr>6Uoeo{6MJFdO&8F+I@b?9WllLt2OBS5d;{7>bMsZJl*)vo9a6dvY#hl|?lntrV- ze^cg_!Y%TO)FEW_cd8L-Pg5uDY0JJ}Bb!LE)X)bHJ~PVjup*ylt;}>i4h5OesH*~b z%8d?zvRnsjv#T*#2dLdW?ggcPx-G~Pvd%H4@XUDyE_dt8>38Y&7Ju>m4C8;(GSoeP zkTzxLNZ4C9t-Zlt!Y}#{W#)F1ytzQUh1H1Y5(aBQ+wwU7FRQ&lnzBUcl>e4*5V-c> zy8!MGHF8gEr~MTrGCDvNNj^JAmT+`EQV517$KfOBy^l#+ry#Tb{d99X6KrR^7v4TO z;SE-Ui|>yd07)xSh%eqp8EXIjQJM_@3OM3|6;%i1ct868Huqj24s9?}(4-?eT{E$e zjs&D{5WNPnwBf>GBoS`_Kyulb&Hpht4G9<@m-(qs2^6ybG>i_Do?7AV;_6K4rZV!O z-q!Ef#f{)2@$FHVn7Nr50UI?$l?;G4xQy86djf?@TgBY=kG7B^cO}rA{u2}k2p|wW zzun@bXVjm(IBhoo%uAPE>qkL2V&myy^AimKia)kc`Du~k{?BWW6zeX~GANmW)W*HR zziT1=`rYq9{n!hL*Lwnt{X+oc8X5y=!-0O!G^t@y5P8x4kvpZ4o|E5oIN${51}*hz zm`wmh?Bn-UAXxh+k?V$*CaCY3x&Q=*x2)eU4@^>>@0x?1i=FoNQgomPLAPjA&f^@?^adOI zq3TIV!;bG|w{Pl@;l*hZioO1)ctEbf(*pV%U`Gf2t`1^aJ^b5K(^^QW{+Z<-5B;s6!1w z1)#l=kcA3zuoxe#IgKBT>?$Tg6d@O2fKa&hGm+g$!>d0YhT5ul3Cb`-oiZyOyzlg) zlfeMV8iS;#rvSSIV=M#&Sn76tWQB;u^FTtwYo+6&&iHSVt5I5Ou4`hNVB}bgZzH;H zMQLRN(9qYu78AV^l3~pdK8mCz0^=E%?+gX<=fsJn5!Fgyh`Vmh6T@P5_n*`aQO}r) z3B(2A>TeiG=YGab=aQwz*}4+kQpj~x0O)&s{Onw!xtH;8AQVjGb!;Pb{oA-yMoQOT z(Ai5bu-!ZJ2NJZQAWk=(6(G%g$W~x0sUOk+QMfVD7+U{#6i;Nmg5zkcg8*b*uvpo! z?aZnVKM(-9O8^OwK1hTeQhRp2Ja+d{sckOK>3E&oy`P^k!Jd)e^Tr`*%X-x)oA;17 z1<;Q;2PxVzcj@&^K|n^xMPLyUiSJEn#tQ2*Tn0#=>#X$d03x}*!~Cd3uVesomV{3| z8h<;>zoI*`@ioA|EyI%D=-!HcjJ zh`a0x225l84IT^g3DnT9-+8a@H8&2v@>f{_P+!2Stn()WA`(Kc|r1`=Wh^IJ!* zAx_$73MO*iT1~B51fx_g6RHTdUpg|Ke)kQHd6oU*9ETH_^W(V4ieAWrU=g59g>3jk zUoO@y>iqavSDTSLZwcf>0;I~O|15K`3vLPZO^bwE@-u}EqSFw?6cL?RtuvGG83 zuP;h4$pX4Qhe1-x=oCNu1y&@0 zzTBo6VhOFk=a?2v{e(w1Momx56HKBibFX2nu6lLVZJOh)#k%Nqp!MuJ>QY-QuUV!z zo#X&Xkp8$lqDVjEmTPo#S~q_d=t~P6!9(>}&a8O)T+Q5Wo&bQ>oO~0gQ8EU~DJ}qC zwj|+|Q=Is{;OY2KsEr=UvKiFpmrF`C zOD448p?Ke^CmYO`2>I45H!bcZpqnoG^lV1%Cj1|CNe1X<3&{{%5gZVb3RD|` z$;9GmW9FA752bE2vqn5EW&2COit3R2b#u+#75wg?(|Go6+-+Pw?)C=W{R-=%hmn!2 zf$YWYpT&1(>*Kn6{lASArK2WBu^Ox^VD)^@+?2PY?ts?>tc0*@<1gV1?O?x4D`Y!u z6j|B^6nSqN0uk~tF-WfIEQ;nU#9=tJX`xHIVRQFp(%-3jKq!HtNQze2jZPPMEJ!uo%CD6R-9eVy@4sI$vYJjNrayvigTDg+(P>H$ow?PDdh7FjAHkR&s zXCXXaf(G2z`Oz`S=Yd0ys>84JGoMjaR)@Z1C!|_Mr)l80wfgRF8~v{3BO+S+|c|Sla{)gv!!cc9l;84>hF!$^^n$)%8Fv zrxpjW&%dtlTi9Ff?!S!j?t)1Lvss{cd#hU9t{llprBZ?~@ zO-l5e{as=w6+0@`AzLti%{Vlb*r=|r@UZFqbS~D~WCwSIgN8-;9V&pU@==Ot7SD?@ z{VV2u(GknGuK^j+YBNzZmnX->)bxBv{QTR((t3x{TyY?4?4?oLsM4OtBhKTc4M4(v zX4&kh`lqp{y4~dDmk|pdv9{|&`gIJ!@7UcV>K-d6=jF5kIM!{Z1Y7|Tl3N?^H$0xB zB{^(U@@vxyE)Q3aHDeHguX`d_tDa_nRrprPWX9x}k)9O+v!3mUl!GxKR=j_GA70?D zW}3`f7qZWj!t>_CO}b`svM;^!C=oR2tpQ=vNZ{3L1lL92OR$}hD1Y_ZWS0*Jaop+s zv>#7Jo7M~Ty2KyCz(#;M%794!T24t^@CJM;&D(gp;@8<~ae z^@AhBzJ>8<1T5ty^29L_zlOsm)AV=X4Fb1ggA>DYWO%H%o_L1}sRVyxZKXo|#VqTj z@vORq-ck;>tgOMnjOS*{o8Z{Wu$%wzB3Dn`%P34mO{Fsw$Wu!v<*H31NB(H!*6;54 zX|x!C=Vnq+vNmN{hU9{o96M!1djkaZO5W`B|NY- zS`aBz7enY0jm7&~^#fDVx!*9EPc7faQ+1vhKVx;?hfpARHa}-FPOo)iJRR*Z4(jtx z(Vlupf~kh&O1PVBsLzg@oJs?1Lqnxq*<9J035C#BIaD*59`>|fzkTbzO?^Q>Moz^j zLV1)ac85t|+PtL5_!)j&4oB4Xn7Z$?E=%pa5$2`5 zg7eHt7u#4-deV`zH;t(@ns{&ivQ0>Qu}-g4!W!n)yUfe167MlT*g5r9Z`nbGC_q5c zi-^#aa~t#2p;s@6&N6!AE$rQzRoQ2QJ~FH5!syq*=HnRul7IJ$d{i<<*|q0#iN*Hl z8n2~EwY8aYWX_|0qlWuH;7jsF#kb(O*rOiRjUHz4$(WSZ>%mGvY+ui_~5gjhw2KokP?qiW15qR5jDp*eyK zMC1WcSej)ydv~b8B-~xYrk$f_;_W+I*+pEpR8=wE2{prhQ7H=w$qv7T$#DKJCg5y?1bK z_3D!JE35C5THGgik(w}!`g-ivL^vMkH+D|-FKY-lfwofW(L!z%7w`ody}6%QpWN%% z=I8~C94i@97TFPIDn`}g0_Ob|=}Yw9m(Q6q>oDtjv1NJFnuEPz4wDxCO% z>|SgG=QP_S?dlo0u&bcjA124}oMsNc?_z08w6~+8;mivyk;gY1$O6Jc@XNt+wO>}+(vW&x2Jh|rWK)g9iV*FKJfg8&s8#1 zhlg{|@;Fs|u%a6+y)QP&DOjUvT_|s|Goz+gJ^lk67c<-e@Rs%Lxp6v&M`7&;4F#-_ z@6-c;^O9$E-nS?vbeGWC(r@1}CGe}N)kEc`&$7M>6N2XQ7tIqCYc~d#>%>46zWilr&Bes9_odo8b8~DJM10CD+ta1oojgLr5~*N6w0W)C z-uvV|@v97OZAhe!X^dOmsA~M`Yp$o^d&3BK=jkSs_0KX`nVcr%OeUF3!DDe5yJqzg-ooRT zM__s?49h@H$)VEt+Kcs)^+_*%bYA*s!t+$WymWW+mT{5w3d{bM@rSgdD^YgzGXi|L z#y2YGtJ$@89@SYGDL>}G7{m{$o>)nE3KIse^{ia_^Gr|f^yx65-1oumc;Ilhn<@<1 z_&z0%5zyuWUV@h&ZQ+s+st)ICM zctnVo-iso68hTXE3|p>S0ke~eGL1jKz;U?6Hq~V|^IPYP z!683pjf%vp4I@K)O=l=APbyrJO48p!UWxhC=H4Fg(yKj*0kw17mO;$f*;$GvS0E7> zx(q*Ml&5#+wL*>oLfnnqdAtDrC;v%`U*G|ZDfTx;`YKecfI|4MfA9n2N^-_bc$<)}0yy6jHFu>dQFg-nQsm(s*y z+QE@C6kn)LuZwxoriY!hvO51yPL3UHDUQ9*gPb>;&o`$=19dN2nF1XDv&Sb%A2HqTV|cTX^3bswsN25s z=;0<9RBEkx;E)IOrVb75fRS^o5X$Oa<;ygiATKROR?B32)@zka?a>=OYvT+h`d}TZ zaqa!oWa_5W-no@A$@7T`EFVKm#=OQ8CA`Kt<74P@Azk(+lZBY?jFp$d@mE%q+F9R4 zo=IxQe{;K3y)?M!YeG9cl{}T^^%xyn&>Q8kUb{TYJ}HEX$AowEoq7d09a3!H)Y$PN zt$GNtT@%1SuF=cO5NtbF0eqxKtIsmqPe!dIna_Fa^;nu%30z~V{HCT-En^&3=I(aI z#T>z&M0}r8w01dEO7Scu*<=$3AnvcLZZFW;4sN6oN5_Q`f{71|^(4cN_AHlg<=ATY ze)j4)?Q7n?Q2xD(2D63rz`&64C79g@LhH}4jwWG71?sA^I*yKo+xVQreyz;R2&&dg zMq67_3^HK~h#b2om)7J7#TZ!##M=q7m3@o)F8q?eHCky@epCgh>kEkap!M?NOQ!ih z54xU~=Dk1U5*(oS5vEcu6*OZJ_dsulO9czlS6`EYJqxs_RR669tQh}1KNna7<^X_E>qbKbbae;!BDRZmsKu+)0 zN`8}YfQXPvA}lF!>L2J>F|9d+*=y1Ns?dH3N?HWH=on#bD_X1caryGiFDK)Bd9b?P z73A|`ter@#JH`mr7`uFNy+?uYNkvU6H13;3Azq`Mwn@4606eDztqyu2#?ftt39e}z6%1BcT;$pz-WwbCz3)a1?mHSTRGeNN&D9g&y_ zgaJv>SbMe7IAug#x(pQ_+8935dANU?v7gHb?g#D>qrxGD#0Y(Ek3jny>>La_>pck9 zuJ&FrmV~FsQ5rBINP!pPlxD zL6S$113xY^aL1?CJwY^o)tkGZY)ewF>PJZ891N~3<=k8Gk|P(`BJ@X-Z_tp^lt4+z z+Ic|b8nm?78ps!q1Zrqc(!sx>b3ZM%mRpKDfSk6Mkt{0=o2tYsu7hLk2PODmz8DET zkm-4xsbjWpCq6}jN~(mxo*cS1AamJ|D9aV=L!_2EM>R~o{Fa|rpC9G538c%JnNddw z4lbq5@=3~5k@AmMAy@gmo8xRVd-fyV)1+z}vxep@XJvUbZ$PAXZqle#PHr1-GbsKj zz^L{b6TdX=NqGRLE(zPiY^o`R)@$Fq>b>Y>Vq#L&cRKk%kKQqvdza2q9aG?R%mL{P z=LazL{S37_3>xDECouy}-B?apthaCME&8(q_pDXJxw@ZHgOWno;3w7exblCK7vsXg zS-_`wj5Q{d=fp(ln}ZGC>xnEa1l{7Hu_28bL>KcM1@=SMGp7g)Q#&~#}PRyd4&Wpyxc3_ImV&2Hu}^fN&Nhh*2TwLpZHyuO)1m9_`USOPC{ z3y*ZNJM7PkVbZ*~))X&-+d^Hu-E|2#sXia#yXL!9$(EBl@p3{PU~SCF|McjTG}(td zoF`TCABMx8q*Ko>eT4Jmq5)vaW$e@1odv?s;9zr!@6rV~CV(ZnP5z>cd{&afG6~rb zM0&6%Bk0%-`K~dijzL9CLBgR5Xhfx(+`Bf zoo$%F2fk|6qXQ`2YQN}E29`QSkl{zEfhVe>!16267$Xy`{ zvgzc{y=xg!4!QvkSLOUSK6uq+b>2M_*Yb)@HC7H(>RrhV?mB(&9i@Z{jBH5r9NrZF z{|I~YcqrHSZ@i?UQ&}RV?E8qaheR@#u``HBAz7j<5rtF^M#$KAhGAxG6|yCD4#}2m z$(A;Ibh2jc_qiGB^?RQ0_xYchx$o<`KG$b?FGu3+bsp+@vRtc$l{`rOTl18T*{v;Y zw>~5$A2e>^C51#_hWCY8z&gGX*w`YnQ_b01@Q+R)(t&dhqYcGyCu^JGoPXGtCHPni zx{Xcmn0TdR$7ZN}wGK&;iAgZBz49i-(Y`t1(u_d`q2JiCLD@u8om>E?Cgsw&8!|QueCMCiB?7`RoOi|Za11)vLUwkxot(=!W=t1LuipQ+r z#JF!Pk9E%0zcP~6%_q7Z*-#QP_6X$eL2In9C^Lu2nBoBhuB@v;jR}?X+QFN{3c3w5 zo|_V^76x!&-Y*|ugM*wDvJp-D)Qn>b5r`PFnvClR-!5qz%DcNbwc)_Q)bgW%EW%QE zN={O~=a!ae#`;dlIbb|)U_{>@8nI`|zqStEvmiBK%KqpiI&Yq?&W#U_v1#1rjJ-Si zCRH&xE-sExcCgJ*#(yN!7fyf&^-G#{9Tc{_(z@j809TWWPQy2p#rP$^p`$~}oq`r?W zMCa!FFa{0ZHbXa87Fv^|n4?H=$v8zl#XiM>SzC|sB4GV!?mM1?3XQlH{u#Er8&8+mLFF1az*kx>Ir;35V z%6_#Q@O}2wqnnuc+H)&es@(`W;H4BdPHrp3Mc_Q*c}SlPJ|9B1(l2UcknG<4ri~?q z8?F{f5nVcsjEJV8&CyQQ{z`Ndizwm+c!xdIFKUss9%I-HZ=cpf4{I11kqJseW+-M# zwrBYziz=gbq8_Fs$$R7^O4$CbbeDFllvlSJelOpFjdeTXW%t?|xXUnYCKY;cH^|bcbZ@(MooPu;fU6!8tcR zw25y%WBd9r9Vd`O$j0(Wm^hHr)Ua?$?eE7ZmrftZUI4$~=imHueiht3h`K`0%GY^YE&DpEYVntM+fl-mPC^aE59}!#KOk^_V*W(KT z0%}Zqs6}J^9MLznKZlNjA-U~`!1tOOlho6aOE%qn0+-fPd?2I08wdB3&UFlnBu) z&^a9Are>hFacz4xzn?t{x#ncz6Hp~PtRK@yq+q7zulQKIA0vZ-nl_v!+ z3u)YdyAj2cGk)syK5w6|*y*!>XJFDqWMKI}(cfp3L7a!=0dLS!;@IKNVF^TE9jG7^ z!4fc0#@VzgVgbxkgF>-Tro7UlNzFhk^^__P6D5r1aYg%1VlQ@=>*Tx`5Sy> z>O~7dHTw=UIPNFlNe!yxuYZ*sClmPNm5?_3cTRRh*gdl}_j{U-kk_M%K_R#Q&zc8L%|2aTZP@%igPcMJC3ovt>#1 zZd(vRaHig>fh}AcZcbkGu|B1TWX##YDj}_j01)bXa9HZq15TyhZ@_QesrU@|KU`yd(9{X!$A*FGwZD}`yYNj zV7}w39^<^YNt8SFWH@t-@Y&&E3B7=IX-W0|kQlb>&G!suAQqG~&W?5HE=&!C$4y;> ztvn3A%8x2?VV0dsDZf8@&1B!#b(Yb33h|J61Nny%kh3jq;XdQLUjH$Y1mJkQf)2HE zNz8XZ2FV~2u2rtpVDKQzj%hP$UCT($&o3ui(Vp zdQM6UNfAd*T)ZPcmK_a3+=iW91!W&BWbVNlR7PQ8Q&AH+qxE>Bszdy!o{v6o?t|H1 zluqDcVKLk>q(zV$YHwGLrW3Y;Y;S4lF%py-_KtR-1z{XD`cX&AV>Bx-nus#>8Pl>pyw0go=UVPsV_XB^y7|B zmkJRP+@dEkhOA$3jbC@1eb$$?7y(sep%ob_W1aNVen~~evjAP!WDWEUfs1zd(5i?j zBAS+TouU!o+|GpBq$6yjk)a~loNh-GANvw=+fSlO=xJJX;q~~yBD8?}%OwOYcTT0W zx#)bE>*ENqkqm=Xv0UAwn?9zn(KR>FV8kD9_I3)=nsC2G*ncNrn}Z6L0}SdsNY6ip` zb`ET`?CC0WhFMl~IXa0pAOm;E18h6GSS>=<-j&|7JW8!^w$t%bX*0rEd^*b923*KFCWMJQ#s5$Oy-Sc6d`i~W_~p?Bd(e4z#mTLoqDf^ZVMf73 zO=ziR6ASRvDJ0G5DHBndk#ronctEC63V>r}6@Z>=ei681qW9DP_EGPg7_lu1PiFt3 zxrE#afFqZN`$+}>mXe??ZKUh2QAa#TsB!E2$8D=H;BG;#d1wV`>5SynjcaksGf4DuP+fm^$Qme>Gs0o!z+Ou_; zt8h@U%nJQC!~M0Gv*niP+WqP73&D-EmtFcbj_ke}y_)0&-aY17I+Dlp2gAUKN)#uq z2cQW4dq-};7#*b~oNh8u#h02;0^z1)i@dE) zh3>OY7V7%k+ANj%$(Casxsh#La2Y(bqR|RV0!ZS!6vzh-I-EbBJT_2Oc39p$+XqzR zn(p2P2qTh%R3bk+#WUcGW%PO>vL1+3_%`9sH%DYulCuc8omXj1cI@TUy|1#5j$h0P z#Ekb#flEbS`(xYsB*dd_FDQ&QlHK<-v^HhdJ^uXye5etBaT&+1M04!Btpa=l@57RW zq?2xh!%zhV-2(l(_aS6NR`3*T_siP>N88i_Q)iQR@^1U}Dr?*d8-Bl0Y9|Ef!_Pd9 zJZhn%>^dr)v^S8J+NGe^W2J1Sp4_(7m;cIj$=93hjLCPVk4uasBo`Y7*CAPC+T8;> zp3ThK#|+aW)ZCzUk_|<*2 zw9EAAUr3hIQYi_B=~BM44(ESexHWkh28#j30A@-?gyrqKk6|TjP$%8Rx4dWXQQz=B z-x=An#qfry4alU-TeI6~5`pK0RMuJP{$yf;!f3|5X1IB~r+iZtZald=4k*`x9)K(A z2Jh#%``|Ad6rH{J7WL@>_4w#s=achgJNKb`luvV+=Ra#2Lsbg&PUO#+XiLn2pF=XT zI~?kt17NSMIY}mO;H~Fad)?l{lC>C$iK#g;=ik2*C?Xe7Tdl=V>S+?J%sf&pN=H%W zI@6nOLK9>cb05lzytdZ2gkbVuwi2j{mHqcsBB-*(`|;N!?6(1xE1_eNGb?G z;v@$a6Q$+z-C8f}exLY6>L(`3?F$a(QMo&N0wH2I3NQ*OpeS#A!;-E&Uwm~6==`$s zO91cWttt@kXS;AS(;Q?f8|x&yf6Pr?>*C1xoysy&W9%N!Vzbs{2d-w)`pC`n4duQhuFQ%K*DtD9_}44-~LP=}WK`P9-3S!On0ROZ9_#zMy zn0{O&>bRsUH((b~#H&>bZ4^@(btt9(bK7E(9$sgv#0umf@TwUQ$b9cHThqv)=+*po zQq`;jh!8ttIb)PTuSV+Pi*3b#;Lq89xy$A4@=MTbb_Nlg=dUX}N05x`8xRg;w8X7$ z>?UP!2RdbCje`~!v`B%TAlp4;X5lE{0Rbf$G2zO8Hr=>$x!{(u_8QWz28GThianhT z1$8@RtKIZBwH79~D*x*52fxz!lmV~4mSd1`vkr;_G*JSS)*g>6KxK5SpEoQyGY~CKYpC%u^uV>H?m4SB4(AzvnmM4;{SuWuGFO zz%$htu#`_SyWmr|4sdEzy9{#O653*=F+D&A=xrEBR8IjYFE#I{`kgJt^FRpR(CM*S zWS@Ca5+__Y_jc`jBA<-~&Ve0*=oQV9?Z5tSf8m{=rT>tm9n8Qwp66@kfgV{w($ucO zphuKf?psU1NaBNO;P#(9&sDnJ`bR1~`_}|Ta_n1@E`#E>cwZcc*ZY^A-#;knYVLVD z1Zlfu?HKHSPA4!5(8p^%u*w#{(5i6DMfL;JLk~au2tUq0$9ycAv8pkyEGzizEG1`nn`5}c&DdJ7AorwBf zp(swz!`%r`?a~94H;O?iOwLUgq zipa5cBd3*IR&}4dx<6rf>6_6hVIfFdIReI}MK0R@5h6+@ zDBRiU#^ZQNUv6voa*vvKJD$>prDv(TO!pPti%nPml_lnw!ON57bqyRX?SZzV^HtVB zF=o_slI;lV4+9%-v7K%{C7b~(ZhYeSeN%_D1m?8+!V!*x>|o@yo`~br4u(?~FF)wb z&#o#AiJaaiq7nIi+=4mCVt6)ra7FNmeo@U@WjJOLv3s1lv3k7belV}uiX`NgO=e$+ z^A6u`Sk39TyFg$rvz-RCda_XZ?RL&Qg5?>CGf--dCq@eFUrCg@V)69zN8BrOm-cgt zOe|MU3-kN{y{yYD6@0FRAw|PK1(UDZ2&CiU-c-nE1xzH3pI=)sn#x7|cN#6ct&csf zWwr+bQkbRJ!7}AS7YJt0IOM_~58rW4_xgoD*HN9Ba?%jZ(B#w&ZkP7+?`;`wy@qfR zogAOW$7VvKz`_hO2PQP=6FF@&k=*swYox8L!cV_Kc zF-XtzJ#Kp8m>=huUlz9QR_TW$c=61oWX~${=7bfQyTYQYoPtGP6&4UH8tuw6t1{J= zm3JG`__Zh4YzD z;&-D!(jRv<`o1Q3kX)UO;5|Vgob`PEN~7L1IUZA?iRkn8k|I>mk_Ml@0GUMsNEJUU zgB?#vizAw=12&cngP$j9C-9r?al>C61x7uGlo^t)O94`!@O_{EecXOenzP+r zldp?Zv?$0GBv3-@WW2W6{=TLrXoSNNBTue8c~fAGmIP^V8An7UY* zzFL9WPF{X&381jKqG!_>#cMLxkXA)Sxt2TT?>tRkOhP3=cR#-dR(iPQiRdMn1&t7o ziNml0HH_(by(xs0>>=m;$A25**UAm^h8~=UhFu3|#P71#5z(qXA?diJYF?1wZe(}mtU60(e;9~_v9tt5Yt&4?$9ilJU4QopJ|eQ8VWy8z)QcCSaL54b`ZE3 zXNmv$_RS@6c66wtNvDv~sHlYoI!YDKgS%^l4duK(myt&p6kSrq4?zW8%p|V&P57lt z=4E#`R|WLU=oxjip}ucMo4b^i2Ez?z9lc{B6ip`W-U6*|GrOoaZSIn#FGEGVt-7*? zba4OSV-9e+8AZkPfD6G@4(r=Lg2MHdXFK#KogpJ93=HU?nNSvcawfEHYMpX=<{81; zARU2PtV&yql3_m}up{2Upd{+&;^kNq)5oiEA3L4(Zad^te9lGxXIyf3{9%8`oB+tl zTD7~2O|(xJ8qK_%Je}(`akNFdMSsYm{h4Q!9X>zl?BbjVZwp#4B3D^zG1Bg^`pY~A9|q##Bbojg^D*U zW4A!0uqWWx7p`BA7M`EBo=qbx?DTt6{wqPtR}HHFFzjB#q5an$ zNGFFUeJ4ip0FdwT?OtV^^huuV*BpLiZ!|-%Z@K5^f7p!8EZu^Xj*au2XTabR4B8|) z%Wmfx=1iu81`J17sD*^}oneBySQPn;vSI*pKvKoh91VOOXSx<+#^=Ic@S*F5vNgWW46;V$xuFth#Q zG_@LugqpzhqS~S#ADr;zeJ|0W!=xRw#1rs z_IrF`>$0wi0DRrJ=I5nCF^uaJKY59d=izY~tMq9BC*n-lOz#2B^PbqwpZ9-Snv|q{ z_*cCABMqMxk)UZ+U8eP{YN#FYx2^yM^%mfkI*LOfn52^ntqQGw$(w_w?R2zAl-q_x zxja_m@i2n5Wx8cNYYRUUk=HzvsLQN_=Wn!VgicLdoQ3R z;BPkmvc4(woh4AvJjQxOAg0lUNRmP)IiyH?S?GRH+t0_%oT71zh`_OvAd*=K zQbsk0ggRHV+?yrp6TncWR{ccf?Ev3jZMQ-rOiCP#l6fpRH9Ys*DNEF1L#sf}Y(~M5fv-Uqn?E}JXDH=M zh=}qp-OQs#IQ6LhhqKq4c8FH+%*@C+_LpA8G`#^(XxHoDSP(y^=zjhbr-{iK8euW& z6Ubx%nOlLoc)LP}MPi(n*3r;BLiVg;S@z?HV3ML|_#8ENeh4}-*MeJG&4#tpFz+kx zcTZWy&sy42hVCsY%D(D})%PcU(?;PP(2D^kw%bdW~xwKCsqwUMh&?i1BU)JTt zHWycOSVh1?bYetW6*l#upr4jEy5IK4^qaMf^aAkbZ#K(n=upCt2X$f(a%W(47PS(y@ab&l6~1s@Ep<-j6<#4Stcm;Xk4_hA zU^B)G;yh6?4t=Ry8afr(lWwkz;T>Qge{kgO-FjgoE$t`*bFKDI;ns)DPpXW7kx;2_ zDC=)%AlkV+KPPqUPMe49st=q0N4%sM3jXA_?dSC6qLo|7>PU#(5+=tI!H zZ>6I{wo0t9?oRJL4o1I#|Sg-Y~56T-#)hx>Atr3t$nE4?7w;bH6V0QP-!w=KTj zH8eG@%}L(nu~B$}QFW}PHPdap=afHFK#Q4JRAXf0Y+s@93G_V^(P-^(v*PC;93UKj zVf^f2ve?h!mUp=qQ&Pk$f_j-RKR)ul1K)x4B~rlfdKlbeG;fc?Ea7u_j_Ou##$Y$H z9p84$20a0<8^E4|(ADew47MM*sf#$CuRAdlMaY=?` zAUG`kU1C?uW@YGNhK-t5T+|kV)OygdB76L;yNFwtWP_?rqki^O;Q_!mOlX9#EaU z(PST*fcic_G0e&$_#Xc3F$WaUYNWkZUG!_dtfr^+bahP?(bqLJ{eH!HPQRYZ@rjvd zgX82*NJQ*!8$8tVk#XzAKJ-Oi?AaJN=g$+SqZO-imKEdBp=4EtyZ)XJrny8ez(h^x^I#Ig8#V4GUcTdR_>bq%qE>Q3 z`x7-W)(*dHE7n!Q<#1e@o8nl7ggAfBii4DZY`4-!HtPqkn2i1PPyv=Ex%jMQ`u5T= zBJVJ2ED|3m%B(LC(39s}qkO!?IVVmCBmF8?4nmiE;rEf}?S**EsJH}c*~w2j5#;vT zBpzrG#q*ptEB%tq`?Fs>1*d|gonX>9ddWkW;xLxqWJ?OrE+0?H)`pSlfc^OI0mB?} zd{va-K4QJ(4#Eb#aj5T6yeyW6Yl7(L(U<1H3>sT7yj#zvf}plrth2v9!qG+18RE|= z-C3#=I5E%l^jgiTSMn)Sbkq2B{7iZ3_gUYg5<^YfmGY4yYO0e zJQsgIFCSVW_8^OY)ht{s>L%&rnamn9-kg=0+7TFDraE^^TA$bt4ent@_wM$=ImH@> z3&I)+gzfEuIWf*UHm&JG{(W=*{d~Cf$!a%BR1=&L}vswUFd56eJj>x)ptQuNkX8^Id32Fzj7K znr}La<)wLpQvcNv2rAgl_xG19O*j?MDvt66prRfbFT4nTK0=9*wigK+i(p}Fbn}m~ z(RP55?0KQr$A$q6f5~{JLDi6>d1BluB(>VDmLmK)EPp@epywI?E>`zN1`3?~w9melItwdCV1U0K-Rq_3v-v4neqCBw*9E&u!B zeX&Isv=>)2kA~8PXKH^I`@t(RPUx~+BdX54jJIs8Drpk0I_Jt9+{l)7^irY2`J3@a z3Jm-d-VKHuem17TqFtg#-DfOaysKm@oogo}mCXpWd3nc`x7YZm*POS>l;;3$+J(y3 z%GKWGDHnz=SmRBb=RO!ag%(Vi1(azFG%?en>dwV@e3xx{+``VmDswth*-Rk*hJF<> z$2Xk=CIW5xVnOe&oa>Za$R0KwT@&nb9si9O$V66zQ6E+$pduyd*qI@(s+nIGG<7Om zF-e;K<+XJRW{cPQnI)=+g}B&u`;pDE?-h<%=RwL!PnSC4PCqs%uKaMVP{2)(vwXL6 zss4y?QN;9aM7#q5L{zjHN{1Fseph3QeP7jo5%yiwDnP25{8c>U`9N%IA$R zg74CMNa*aBzpL66&p5vZB!wN4eyT%8ji>RbdFc+ z-tUG@T@%NDt{C)R?zumKuUTVO4v&6C!UkAM8)d#eR%*Bqn;lt|o<;bwr)A=#O=Eh| z^M218uPUCu(us<6(*|h-mox=4{#P;edg!Ker71EK#2$#qujE|P^Geh10iL_S$7o9y zss%7eCKs=E%m*1Uyv05Ik(Jfy8M;sDzH?$Yj_b@<+Gw>P&iAXY1vv7CtQE|7Uw2u`kslfEMw13zzwD5yPYjg&Op&#fgagz1;8L8|E~6|-Z?50?lY zK#N+fNXq~9vRhJD*-3l1<8Enz{jFEt=IV1DsUwbZ2{~H+b5$s(#Hun{dg^wWoq=sy zP1YsyZTq3?aVo~vR(9C63ic6@>dqN(Z&RH)vlz}_IqmJ&S?=JIK3ylAYwTZZ{&-P9 zp6D~>-c~*z8di>xUGk}CNhOB-qbfjAc0Fv9Ih0%|ZvAn@`rRoqq8f?J z;{u1*i}R^L0U!ELg z#@6Je{t5OS;X!ZAYE!(_IulqRnU(UH{@UNfYy|0(n_a-lsf+$=d5rz5&L_#TcCR0* z*}2M3#vj9~B@7pai60wTu|FvrhU8&!vm+F*5Rx31xP5v zXl185;qY^V;V`mIbYawT9`QO(N2E+srStQp?0&E8#hd6JL<9GmU}5nag-Q3lu~uhx zq&%#-m(KfsR;Wgp{@Ju$m$RG#J&)MG9!*|nvrL|ghaR>HfXPOMrEO*((~_@wWW*kx z92POkGsmujtHI%?8SvuaD-}u7fBkXeqWx-%7M+VeUZ$7+7nn9sTQuB|@9>6MoD`y9 zB%fV)fo3ikjNE=JYiD|Yr^g{fLuOA-w>gcfSPGPVraV8PkV$K(0 z+h^apkP0l0ovUK8lE2kO?5R*LTT1{N1(l!nnvA` z9{Zn`;oTj-evx2{9+mwA@&F)gvfx2uz%>-NjAh?Gy<^v2VTGARAgJ%4r3f*yDiK#M zHRk2qJ_)_8r`nMO=7(khbF*C^WtHB8qujB3_M&PUZCt{Gn^=HYIYOrLm6#@dZLLzQ z{^AJq>zxpPy5!uqt@>scj(_IjsI2*M{#DfGXbJ(!GaBn2%V|t>x*aGUe6U_M<7A*w z)AGmea@vpx%I$Bees3HWzr~teaMY6@imC7#@GMXUZ;<8?Ccb8vt6uO4#-tpk)W}S1 zW8LK0ZVTK{T-CmF{B7XWN%3eA8am4GVOK&Hg{z4%T?&(20noEVnB!F2-79@I2b6Uw zUS`y7|2XL62D|Y7u0Lk?7uUNc8eAJO@m*4xr^dqrHFK8NW%qq(OeC_`zEkqEK$9_4^_-4*P z0ACkt@K-B|^KXtM+WY>CDLZzb)B{t$_4X}7TlRpdLr?85Lv)o1;;A|>uv!h{z|fAr z+6HTn=mu@Q;Ees$Ati_FDe*)r8Z~{Q_qWgp%H(?(IWZd|N>3Gl+rmuIR=~t2!R^bd z9ug&5|Fry%H^Kk&FTL*=&yS2>S-1+0+6P z(`J}WUX1`BRy&2UF6RFQa8W1j1HPS!=Ee4I0XH4d1 zkC|_8GZbLzn!N)*3P!;#S~<{Q=mao?blF7yL<&BG%jav9S4H~$>Nio(^edr=ZFk85 z^eGjjXuz-mkDzdRUx;$93?W0CTRc9c3=(xiokhPkh+}I%`E16O4PY85Dp^1z)&bTj zw_W6g&#Oa(tagcgsv$|>McKas?f9qkZ%sDAHG`60wavO}HHe^Eu1sYKnL74s! zGQhqPKwzxQj)lesl_cF}+%*`ZreV^0VvJoigdMFJepLR+cuoyf#>S-U0HMZ2=3(^j z5j}3kOWmY>6*osYrGXx|s8!FfW?%edS-hIhW(iU#Xu*Boa*d#mYb4wTIYv0?Lj2HgSI-s+U$_$)ny1t+B!qbMS7H;r6WK zOYF(JmRscK#WVkL&F=wRA%l!K!0}pk>^MxcU3nS8ypN0BOuoNz7 zQbgBc7o+vvoj@@_edbIZNFreF&Z@Q<;&4vR&U^}#6%p2N4;~WKn?hOZ6U6{0uv z#kgb?(ewdrwA%n?deL8)TS~+Dq^5Ly$K2-AYzt(hK2C^Y!2@SlSKOUSs9jxjVSex^ z{Iro>jA-wG9)m5jiuhq)9*H>!zwhZpl}>8eEe;L-THC99aC(5h2dL5wU5H>_ z9|`zX-&&ZGokrC-u;oK%pm)s|jI~QE9$B8ziItF;-rr&|3#_o%*4UDi!Uh=?v9GH(SPe{Ev`EQEPaH3b%-xe_js|Dyj@+RpNW`T%xr;6f+u*E|;^9mSp z+l~V;RSu6S)P!n0J+49PrzZa*v9s_tq6uhKnY&va%_5A!)Ob`7p!`;jkBws!ubyr9 zxM4PFq88h@>&VrM@2ms-YOxj3gx5b>jv>^q>xAqK3f$nnHBsBlN@7GUEb@g7L&*p9 zPQDbB>0VC7dHtvOIfTziRY(m={p){}`WxK2xNX)MI|hBw;$}KJTI`PW9{`nf(EJ{A zEc8U?D~8;W{&)h?wLEdDLfim4d;wjMF0&u`05`0&_p(P&2>JaehV#A7UEC-M|L?%V z-S>Z{7}Dk*&^5EV5i9@SiybYXL+23Bq^;vvksEhV)E&c=(U_IHAQ3wEfA8fW;_KsX zBPPN*{f-v)xth5Pg8_p8kZKbD1Q)uN|g;iccaV1g+G?pkm2OVJGB!3OIiY#{XrZw$K=J74OQK(r$~h`bbWmap0?ts z@!7xkilA~V3x&1FFmCiNbl5I%th>;c$eY9R5)5fNb708hS6H|TL%!iJk;+eN%=C`M z|7uh^fs%U%sv0gJY84$5uJ%P1X!K+`M}!K9{qm;R{XC~?r)IOZZ1@qmH&d1tn$Xt5 z{Ou1o@+7!zn}(Ay|BvhKAhR*ZCnA@J_czJvvkXeoWh?C_a{%n=6Gp`XFtqvk1<-WS zwq&_XTm05FFcw9(Ye;tO-V$bSAlAfwidTpR&J=f$e|$1z1uqmLJ~ z8LMe@dG&W#+@dq=D88a!C9VHG7;^{YZ?rx}vdf^#YL7d*^1#qM`H!J_=e>x!y#5bek^V!LApU8dgo~CP_CugQRT!kiaFg&zBZd1bOCQbZ5~}3V^F5 z#K+RAmC5*$ptS`pP$MtBQYry>r?jo8)%L^3we@b$XtfZRSg>r7b|T!PX4X#CGgt402B7Qbvuk)JJOzL2mefIll+R18|GKQO2tGbmfox8W-^yBN6CQoR8(pW|I9bZbS^BOY#kP9#J zL|Ipb^}YKK-Rb5)uEVMh8lQQF{wT8;GxQML7_xv?x#pdpmt|ITK^CV1U@iWfm@2f5 z4zo^_8SaHU+9$h2F&+PSgiXLW{@6_Xe-*YN344^k+f(VKfSGN3rls%;BJHPo*-5j# zbMjfjt?-cVK96>iqglWA@~Z}zrJb%3u8(PIh?lN0fFC~a#}8LUpi$gurN?8SxifmC zAZ3E}fN>#b!%p;BKNus*9$o@;=_T(jV?7dy@TDX=owFH1%=hy!Tj35GWXHU*2Jg|{ z)WU9QIsW)*4-K@HM)$+npODSCbjA=Erzd7x&+5cVN#T{A9z-~x7_SFO1y2ni;BpyB z&eW{U6``5Z0OK$WnLvuDwY5?I**Q7#ywPvmKZy;&8m@~nRJ*>cC_18{L(DDhDg|X6 zjvD?qdNw{>ZH8J8)MmPwmORvb7E4ynRZKy|y*ixgeW7?b$+@@RsF#!p7=4)_&dM5@ z$-S|^%Q|){j8*2@OBA^`R0J*SQvT`hr#P^*dml`(7slBe)W+lP|LuNdNXX;b-~O@| zT#c+)H4T|wy6&24;FQSTHgHu$->Gcj9PgTeQLoux`5#;wJT(oQj=*Gy3+1d(R#In} z%hjy%ZbGBI8PRVo+!sQw+^^G3# z>Kf2d;r}};d&>9HvWLHbk~#kYKEZwG;Vy@)gtYm;8D(%M5&w6WT-JeHjx!S`Krg!B z_oZ^Hc;u#=?3hir3J}BycM$FA%Dqa>r1-w4P@wM8uDduuoWPKbnB=o5&AxkIL{ z0|4ko1)(=D0lJ^dvCh2RsI1yIq5G@iL_bDNV4JoX62*1bsftr{+2)NcIbh9((-vdP^IYh6MHBUX`0U3eaMhg;_X;?qeJFFrNa>A9u4L8JP6^JGA6B+fa8@{-LI) z|H!d+o#B=aG-#S@oMfYKZxVRdUk_x{TertJw)Wo!yB(frcIwk9y3Qz-$Du*bR@Ude2k!zUb69DYMAtK0?bhGF zMuz%pWV{(iRH5Ul4m{v{JCHXv!*MUc>A3_^06#ya-a|peKTJLa7c9hwAEzz{Z(SrMX=~M@1zerL`$%X z(K`ZEP_4c`=T@b~LaOz~Czm$y#$2MncMK=RDH_dsiO@qq?0bF}>P2@I0d(<9i0~T6reW^%O zLJq&@_nPmat~pD8h;-%;I!2Dz2noJ3z6!5+^hBY>0yjZZ{?lHkr}EvF9^vv};oc0e z@s2_tys^boAtpd6TuDUAG1!guM}0>#hrusb8tFoSxqPhKWOzpz$rd7c`%|h&0n-1` zJh1VMu{nzPr`buMIy1(>$()_*eburFz7k29Qf__@9ftf^4k!wj&Im^f>B~a1DlXQ* z+JBKa_BQk0I-L>HF+`#N$T~d`I-RCFWz+)loVbO4U(+#EPKZdwVMldedU(7Kj~lKX zZdz_vG==vGdd-*PhXx>-&4G?3`jY2lZ`W-Ew+#d3;rR1}gsCWF!NR(>p6~s~J!dfA1iI)@2o) zIyXe@1<>KQOoZ@T0WH&Sk}9j!Nzhd6Dv8`O13;ugxYy16O8Ub~_*h|EW9m-uzoVly zu@TNI6uwV7ENK^VSjH*FJYaRM#WqsOK7JS-8+t$}2@xQFj>HEzStksja5_%E`2)F0wQ_8CmuruxWa3t!xQ^blJm-$7TkRZNo*lFOW$+IZum<^K zo&xy3kBrcc0Ydv8k>s|oV>Oe8${~FbYN{3lOs98x(JomV<30`zN(p8Yp_J=)ea6YT z-zpjvOW)gsne2pe_`}DKM7KmLuQ{+V$++}N&%pNSCEm7Z@6OU- zD2D4dc|V83*KN05A|HDGG8OsjKfF zd5KN*l@(cyBgq#x>7V8ew&k|t7QORy%JK{IncVUiWJ3N zO_Lz<(=t`+N`wzATnj?#o?WmZmX>xM+Z+o(B-Oxdwe6)r_X)e_xk~OO*B?6-_GPIY zq40bpWFeYNJPH(cN{?dvD7iP>ht!$~!RNzKzu7jg?&_15muAOupwZ*DUu+@)_^o+{ zM>n}JU_7_ZZ$+w})PJg?1?zYr03XviiWS^WYm)fF<|yZ9kKqW#sS`VP?|;yi^o#Ru z0_rce-M^3ZhUi8Jx%QeUDqjY9^|}d59PrQX9z6FGdZ{h@t69eZXIb1JnEc*>WK;F6 zw4{VTX||sU<`Q4kgOh#-!By%(TN((ktk8ii7)$@)-vfMiGANvv=?sg3&V#h`6T`$S zUmpcmiY~cI96Llc+^`#JD{^b@qpny|U)0}faEx$1Q3q-GHH%#NA*dl_Wu%opa1!p4 zrg3)5p&rPQFCUQKlzxEL>s@NjUss^XHw~exE(LNBLN4k^BHTzXb}&dIP7W2oZE1d+ zc~;cfLG=nWN>p`;>|47A7-Z{Pl2Ta4&DJV(0*G`~d7O$95^|6ho-z4A7sMz7l#NF{ z0G-nTxz6=Yg6X+SLDTGsAKRcoaM6RjXN$Aj8BT2w^i9?WallJX_$ypz+Zontv+zKO z0f@T$GQvnV2SbGQdBi&lXR+e1ctb+|fm7iKlbj{3K$R@!-*PCEszBfFUA@}rFVHr{NZ18{QBA|$TdPJ#HCYU* z9+A8yL&%a;oK5_8vd!3o;TE0597vodAXJfj`7#)$J&GwhX7k4;##0{wU!6;8;ZyNs z>EHpAXY8-k?L0sS9zGA+l7jOkJ2z8~SJae)eIm^zLtv4z6BW1G%)39ph$Qi1CVo4tGQ zp%X7;M*ur4fw(}(1533Tyxg&-2TE6pwByAurydyjn7w>tB>s|}ML8OJ04XN{DE7S)T#8=#Js=2)?7S9mRBB4v{jJgLp!ohx zC9-Waq0}TWbkMZLny1YW7IgN}JnfUG&yG7vtcSsm_ZWHu$AJtn_O2lLu(JQN#8W=* z=J(`o{&%qh`rl99gnC?WUk&%rjra}138xa_Wj?2#&y_vEVIuUO694EVO4g--^TgK#ivN<)mR}+xiE2$t`8n@6bs#jtY3c{jdyM~WGkyEn z5Hjf2CITQOc{k_H7vIXiM4)&34BY6JK{yuqwxWq*aLWHjjfVPpSXhmpmh3c~R9Oy0 zx$5sB>|cg_QQnYR4JGtL$8ud81X|>&V?D)J&X_iyokOy}?tut@t$PJ%V3zXz!7P=g zCOnSi2PXP=LHc-Vn6uR?ax@gkE)ldNQz2_7Kf?gpy;J`J0*59pv zmd&iGoN1zs7Cz)msfni&Gu*zLp{#b8zBWs!>*Vql?>WIb@LDLC8P?Vi?x9N$T5zx;twzqyCPmW zN_aBd4Bfnom7Zfp|1VlMU$2fQHhM4rz#Ww!+IDdg{-6x>K7$cQ5TXpC0MntTh@e-K zqWCo8&<6CK&X2zxA6tv-68YmC?xVI^WZv#e9>fx|*l+&lpg`-S#$A9MX(=QhfrDs) zL!7#Ra3}SpK2i?yYw@seA79?nt&iYnLfQ&i3vzE{cAOY2FoZcQd-YtE z@fO_@q6BCQYI8fDAud7sav<+;rdkUq)F8?h3Flne&y4>IgCwrypXIy%9>3N-MfF$T!Y;jxDq+`PUF10cR_6WPDX@pxY!?|~>E{U1cC z^-0Wa@5)i()lM!mC}(G0T=$`qmIs4wAn+v}ma>1JW{T&A9Cc&?I1hU$Hc(==;h(|5 z>sjZrbDi!-idL`s)3K1?eYgz=0uk1coi2gl=P-zel<_;ZC(ENmlqhpmv}pw6j(V z=@U%ru@(w<2aSxRQ4ygWe(#iRsQYpNl7MMV3&__sBg4%7Al2O$1M21p$UU@2giV+D z%xL$~>vs@ZRWsD9=Fx|lkmG;ykK_M>dU?M{ju<7~c~pVOOdbrx(8eAxu6*dO%i1Kz zs@hrZ3{z53d+2vQT?HeZfNSax4qbw=oRLSTp~g_cW~nv|FIWZ{I)#~}+=FsjPx0jM zZJV9Ck;3e`9?#6dx=k`3s()r4gNTfE%Mq*M_xR_~+5^cF83cUd^Ceua8J5VtPmt#Z zSQYdsvsvGEE3C$mDj{ocLFA8NI1@vsSc8W){;Rh+KTS1@!{zvyf^iJW>a ztjR2Q6{ucy{lv``UGVXqy;z8tL==R|j6xGv2B9J*LKAz0G6Q31xtisH@6Es|bPOK2$D=$_3fPCHp-fVWJz(j{M>-$%g@4yLM z*G{TRM~z_Hp#dE2D)qkQT1S4!nA=z$y-`)+ia3tdP;5GVHOe)EsDDZrjT zy<7z)dL6>>b7+WVHV~C#6!4>kekNe>r>;CtFk;vvHue{f(;8s&6JZAT;VyPR8ThAN zdql+gl4YFZ7ALHKIqjGK$_O?2!uxKM!ir&bWO^*HUs(-qDd|6MDFt{O);rV*gxHkK zV3$ar&7f)Q;4l;2mqg`=NYUVU7LV%KOP8|%r}rNFD- z+4`&Hn!jyr8E8+>Csa*UHhqJ!y(ujqeX?FvGnjO4)1!z4s9V`>9J={q8OHzr?)UnPr-vi{-6)Clfjj4OxEq-3 zfoGZvAO&psO`-MPeDfi1guBn}Sv+kFZxNDBCtF%HWvBU$i`;q#a0J&krXjiH^F!Ok4-mz5qc&51zjP{qK8BSty zlC9eXvjKKs8=oo$A_n#^cd|YBc0t}|9A^5G2%Xa0cZ|rFblUXm%e;) z#}#QI1lC2U7p2%idR1Dl$z1@m$j9T63YywKX5S9xnH^95QqOguh0Us7YkO(y+-2zg z+mF7@v=5hgccQ3zp54%xHB#mMfiHBF6#>7wgEC83eMU2bucncSaTbm0<2N8zvHv(e zFqm}X0q%E5T<WOnupd%~r~3+zJ$cRRHIfT-{lTa8tfkrv%FJ?z#DOA{2aG3smb!LZWhD`6 zt8pqm&_!_X(%g$km>@=Wvgt9*28fuNsijCg4(Gl`LU_+9Z98(}Dm8n_1)uZRNC>Lr z=`WNo$ru~XOkZ_b62z7gFK0QUwHQ;2QjX(orh37T)0 z75m}QK&rchuygCR6;Nj?ad9)`YEZf`n2q`%2Lii>Pcqk#=&x>sU5J~;bX5UFt$O+O z-0!t#hFfZ=yhDU6H=k$nSXPK8?I;#t9$JQYKD6daQQL~g1$ zSj%*Dh2#!Nai8z%XS_;$Wd(mG%gZchG9u(5znC3z=>KEy&BLi&-@oA!k+!4^wNyxj zB}0Wmh7>Xn%TP;(W23&4ELN%iB-jLV} z9c8i*N!V4?CtRw1b2jDpmXCkfzCF%s4f|B2ln@&SgoNia6B<&3(<0`3#F)eP+`|TF z`}g|^6fVy!FC`fmK4aHg2ebEnr{1o_^D8*FQHyLI6(Ycas^A}0Icc%6Sb4Rm+4kNs z)6yPEdOGN1zWZ-4Cf?X1=II{x&ODH8jMV<0U2@YjF;~)_z-AtY0eqLBx>*#1Ak!qr z+`Dz`>8ftd7M~@?hUz!pU$jEbz z=4fy{I>N}e#{eHRqkDPX=k@BshG|jVfB5Ox-utq3Sekr3r8sAD2ZT||nBfLXL-&iO zTfn4~^Z#De6@zrt>iLrk8Ygp>eu|~7Ow1$P<Huf;1zKS(-s`X1&Uf7#Mror6oG> zZ5dbsb0_dTf!N?rRknti&ATN!w~cFNe>RCqOtW25IZ5B*W;4!Obw9GIyx0n{p6~9H ze=CUlJ?ZL}zqY$t14tgZXx!N=W1)er5SeV{e+7n`G-Z{;+^Ac(Qjh&6S_gkM*?`eJ zcbqiK&8r*#?A}wTg?3i)4nIrlwFkdq1CP{9-KaIqZD+yNSa0LzTe+L$o$g1t=}KIC zC>b1P>K@#d${d(izfXEJ0_NpOz`VT1Z`xXO`_-#AOV(T-hoVng?{zHr0nPhCG>HN4ICpX>&n$=a+rIbP&gO{l>>+350#Y zB6`~vlDMOBaSNZke6shnvj0WZzY2ypK#P>|XTaZhO*ISP+A?h-M6&z8Y4RvwBPApE z&Q*rvBlj9NgvpSiF6+)^{Z5V7+=Rylj~h=$@5J#sd=EYswM5O&|LLZ!!+#u;(SO>C zrA9Msc3dYy3?G?sobRJB-^^E;)>Nn7zFO?e1wWuMxvJ>ZV5lSL=gNg zhx79${ptU8e%|{i8Plrsg1bpKi$rN;*Do?A6GyM&&l0!kLh``nTJX-wT}2HSKI)6< zEuMD^=D@i>sh%KmM!mJ2xzlnyOkzOyJ=uQloLR2#K`}o16m)2}pS6yP9B7KH`JH;o z=uBdHybDEerDp3VRvd6xw+4gINW=9B?~5OEQ9p1 z{(vcHxi*U)XRrg`TE2v)u@5YVmnyH0$tD{Y@rY`S>pGcR3QYKK9W5)O|f1o33=T1fO}Jk$S`|b)V}If;e^)*MY4K85LVB zXVoqn3@m+dE*>3L^~oQZZ<_ceT=2zoO3}mRTDe7|M1o;=1=aqHzuYagjIIEc~ zA6Y5$YA`*

#8w;M5(l%v&(!Z+)o@<^_Rg6%uSZ)Fjr)EH|>^I8A={3;LZ}Y%d_n zQODb7dE_zM)4uM+Lx{D#;CX)8#(_heYZmcm+Oki{5@nC27*daEKaGBxo$laK+|{ea z)i}OB|CB5r*8%yOeM+gHgi1~=|H_ewZLmMkXtj2JbEzlIPIu8p1nB9~&)WY;3fw8> zm0d<3wJaskj^ih-E5pMiopZEG=i~{0W@Sffxtnw9$a|U8i#}tSt*ajNkuObr6~UvvYBlRd^;H4xtQ;h4;hpJ#{W;U~c2zm%n@hK0 z6jS_(ZK*P?UKne+*gAJ|zrbmO(b|k{q5fl=oa!@<{p_I*^EkP1^ zKONDda*LIBwehnjZL4mcNXyHvQmLQ}aK=7i5ekjE?>iShlH;#6I6dotg$<7*@o`yG z6&~7}mBkg$IfS)Yhz3%dqPaH^79>AR*#1$?hQsKjO+}>c#^^6XSaHqCQByCjB8t>% z&goNCG+S9JU#!z?1D>o=vt*|jMxQQEvGM?{OTi76v~F#YY%R;Vbdz}#@5WO?MGFp@ zzC25AO-hfm!-ah0T{;dGs6L)AU^yA>MrfM68d{7~`=wbCqcs^FcIrXXL5zlWb9*Mq z;8Oh|mXqaQW@)DLK3U`(T6&C^{Nk52i`jrG)7O<7oZe%88HiI^IE4M-uX9rBvgSt) zQ@NvVJzZQbSDGD=f79uK*peM^FE4d*T1o*f zJeuWj&bq|YV|g;*o)*P^SG4UR{QI%t`uGlGE9$ro$8|T`oC>s7i<{LS7os$7 z_K8f;T00w5`<)d>qeo#PkcxVqH_Q@d6af;?;KMHQ61a_v0QH*DGo}^+oTcU_ z7y0fvIA#QsY#lL#KslAUFF6H2zYrF*jC9RRI1~X*keTF5TIx+o%b-avk#A3Rhz?mD z4%pjuQ8%e7y*mOjjR_gum0n5r41)}C5+T#1lYU{Lf63@>L`IcXWK=iiUAWe`(;?0B zmO+ou54ADp_$W>UnF;)^MTBSW>UsGy>CizFdLlS`U+*mm1LX16 z3;ijc;JJlM>bIdUk7MYzVCk=v=0`9<7K!*smJ(*QanimK*Zq7|p47-_WNX$=d3D(7 z){FXxXIwc0ah}6jf<%Z{ZglYL7{ImEg?5aI1O?{WKTIEmJ3;g4*_T*T7s905}2a;_UVGOax0NJ+)%zqAJ}G@AqKGE`Jl_T%Le%tc#y%`!J+ zhUik&?5i%&ONq{K$#*oAVe&FOI?Pg?4Ck)S<$Z7ucZhHhsxqIrK%zD{f%z`9&+t{oAY-=}> z<@3>a61XKuPxZEmaGHz=#x<3G)=SrhZl)62MpK;ZeH-Ie5oM#iH2dA44@XP=9Os*! zYNvf9xapE9?KqFn9eQ`1bbtC3y_@JXJ>P`trOx z4@sH!q1pXWh5c=65Z6=bI?mW${)Mn6kz(1r#Kdbz2-&APqgZg?rXH_}Sxx+9>|{1a ze#CjgOb)N>TkPT$m%FS;G49N~#xs+1oD%Y;x<(-l zEZx42i4Wv*^)*EY<+#i=*G~Xozl&lq-gI;oHEASh?ECu?hesHnp8^?{CmNyExgeay zRLX7SmPOF%B@q_dAYS_Lq}Y_v=)|(chm2u@=C=%u)6qy;!yuu0>LQyP?W^vBczhPG zy5&`)=LTbJx!03u3xQf`1EX$(#q?{jq%nt@1B)Am?Bg6%GEY(q66asYpz2oGm>1C% zmOGikOt90n?U~yBn%xj1k|v=R7x$xCrtUa*kapP+WH(9V^H|?da{u{kUVRBOn!UIX zzx{9mfH6xnKP?JuW3%=ex=xST%64RaXIhUy@N}QqL*f+C?00Ja)Dnxacb@E@ez{Ly zvly5}>B+^{<5(BXp7m!4C3G8o8xW7J%U@EWm-Foq@2GWu+5f2b_oSvXp;<0h_w;?W znAe-sD5mu+d~qM}Q?-wJe^~sm!x}FCa*~@Hq@)izVB<&64qV3MrKcwA8l_44V3Ve4 zf!{I2A4wN7oB%{FL%nl^#-pwD__ayU8QN6`-W092xzwpQ3eb$zTu>bfzKUkL^}xkm zEkB3g6qYJSJ$ig1@$?aiqAH zMjXjGy2<>#Mau-_XHXE)l z8+5o$OTA(&+IG{w-xjxuFm+cBYaklrepsl=-YZGVGFosSU&8*3vm*f6b^hF!?&Daq zTdhYt?5g*UylHL~MU!t)O)4S$Z1*|ssM0a~qBHB82E8&eHsC{^b=>0)dJ2rD9)puQ zey8T_ixaque}8EdpmLOH%|#nFPY_ro4Q!|0{~CNi@5Z-3n_PDRsv>1pYLvE3*6Z~| ztErD4&sRXf^5^^?L`Al7ucWa!&OSb4hq$YgGcDu2)t$??j~D;w>`H5YGxpamQxT8r z19)7yJZ)LRN52`nV@%wKGj?6x6neO>KMK-UZ;DPECZ6e=sqF;5M$}N74=BRvpaA0~JO($-?s) zepLgbMkOQPBlCOPb0o-kZb~Wr_Q|HZSBYW|D;GXNJ&R5nrQgrsOyzwL!yoG3hzZ1} z8{rc8=1C^G_LnhJKVz9rn=nkEVPVCw;yl=JoMEZ!L&O>bs@Y6yHfd>w{(khy(Y0Ox z8@hvF#$PL^O{q3>lCW+mYg!a^^$zu~_p-yhNFy8ek}^q+(n0KA__(IFI&m45SN?T> zvYcdDg*CBaPLpJ)Fp=Hg40O zGuBmZ!szc)!j4d?eBoj0XV>56Z@Asm^=>4gVFzko$9aWgcLJeyY~AK@Uah&*^c^15 zSw#Y2{Lf0ogCIo4e=y%*^}t3;3cX#)JfOljNSPZ*x<@63TWn-2`!lqDN2@15yaS<* z#(GB$r^`yEcL+NXj~!mj!8m+#He9cU#Bd=%Uljr$gmHdrUh8}emL0x5g}Q z%2-D!&70wvMuVr-tV>4zcENU;Z|pKn#rcr$ugu|PgjLhbcgd6rt#}LfB#~2Sbdhbe z5hrfX>x$BlSRB)IG6Zm$d*QTMhha6c6RTK4#Wv!NH{-npbF@u?kDc)<@nFVASK8^G zew6si(m>=vrq+FD_Aa=pUMb;gpCb_smXh&({+P?g_nhlwKxY#g!`3Pu=24rYQlF{JB8V<;70YhMjai+2|3KcQnS%eA>%A=#T%}Csv&6 z=#J^5$SGyunG3h*?xHsSR(}q2ieAz&IcdQrz@3=``{~ zuIRmW^}@)n2!PgR=DHoJC6DKkz2g!`ug!Bk)AOi6iQhx_>4=L~W&nJ#+)Gsaf0pUB zuZX`jBK~jz+(-9?g(2JW3OCok2Hiz?a`bqNJc!wmi_QVHOGX#%hlyVpcj=tu#?$_#K6PJ?%_mL%%)a^$PfVl7T-kqm~iu+CuM04`!^L;Z++kiam5h^)*M$2G? zGV|r841EbQIJWkCEo1r1s4AiNt$dC8DaKtFE|+~uwjBdci)91p z_uZeSfyl(4rP=*Vb;Xlmuj9TO3^(kx%gBfFl9+MwG$NQGgSMXT#y>pKqYCC zak)0FCpmIas6Kf(){_rZ7H!wYuZHCu$DyjG_1Yl?^yuXdCot|;Qhck|470VRk63DyDMr+y=uRh=6 ztx|rWMF7^Fu|&_QIRmzJuVthsQ`h>dw=uMzfuOzzFP=WtznOLFGXGhl1dG0tKLQ`Y=@C+^FcAzuT@ zgh+aT{OgyBaD%C+lIj1f(~IgyVM|oWuwJf!eKEgP*|Pc1E!#i{ICnF*Wqo67#vd!& z7=`LBt%6G(wN>mEw=K3>iyK0wk<7uAax*0^c1`q1g+a=ZKb%*x7N)XNLta?EB$+$d z6a5ZoK3^peg%$RHg=b1ajP%N0a=hhZ>d#>BnR8UzqFUYb>^+5b;*w2yRs9;S7}kyU zmZ&LQ{BKlzV&KsG&8+L)M4)X7PDK+|P$+`xQ^3oj3(739#JFV@4&vubEMP}dMRPEl zZt05pt7oHHzxi|JiUCPZLA5?+o2eT3z;^>jK?nS#weU zvvo%N^NMRtp=TSIU!!U!J1$`sb7|Y5o!#l~A=< zdE}{Ca$>FSE|8fM_eyn?*VW+lgsC%6Aee~x*ZwQ6R#ksr8dNnp^HD5mm$!NtgjE&m z5oj5gGt)Xd|97{L*X2kcnb%lx7fZnUjr#Hy?832*&jvM}S8l5sdRuu%)c!C3TzM)R z4uG8nDr0XqLoi>`3WTW<8X>Tf8Zji@(}KXYZn1OcBaKNBYX`2q0i4HKfbr?HeqWqS zSIbNK{X0chOEN{CY7)U-QedGeWNNm-ExXldr6ug65@KK1nK)kY9Kq^+LHwt$N z@K&AxTW^DTCsvjrz*^+6067?Y1iS-4NgI#tl35A1ev2Mfe_ns#G0^aFiJr71kyXFe zH8E}BXx;k|Z=T6xO}92Eec<1F!j4;I@}Wsz0or zVs#K{HG(l5Cp}wE-_6^xP` zffT~3$c?x-9#c*Q_u`%6gZ7KSP^572^R}68&f>C(gwaAC6AafBfKam*y-~h&ciZJwI~AB8^empC9f>wpi5mvRGIKhKkd!Z zH=(7J%r>C=N_WnuT)=k2#@oYoK6ZwQDfty^14ajJjMFa?f zP`UxW&o;j+Z!HdJ_(?t^SCd`>jaZJyVQZ^|3E16B=FJ*{6ntJ)Xgo44%PIdl^qN^) zS49H=uiZL_8a(jq>LtMI^DWrqEsBILF&bUUeruFW&+r~< zPLmHatfedl46oRq)iO}KNRhb>=}GD|X;D)BTP5_WoP(v63_~rzW#9L@ahU8{i;P-B z22w8eYpc1*EChukJRr(2P_ol~ht#PhKx^dt-YA83sG;??v#;^*aVN%;mgkaOZokj* zTXYg=urKbsR=!}3&{b7u?<4w9ryBsYJOsr7M-|ev3Os%Y*wmgQn54zu$iz{ElLnG$ ztxdzw&?K}Ni-*Oy$KAQ0CF-=azprxFbe$P>STmii6+I+G(RZjLz?4@yp9w(KO9lxdfm3>-IJFq?%a1!Y37A@m=?yNyO>kyxA<-!hw1e@w7+-*J#tad z2ZEEl#0e#U9CluS4l>7tpE!Is0=WxPK~MvJFOWnYE77ta=z`G;>)tP2u} z4h~Gz-f`nC4CXy6kW1f2+>Q4$(5_Q-&vw)7!=1pn>@t;VKIU}FAi$-j zp3g~?3r+`aqmkWgniz@iA>1UnDy=?%BAq^_@UimFG*vLNqDKJgy9FWLYH zUN>KCYk7Qi5R0T*rKH!YR%tvnLq6j{8R=_3^o%|8nlvs7CL3i?IKN`~rcBt>w?npj zK8BR%P)gFx=KRI`)8VvlG`{V7d)C(}q2)25PPL>PKeq~~XZCyDT$)I_woLIi4{&iX z5zk!E@zD@@l%3@LI>1~(wQz3+?P($&H0forPv@qn6Mzh@oEaZV~?Isea` zpMG2!e`Of%Af>HAq_kDr_H=sq$uU`?lH+99YYrcW#thETSQDDS4+M=z5p}tlk`Ua#}1#0=FMa7dxiW%w<|Y=x(8eFiqDzsLm(S~ zu}ObQ*DKX}WRQ5cbvu4F@E{F+ZR)29c&oK(;$F(_HYI#ct%%AAXURl(kX0fKL{oTq z2evJFJ;2CU8Em}?cCaIGK=pRWT5itn-yfd9$g?&;U=M3up?~yuEa%1&2H7=pQa%$+ zj&PoqD>Ws+gb>A2lHD;}jhxqU=cWv$=~1Q&}T%14|7m$7K1^ zE&B?%N3-TyuUpToSRx z$~iB3;fa=`H@&?dCJitf%vcI-TW{I9!RjT2_DkP21lX+NJp1*C@U@<0BsRJ8LmKX>?!L@^gbNfN3pJrY3Usr$455D}|GeV2_(TCsY#6v6Ox0 z7<~u#Snfe;QW)Wip62Nw(nYOmgoj4SR2NqucSftyC~2?t!Q(j+_Qqe0@yqtIz1q8> zJS!hQIa+rXd%R)t7@u{$7fEm4SnIuMZMqXZB>M@srJRO-{T6fV)ZMg&7(ZQ`qQUGW z7>Qvn{m|S*ob0DDnlJ=9KZFnUy1@=0#lnG5J7BX{w!G z1%bQG{W3BO#@Ulrn(k1hy6u7Y7phgnAGUuC`#?7HLQC`^z7>=F1i2UMG3dSUiHCbJ z6$UgDeG2C0zj_3`nqsbmZZK6Rny6wiTt9B{mJaVauQ}#=`4%tviYTGzO_WDrie0tVAl5Bo#3YUubZ7hjPj-3qhFMZG>Dpb!w;N_wSTVdOyTCn1xv7f-Cs zy;r|OO_$u^UfMsYq$u%VvQow(o9q|Ia)R8yqls57)OV6FTs5UK><1kr`FFDL{iSW1 zUEgh%sWa;tdrj)}JEXd@*Ez^uI|-tDEtLR!?Gaq1GG`ES)+ws8plnL21o~Hop}c!F zWM53S$!2GQmZXD0GKi3EbHgDvtkL1I+kzt0ORf74X1J^;B!dUx*tJNLT5qV=pI)jXS-e z`_{9};AISK;?`!R%6>E|lIpZ=yt*>JVWoRL#4{U#Q3x;I_0$;}{y>ded= z9`^|?Ua^{Ip}1sTXtQE9Nr1cZEw(JG{8AmXx$Ltl$1hRsoBa)4$GLpAHuWdj%Mxia zUIVedJA52Zkqtnu63qmxAC!vP8W<>lyS-Vd?OD$t=S}L*2$IBONltr!Y_}*`KaaDk zl9Ao6(uK4EuCKAg!60L;c!|A}M^3r>m2WKdwUCajGt)4Tc<>lr;E_osSx|TJ>C2hAu@ElXw=yS_;i@e*9G|{GNRlYULW#y^(p7y0kLA*1YXSiD+ClN4`%G|k9Pn}cXDvR+U-_zjaDP^u9k#eBQe(B+@UzI5 zTFdrhV_qRzrxzhub2B2;d#TgJS{=q*)B^Xo$o8g(lM^webv86{sjb-CwRuxUVK}>V zr)RP{1Ye*O+}sW>cxOr#)TR=9V$aL=wr50K&21E#D(Dx-hO*Rae8olwl6Ngwx@Tr+ zy`S@F!di1rNvXYl)--MZ<53&l;cXP@K33S&wjSe*pQ)MI9Aw~ywWEqB;~w@lN7j{+ zB>Zi`SydWZ!kp}1b1LY4S+{%PSzW>YYhk`JZ@JQ$>1qiNY|t z3XkhO2I+RRO^d14aLV~{dw!&|&-vsf1&aDeG~E-CilqF~H1fjT003?$Zd^BnyW*xS z((QB-_Ojyw{+xA)(t`Z0$pe&IgjJfhwOQ*(a>K^$Sx8&83MGfyACDCio9_7*M!Pn%BE zHym*$ENuZBvQb`%+(EVEPNiMPYa93&hkRMM%WZ$VKIQI?Hqzh%Pw0G#);Rrsl<)A% zFkfy>A3gz9t9E?~nT={40+4z$VV>m&A4}f5!wU)=7l%X!Z z;}2n{ZL57o%r@zH~P@}`mCg^@`Zo+Co~F(i0*bo$=Rw=4)R(HzBi)P#hl;XEF-tWhoub&oxWx?ewGjQLy6Ffw69g+ zvXTYwqDZ|CR^|Qqmlko{^1Ax`fDN0(Yc>l$j!J9yi)==7(Fhq&Yzdb8b*2nI~u< z#uesQY|G(&?)9*aZMNEAv7V}0{-J!{u;}jA+3ya#<_%VAGxeU03PH9AVgRrGo$g_< zd5SR6(`A%5lFoy}DDp`)t~EmsU|~}`nj&AtV&iX@sbR}8yym?9u)%_*|LdJ>$CXD4 z&pN##$P{B~d0vyF(}5DD*$z5w%{eF2lIApHe3Vm9-}5DTT$1CfmPs8J=m;}38h66Y zo$X(ru5rLtZTlN`JT32PD&ZbAN64CBtVhuQo!ZKo%^NGRZNkG+h4=X!*~m5fR?{p^ zz-{^50g7VU6@$L}(k(2ae443g`)zz+u%J+lNxkG+jliUF!IGTbblBb@;x zKc}7KkJx^euOm;c^9Fy<&wZ~?QDe+0K9yF(ZxzrQ)cNxSrx(&0TVU}8x^)_hlM9z9 zsSaM81W4{8Zj?Su2K@S$5Dvwrq|9CtuOua18wuQMxq09 zH90_s-Y+N5c8v2dq0)%ZuupC(D>0SlYNmIQm^^k>JanlcsT@jrMgt<>N^}UBd$N5T zUS+W@Oubbv7ca@NqYU5A{%+wiPf{BJ9daJhA4Lo4BU`Nrb*XGEi>ZCvtsm6^_&UKt zB%Q?|=qc@&xrhn9pXk;%77)E$(p8(;F>j-kW4ooaPCUD|>muMmw0+{4)jgyls0(Ev zzH=)oCi?3w3BMg|68K#w*PoL#`$b6C9pnL5-tFl7laMKGxBT`7=3+M$+utZs5Z_YA z@)dvXR7jfSaq)sJUhcL*7nREL>&;Su+ml-M?3b2pVctV`y2f{8uw+(}mRfY;QQJ0i zZl9E{hoAxiBGyh${&am-!i`b zlH~Ohkn^ij&`F8W>i829XCrN78yfi(UeuAvVXqHF#_N%~&RuWWItx_nQ?Lb*bfbp&hKJ)Il-G|L@}r@Xg+Jj{o|-fPU*t8(7Ni1kx$ z|8L9@;wEc#wUM4a*Q1g`#+usr8|=&OWxCS6ugT0KdpyI@w^GvlxrtfI(*2gWCw5>< zV>k!>dUd|ltVB@Wh(OgeMuo(4EbsBhM`9%s_oJ3K*ScE-CDW!Fb90M;Q}`Lcn+@_B zA1V0U7u3(QysXAo8=jx(>1CJ4PxHqzKn@n|Vl5k13ghBv-Awn}3=5JP1_ad~Y}DCV zAQ>D@G(6#QdV-pn8Z;*FY^sUVmJd~^vCC|#)or(YnQcBP2d_wM@MhgP`Ao;{ z4vo8FY5-)>t6K-tHJDHnJ#;CQl92C6HQZp2G5zCRV$I~~sHUp%Z{}NBC0o897t%1WZuJ8whULOY36%?7dfm$Pn%RNkIxg@t&V~L1J#l{~ zgazEBdHfD!SnY-rn`H z@I%Rm&~+bD35{D3aRS89y<7B4m*xla&r}9yLObIiQJUrke+Z;c`t&MCGy7N<3N{ z;LdXhKm5Mn81d_dtyEoOvNXC@<8zSN+oq--4pj2efBt&ivPihxU6yZG!7>QsPD5vB zx6iGzJ}AP@5NdlkBu2Tk?;qt-_zEBB%lCwT+M(_IqVmx5TZ>=W)t)acE|ete)c-~h z!6opW&LbT~*^d#zH-wvd8ku^SVM6HmGTnL2c4pLTy-@NX-#I^qU;)mNfohucMTJ3K z7~p|sVHFJt*SZ|Ex4GV4PxPHB={^bE!%dkr1C3e8NR}WA;LFHD8poU`5KbzfE}<`Z zH)fhQS zrS`}pbTt&9EgPAFf=P4nwb79!Zy?ZrjtsQtm%^W07<;db1~M8{Dpun;K@obwjC*WZX0wm-= zczzVE}C!U!vDycqxb_rpy09jg==&LD)j5fa-ZkKWzjA!6S5Ibk^d>gN~GG4j@H zddLCr&;Cfh&o#$y(D$Z)_S1)J7!pAhWfuF16Muy&t`K3nZ6LYNLyJs|)>E2F9ygtY}&b)*)W3d+tyE&ODMQ}+cmRdun~AC53oDd(>;{Oyoe z3ywV5EFO%z1vS2XlghIBrf^M%T{xd+$sue)k3keJ6c z=I32{%ifQtw(v7N3z}<7@IjA~_N-Qh@CNn|sgsojG`CM~O?_Br~RUQBbIt{*;Kyh;z zIA*-E7jJXd7L8wZLv#Zj-8qBq00&`o!_)St%fgp?0QR7@qv+}|EV@AC=$$mrh%7}} zq2w%+GlIzC0t%U;*j2a~xX84daKNBsZ;=tgO;C83VD`^*Af-`?&Ld7ekSjj}Rozr{weOR+m{9To zvA)~LO)y4ZpR9BsbOp+F3=!HtNC>gd%L_HR=c^y*ku^bcVr_l^p>ah%P=Fp&4$u=< zo7j5o6M#~8`Ci(2eu>MWgV1R}jz=dx#v;u3v(C4PQx-&@G$>6C%&Ci;rmbfqR`_A4$nlt5xaW(JYte60K zr0W1ZtLgPjU@~%zIY|3CNeBlKjDB&wSOO_FXPBv|;MgE8VAyDS;`g8YQu&NS5f5fM zYErstgdL3Q4erf#u;yFYaLBN%A+X^d7X=Socp$h>9-<~hOq~Ygkx@|)^)*n9f8F>ObCOX;IFRHdmWGJOaf zN&f{Xkv5=Wy?{*oOI-A?{BpIi4{?yxCs3d2=V%3W1$12p@ex=ZL=tLXC{Lpuup2S3 zK!9Da2j*R0d~->U>)fgYMMW{B=^=0v^8nU1K5^KK-l<(!hH%TDdCP$~2<;8YFx^5r zdH(aYva?`S!vQiRL0fM_x4Y^8`Sz1Xh*GXV_x;vzi`+IuO~IoF045JhnFrEmB+fse z@?w8kq}ucVB9FDI>U~;Y?n5{f@ZUod8Ge7Go2Gz_%_W)A$w;J@=QY`HQk`w?Tn*+kwuOP? z7e*2L2QL)Z9Yo)({)&Ty*%hkx&G(Cu#3 zQb@!y(hYr*d8KndpDJALI*ibJb>bFmyocr6fg9kv&7kKDs4wiZZH$JPucN5QC^2yF zfAjJj5vnVojCzNR%i$!OLCrZQmNcURJ<<8##}_3f0Fkc3`6^yz1hfcb`mQNaK<1QtDwf`6XIAJ9F}1RixIRF#x1H-{!F!EPch=u4rcUw0{a z$f&jJ)0z9WrR|J6pq`GJ$)3ShMX zeuZ*K3>pwp0i9c~Wd6lyto~jmLci2F2}56VO$b`MyPksDKP16?=si@k(T5lUf?R#* z7D4a%RPKQn0Js%>fz*qBW)k#plLbm49gEI3+Ukl zQ1t6Oo%uDrKvj#VX^)xPz11;RKs7k1YDixl>GJnyFxUEToHO|W^xV1Qp&O-Bz-oQ6 zAA}H;g+r{k`+9!9<>7F9E{-tv@{WX>+o zGMxPSlv71@liGvFku-tMsqdL#75}qZgvJ#Wh)O(xeGkb(G?Nrmyr-#Uq75Ie^cLFTt>FZ26PKd;e zdzsz6HfS2e=-+7&hkcU(*v)VHicDTZKw=1%7lAT6x8|Pr1eVfPaCJKs06kc)0wV%l zb396rXob&W0|Tb~>l&&&|Cw?_lqaw?wr#)G@vZ3*0)f2*>dq=lAZe8@E2TCCScSAf)2MC!(ihP1w91w=SHuz zw`~7*Aj!2P`{c_#BO)taIdM1P4diXL14KB9d03lit<2f zIFRkaWi__Cc&RCnmembmM4-76C4qsDT0*g zG;Bo(Aewk_bA{I!IYx|$aL7#K&LSw|?rg=aM3#V2gBH0$o0B(_;}2d=NP>!%2j@dlc^xm8!$5d1HThM6tjr%d|3TixvnX|#a zI1|LEUtAkGWf_7nwNiEy)5j^`(dD1}xXJ3dT>yeME>%U8s;u2By!=Rad3k%;f1G1M zSX&`FZ0uBqIW^`;C_JdN0n%YTXD>3e)dp0@bgW`XDsp`o9d{C9uh{zw3pSz&OEg_U zA#p$dNBIwuS_25JC*V-LJ!U=usw!3fM6kCkG&w|{WoB|6(Kj?su4cv*%y7oBe+?6s zA@9z{Jiy~<{^_Z@z_*R)YBpx95(V7B>|QnFY@5=NX03mCHYgMd{#Uo>| zFUGsxasG4c-N<`j#>+#7aw`o!d!+lm=Ra{22EgHYzGlOrSD}x`iQwN)zgJtL@#-hQ zgou0?`CQCnhV=^U23+&D8GJzrXe-?5-eksCvf?HY-Of<(s{%Zk?hsJVcVXmU9QT_t zph|9or0Y3k!pwRS9*-Xa-Qy~n?(*lD7UWXXrL6Kl0qJ_=_4GZly8m-tqv>HWtY--5 za-PG0+uFt-$UxA|`t*x1J*RAnUjIHctr%vLUz+{(%{Av*C1Pe)46bl&F2wM)+$)P+ z&DsX-u@E1%L5A77Wirclu?l#j&GyCCt;hedD5f@d-yr@FjdC#BT_jfbe~4)y11nJW zi|wHN<%&Qym(}jkZla{g3Ak8oFmlN%9r&}pMo)v!w&DQ!+f{AZSLwW>V>#TR1Op0< zC{Lt%Ho(-qQ^MvFv0jh?y#!2?ZiFl(c!t)4>pk0_+F@NynQ4fPXzX6s`d5(wX#j`h z%0}kh;p8)W3Mt=Fn$45v?)TIPEBMQZ)c3ZX>@{(=i3%1#dgc>eFv+wRw}0<}2XwdV-pKx|}RH6KL564v9{P+hN8Phsw?9(G%A- zSFiw(IE(4}r*o$mJD%_keb0z|wj<;kEi2z@ zF}`abdbReIu$3^PRm9e5A%}pBKy3pb_7dRQb1C_4*yfQB?ERvDN`Sg3*o=OxV(C^(!-FnpJ?dofHxsY0!H@gX*qkn26|PFL+uZ=k0YeZWkchX zemp!n1nSsu5M$|a>VI0Jes5L-I~W9He0?j77#+#{g^cWgtfSjtBoKTRymLLn*t!n% z3yZGFVpjx>OY+q{GG~l*2zQhpAlQL*)&X%1O7Ql&-LC@HI1@q@GFwvQsxl#FyML{4 zZ`;%BMATruK^;~>lBlT(`W@m(&u-wP;Ty zSIcfPqaF)l#Cr@a_X-~rZA2_l95_=*yryuqo9K)xv4;#A4e0bNp4pCRQhfqh7RnXhiYoPGdkHToD@H5ZlK3++!2tuQpmJwgooZqKB-2j07*I>NYXYuv5s z`4Q>5j;e>AsY8ypQ4czl(Km`nSEqQv-YG;iVxajOsTsD%mPX=NWSs}BC-NuwikK&TC;*~T$~Ah@6JAQ|52|zbe}PA01Oj@=tI%3&0AQ)f7T$=w$c~l}o^=R^ZWW zT`;hi*(w_{ZGq$d->z4%G!xWdK3NhqkEm?};fI>W6G+S79)mbuJtr{yo`805gG49B zX}K-|=!bUl^$P!G%07dEME)pt-;PI63LHm-Lz0o98=hc6-u)bc435-^=dYmrwRK^O z?|a~&yS7_#Wxs@tp=7~euVCr8XNFX|?JWXefJ7~{V%o!mmTLy5US2S8G+2bzvtjtB z0J5WBpevJo0lK^F78ho&9eG5WLo!63FnhTj$|cs@clgepMzT7?FmB`pWC@Crh9T2F z(mll=NcLiDUC}EVD3u3iuS==0*;)sl$4Y-wGFn@d90_><*8WW_1DuKSB+-6vn(jNkl1b_0`5aVh*_Y8$Eu_I zv%2$_93On(aa^+L7C>9SYVya{{JREx2G{Mk>;r|05bs66yX7HC$mK4W6f*>_A{8hE zTU8f^Z~}Z4X4W@{Rya^r|1b95J1VLy>l;=YY{0e~38JWo0g)geQ8K82l4BttL8U;l zfPiE{F%uLDkgUWiqJ$y`133v4AX!B~zybjQ$?v`uy62hq`DT6Jzu%gfwPsDzx9&Y> zpM7@z?M-UIv{!@f*1K*4;8?Cbfv$m(0Oir>8s&rjN-!w(b((tg0K`t5AYWixf`wpD z$tHvpi%Iyr75xKJuBk9qrW^dItFWPKW(Lr!E|;-@6hKAJid?6HZLJn&P3_R%L%Joj zv;zkZv+YUUz!uQkA>&Zu6auOf0N4rfugJ$a8<1pnh=BNy{xYP;rMh2D^7(H|xlNQ7 z=?&q4DO@s}06wurDurh*p0>#NOa50a zoe{@@v;_!z4}xKS9f5}(1{BP!4NR1HN-Ss*bTUoQdLL|n4-zfK19*lLU^O!B090+DNzcE%H;-&ya zshV?HB}^JU{Eb_b%fLixa|CkYl&1_45lVKf4`rnJ@+{RR{r=x4hsD)$s-)f+4BtCM z5+d+FG*njoYMt={VO4P&%-d6^ugOCqa87%6oP87vKDRu6uNCOnoY1X&URCs1y=}(@K!eDIPJ9BTn5I zl^q<^9lrf{6!z3}O6k_K*^AII8`{dA3jbio6`141K8nVL%H4=uOdm5 zazAv-pnTn!QoaQvV7n0sZp?V%b+}jjnl&3i07@<7Ud-7jtQ#BlYye8ua?qhr({Ql2 zKo>>KNz-zm);!U(>QZ#qL(h}$NRE(sU!&_SgEE}WUV<#dZJZ7Z$R*?r6!@;X&J0(0 z7RMP?nL$3VT;!k98`jH|SJXAj8iw+zhR5MTnFBRCGTKN0+#BLqPT0h`sE zBV>rZ@n&(4#K&aLiGp=z4Q{RGofHQm`4WUNkpIU1oJco@(vMul5oiI%nEvy96>=xP zb*2!Zxp`ypZ(x-*cl_N>B9NlFIKJIa!y!k`9e6Axw3$C~|f!TfdL zCs?QUzmcqRAU_oHgj3G)oFvNwB(hk*oGE6gtrWR$TTqW{Todj+Qow=)6_5!YiE#G~ zdP8x`95$UJb~A51DA(eS;xh^n9{WDfLsAS7U{_AU^>qd}AImfbjNxIJohbohTVvJx1TOM*#Bjzt z^g~0x@4Y#ub$m4d`9+-WdRx7Ub*s*i_xx?toewigMipndeD-4Pqfmd+H9H@o}J?j+;y+29v_MC@)X0$MT7-1D5|M}fMP?>5! zEey$?c%l0R3f;CT3&aLLTTn|?Ab$z;J!rGHi`(Md}_}6^rhd$!AFWQ1A!Kr z1*J?Lqkf)o5{|KLaguo_-y##XPR2Rtb<6iS#NYDClHwtI8;HvhLg{SK^Uzb!U1dk$ z{eKY-PtKj9<7+#lqAaA>rGyM9L!!R3tPlpuLjq(fAskTpYQp#=sqa-C{%^Keps2>C zQKLh9K@8kHN`RDH(4Q%wfBhHZMn58H0If6Dt@|af*#Umj1Gv7Au#QCSpnfgWHpxyx zOZs8fMEpO+Ph!;mBgaR|2q5`Mga2iQ(4N3s`z@U!NfpCQ$VTOXF1TA!o{G)HT~0SW{7fJt~GnGunMu~J7CzlSq|{t z-s37C!IB$I4}xKJZaaHCs}~OsLbx~^QuFg1I%DRbLhJcnBOWyAuIgN*Ea}`xEE3sz zj4Wv_4-{w{lz^sKcp1}1icG1>wK1`(NO<&>u3)b**by(SYppjk!^n;vAdM+=dhkMv`{j^+Bw3*V_7DCxgay2xN!avCn}RZPA}qtTWJsyAP~24>-LuXO+7aZ#a7xMTq_#{Ai*WX55|%E+-J9 zW~YkkxZAS=M!OOJbw_p;QbQC-vE=rAgc8O38CT_Q_#Ej3Nz#~sWOpB$s(n&Pg=yx{jx;5uOVVlX4>0*yPHN;sbK^Y25F&oC zJyc=qHmRlu5_EZ&+rX%dOW&~VTt^by?t$pef3nbU_KHfBi^1qBst1{EMSg7N+mY-L_AcLO=O4SFw@ zbmT{p&sWUY3c(0_VQJiLcBvT~TM5MU5Jv-&+$L|9mMYgOU!G+JQOTpj`;-Ib zLqdC=+Mkl#3W2Za#GkMW;gQLl5OcHw`Ec=2I-ql;dK)Z{|kn&ho-AKs@u?j3o5u<;+YagRbI=JFS3 zq3x9wPpv`K_v{F1eUuV1k$hsG;x_b+Anp^H%S1+*dO#VYXOVGRo=wk4EPp-H0}s(< zcNE`HiS41UcNh#*q1tm3A;Y4eS^u{TxC=C2{eqde^Y_pHq&iX6|!62GFU<> zkBERD*ecpl)FN2Xdgc$9iU_4w>TzPW&(~Lw;e|-|6cG3wh^w?hs78jPN*DqtugG1fl!HBzUs`5mdIKqhcGuJ@Wcmu+j61IMJ!wEokk zSe8QBh0?{dJT&x0jKb1Jua@}LYcE4REE)GwygcOP7au755= zqY~M2;d^i}O#c|u^Bww2=gJ2#gR$WKLywu*eTsbpx_n4Bi$&|~8yJ9S6({dkzHU5P z?KhmsX932ZZHaal_mkz5W;UVJSPK2uqZEupLqdbpUV6nl!L7fQHj0a#$x5Dpw4M9$ zWJuZ&VA{nczwx&jFsr8lq6`M-`EjFSS<5Jk<9NiPBl!c5c`1R3R-XuyqaVU)o`%{r zwnSY@EBw<=e&fe^Q?MHy^WW^s^+RIRNMn{7NoXn#Md9y;3~GwhdWo7@hyRADFkLjI zY?oN^bnH0O9T1i@rH$?qXGnpq-hR1{R2PPZ(hHUG7yAxP)nrZ(+C3@kBh{oXu9B6U znolTFiHnz_F;ZtwhTc`=f)z4;J;yWDntC_(|4y9u4k%15pEOsZ6g5*UzT?ezCmjkx@9gWEGJl|Fa9~ zsn|PjGlO@t$Nr08Vo1P8B+p)Xs-qrL0VTQ&8B1AE8`7)op_`szfR_e}@dSBMlsdZd|t^F<^v6zm&S?qJM7`aJnSmO^Go zV%89(vdc_8xjGP$`OheV^YJZJJ$Bk@l5G}9w#}yV(S2)luzFwN;v1dE5PNhW=Oq=76V3! zW0Xkl4jLjwSInEzK+GCUvt5-p0*w#@Vx-bb0!FEq)Ds|L<_NZ$kMR)O2@MYa&Hh>XkAobT~h1yc% ztR+#AiOnUm1XdOIwPVs7FGk8(0+a+5$8Pj0Z{4l7xU;5af;5pI_-C}Ej z9c;Fh>1ujYhu6WYYxQ!M_Yp%(CI6`{eoVQWhN&{o%V~LaY;lP+yg30WHN5m<ID5 zvtR_q1G?m5jw^>(PADae)5E}u?$!2 zf3lJU4}mNTcFVD~Z9tByJ44%fPb?Bw9o?MuBqG(qh&rXy_kLIl#kylET1eS%Z=sF0 zDfkjdlwfQZp;Q^746oH3{?5v>>QSN$;UyBYoW@?Xe@)wefhF#&%PUi7zNzQRJ`uU1Y3eF*t%BT}#WN3c*S)@%1Zfg4rZ%^@**ek!4dY{AHINca-{ z)4lms)up}{z)YAB<4gLK=e-lR+fKk*Q>bbF zAy#>e>l*^FxQz_4DB_@*ICL|V+4%8}C8Pa6-2di!mP|2eHIkSE8r+VKZS3fP`TryXYrryp@ZkZ96I<#M7L?z z-bTJmDX1Dn^okqFpMvmVEh_iQ6bVP>SVYPF7uY#-xKEPH!4Tp6uNpV&DDCCO%A(|(Xcs#TGgu4` zzWYZTvr@Hj<(sp&=IQnyHr%kkuqW<3NYx1>hk?Mjppp;`4ZI9e4Nn-$|I-}XsQV15g>|jIL-xfeBMPofLk2|aN5(1uB zjTqrK8v!$;-Z*o0_1)S%{6BkAT-%V$R$&9DPmGEtk}2?jqOHkhsLgV7H?G>18 z+B?LVBPQCVp>EgHJ^9PZQ*}Dm8)s7Ct-s`69S%O*XCk|PR!Y>hi()~w1Ko3g)aj0^ zN7@5U^mq7sAXCGO`x>LF?|o>Hn|0wa7tP8>O6CqbM_Af5TuKPvv6F_Dp6Rmg&Nq>k zYQn{@Bt1ih^%^Vdb^9h?(zhPZl!B7MxW4M;V*O{n+v}swT7J{|+DnN!mA5%Jht*^% z?~`A^mH(Y4h!@^r*FUF69Mn-f>c(=fR+x5TFM&S&{u%b7=alC@urX;cq|@rlv+Uxv zbSH2<&`jaq7o#e?1aPSGRVdh(#Vla8+;J21s%PIC%=I@vnd}_RDSBs5&%~v82VT-5 z$Q8p3uZY?d?5P&tnU$iki{>3$=IKmqPR+Z77baTBL$ve)CW^w`@Zjk+!CRY}blIT; zk&V0Q1*#Msl{6hU&DIz)A*znNvbKwn9@Z0tO{sAWVkq=a%3~p zYbncOn0aWm-P0-5qbMauwgEv+o)o{=)z?=!AuiK9>aZ={?h@L4XzHTsz)U>ShA38M zcuK3yJ=ZN^IpA>NZo)nSp?9?3=DsI0c@M?A$)6lhpo_}fW=I)X>n)L(3)o(dLt0qc zpH`>merlqi1CXuDZdi3Q+~zwZ>eKc}+xH7MN0Wc-0)Xc#y?)$1G9mw^p$k8u%Y)uAiWk_{+wRihYsax1p z&xbgR*+_T4z`GB}j$Sd<(KijSPaIsMP|`u0Oi9N%&V|{-Ofd#B)1V%mJct8^o70&t z?~Cd5M}a!A-GItyI}_&BdNQjn)bIQC4zWdq8D&51{ObniVoZaE?>3#_&A^j@0H|q`f}y^RREu$}GX$C2?BVn#%KNHjTl?}>LyGCd ziZII{gq)oo~u)ARd*Eohu8rYhB}Ti=`0|LwV# zh28MiDpSA$G$=?kNcXu&B8I-RIHu&T>C)&=gVS`TP4#~padP9+_OxJ6$S-j z)>dk^hd1Cn?X_+Ou{<@aTYx`QZ7$jh6z1@U#VLGd&_28LaTA)c#*{wiA;yatSsw_u z4Z_+^s!l8+JHS_&=9sY+{+zeskui3Af1J}r-K_Z+TI&$#fzcmJ{VJZolv#e7G^r8L zeA!fbyOf>#mqzWadeOw!tM7T_Upr}tShyx^$f^z)tu1`FQK%?;cX;+g{8F}!BZ6noJ3-DW7GC4^rm6PJy0c+iH>gEg2i4w_! zaBtcMN0#x+>d--Kt*5x#4-qK#pRB}nsugVpb*AQf}LP zsAuH_M0_n4GgakqUYj!(qrC-9&w386u7)`Kj=csh;#~jOAaP{1QFVYribuNhQEUsC zwr!41ZgCUL(WUPPeDsP|O-I*IZk>wbKJ=>NH}6bT z@M{#uyM2|#{$*gd8tIlWdn?2KnosS{-^F+pq4$nOc~10Ccyoc@CA+Ou$D}nvMr;Sb zS=&xKd$r(0Z0+NZtduqs=dB~1p_dg@Y2IbyyZ|me0VVV zap~@g@Fs$FF$U)5wfm+13+) zOM1u*Lxbn~PM2@Vrq_8l!WxJ#qVMqhz%#WeI1e_lb| z985P2SdTp$|M?G_8vzL)&d_zaS;<%o%*1FV<7;f}X69}czR4nA?wvH-JX|rc!Uyfv ze@8oRCM=)lvZyR<1a1;)SKT~;%Uux21(dwl%Xgaw@NZek9 zD_qgn!93&(fy6{R?!IrABpB1vUmTHg_GMY)-gt4#D!3D>fai-XdFINdG@S=_X+(q~ ze{lS05Soo^_7TaeKg=-8ymsYB5c`$&d8C|B;(d!>v?BkCuC1G$R^19~##!Y>$bR3< zGt`*UK9_R{VXHw);T4Ba+7_d-X2WGWm3r~9xZ{hi8nM5{PGeA)PJUnvWeIA#oF zw2@mHzZrJZ5q0+~13E)?oOcjox>7ag7l+WC*wKs&l^qxStQWfRePdD~V135-j1a%p z!upkUC(jDg^}=l7rsn7Oc6LCY??J=MrgA5HLe)yclfPI?Rv1TM1XkTqdgB*QMV>}6 zL^1EE<5f)zipnuy;3`+r@Q>0t_1UzNg@*fw`VWJ(B&OqU&aZ`6kHA<-+m07EUsuow zY*vGD)VU=v*EAh>{Lp9mHmEiZo=Cp?r=3yF_^*+b>b~y~dLibi3ma;1&OcNwT0DBy zVxVhJO4lgu0pli&yv>lC?}TWQ?POziL#n0S!0NLZo=AbkWBgAFVo!}FS)R@@TK4 zmCYtp`(iEH?y>sf7U;jUBD*T z_dWON^!=UviD#XB4Eyr$Niv8v=PfSM(v7Qc$$&NCZd&o>xuYI~2=jN@AAx^4XiH)1 z4J0345)LQ&4r8>D&cS`Cs1S4hNlnGu(G^2!RVqRG_Z~|*-AXwqZj6aCv_~x9KFtsU zS9zndVAq!MO8l^;-7P~Mlq>hmrZe8{v~uXW_9x~)Usd5fwM2y)==2L)!>)*a)<+#f zZ*75sGAVh^pA|2X#KM8rvTm^rb+8f0dMVPdFyd;}9zP+LVH;pd)AiI1?=;c4+`z1Z zSwVVMTbRD?+jG8ROoD?IEn~s3o5foW1p5{H^O6GoZ&?!a@r-uQU8e20ulT&#ZT_M9 z#P#?U_b{iJ3e5Vo*!k?w#l@R)i18KkZtWu3jQx2WB;+TPL4{NHGjl~H`jRotDbs}_ zsfYTzBH1LQm~&599gr)}ief&A?$MHrQLDld@}@bl#*vErswkTiPpAC7w8JrMM^jYa z2Ho2wwSbn{X<{tZwI``y;e;u3p&s{DozR6kg{$hQH&1&%a8&h{_UK}G8H+5m8qS(ZYi_eEpfJ~`&}v=PFMa1mns zA$E;6X$6rCFkYQY#3R9IUCg#uLp|oE)r@1Yj><2p&=ll}O@x2vC|J1oywh2cuCosPhvf8TcqdW@R}0ooXVth`#oaLOJligH z)CNTtMT|Gz`ywYQMY+F1361}WcE?Z;_MD+-HWRUs_@)jY5!L;PKd zA-&#DYPMIUnSHvGc%cl1l0_>UhU$sp4r+w>bX~3%;HhoaeJt$x4(D)vCU-xdUo&Hx zgGKW*Htr1R9_)bkSzhx5!_&rw7&gH)`dg@i6F-}E@zG|V8I5ij-3-~S&8)(Gv1QxC zNYy zlo$ld8xRiypYQSZQ9RF8ODuO(U&!us=KCiIQ=G#+)?@e?^dP$4X9)Mwc4EBL(yn(^ zH#g2wg#TUF?EbGcvmZg>wyxiMIvliYyc(T+O1T4(wRu^qkv$iA*Ai2*Lk|+2QB{~& z-8%8>7&fzrb6wfyt;j*Lf2zE zTTo%XP1Q>|&Q(rz26*)l|Gk{-8t8*PsX|L*xfYKlJxjv<=AT$Nyo)4QG&hG6*!e!| zGAfMDdX$*%exf&nll|=6HJwSgt9D9!>f)yejL5Li_Hf_H`LS~Z-lKQj;=sdyp_Rq!0g8rtrqwp~zu!gME#826#Z?rvrd zcO#YYD-kGFM?zFK$9nDW@*nn{(auSnjrd;Rn9q7Hi92T6DWmmqwh!F8gXLE4hWM+S zP$%rXJ3RXIl_?|C`Uwp+`WHC?Y4j_=YenJSg5`|}&pi_saZhkf9p(MX8J(S~ZW(b? zWRqi0I-@Hp8tt%XfmiS_(#2$qVsfIqI@B%Jy7n~itehJNPa*6WykTnlHZQvJE4RIJ z)020lvBpw1j|*)K4MHnV7WXaW2Ren6xa`qTaGRTmfnn{o3Z{16evSbh;CjbS6jW1O zZyA1s%Z#a9)~WKwV0t`NLOayqH@>=vioRbk0JuL_u<2QjBDO+QUAQO5D^h3$y<309 zgy9aeY+A1NxmZ`L9=+=P3@1bUTlAnlRv&}W#WG-y^A{A1pXN6&&$6**NGW)G!aZ6jW!Vl5y>#u$dsaCt?(DbJHtV1yn0Ei3HjKA8fd`Y*ry4veVZ<(jZKp` zDdQ=x+fZ*S>RcCIlvtGP}!2(-?%Ehdo z)K+VK#~j$`k2`zI3C*2q`1xZQs9B;|?<0r4y^Q;F9O z#A`Q19mjBJ4GS%=dZDbos=I~~ikqTWjf1cbGW<))3%6>K;E|8}xSdgjNk#1STZ306 zi%vQ7DQ6bCW}4Pk_;eZ5U>CL?$bG-6i8tal_ytibJnBUChOD+VzjC8Kt(@{#8_*td z1^0p&cp%Q=mR;BsRR@8h4Pp|vZ&|y0Zg=hw?@{HT0;llkzHzJrdZ~vg2x~l{>dB@P z)Z)-K!^wLh;N@2`IiDp8qsOj){Q;;K6BP8xq z-xJ#0G5X`aWYLlz_^nR@f*u{hj_M~qj~dOm{{fSGEA6CsdAaGA^5Os_)-L9yXBz5K zS6$&avoF29d(nLUv5{DV{POF;=&#U?8l}iBs}X{8U{@by$^Y7Dhy8^fS`d3jdkk^I z`UE?33=JUb{(hdMo#%(Z-?K*)0+uNX|&y zZu1{O_Ne*x^E(!*hjv3L|IG?Vk*p52z_ojZNWf`e74A!7XqW38a`KAib0LL4?9M$7P@5B6x7{}&3C_LXn>8t7T>t45C_esZZ^@>F=VaQ75 zqcNgWI>tS`s!OVl>OJqey3<6$W0+rjqfE(+LYdYwoe~YRJzvs$s9>nck7fYx$V=$tO|e zqnw^6AOC>boy-1FJ5DY=DBwHIAM9Gq+-&7zfiem)J<~HR662OQE?{HqSyez36_0Ly zoMId=xe(*w|F!86Lk4q{M;PrR21~tY{<=o>H){(9#YIPpQ|slLgg#Gz*%@CptM)tzN;anpWgVsQ`NxSa!)A1FoW0adm7)%h$%t^xcq?qpJ=Wo$d z=U@EPljrX)V{@pRfv~)y0U-@1iQD3&06Q0w^8x{2eco|9IXkDu&z~Nu#%pX#n9yum zj1D&ZCcIxOyZhRjQ?j*I;VnJ!5L9q9N{-BE#Mp-n;-c^HY^QzlV({@OcTAa7pI?dD zfEcht$wzb@3Z;!;TCNKv+~kg8U_TkG6EP5v(P4;Z{sTN-vs^+n-=gnNKi(aQ7_FHkp4=*~S{cq*XL$CHSv(J&Illo&lAzm6lYygV!p|w1B z_p?`2N)9d+lw9;*AL`j>KmEYF2~=2j6)COuit-uae6DCD~oj^WCyoeEEhx< zn+~K)G0TSJYQ;`$idRa|)!6hiM`27DI_#ti3?&Bj^$+4$Q__1&g#O$yu}iZry4RUi z#?Jy!Hg;iV!@u|3gqt0{1>!-9!aZWpWWcADmb}Y5I|BDmV&KpHFK7(lq80gqV@mW} zeJ@pH8`IKf=PNq$KZGbBqqdpZakA0iy^oynksGz@j+ehm08s5n?$ZKkI}WKlZbi#} zI#SSouAoDKNclyS5hZu?W)mB}%KS&rNhYojS>((O(S6~p6!}W!cTrMKN+qS|CUysT z8q?73JElX*U$Bc{uS8OqfPs86;*+NR)0t9tD-P|XQ9nw4Ec)7WTUzqlgfCJadtllJ zk8xGN-_=i39`jzXgD?1Q6){45N`6OqtwV-ApSTF zqhp%U9&;nZ-)pwFWPFt~V3)NEl%&vjQlJUN)fb?g$3S3(D49xM0~Rf$n9nh!cPfj!LRtXP(~=@KPP7n4Cb z^Mz3TMqf2yA@IEvyTwR}j=mEVM~uFekHUesS58?~E1wcVR=huOASiZV&AyAur`D8A z-oYU`lxo{r=NCVq&t)Mp#o9b63{#s;Sz+s;d}+JhFxENbWw$9W8~OK_ZOioW3&hkbY5rlaooE%c;;GBCND%lk4(QLB=tx8G(SLUKR){t2U#SZ_$*EgQ%k z`4g7&@n|5~h?No3jHl2bQ>XSZmf@~B*@{ig@g`A?I~@1polZ0Z(>Eh>pxU{nu%Vw~ zyBU&L*d&l+Yp2=U%%8v?hFh1rC244*bd&Gb*p$yg?Cii~yu@De6@JHra6C#rMz7I# z{}C0FOLuDRDCgaiSmmus3GMJARL-#yP7C)+I&!cbU>|`~l*{l_^sNjMk42u`Ceazi z7gHeJta=d05T&0?l0Jfy+1-3(GF#FkR3<=!_pFrrKXg+0iAQW&2t7FT|b@(p8z@lDn&`TmWECpI2^c7-p*=#p38e)e=_2t^bn|5{Cz z(t!hseqD3Wv8Scf`0zN(Q}~zf_Fk^9u~{QzfyOkEh5_El67vUWP#T*1N5M*wvJd!6 z>T9@1PnXtV#usyusf$PuyNuR))5p5Z`}I;UIDa`Ux4EgzQ$BHXZ_2JxVpF_rb#vWQF`Rq$}si?^gSF zVtb7Ico5dQ!Ga9gF#}gWOFfG!R0Wu{f7Q#<~HsGMt{3+%Am6o} zH13e{J^8dEKyjNr{j9_IGKdX+HS_bZE@`@rDwE058LCNTDT;_}JF^`I z5huG<=srls-SH3E=A8|fG$<$fnIN6ZPX7!gV8 zoujQM5y^*6KO(WVr1SKN_=TOYf}J!6;qR?~kq;RD`3-^9Ex9*9k;k$*pzsj(493w9 zs_@}0!hAY2qgjb(#>^>No!&vfvW`HTWU{@pYWSea*pm==r0evYJ?VZDK9L=C$I^~XMfV7G$ui)Z zGiwmpG#uSy-OuA(j-nV}+0bF`lCBsmnzTz~F@Y{5S<(6?()2OuUpg2n3Ktvl=7C@) z{WF0Zc@snWbot~`6#QlPUTC#+Z&;anKHn~RIO)b2!fzC^C>|k; zd|a4Jixl!B%LHi10+os6*E8)W?T;bdR{}YI#W`@Cx95wt4@76vNk z*Q?u8KflOKwkH&1pUjzLAhwekp?(~H?q|5qdA#|@0M_T_Ld`R-I z98AI2b3DgGb;hz&k+4Rwk+8U$UJwqD)_Gqwll2h~_CpO!a16L^M1ArPT~c6&U4saH zMIq*Len5dgFf!ThVF_LWqIeK=q1A5@=QJU?$?$^9`$*1to~2!+RZ23S>1MNs0zWQl z9N&w0$O1{=zOL3^ctUkwTS#7;BZr3jBY7w{l|b5YMr^kQq^L6B2pJ~#2Tc#cqxrJn zERG-8eVQ6v5Qrd!WekPIn~RNg(!$q7hG$7Jj5=`C_u0%flY<^R0@0)p9-d7jWC7zC z5P>q_O|5P{^80`aIK*49^v!;0QHqOf*+_DbLwD&%1n~tE9&DU>NrcyYtcE48-*2b7 zv6q#R4dVJNaV@6!&GC+UI>TxGoG+?R<-w2tR0Ho24W#(Aik75YM{&N;A8|by%m%9; zj)Cis#Mw|b0!#NOPvu9DzWX)^tGm{QeA#oM*HaLP)e7-&aSLZ7s)PQAGU+H{p`h)Q z+EMoHOQbUreq-R7 z$*MQn6t;5`7N)ojMBWN9qpn`~0_mynH-vBbj;JKHqf1BqWo?04Sr9WZN$VuAQyl-= zc2d-Yf3HT!pVzV4m|DK=5Y9BG9tazH&N{gwVGE|1Q?NupTT?N99nfXp*evevqhm)LejI&zIs>j5UsKue7>C}b@0O>by?=}=hNJnSEf4kNsu2nr zU|>HV+4Pd;9)xD?<1Vbds7TNUj=2LW&7??iVLP$dEiS!ZbXT_1w9mHQFS6mrj8%fOVEeuSV zfIu-^`Y|I*ed-m$st~xF`2D?#66W0W;QJ*Hh^(o=jTAYLJx<;3VM<(G=C(TU=-n7C z+tZ8EImE{7mB2|?egi7*PYMjO$? zo+Fqr`H-jwR}TX_#NI!Doq7oBP-BQ}ao=qjzSL?(-{_Nymbt(JWv&(bscRReeD9Hy zQ7|F;hOk)hjzDCg#8Vj&l&}?PCnU(t6tU9S54Z5X1`$51#YhOM1vt}WqO((idI(=A zB>OU3gUzip)^UfX7$uV|2AXx#b-?kLfa7^?P@|9Ze`t_5vgdpH={Cb8yX4<8`f2&# z`-aj#${}!0TIR^Rg%mH^@p39t~DLikc^)2Q3Cr z_+RdWSot22&{&DRmmT3`?ZWa#sC%L&ISNPQe)Ar+Uc%BE{>Gu9$t|n{4=VnyRY;9W z$x$FZiOvOR*-dNYd|S@i4P3DN2$Z}UmefnlbgWUrSAN50ze(q<8T`f}@Sk6IG|@jL z8RxLBUIR%ax%ack&D#wbhSNKSxv5|Ojg)bq)&H$jm%aZ&Fo7LAH$aPmIB~J`bu15+ zvl1wqm+?|HyqViJtr?LjULz8|M|XcRr|QAI)&NznjuYb>Gcm80H8KhpU*D8XjvP6$y2zX9c5 z@>R}9r?tef=?N`ik)qYq%<{WC0zX3x`M!(?6%5AoO`psfOI{D8Rt#tuvD$Uxb+o%6 z(8)o#X)8e8h7)ChP9<@>HTjdn*NU#0Yg>9jtIunu{Uk`s9&}TT6 zD$f?p&T^R#cB{ANLH*zRfQwDK4sFEDquulls3bmd>_XC-L3gJ2>Hl5lz=dw~%*W?3 zIP3%c;SWK5uh$seyo$Z<7;i#^7bjfkvOEmB>oConNmW5K|MvFQ} zxe4!cvRUF_h1D;@lUTli8tKW)E2MJbeM*||zW0jV3Z51ez`i6&mKYFQ;^HqPsoTI& zSj%u|TDt66EnERsJ;?xi2MGEW`<=I;%E@esbTg@wF-^jA=aZB%{&rQF-p)4$X}{$S zQkhVbeDNd07--_?Gqur={~r`9Z1S-uUVe#>D{~st*JNY>-+T32mMQf;|B4JVFH!7Y zW)Ci?%j16~Z#cYoer_8;YiLq@nRI4Pp~{_vIy^}KJsH^pt)a|AqQPCbi+k)!#gHgB z@?K#{GI_;I2HS$|SU(D6@?~Yitus3wCT|4F4tFEpoyyI^!DMkB`{n+;9?c+|PLf?k z(1?NoaDk4hRt5EDZc+ej{&Qz+c=BJ?swf%uFGsUn)?K%ga~z9 zl23~&MtL*>KolCx7e-n^OeW7Vl-e;z$M|(%rh$jX6ZcsX^l#v zamte|Ue0!GfmO7iJZVv@&Odc74yPJh!)rytLh)2dw~rE)-{-kqCKBj~FwJRs=fpyI zXZt5XO3spL|3mVwIU3t=FK8uLbT0ad9#yb_8)5fsEte{ZFPl@?v~ndQYHy4&@ac=G z%GW%=3flQv>eM_yDdkxoNr?;d%zYS!=nrnNJ zs~4+J%v)HoBZ?|g3I=!5jL~HSI!Y8CJ6*^^10`rRo?pb;mB2z04?KQ&>f&9(kCkZz zKug_J6-fu>C^}l~_pT}2S^>f8b*bXdKyO&WyZ!b7d4FDyrvh;=SCWZ8JulCyvGj0; zKvJiBu79F~M|pB68MA$qRF8M%Uv&C%U!=tY)j$}&+kg8FLAd@BKl#IfbrjvlIrk2| zJDmh(MZhd^ZV5A2+KO}Cb2xow380QX0h~<5ZBk~mtPCJ%ABSelR|cMiju(CgkfWbl z5KFw}_f_-7*gmlipOM8_+0Q>=b!}qUidi+oD3I!tMZ1 zsP;l26+^%3C7l4(!S0wfY4PhVdn`aS8%V&}2^#j&T70{Bp5*p7RRz21Uv)wf05RSh z%WpQNwpi}K*4IC^n%_Mc=gze7X@CSEgye9q@3-?u>;zq(J!$pEO(0XO7NHz5cNL1S zh7KX%Yp+nR*6R{Bax=zpNGUW4nO5w!{WC+s5rIle+MIoNAQnO9w&{ktMiwVAq+4?a zl~wBsiyJwyQw;iDU>psch~fD0v7PYKm%;6Od@bMa`X!}xr%B+vHn!ZG!4n8hu>IxI z?R2C?9|_yI%!nY|yeswmGDav%zfj=Ht2Ddw_d_2l0b7-@B-RFXvXdT|Yc#o+8V*Bo z-UNdE8VF-Sjj#OW_w9V#?*PvY!oFByUsQQRxHe=)k#74F5$-6~0-0MV`P$O1@;C4M z)LH-=xe_0VHv|tlo|=G+yQvBCqXKfG!LSh^Yty$loD&_X`_%Y6!fEQ*D`I(eH*2#k4oNeV82 z*Zzq$Slg8{Go&B@MBDy+kPEeUGp1UE^1I&{RC6Im>>PUg{c$Jk;x?2s_f~oEo33sl zJv)%8hOuLuE2V9n0N#Go_u}N)UvRWtPRi4ZyO$K;=?SR!_Huj0l8`Aits5qdAMW0z zkMvH6yMAQ-V;l5LDC}Dm!Iw>RhWYST$@$4M7XZ+f?%t?ok8%XVL=?A{ zYQ|SquBb?s>n;M!Gk;jiB$R|i?r-i>!Y(QZ28RKRR8$hcs+$8uS>7vK0L*hHw#=|3{dzxfD??`3d{X`L0)mw_ z`#k?T%rAS7UBU2dx@48@BF?DF&LIR_$y1zxK-|Pf0n6Uo{$ge1yS1eKtao391P@?P z2Nbpu48afrx~tw87s{)eJv4b%)eW$&uEZy9{(xoQNbaxl@ibNE$x_l&=kWt7%D&))gcuW*Ufs7|Eqq=$xx!nNB-Ir&n{?GvY(hZ>g) z;w7sY#23~Q=&{Z5w*l~3ojVL%wL0k17C$iU`|Y}antAs-&gJ|GIe?%JT&et~EHn`6 zR)0-Qc*riGSORUC|@m;pLyDgb6=K*PP+rq313(EM@ zpcvNc`}noE@KjTJ`K)+Bmq+Si2*#cVKM#0(r^RNYYFFZ9TCNVwV_FpYiO~w}Yk;&X zlUt&>yW>XT>W7AnW;~u?ANw_ShtO4X#>L5) zbcM7U26IHk;A=L4K?u9Tr9?P0q}pr;EO;z6lV!Y5u(oYRN<9nb9gFJ-Jj8v4!%v&0 z0g*^VAAis^xtnJfb7@16hM9wwdcxvu8SlTC^T(LPSMgJffp>^qX}Ne3Lc+&qlNcI%M2r;y?7a5 zo7JBtAKjKRVyNZZ`1E@Te}c6f^C^tsndq`s^nl?tuX5vRf{l3p7|6|K0>l3H+x4P& zaqoq?>THju-fV|Qt++xbVZ(ulJJa!#vNr1%@Y(m&bYkoy=Sz`R08R%s0Hxl!&p_zt z70KHdf1yzAvNOgVc1Q(x34iH=Rqf9jTFmg8((swq;sU-?M>X(9)=MVF;?YUm`)gg*d?mM2Z#bp!ba;)v-jzfuk;L!FppH3OQPYqPJYCw0|i-=21=;?Ig(C?xf=J6HQux!=_o zKguoK?)S_s)w<`~-iGY<<} z{<>hv6ND|8&KMlOCkM?8`F5R=f#-AHo-cX5~UtS9Af3{bvS z;ti!j{i_85IQ<0N9y4YK^x##glRd1>(eohs%80AXvE12ws^@}>PAexD9y>L-X}$S8 z!7XW0cG6ZmA%i9=2fgZ-8{rtvpvU|@we(i#CGm8XyV&eM`LBXdT0Ak-{4D?2plC${ zJ0LoWF?#*w7&cL<=H;f&awP!W+^79Bg*dlMA8n{thQfM9j)oI%bZKMmzh?^AieeOyDRs5RSq;#KcK%C<%<2U%uXO{eLt}_g{I$v4B8JVp_4f*s> zn7(~uJojSHL6HYXdP(h1q%@93Q;j#OFV}K3p9Sz&hmi&y7JV;ih0;E_Ro}C5*P+2> zFF=b2Med8d&Kjl#+(M?$5c1jstZvjc)GdBY454P9+2s1@wts~*_e&$12vxahU3-Ni z>=&$B46xCW!lzz-`1Xv6*W$gKRh{wi{l;S~v$|qvBMxL`bEr9H8i!@u@qac!S;-+iQ0~45+}M?R zX-EFxUpyeHrD|L^k$xk@=6qUt27i^XY$s_FxZdTDYiHC0n)%XP2A9eQXTC%V7aU5v z?1kS#@BWl}eB~zo7kcTsv7@BZ?D(ek?YAtwZ+lq1N9~@i7&p}9t@MznT|E)9NQnky zfeml6d8YRr9pm>4!We1gvBZt2FCUH3E|{zN}daj5iKr=8RmisnZFH zV@=gcmYBw4EOFV2Gx#Y%&$1SiUpgp@5WEz>*B2D3_*TFcoLjY--y$>?$nn!WSL`(Z z%jFHBbkV9(bXKa~u}^(R)F+oAT(0d?(NKo-72V9Fg{Q%M@q|IDnev-jgZ}HS1Yr|oN8=QbH$aQ2NQWG|0)bQ-^ghnKtOf%}O1MhcEU^#c3o_?0kcLQOJ+sO;+IwHB20Vr+s z(iP8&PtbmVXVPdkW0tX?t^WAR1+1-vV}RP#Ey6yTU!U%-lp9=OuAXR|F1hz|*zrDM zzxJV=Fp@UK`t6txxI;!213d$5VQI;0rSk5(TPLwE#-gr)= z*GcUDruSFo`iTnGIYIs#)_c;MmtPl&W*JFH7pU>yXE-W?A8&GD_%2e<5K%BaNy~Fq z39-=g<$3^oI;4MS@_+_L-g^8BAWSDrJ=qLVfoV*)LoY!Tzddgj)tMd2@7{YdM0EmX zd=;owu)yn=P;`xN@B$0d|7q{q!=YN&xUEz}o!WBQMK-(0ZJMc2WROaGkZI3k#7GF0 zj7m{csF7-?T*A17AsQOSZDPdCp2#j4_XrUsCc;RO&dApJ)~uO*I?p-(?{l7$zvl6* zwZ8Se-|cBg#HBH+*FmTVXODkWq8nt&Nl%E~ZW1 zZ%o_s^1crWE`W6~>ob9h6QOt|4%o@&5`$U0a%2&~pd%yiQgG~qy69xhK94lAt?&80 zl;i!$uk;T_-SgJp#a_uD!X7sV%k5=3u3iIUzXVJ;Nd2P;=qB~@jdyhtIx0Js3Njgg zbCORC-Tu-ZO(S2WgvF$R&rpckXYGK9L4ixq9lZ_{BhRW~!3--D+akzmJ7H8F#ugY@ zzq@k&9ylM;e2Ow#kG~jAQc;LK?;i{)i#CDJ#kHArG;9ncxyTn-eni!=6$oz+(KjT! z?OAX2{#sY}=FF9Lx~DbQ41z&+YAJTV*Eqn^b39@eI7aek`Hp#dB#5^Q5E(AkpN_@_X?p+#O>;4EpRdW;L#TdDdY~MN@x8 zfNYkTy)2ViF9uj|>mdeXfyxtgv5Kb^ggkcUX`lIY%?%1OJEJ^spp^NU*EuM!F!QJ) za&=$!>%R51s+>q~CO0t$I7;fD@o39lu|yVz0D@HO--zjy z9&-Qd4RbXSr>bzGG{0Y`BC7+g+fOFS_d5IgX8cGAe={4h1&=U<5DmnwE^+_7s zQz7o?XLWOi5iX@6swnb%*P28cUP6XK9%Y0vw{#Rn6F5hN4MEK$#ppAc7iY2-W9}Y!DJrmzWBCzX@VocTyrJN_e9|TT5T>c>Rd*vV96iJ(W@6=xfFM?@q@f09 zQtB*o$sS8lC#XuSa#OFy%c#(SkfobBuHuA4FQC{fxHikO9zBJevfdpY0#1*^BCRJ; z4;vd?X9IMANCvcZT(p>v z`x=A20`iHgC%TV{<-j*(bZsoAQd@XxlS9`7V*(55bKyCjhSn{dJG_bxV?ib@yWw)S z#+9KKe4QSt)c$)B{sfncVRqyi4*%SgAL+*kGvAFUt13TLRq*SFes-1KZ6Yx>G5a~9 zNI6AU(};}Qrd&cR#NAu!>pyoVkU(L(`u|PWy&=CEu4pI9(v4YX3q z4gmon0k*oeT#)pDZLxtg&z zml)Qj#`|YtSt}atYYH`Uf`yw}8j>EU^Ij>CYbZ4x#C7UlR@ZdAuCq~sbF@@1kbJ$e z(aFtxFPNa_9~2<%w7IIhexXt@XT--DD}RkO|0JnM(^tcd^N3M>NUS&C z7A&_sfl3WEZ?Vs7TR$%R2*&x8uz}1Udh;Yfq^!jQA`u796fMMnX`s~WhUo$lVx5Xn z!=KFAdr)2-P)2W_ufftC!TymkeQ5fApD3`4yRi;(=F?|xR%FDYy;UWsRu5mQ=#_`9T z$Z*$RlF8wNM91I?rmnXAtLGSjnIL^F=>Z40q3%6a6WC~kTqclHGX?vVxO2nCqgK-! z9|W~QW*2qGF5($c}rZa2O_+xV}jZktW4!0sgVkn7n4G`+9rG@_>1@O(`8-k{4oJ~_2vA5KLTScnGqcIb!@&a$a&zMgA=*|$OY9LJ8`A@F~%5`DB+>u21hRnzdhB4nGZ}vH)(D;q_$j4T|Uq1wpTbU7bxR+B^C<%cAG`b%H(4f(1rWW!hIBu^k+Cf;-s2%ruFuAIO(UZ3_ z#wft5)}UfKN#$`1r(W~9#Vv9fq$pCatVVnIWiTS26)3pS+c(p-4Af}k!Ff?={Ei;jFfX)W|JVDb8f>jEQTo|Ru3m$ zRXrVBXhkbt=q!H8e#ksecfFLKZZxp(Q@23Cl#mg(C2Q2~l5Q$A0Cp7)m5g?|kffKmCP7rJ)X#lDg67ZzKvKc5+v3hoA z%b77JNoaSlVukE700wukg8E@>&QRq^(gVDHm{A{UU?2R(0{6+)@DndqZw1qs05{Ic>vI%Ji;<-0c;%_V0Mq(~e zn0zJdv1^yG!Lw*<+}6P3)v#7d13sL46agC_Z31lHYUc=%>;Qx(C*^I*<=xSfn$Q8@ zfo}2C)}?TN1yaTzZMcXpeG3R0p8W_&;*vqnI(iEx*v88cMX2_q$#58=)k19~N!v(T zAE7P-`t1O+m;2wS#&(Qj z;h{J{PtDbpcDogHafil65lpOJpaREN3)16nIl{Bq_!?MS*hRLRVKW&@nYPY!?yR(8 zV;VSXqg`E6Ht%dAp{L)&p&eH>Y%5WL0HL zL1x;tmD(O4*`Ltsz$PDO=RlI3VN<}3ZffbF!7j(2n+W&;P&R{hS;4HVLKsxs>VSjf zatT-e>uV)pNxgSyCj?cqi@&;1daD|A$SmPrmz@Xy zyy9!d@1KoRF9SzR?cqc=ythhEJIU)t7@YdIHvjYMe;RzBFDEBojorC@pQOG2-L8Cl kec25AD<;xkYW=wjI^Sb_)|@REkOP0%T~0fTY&|3X0p(gcIRF3v From 9bde981c44e9bdfb459d8e1a031c5462ee563daa Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Fri, 28 Jun 2024 20:30:26 +0300 Subject: [PATCH 091/131] feat(queue): fix static check issues --- apps/emqx/priv/bpapi.versions | 1 + .../emqx_persistent_session_ds_shared_subs.erl | 2 +- .../emqx_persistent_session_ds_shared_subs_null_agent.erl | 4 ++-- apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_group_sm.erl | 2 +- apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl | 2 +- apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_proto.erl | 5 +++-- .../src/proto/emqx_ds_shared_sub_proto_v1.erl | 3 ++- 7 files changed, 11 insertions(+), 8 deletions(-) diff --git a/apps/emqx/priv/bpapi.versions b/apps/emqx/priv/bpapi.versions index bc3e4f1a2..02dd84f03 100644 --- a/apps/emqx/priv/bpapi.versions +++ b/apps/emqx/priv/bpapi.versions @@ -27,6 +27,7 @@ {emqx_ds,2}. {emqx_ds,3}. {emqx_ds,4}. +{emqx_ds_shared_sub,1}. {emqx_eviction_agent,1}. {emqx_eviction_agent,2}. {emqx_eviction_agent,3}. diff --git a/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl index bf0798e1a..ad00fadbd 100644 --- a/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl +++ b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl @@ -364,7 +364,7 @@ agent_opts(#{session_id := SessionId}) -> now_ms() -> erlang:system_time(millisecond). -is_use_finished(S, #srs{unsubscribed = Unsubscribed}) -> +is_use_finished(_S, #srs{unsubscribed = Unsubscribed}) -> Unsubscribed. is_stream_fully_acked(S, SRS) -> diff --git a/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs_null_agent.erl b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs_null_agent.erl index 5bdae08da..d984194a8 100644 --- a/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs_null_agent.erl +++ b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs_null_agent.erl @@ -14,7 +14,7 @@ on_unsubscribe/2, on_stream_progress/2, on_info/2, - on_disconnect/1, + on_disconnect/2, renew_streams/1 ]). @@ -37,7 +37,7 @@ on_subscribe(_Agent, _TopicFilter, _SubOpts) -> on_unsubscribe(Agent, _TopicFilter) -> Agent. -on_disconnect(Agent) -> +on_disconnect(Agent, _) -> Agent. renew_streams(Agent) -> diff --git a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_group_sm.erl b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_group_sm.erl index 3932aa6ce..aab47802b 100644 --- a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_group_sm.erl +++ b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_group_sm.erl @@ -372,7 +372,7 @@ handle_leader_renew_stream_lease(GSM, VersionOld, VersionNew) -> %% Unexpected versions or state transition(GSM, ?connecting, #{}). --spec handle_stream_progress(group_sm(), emqx_ds_shared_sub_proto:agent_stream_progress()) -> +-spec handle_stream_progress(group_sm(), list(emqx_ds_shared_sub_proto:agent_stream_progress())) -> group_sm(). handle_stream_progress(#{state := ?connecting} = GSM, _StreamProgresses) -> GSM; diff --git a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl index ecd06846c..ce38a72f9 100644 --- a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl +++ b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl @@ -40,7 +40,7 @@ -type agent_state() :: #{ %% Our view of group gm's status %% it lags the actual state - state := emqx_ds_shared_sub_agent:status(), + state := ?waiting_replaying | ?replaying | ?waiting_updating | ?updating, prev_version := emqx_maybe:t(emqx_ds_shared_sub_proto:version()), version := emqx_ds_shared_sub_proto:version(), agent_metadata := emqx_ds_shared_sub_proto:agent_metadata(), diff --git a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_proto.erl b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_proto.erl index 53a6693b2..0b1770f3c 100644 --- a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_proto.erl +++ b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_proto.erl @@ -244,8 +244,9 @@ leader_invalidate(ToAgent, OfGroup) -> %% Internal API %%-------------------------------------------------------------------- -agent(_Id, Pid) -> - ?agent(_Id, Pid). +agent(Id, Pid) -> + _ = Id, + ?agent(Id, Pid). format_streams(Streams) -> lists:map( diff --git a/apps/emqx_ds_shared_sub/src/proto/emqx_ds_shared_sub_proto_v1.erl b/apps/emqx_ds_shared_sub/src/proto/emqx_ds_shared_sub_proto_v1.erl index 117b34e98..2dfc8be65 100644 --- a/apps/emqx_ds_shared_sub/src/proto/emqx_ds_shared_sub_proto_v1.erl +++ b/apps/emqx_ds_shared_sub/src/proto/emqx_ds_shared_sub_proto_v1.erl @@ -14,6 +14,7 @@ agent_connect_leader/5, agent_update_stream_states/5, agent_update_stream_states/6, + agent_disconnect/5, leader_lease_streams/6, leader_renew_stream_lease/4, @@ -30,7 +31,7 @@ introduced_in() -> emqx_ds_shared_sub_proto:leader(), emqx_ds_shared_sub_proto:agent(), emqx_ds_shared_sub_proto:agent_metadata(), - emqx_ds_shared_sub_proto:topic_filter() + emqx_persistent_session_ds:share_topic_filter() ) -> ok. agent_connect_leader(Node, ToLeader, FromAgent, AgentMetadata, TopicFilter) -> erpc:cast(Node, emqx_ds_shared_sub_proto, agent_connect_leader, [ From b4a010d63b7f936e83298e86cd966e2a854fc989 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Tue, 2 Jul 2024 23:14:42 +0300 Subject: [PATCH 092/131] feat(queue): implement unsubscribe --- apps/emqx/src/emqx_persistent_session_ds.erl | 10 +- ...emqx_persistent_session_ds_shared_subs.erl | 215 +++++++++++++++--- ...ersistent_session_ds_shared_subs_agent.erl | 30 +-- ...tent_session_ds_shared_subs_null_agent.erl | 10 +- .../src/emqx_ds_shared_sub_agent.erl | 63 ++--- .../test/emqx_ds_shared_sub_SUITE.erl | 115 +++++++--- 6 files changed, 326 insertions(+), 117 deletions(-) diff --git a/apps/emqx/src/emqx_persistent_session_ds.erl b/apps/emqx/src/emqx_persistent_session_ds.erl index 62e6bdd26..517681f9a 100644 --- a/apps/emqx/src/emqx_persistent_session_ds.erl +++ b/apps/emqx/src/emqx_persistent_session_ds.erl @@ -621,9 +621,13 @@ handle_timeout(ClientInfo, ?TIMER_RETRY_REPLAY, Session0) -> Session = replay_streams(Session0, ClientInfo), {ok, [], Session}; handle_timeout(ClientInfo, ?TIMER_GET_STREAMS, Session0 = #{s := S0, shared_sub_s := SharedSubS0}) -> - S1 = emqx_persistent_session_ds_subs:gc(S0), - S2 = emqx_persistent_session_ds_stream_scheduler:renew_streams(S1), - {S, SharedSubS} = emqx_persistent_session_ds_shared_subs:renew_streams(S2, SharedSubS0), + %% `gc` and `renew_streams` methods may drop unsubscribed streams. + %% Shared subscription handler must have a chance to see unsubscribed streams + %% in the fully replayed state. + {S1, SharedSubS1} = emqx_persistent_session_ds_shared_subs:on_streams_replay(S0, SharedSubS0), + S2 = emqx_persistent_session_ds_subs:gc(S1), + S3 = emqx_persistent_session_ds_stream_scheduler:renew_streams(S2), + {S, SharedSubS} = emqx_persistent_session_ds_shared_subs:renew_streams(S3, SharedSubS1), Interval = get_config(ClientInfo, [renew_streams_interval]), Session = emqx_session:ensure_timer( ?TIMER_GET_STREAMS, diff --git a/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl index ad00fadbd..94bd2c82f 100644 --- a/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl +++ b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl @@ -24,8 +24,24 @@ to_map/2 ]). +-define(schedule_subscribe, schedule_subscribe). +-define(schedule_unsubscribe, schedule_unsubscribe). + +-type stream_key() :: {emqx_persistent_session_ds:id(), emqx_ds:stream()}. + +-type scheduled_action_type() :: + {?schedule_subscribe, emqx_types:subopts()} | ?schedule_unsubscribe. +-type scheduled_action() :: #{ + type := scheduled_action_type(), + stream_keys_to_wait := [stream_key()], + progresses := [emqx_ds_shared_sub_proto:agent_stream_progress()] +}. + -type t() :: #{ - agent := emqx_persistent_session_ds_shared_subs_agent:t() + agent := emqx_persistent_session_ds_shared_subs_agent:t(), + scheduled_actions := #{ + share_topic_filter() => scheduled_action() + } }. -type share_topic_filter() :: emqx_persistent_session_ds:share_topic_filter(). -type opts() :: #{ @@ -44,7 +60,8 @@ new(Opts) -> #{ agent => emqx_persistent_session_ds_shared_subs_agent:new( agent_opts(Opts) - ) + ), + scheduled_actions => #{} }. -spec open(emqx_persistent_session_ds_state:t(), opts()) -> @@ -80,32 +97,29 @@ on_subscribe(TopicFilter, SubOpts, #{s := S} = Session) -> ) -> {ok, emqx_persistent_session_ds_state:t(), t(), emqx_persistent_session_ds:subscription()} | {error, emqx_types:reason_code()}. -on_unsubscribe(SessionId, TopicFilter, S0, #{agent := Agent0} = SharedSubS0) -> +on_unsubscribe(SessionId, TopicFilter, S0, SharedSubS0) -> case lookup(TopicFilter, S0) of undefined -> {error, ?RC_NO_SUBSCRIPTION_EXISTED}; - Subscription -> + #{id := SubId} = Subscription -> ?tp(persistent_session_ds_subscription_delete, #{ session_id => SessionId, topic_filter => TopicFilter }), - Agent1 = emqx_persistent_session_ds_shared_subs_agent:on_unsubscribe( - Agent0, TopicFilter - ), - SharedSubS = SharedSubS0#{agent => Agent1}, S = emqx_persistent_session_ds_state:del_subscription(TopicFilter, S0), + SharedSubS = schedule_unsubscribe(S, SharedSubS0, SubId, TopicFilter), {ok, S, SharedSubS, Subscription} end. -spec renew_streams(emqx_persistent_session_ds_state:t(), t()) -> {emqx_persistent_session_ds_state:t(), t()}. -renew_streams(S0, #{agent := Agent0} = SharedSubS0) -> +renew_streams(S0, #{agent := Agent0, scheduled_actions := ScheduledActions} = SharedSubS0) -> {StreamLeaseEvents, Agent1} = emqx_persistent_session_ds_shared_subs_agent:renew_streams( Agent0 ), ?tp(info, shared_subs_new_stream_lease_events, #{stream_lease_events => StreamLeaseEvents}), S1 = lists:foldl( fun - (#{type := lease} = Event, S) -> accept_stream(Event, S); + (#{type := lease} = Event, S) -> accept_stream(Event, S, ScheduledActions); (#{type := revoke} = Event, S) -> revoke_stream(Event, S) end, S0, @@ -118,19 +132,23 @@ renew_streams(S0, #{agent := Agent0} = SharedSubS0) -> emqx_persistent_session_ds_state:t(), t() ) -> {emqx_persistent_session_ds_state:t(), t()}. -on_streams_replay(S, #{agent := Agent0} = SharedSubS0) -> +on_streams_replay(S, #{agent := Agent0, scheduled_actions := ScheduledActions0} = SharedSubS0) -> Progresses = stream_progresses(S), Agent1 = emqx_persistent_session_ds_shared_subs_agent:on_stream_progress( Agent0, Progresses ), - SharedSubS1 = SharedSubS0#{agent => Agent1}, + {Agent2, ScheduledActions1} = run_scheduled_actions(S, Agent1, ScheduledActions0), + SharedSubS1 = SharedSubS0#{ + agent => Agent2, + scheduled_actions => ScheduledActions1 + }, {S, SharedSubS1}. on_disconnect(S0, #{agent := Agent0} = SharedSubS0) -> S1 = revoke_all_streams(S0), Progresses = stream_progresses(S1), Agent1 = emqx_persistent_session_ds_shared_subs_agent:on_disconnect(Agent0, Progresses), - SharedSubS1 = SharedSubS0#{agent => Agent1}, + SharedSubS1 = SharedSubS0#{agent => Agent1, scheduled_actions => #{}}, {S1, SharedSubS1}. -spec on_info(emqx_persistent_session_ds_state:t(), t(), term()) -> @@ -149,9 +167,79 @@ to_map(_S, _SharedSubS) -> %% Internal functions %%-------------------------------------------------------------------- +run_scheduled_actions(S, Agent, ScheduledActions) -> + maps:fold( + fun(TopicFilter, Action0, {AgentAcc0, ScheduledActionsAcc}) -> + case run_scheduled_action(S, AgentAcc0, TopicFilter, Action0) of + {ok, AgentAcc1} -> + {AgentAcc1, maps:remove(TopicFilter, ScheduledActionsAcc)}; + {continue, Action1} -> + {AgentAcc0, ScheduledActionsAcc#{TopicFilter => Action1}} + end + end, + {Agent, ScheduledActions}, + ScheduledActions + ). + +run_scheduled_action( + S, + Agent, + TopicFilter, + #{type := Type, stream_keys_to_wait := StreamKeysToWait0, progresses := Progresses0} = Action +) -> + StreamKeysToWait1 = lists:filter( + fun({_SubId, _Stream} = Key) -> + case emqx_persistent_session_ds_state:get_stream(Key, S) of + undefined -> + %% This should not happen: we should see any stream + %% in completed state before deletion + true; + SRS -> + not is_stream_fully_acked(S, SRS) + end + end, + StreamKeysToWait0 + ), + + Progresses1 = + lists:map( + fun({_SubId, Stream} = Key) -> + #srs{it_end = ItEnd} = SRS = emqx_persistent_session_ds_state:get_stream(Key, S), + #{ + stream => Stream, + iterator => ItEnd, + use_finished => is_use_finished(S, SRS) + } + end, + (StreamKeysToWait0 -- StreamKeysToWait1) + ) ++ Progresses0, + + case StreamKeysToWait1 of + [] -> + case Type of + {?schedule_subscribe, SubOpts} -> + {ok, + emqx_persistent_session_ds_shared_subs_agent:on_subscribe( + Agent, TopicFilter, SubOpts + )}; + ?schedule_unsubscribe -> + {ok, + emqx_persistent_session_ds_shared_subs_agent:on_unsubscribe( + Agent, TopicFilter, Progresses1 + )} + end; + _ -> + {continue, Action#{stream_keys_to_wait => StreamKeysToWait1, progresses => Progresses1}} + end. + stream_progresses(S) -> fold_shared_stream_states( - fun(TopicFilter, Stream, SRS, Acc) -> + fun( + #share{group = Group}, + Stream, + SRS, + ProgressesAcc0 + ) -> #srs{it_end = EndIt} = SRS, case is_stream_fully_acked(S, SRS) of @@ -159,17 +247,22 @@ stream_progresses(S) -> %% TODO %% Is it sufficient for a report? StreamProgress = #{ - topic_filter => TopicFilter, stream => Stream, iterator => EndIt, - use_finished => is_use_finished(S, SRS) + use_finished => is_use_finished(S, SRS), + is_fully_acked => true }, - [StreamProgress | Acc]; + maps:update_with( + Group, + fun(Progresses) -> [StreamProgress | Progresses] end, + [StreamProgress], + ProgressesAcc0 + ); false -> - Acc + ProgressesAcc0 end end, - [], + #{}, S ). @@ -222,14 +315,16 @@ on_subscribe(Subscription, TopicFilter, SubOpts, Session) -> -dialyzer({nowarn_function, create_new_subscription/3}). create_new_subscription(TopicFilter, SubOpts, #{ - id := SessionId, s := S0, shared_sub_s := #{agent := Agent0} = SharedSubS0, props := Props + s := S0, + shared_sub_s := #{agent := Agent} = SharedSubS0, + props := Props }) -> case - emqx_persistent_session_ds_shared_subs_agent:on_subscribe( - Agent0, TopicFilter, SubOpts + emqx_persistent_session_ds_shared_subs_agent:can_subscribe( + Agent, TopicFilter, SubOpts ) of - {ok, Agent1} -> + ok -> #{upgrade_qos := UpgradeQoS} = Props, {SubId, S1} = emqx_persistent_session_ds_state:new_id(S0), {SStateId, S2} = emqx_persistent_session_ds_state:new_id(S1), @@ -247,10 +342,7 @@ create_new_subscription(TopicFilter, SubOpts, #{ S = emqx_persistent_session_ds_state:put_subscription( TopicFilter, Subscription, S3 ), - SharedSubS = SharedSubS0#{agent => Agent1}, - ?tp(persistent_session_ds_shared_subscription_added, #{ - topic_filter => TopicFilter, session => SessionId - }), + SharedSubS = schedule_subscribe(SharedSubS0, TopicFilter, SubOpts), {ok, S, SharedSubS}; {error, _} = Error -> Error @@ -289,15 +381,25 @@ lookup(TopicFilter, S) -> undefined end. +accept_stream(#{topic_filter := TopicFilter} = Event, S, ScheduledActions) -> + %% If we have a pending action (subscribe or unsubscribe) for this topic filter, + %% we should not accept a stream and start replay it. We won't use it anyway: + %% * if subscribe is pending, we will reset agent obtain a new lease + %% * if unsubscribe is pending, we will drop connection + case ScheduledActions of + #{TopicFilter := _Action} -> + S; + _ -> + accept_stream(Event, S) + end. + accept_stream( #{topic_filter := TopicFilter, stream := Stream, iterator := Iterator}, S0 ) -> case emqx_persistent_session_ds_state:get_subscription(TopicFilter, S0) of undefined -> - %% This should not happen. - %% Agent should have received unsubscribe callback - %% and should not have passed this stream as a new one - error(new_stream_without_sub); + %% We unsubscribed + S0; #{id := SubId, current_state := SStateId} -> Key = {SubId, Stream}, case emqx_persistent_session_ds_state:get_stream(Key, S0) of @@ -347,6 +449,57 @@ revoke_all_streams(S0) -> S0 ). +schedule_subscribe( + #{agent := Agent0, scheduled_actions := ScheduledActions0} = SharedSubS0, TopicFilter, SubOpts +) -> + case ScheduledActions0 of + #{TopicFilter := ScheduledAction} -> + ScheduledActions1 = ScheduledActions0#{ + TopicFilter => ScheduledAction#{type => {?schedule_subscribe, SubOpts}} + }, + SharedSubS0#{scheduled_actions := ScheduledActions1}; + _ -> + Agent1 = emqx_persistent_session_ds_shared_subs_agent:on_subscribe( + Agent0, TopicFilter, SubOpts + ), + SharedSubS0#{agent => Agent1} + end. + +schedule_unsubscribe( + S, #{scheduled_actions := ScheduledActions0} = SharedSubS0, UnsubscridedSubId, TopicFilter +) -> + case ScheduledActions0 of + #{TopicFilter := ScheduledAction} -> + ScheduledActions1 = ScheduledActions0#{ + TopicFilter => ScheduledAction#{type => ?schedule_unsubscribe} + }, + SharedSubS0#{scheduled_actions := ScheduledActions1}; + _ -> + StreamIdsToFinalize = stream_ids_by_sub_id(S, UnsubscridedSubId), + ScheduledActions1 = ScheduledActions0#{ + TopicFilter => #{ + type => ?schedule_unsubscribe, + stream_keys_to_wait => StreamIdsToFinalize, + progresses => [] + } + }, + SharedSubS0#{scheduled_actions := ScheduledActions1} + end. + +stream_ids_by_sub_id(S, MatchSubId) -> + emqx_persistent_session_ds_state:fold_streams( + fun({SubId, _Stream} = StreamStateId, _SRS, StreamStateIds) -> + case SubId of + MatchSubId -> + [StreamStateId | StreamStateIds]; + _ -> + StreamStateIds + end + end, + [], + S + ). + -spec to_agent_subscription( emqx_persistent_session_ds_state:t(), emqx_persistent_session_ds:subscription() ) -> diff --git a/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs_agent.erl b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs_agent.erl index 72b4fa22d..b49ceabcf 100644 --- a/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs_agent.erl +++ b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs_agent.erl @@ -59,9 +59,10 @@ -export([ new/1, open/2, + can_subscribe/3, on_subscribe/3, - on_unsubscribe/2, + on_unsubscribe/3, on_stream_progress/2, on_info/2, on_disconnect/2, @@ -80,12 +81,12 @@ -callback new(opts()) -> t(). -callback open([{topic_filter(), subscription()}], opts()) -> t(). --callback on_subscribe(t(), topic_filter(), emqx_types:subopts()) -> - {ok, t()} | {error, term()}. --callback on_unsubscribe(t(), topic_filter()) -> t(). --callback on_disconnect(t(), [stream_progress()]) -> t(). +-callback can_subscribe(t(), topic_filter(), emqx_types:subopts()) -> ok | {error, term()}. +-callback on_subscribe(t(), topic_filter(), emqx_types:subopts()) -> t(). +-callback on_unsubscribe(t(), topic_filter(), [stream_progress()]) -> t(). +-callback on_disconnect(t(), #{emqx_types:group() => [stream_progress()]}) -> t(). -callback renew_streams(t()) -> {[stream_lease_event()], t()}. --callback on_stream_progress(t(), [stream_progress()]) -> t(). +-callback on_stream_progress(t(), #{emqx_types:group() => [stream_progress()]}) -> t(). -callback on_info(t(), term()) -> t(). %%-------------------------------------------------------------------- @@ -100,16 +101,19 @@ new(Opts) -> open(Topics, Opts) -> ?shared_subs_agent:open(Topics, Opts). --spec on_subscribe(t(), topic_filter(), emqx_types:subopts()) -> - {ok, t()} | {error, emqx_types:reason_code()}. +-spec can_subscribe(t(), topic_filter(), emqx_types:subopts()) -> ok | {error, term()}. +can_subscribe(Agent, TopicFilter, SubOpts) -> + ?shared_subs_agent:can_subscribe(Agent, TopicFilter, SubOpts). + +-spec on_subscribe(t(), topic_filter(), emqx_types:subopts()) -> t(). on_subscribe(Agent, TopicFilter, SubOpts) -> ?shared_subs_agent:on_subscribe(Agent, TopicFilter, SubOpts). --spec on_unsubscribe(t(), topic_filter()) -> t(). -on_unsubscribe(Agent, TopicFilter) -> - ?shared_subs_agent:on_unsubscribe(Agent, TopicFilter). +-spec on_unsubscribe(t(), topic_filter(), [stream_progress()]) -> t(). +on_unsubscribe(Agent, TopicFilter, StreamProgresses) -> + ?shared_subs_agent:on_unsubscribe(Agent, TopicFilter, StreamProgresses). --spec on_disconnect(t(), [stream_progress()]) -> t(). +-spec on_disconnect(t(), #{emqx_types:group() => [stream_progress()]}) -> t(). on_disconnect(Agent, StreamProgresses) -> ?shared_subs_agent:on_disconnect(Agent, StreamProgresses). @@ -117,7 +121,7 @@ on_disconnect(Agent, StreamProgresses) -> renew_streams(Agent) -> ?shared_subs_agent:renew_streams(Agent). --spec on_stream_progress(t(), [stream_progress()]) -> t(). +-spec on_stream_progress(t(), #{emqx_types:group() => [stream_progress()]}) -> t(). on_stream_progress(Agent, StreamProgress) -> ?shared_subs_agent:on_stream_progress(Agent, StreamProgress). diff --git a/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs_null_agent.erl b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs_null_agent.erl index d984194a8..8156db76d 100644 --- a/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs_null_agent.erl +++ b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs_null_agent.erl @@ -9,9 +9,10 @@ -export([ new/1, open/2, + can_subscribe/3, on_subscribe/3, - on_unsubscribe/2, + on_unsubscribe/3, on_stream_progress/2, on_info/2, on_disconnect/2, @@ -31,10 +32,13 @@ new(_Opts) -> open(_Topics, _Opts) -> undefined. -on_subscribe(_Agent, _TopicFilter, _SubOpts) -> +can_subscribe(_Agent, _TopicFilter, _SubOpts) -> {error, ?RC_SHARED_SUBSCRIPTIONS_NOT_SUPPORTED}. -on_unsubscribe(Agent, _TopicFilter) -> +on_subscribe(Agent, _TopicFilter, _SubOpts) -> + Agent. + +on_unsubscribe(Agent, _TopicFilter, _Progresses) -> Agent. on_disconnect(Agent, _) -> diff --git a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_agent.erl b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_agent.erl index 0e8d17614..70b203661 100644 --- a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_agent.erl +++ b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_agent.erl @@ -12,9 +12,10 @@ -export([ new/1, open/2, + can_subscribe/3, on_subscribe/3, - on_unsubscribe/2, + on_unsubscribe/3, on_stream_progress/2, on_info/2, on_disconnect/2, @@ -47,40 +48,38 @@ open(TopicSubscriptions, Opts) -> ), State1. -on_subscribe(State0, TopicFilter, _SubOpts) -> - State1 = add_group_subscription(State0, TopicFilter), - {ok, State1}. +can_subscribe(_State, _TopicFilter, _SubOpts) -> + ok. -on_unsubscribe(State, TopicFilter) -> - delete_group_subscription(State, TopicFilter). +on_subscribe(State0, TopicFilter, _SubOpts) -> + add_group_subscription(State0, TopicFilter). + +on_unsubscribe(State, TopicFilter, GroupProgress) -> + delete_group_subscription(State, TopicFilter, GroupProgress). renew_streams(#{} = State) -> fetch_stream_events(State). on_stream_progress(State, StreamProgresses) -> - ProgressesByGroup = stream_progresses_by_group(StreamProgresses), - lists:foldl( - fun({Group, GroupProgresses}, StateAcc) -> + maps:fold( + fun(Group, GroupProgresses, StateAcc) -> with_group_sm(StateAcc, Group, fun(GSM) -> emqx_ds_shared_sub_group_sm:handle_stream_progress(GSM, GroupProgresses) end) end, State, - maps:to_list(ProgressesByGroup) + StreamProgresses ). on_disconnect(#{groups := Groups0} = State, StreamProgresses) -> - ProgressesByGroup = stream_progresses_by_group(StreamProgresses), - Groups1 = maps:fold( - fun(Group, GroupSM0, GroupsAcc) -> - GroupProgresses = maps:get(Group, ProgressesByGroup, []), - GroupSM1 = emqx_ds_shared_sub_group_sm:handle_disconnect(GroupSM0, GroupProgresses), - GroupsAcc#{Group => GroupSM1} + ok = maps:foreach( + fun(Group, GroupSM0) -> + GroupProgresses = maps:get(Group, StreamProgresses, []), + emqx_ds_shared_sub_group_sm:handle_disconnect(GroupSM0, GroupProgresses) end, - #{}, Groups0 ), - State#{groups => Groups1}. + State#{groups => #{}}. on_info(State, ?leader_lease_streams_match(Group, Leader, StreamProgresses, Version)) -> ?SLOG(info, #{ @@ -152,9 +151,14 @@ init_state(Opts) -> groups => #{} }. -delete_group_subscription(State, _ShareTopicFilter) -> - %% TODO https://emqx.atlassian.net/browse/EMQX-12572 - State. +delete_group_subscription(State, #share{group = Group}, GroupProgress) -> + case State of + #{groups := #{Group := GSM} = Groups} -> + _ = emqx_ds_shared_sub_group_sm:handle_disconnect(GSM, GroupProgress), + State#{groups => maps:remove(Group, Groups)}; + _ -> + State + end. add_group_subscription( #{session_id := SessionId, groups := Groups0} = State0, ShareTopicFilter @@ -209,20 +213,3 @@ with_group_sm(State, Group, Fun) -> %% Error? State end. - -stream_progresses_by_group(StreamProgresses) -> - lists:foldl( - fun(#{topic_filter := #share{group = Group}} = Progress0, Acc) -> - Progress1 = maps:remove(topic_filter, Progress0), - maps:update_with( - Group, - fun(GroupStreams0) -> - [Progress1 | GroupStreams0] - end, - [Progress1], - Acc - ) - end, - #{}, - StreamProgresses - ). diff --git a/apps/emqx_ds_shared_sub/test/emqx_ds_shared_sub_SUITE.erl b/apps/emqx_ds_shared_sub/test/emqx_ds_shared_sub_SUITE.erl index 3e80b44a9..defc90c78 100644 --- a/apps/emqx_ds_shared_sub/test/emqx_ds_shared_sub_SUITE.erl +++ b/apps/emqx_ds_shared_sub/test/emqx_ds_shared_sub_SUITE.erl @@ -221,35 +221,7 @@ t_intensive_reassign(_Config) -> end end, - Messages = lists:foldl( - fun(#{payload := Payload, client_pid := Pid}, Acc) -> - maps:update_with( - binary_to_integer(Payload), - fun(Clients) -> - [ClientByBid(Pid) | Clients] - end, - [ClientByBid(Pid)], - Acc - ) - end, - #{}, - Pubs - ), - - Missing = lists:filter( - fun(N) -> not maps:is_key(N, Messages) end, - lists:seq(1, 2 * NPubs) - ), - Duplicate = lists:filtermap( - fun(N) -> - case Messages of - #{N := [_]} -> false; - #{N := [_ | _] = Clients} -> {true, {N, Clients}}; - _ -> false - end - end, - lists:seq(1, 2 * NPubs) - ), + {Missing, Duplicate} = verify_received_pubs(Pubs, 2 * NPubs, ClientByBid), ?assertEqual( [], @@ -266,6 +238,58 @@ t_intensive_reassign(_Config) -> ok = emqtt:disconnect(ConnShared3), ok = emqtt:disconnect(ConnPub). +t_unsubscribe(_Config) -> + ConnPub = emqtt_connect_pub(<<"client_pub">>), + + ConnShared1 = emqtt_connect_sub(<<"client_shared1">>), + {ok, _, _} = emqtt:subscribe(ConnShared1, <<"$share/gr9/topic9/#">>, 1), + + ct:sleep(1000), + + NPubs = 10_000, + + Topics = [<<"topic9/1">>, <<"topic9/2">>, <<"topic9/3">>], + ok = publish_n(ConnPub, Topics, 1, NPubs), + + Self = self(), + _ = spawn_link(fun() -> + ok = publish_n(ConnPub, Topics, NPubs + 1, 2 * NPubs), + Self ! publish_done + end), + + ConnShared2 = emqtt_connect_sub(<<"client_shared2">>), + {ok, _, _} = emqtt:subscribe(ConnShared2, <<"$share/gr9/topic9/#">>, 1), + {ok, _, _} = emqtt:unsubscribe(ConnShared1, <<"$share/gr9/topic9/#">>), + + receive + publish_done -> ok + end, + + Pubs = drain_publishes(), + + ClientByBid = fun(Pid) -> + case Pid of + ConnShared1 -> <<"client_shared1">>; + ConnShared2 -> <<"client_shared2">> + end + end, + + {Missing, Duplicate} = verify_received_pubs(Pubs, 2 * NPubs, ClientByBid), + + ?assertEqual( + [], + Missing + ), + + ?assertEqual( + [], + Duplicate + ), + + ok = emqtt:disconnect(ConnShared1), + ok = emqtt:disconnect(ConnShared2), + ok = emqtt:disconnect(ConnPub). + t_lease_reconnect(_Config) -> ConnPub = emqtt_connect_pub(<<"client_pub">>), @@ -364,3 +388,36 @@ drain_publishes(Acc) -> after 5_000 -> lists:reverse(Acc) end. + +verify_received_pubs(Pubs, NPubs, ClientByBid) -> + Messages = lists:foldl( + fun(#{payload := Payload, client_pid := Pid}, Acc) -> + maps:update_with( + binary_to_integer(Payload), + fun(Clients) -> + [ClientByBid(Pid) | Clients] + end, + [ClientByBid(Pid)], + Acc + ) + end, + #{}, + Pubs + ), + + Missing = lists:filter( + fun(N) -> not maps:is_key(N, Messages) end, + lists:seq(1, NPubs) + ), + Duplicate = lists:filtermap( + fun(N) -> + case Messages of + #{N := [_]} -> false; + #{N := [_ | _] = Clients} -> {true, {N, Clients}}; + _ -> false + end + end, + lists:seq(1, NPubs) + ), + + {Missing, Duplicate}. From fada2a3fea6e15260f6e92ec3ef1dfb46c6f7792 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Wed, 3 Jul 2024 17:12:24 +0300 Subject: [PATCH 093/131] feat(queue): reorganize and document shared subs module --- ...emqx_persistent_session_ds_shared_subs.erl | 590 ++++++++++-------- 1 file changed, 324 insertions(+), 266 deletions(-) diff --git a/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl index 94bd2c82f..6709eb37a 100644 --- a/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl +++ b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl @@ -2,6 +2,30 @@ %% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- +%% @doc This module +%% * handles creation and management of _shared_ subscriptions for the session; +%% * provides streams to the session; +%% * handles progress of stream replay. +%% +%% The logic is quite straightforward; most of the parts resemble the logic of the +%% `emqx_persistent_session_ds_subs` (subscribe/unsubscribe) and +%% `emqx_persistent_session_ds_scheduler` (providing new streams), +%% but some data is sent or received from the `emqx_persistent_session_ds_shared_subs_agent` +%% which communicates with remote shared subscription leaders. +%% +%% A tricky part is the concept of "scheduled actions". When we unsubscribe from a topic +%% we may have some streams that have unacked messages. So we do not have a reliable +%% progress for them. Sending the current progress to the leader and disconnecting +%% will lead to the duplication of messages. So after unsubscription, we need to wait +%% some time until all streams are acked, and only then we disconnect from the leader. +%% +%% For this purpose we have the `scheduled_actions` map in the state of the module. +%% We preserve there the streams that we need to wait for and collect their progress. +%% We also use `scheduled_actions` for resubscriptions. If a client quickly resubscribes +%% after unsubscription, we may still have the mentioned streams unacked. If we abandon +%% them, just connect to the leader, then it may lease us the same streams again, but with +%% the previous progress. So messages may duplicate. + -module(emqx_persistent_session_ds_shared_subs). -include("emqx_mqtt.hrl"). @@ -55,6 +79,9 @@ %% API %%-------------------------------------------------------------------- +%%-------------------------------------------------------------------- +%% new + -spec new(opts()) -> t(). new(Opts) -> #{ @@ -64,6 +91,9 @@ new(Opts) -> scheduled_actions => #{} }. +%%-------------------------------------------------------------------- +%% open + -spec open(emqx_persistent_session_ds_state:t(), opts()) -> {ok, emqx_persistent_session_ds_state:t(), t()}. open(S, Opts) -> @@ -80,6 +110,9 @@ open(S, Opts) -> SharedSubS = #{agent => Agent}, {ok, S, SharedSubS}. +%%-------------------------------------------------------------------- +%% on_subscribe + -spec on_subscribe( share_topic_filter(), emqx_types:subopts(), @@ -89,218 +122,8 @@ on_subscribe(TopicFilter, SubOpts, #{s := S} = Session) -> Subscription = emqx_persistent_session_ds_state:get_subscription(TopicFilter, S), on_subscribe(Subscription, TopicFilter, SubOpts, Session). --spec on_unsubscribe( - emqx_persistent_session_ds:id(), - emqx_persistent_session_ds:topic_filter(), - emqx_persistent_session_ds_state:t(), - t() -) -> - {ok, emqx_persistent_session_ds_state:t(), t(), emqx_persistent_session_ds:subscription()} - | {error, emqx_types:reason_code()}. -on_unsubscribe(SessionId, TopicFilter, S0, SharedSubS0) -> - case lookup(TopicFilter, S0) of - undefined -> - {error, ?RC_NO_SUBSCRIPTION_EXISTED}; - #{id := SubId} = Subscription -> - ?tp(persistent_session_ds_subscription_delete, #{ - session_id => SessionId, topic_filter => TopicFilter - }), - S = emqx_persistent_session_ds_state:del_subscription(TopicFilter, S0), - SharedSubS = schedule_unsubscribe(S, SharedSubS0, SubId, TopicFilter), - {ok, S, SharedSubS, Subscription} - end. - --spec renew_streams(emqx_persistent_session_ds_state:t(), t()) -> - {emqx_persistent_session_ds_state:t(), t()}. -renew_streams(S0, #{agent := Agent0, scheduled_actions := ScheduledActions} = SharedSubS0) -> - {StreamLeaseEvents, Agent1} = emqx_persistent_session_ds_shared_subs_agent:renew_streams( - Agent0 - ), - ?tp(info, shared_subs_new_stream_lease_events, #{stream_lease_events => StreamLeaseEvents}), - S1 = lists:foldl( - fun - (#{type := lease} = Event, S) -> accept_stream(Event, S, ScheduledActions); - (#{type := revoke} = Event, S) -> revoke_stream(Event, S) - end, - S0, - StreamLeaseEvents - ), - SharedSubS1 = SharedSubS0#{agent => Agent1}, - {S1, SharedSubS1}. - --spec on_streams_replay( - emqx_persistent_session_ds_state:t(), - t() -) -> {emqx_persistent_session_ds_state:t(), t()}. -on_streams_replay(S, #{agent := Agent0, scheduled_actions := ScheduledActions0} = SharedSubS0) -> - Progresses = stream_progresses(S), - Agent1 = emqx_persistent_session_ds_shared_subs_agent:on_stream_progress( - Agent0, Progresses - ), - {Agent2, ScheduledActions1} = run_scheduled_actions(S, Agent1, ScheduledActions0), - SharedSubS1 = SharedSubS0#{ - agent => Agent2, - scheduled_actions => ScheduledActions1 - }, - {S, SharedSubS1}. - -on_disconnect(S0, #{agent := Agent0} = SharedSubS0) -> - S1 = revoke_all_streams(S0), - Progresses = stream_progresses(S1), - Agent1 = emqx_persistent_session_ds_shared_subs_agent:on_disconnect(Agent0, Progresses), - SharedSubS1 = SharedSubS0#{agent => Agent1, scheduled_actions => #{}}, - {S1, SharedSubS1}. - --spec on_info(emqx_persistent_session_ds_state:t(), t(), term()) -> - {emqx_persistent_session_ds_state:t(), t()}. -on_info(S, #{agent := Agent0} = SharedSubS0, Info) -> - Agent1 = emqx_persistent_session_ds_shared_subs_agent:on_info(Agent0, Info), - SharedSubS1 = SharedSubS0#{agent => Agent1}, - {S, SharedSubS1}. - --spec to_map(emqx_persistent_session_ds_state:t(), t()) -> map(). -to_map(_S, _SharedSubS) -> - %% TODO - #{}. - %%-------------------------------------------------------------------- -%% Internal functions -%%-------------------------------------------------------------------- - -run_scheduled_actions(S, Agent, ScheduledActions) -> - maps:fold( - fun(TopicFilter, Action0, {AgentAcc0, ScheduledActionsAcc}) -> - case run_scheduled_action(S, AgentAcc0, TopicFilter, Action0) of - {ok, AgentAcc1} -> - {AgentAcc1, maps:remove(TopicFilter, ScheduledActionsAcc)}; - {continue, Action1} -> - {AgentAcc0, ScheduledActionsAcc#{TopicFilter => Action1}} - end - end, - {Agent, ScheduledActions}, - ScheduledActions - ). - -run_scheduled_action( - S, - Agent, - TopicFilter, - #{type := Type, stream_keys_to_wait := StreamKeysToWait0, progresses := Progresses0} = Action -) -> - StreamKeysToWait1 = lists:filter( - fun({_SubId, _Stream} = Key) -> - case emqx_persistent_session_ds_state:get_stream(Key, S) of - undefined -> - %% This should not happen: we should see any stream - %% in completed state before deletion - true; - SRS -> - not is_stream_fully_acked(S, SRS) - end - end, - StreamKeysToWait0 - ), - - Progresses1 = - lists:map( - fun({_SubId, Stream} = Key) -> - #srs{it_end = ItEnd} = SRS = emqx_persistent_session_ds_state:get_stream(Key, S), - #{ - stream => Stream, - iterator => ItEnd, - use_finished => is_use_finished(S, SRS) - } - end, - (StreamKeysToWait0 -- StreamKeysToWait1) - ) ++ Progresses0, - - case StreamKeysToWait1 of - [] -> - case Type of - {?schedule_subscribe, SubOpts} -> - {ok, - emqx_persistent_session_ds_shared_subs_agent:on_subscribe( - Agent, TopicFilter, SubOpts - )}; - ?schedule_unsubscribe -> - {ok, - emqx_persistent_session_ds_shared_subs_agent:on_unsubscribe( - Agent, TopicFilter, Progresses1 - )} - end; - _ -> - {continue, Action#{stream_keys_to_wait => StreamKeysToWait1, progresses => Progresses1}} - end. - -stream_progresses(S) -> - fold_shared_stream_states( - fun( - #share{group = Group}, - Stream, - SRS, - ProgressesAcc0 - ) -> - #srs{it_end = EndIt} = SRS, - - case is_stream_fully_acked(S, SRS) of - true -> - %% TODO - %% Is it sufficient for a report? - StreamProgress = #{ - stream => Stream, - iterator => EndIt, - use_finished => is_use_finished(S, SRS), - is_fully_acked => true - }, - maps:update_with( - Group, - fun(Progresses) -> [StreamProgress | Progresses] end, - [StreamProgress], - ProgressesAcc0 - ); - false -> - ProgressesAcc0 - end - end, - #{}, - S - ). - -fold_shared_subs(Fun, Acc, S) -> - emqx_persistent_session_ds_state:fold_subscriptions( - fun - (#share{} = TopicFilter, Sub, Acc0) -> Fun(TopicFilter, Sub, Acc0); - (_, _Sub, Acc0) -> Acc0 - end, - Acc, - S - ). - -fold_shared_stream_states(Fun, Acc, S) -> - %% TODO - %% Optimize or cache - TopicFilters = fold_shared_subs( - fun - (#share{} = TopicFilter, #{id := Id} = _Sub, Acc0) -> - Acc0#{Id => TopicFilter}; - (_, _, Acc0) -> - Acc0 - end, - #{}, - S - ), - emqx_persistent_session_ds_state:fold_streams( - fun({SubId, Stream}, SRS, Acc0) -> - case TopicFilters of - #{SubId := TopicFilter} -> - Fun(TopicFilter, Stream, SRS, Acc0); - _ -> - Acc0 - end - end, - Acc, - S - ). +%% on_subscribe internal functions on_subscribe(undefined, TopicFilter, SubOpts, #{props := Props, s := S} = Session) -> #{max_subscriptions := MaxSubscriptions} = Props, @@ -313,7 +136,6 @@ on_subscribe(undefined, TopicFilter, SubOpts, #{props := Props, s := S} = Sessio on_subscribe(Subscription, TopicFilter, SubOpts, Session) -> update_subscription(Subscription, TopicFilter, SubOpts, Session). --dialyzer({nowarn_function, create_new_subscription/3}). create_new_subscription(TopicFilter, SubOpts, #{ s := S0, shared_sub_s := #{agent := Agent} = SharedSubS0, @@ -368,22 +190,97 @@ update_subscription(#{current_state := SStateId0, id := SubId} = Sub0, TopicFilt {ok, S, SharedSubS} end. -lookup(TopicFilter, S) -> - case emqx_persistent_session_ds_state:get_subscription(TopicFilter, S) of - Sub = #{current_state := SStateId} -> - case emqx_persistent_session_ds_state:get_subscription_state(SStateId, S) of - #{subopts := SubOpts} -> - Sub#{subopts => SubOpts}; - undefined -> - undefined - end; - undefined -> - undefined +schedule_subscribe( + #{agent := Agent0, scheduled_actions := ScheduledActions0} = SharedSubS0, TopicFilter, SubOpts +) -> + case ScheduledActions0 of + #{TopicFilter := ScheduledAction} -> + ScheduledActions1 = ScheduledActions0#{ + TopicFilter => ScheduledAction#{type => {?schedule_subscribe, SubOpts}} + }, + SharedSubS0#{scheduled_actions := ScheduledActions1}; + _ -> + Agent1 = emqx_persistent_session_ds_shared_subs_agent:on_subscribe( + Agent0, TopicFilter, SubOpts + ), + SharedSubS0#{agent => Agent1} end. +%%-------------------------------------------------------------------- +%% on_unsubscribe + +-spec on_unsubscribe( + emqx_persistent_session_ds:id(), + emqx_persistent_session_ds:topic_filter(), + emqx_persistent_session_ds_state:t(), + t() +) -> + {ok, emqx_persistent_session_ds_state:t(), t(), emqx_persistent_session_ds:subscription()} + | {error, emqx_types:reason_code()}. +on_unsubscribe(SessionId, TopicFilter, S0, SharedSubS0) -> + case lookup(TopicFilter, S0) of + undefined -> + {error, ?RC_NO_SUBSCRIPTION_EXISTED}; + #{id := SubId} = Subscription -> + ?tp(persistent_session_ds_subscription_delete, #{ + session_id => SessionId, topic_filter => TopicFilter + }), + S = emqx_persistent_session_ds_state:del_subscription(TopicFilter, S0), + SharedSubS = schedule_unsubscribe(S, SharedSubS0, SubId, TopicFilter), + {ok, S, SharedSubS, Subscription} + end. + +%%-------------------------------------------------------------------- +%% on_unsubscribe internal functions + +schedule_unsubscribe( + S, #{scheduled_actions := ScheduledActions0} = SharedSubS0, UnsubscridedSubId, TopicFilter +) -> + case ScheduledActions0 of + #{TopicFilter := ScheduledAction} -> + ScheduledActions1 = ScheduledActions0#{ + TopicFilter => ScheduledAction#{type => ?schedule_unsubscribe} + }, + SharedSubS0#{scheduled_actions := ScheduledActions1}; + _ -> + StreamIdsToFinalize = stream_ids_by_sub_id(S, UnsubscridedSubId), + ScheduledActions1 = ScheduledActions0#{ + TopicFilter => #{ + type => ?schedule_unsubscribe, + stream_keys_to_wait => StreamIdsToFinalize, + progresses => [] + } + }, + SharedSubS0#{scheduled_actions := ScheduledActions1} + end. + +%%-------------------------------------------------------------------- +%% renew_streams + +-spec renew_streams(emqx_persistent_session_ds_state:t(), t()) -> + {emqx_persistent_session_ds_state:t(), t()}. +renew_streams(S0, #{agent := Agent0, scheduled_actions := ScheduledActions} = SharedSubS0) -> + {StreamLeaseEvents, Agent1} = emqx_persistent_session_ds_shared_subs_agent:renew_streams( + Agent0 + ), + ?tp(info, shared_subs_new_stream_lease_events, #{stream_lease_events => StreamLeaseEvents}), + S1 = lists:foldl( + fun + (#{type := lease} = Event, S) -> accept_stream(Event, S, ScheduledActions); + (#{type := revoke} = Event, S) -> revoke_stream(Event, S) + end, + S0, + StreamLeaseEvents + ), + SharedSubS1 = SharedSubS0#{agent => Agent1}, + {S1, SharedSubS1}. + +%%-------------------------------------------------------------------- +%% renew_streams internal functions + accept_stream(#{topic_filter := TopicFilter} = Event, S, ScheduledActions) -> %% If we have a pending action (subscribe or unsubscribe) for this topic filter, - %% we should not accept a stream and start replay it. We won't use it anyway: + %% we should not accept a stream and start replaying it. We won't use it anyway: %% * if subscribe is pending, we will reset agent obtain a new lease %% * if unsubscribe is pending, we will drop connection case ScheduledActions of @@ -440,6 +337,134 @@ revoke_stream( end end. +%%-------------------------------------------------------------------- +%% on_streams_replay + +-spec on_streams_replay( + emqx_persistent_session_ds_state:t(), + t() +) -> {emqx_persistent_session_ds_state:t(), t()}. +on_streams_replay(S, #{agent := Agent0, scheduled_actions := ScheduledActions0} = SharedSubS0) -> + Progresses = stream_progresses(S), + Agent1 = emqx_persistent_session_ds_shared_subs_agent:on_stream_progress( + Agent0, Progresses + ), + {Agent2, ScheduledActions1} = run_scheduled_actions(S, Agent1, ScheduledActions0), + SharedSubS1 = SharedSubS0#{ + agent => Agent2, + scheduled_actions => ScheduledActions1 + }, + {S, SharedSubS1}. + +%%-------------------------------------------------------------------- +%% on_streams_replay internal functions + +stream_progresses(S) -> + fold_shared_stream_states( + fun( + #share{group = Group}, + Stream, + SRS, + ProgressesAcc0 + ) -> + case is_stream_fully_acked(S, SRS) of + true -> + StreamProgress = stream_progress(S, Stream, SRS), + maps:update_with( + Group, + fun(Progresses) -> [StreamProgress | Progresses] end, + [StreamProgress], + ProgressesAcc0 + ); + false -> + ProgressesAcc0 + end + end, + #{}, + S + ). + +run_scheduled_actions(S, Agent, ScheduledActions) -> + maps:fold( + fun(TopicFilter, Action0, {AgentAcc0, ScheduledActionsAcc}) -> + case run_scheduled_action(S, AgentAcc0, TopicFilter, Action0) of + {ok, AgentAcc1} -> + {AgentAcc1, maps:remove(TopicFilter, ScheduledActionsAcc)}; + {continue, Action1} -> + {AgentAcc0, ScheduledActionsAcc#{TopicFilter => Action1}} + end + end, + {Agent, ScheduledActions}, + ScheduledActions + ). + +run_scheduled_action( + S, + Agent, + TopicFilter, + #{type := Type, stream_keys_to_wait := StreamKeysToWait0, progresses := Progresses0} = Action +) -> + StreamKeysToWait1 = filter_unfinished_streams(S, StreamKeysToWait0), + Progresses1 = stream_progresses(S, StreamKeysToWait0 -- StreamKeysToWait1) ++ Progresses0, + case StreamKeysToWait1 of + [] -> + case Type of + {?schedule_subscribe, SubOpts} -> + {ok, + emqx_persistent_session_ds_shared_subs_agent:on_subscribe( + Agent, TopicFilter, SubOpts + )}; + ?schedule_unsubscribe -> + {ok, + emqx_persistent_session_ds_shared_subs_agent:on_unsubscribe( + Agent, TopicFilter, Progresses1 + )} + end; + _ -> + {continue, Action#{stream_keys_to_wait => StreamKeysToWait1, progresses => Progresses1}} + end. + +filter_unfinished_streams(S, StreamKeysToWait) -> + lists:filter( + fun(Key) -> + case emqx_persistent_session_ds_state:get_stream(Key, S) of + undefined -> + %% This should not happen: we should see any stream + %% in completed state before deletion + true; + SRS -> + not is_stream_fully_acked(S, SRS) + end + end, + StreamKeysToWait + ). + +stream_progresses(S, StreamKeys) -> + lists:map( + fun({_SubId, Stream} = Key) -> + #srs{it_end = ItEnd} = SRS = emqx_persistent_session_ds_state:get_stream(Key, S), + #{ + stream => Stream, + iterator => ItEnd, + use_finished => is_use_finished(S, SRS) + } + end, + StreamKeys + ). + +%%-------------------------------------------------------------------- +%% on_disconnect + +on_disconnect(S0, #{agent := Agent0} = SharedSubS0) -> + S1 = revoke_all_streams(S0), + Progresses = stream_progresses(S1), + Agent1 = emqx_persistent_session_ds_shared_subs_agent:on_disconnect(Agent0, Progresses), + SharedSubS1 = SharedSubS0#{agent => Agent1, scheduled_actions => #{}}, + {S1, SharedSubS1}. + +%%-------------------------------------------------------------------- +%% on_disconnect helpers + revoke_all_streams(S0) -> fold_shared_stream_states( fun(TopicFilter, Stream, _SRS, S) -> @@ -449,41 +474,39 @@ revoke_all_streams(S0) -> S0 ). -schedule_subscribe( - #{agent := Agent0, scheduled_actions := ScheduledActions0} = SharedSubS0, TopicFilter, SubOpts -) -> - case ScheduledActions0 of - #{TopicFilter := ScheduledAction} -> - ScheduledActions1 = ScheduledActions0#{ - TopicFilter => ScheduledAction#{type => {?schedule_subscribe, SubOpts}} - }, - SharedSubS0#{scheduled_actions := ScheduledActions1}; - _ -> - Agent1 = emqx_persistent_session_ds_shared_subs_agent:on_subscribe( - Agent0, TopicFilter, SubOpts - ), - SharedSubS0#{agent => Agent1} - end. +%%-------------------------------------------------------------------- +%% on_info -schedule_unsubscribe( - S, #{scheduled_actions := ScheduledActions0} = SharedSubS0, UnsubscridedSubId, TopicFilter -) -> - case ScheduledActions0 of - #{TopicFilter := ScheduledAction} -> - ScheduledActions1 = ScheduledActions0#{ - TopicFilter => ScheduledAction#{type => ?schedule_unsubscribe} - }, - SharedSubS0#{scheduled_actions := ScheduledActions1}; - _ -> - StreamIdsToFinalize = stream_ids_by_sub_id(S, UnsubscridedSubId), - ScheduledActions1 = ScheduledActions0#{ - TopicFilter => #{ - type => ?schedule_unsubscribe, - stream_keys_to_wait => StreamIdsToFinalize, - progresses => [] - } - }, - SharedSubS0#{scheduled_actions := ScheduledActions1} +-spec on_info(emqx_persistent_session_ds_state:t(), t(), term()) -> + {emqx_persistent_session_ds_state:t(), t()}. +on_info(S, #{agent := Agent0} = SharedSubS0, Info) -> + Agent1 = emqx_persistent_session_ds_shared_subs_agent:on_info(Agent0, Info), + SharedSubS1 = SharedSubS0#{agent => Agent1}, + {S, SharedSubS1}. + +%%-------------------------------------------------------------------- +%% to_map + +-spec to_map(emqx_persistent_session_ds_state:t(), t()) -> map(). +to_map(_S, _SharedSubS) -> + %% TODO + #{}. + +%%-------------------------------------------------------------------- +%% Generic helpers +%%-------------------------------------------------------------------- + +lookup(TopicFilter, S) -> + case emqx_persistent_session_ds_state:get_subscription(TopicFilter, S) of + Sub = #{current_state := SStateId} -> + case emqx_persistent_session_ds_state:get_subscription_state(SStateId, S) of + #{subopts := SubOpts} -> + Sub#{subopts => SubOpts}; + undefined -> + undefined + end; + undefined -> + undefined end. stream_ids_by_sub_id(S, MatchSubId) -> @@ -500,20 +523,55 @@ stream_ids_by_sub_id(S, MatchSubId) -> S ). --spec to_agent_subscription( - emqx_persistent_session_ds_state:t(), emqx_persistent_session_ds:subscription() -) -> - emqx_persistent_session_ds_shared_subs_agent:subscription(). -to_agent_subscription(_S, Subscription) -> +stream_progress(S, Stream, #srs{it_end = EndIt} = SRS) -> + #{ + stream => Stream, + iterator => EndIt, + use_finished => is_use_finished(S, SRS) + }. + +fold_shared_subs(Fun, Acc, S) -> + emqx_persistent_session_ds_state:fold_subscriptions( + fun + (#share{} = TopicFilter, Sub, Acc0) -> Fun(TopicFilter, Sub, Acc0); + (_, _Sub, Acc0) -> Acc0 + end, + Acc, + S + ). + +fold_shared_stream_states(Fun, Acc, S) -> %% TODO - %% do we need anything from sub state? + %% Optimize or cache + TopicFilters = fold_shared_subs( + fun + (#share{} = TopicFilter, #{id := Id} = _Sub, Acc0) -> + Acc0#{Id => TopicFilter}; + (_, _, Acc0) -> + Acc0 + end, + #{}, + S + ), + emqx_persistent_session_ds_state:fold_streams( + fun({SubId, Stream}, SRS, Acc0) -> + case TopicFilters of + #{SubId := TopicFilter} -> + Fun(TopicFilter, Stream, SRS, Acc0); + _ -> + Acc0 + end + end, + Acc, + S + ). + +to_agent_subscription(_S, Subscription) -> maps:with([start_time], Subscription). --spec agent_opts(opts()) -> emqx_persistent_session_ds_shared_subs_agent:opts(). agent_opts(#{session_id := SessionId}) -> #{session_id => SessionId}. --dialyzer({nowarn_function, now_ms/0}). now_ms() -> erlang:system_time(millisecond). From e5547005eb1cca333d5903cdbcda764e5befdbba Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Wed, 3 Jul 2024 18:14:16 +0300 Subject: [PATCH 094/131] feat(queue): implement resubscribe test --- .../src/emqx_ds_shared_sub_leader.erl | 40 +++++++++-- .../test/emqx_ds_shared_sub_SUITE.erl | 66 ++++++++++++++----- 2 files changed, 84 insertions(+), 22 deletions(-) diff --git a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl index ce38a72f9..8f6b7c683 100644 --- a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl +++ b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl @@ -390,19 +390,25 @@ select_streams_for_assign(Data0, _Agent, AssignCount) -> %% Handle a newly connected agent connect_agent( - #{group := Group} = Data, + #{group := Group, agents := Agents} = Data, Agent, AgentMetadata ) -> - %% TODO - %% implement graceful reconnection of the same agent ?SLOG(info, #{ msg => leader_agent_connected, agent => Agent, group => Group }), - DesiredCount = desired_stream_count_for_new_agent(Data), - assign_initial_streams_to_agent(Data, Agent, AgentMetadata, DesiredCount). + case Agents of + #{Agent := AgentState} -> + ?tp(warning, shared_sub_leader_agent_already_connected, #{ + agent => Agent + }), + reconnect_agent(Data, Agent, AgentMetadata, AgentState); + _ -> + DesiredCount = desired_stream_count_for_new_agent(Data), + assign_initial_streams_to_agent(Data, Agent, AgentMetadata, DesiredCount) + end. assign_initial_streams_to_agent(Data, Agent, AgentMetadata, AssignCount) -> InitialStreamsToAssign = select_streams_for_assign(Data, Agent, AssignCount), @@ -412,6 +418,30 @@ assign_initial_streams_to_agent(Data, Agent, AgentMetadata, AssignCount) -> ), set_agent_state(Data1, Agent, AgentState). +reconnect_agent( + Data0, + Agent, + AgentMetadata, + #{streams := OldStreams, revoked_streams := OldRevokedStreams} = _OldAgentState +) -> + ?tp(warning, shared_sub_leader_agent_reconnect, #{ + agent => Agent, + agent_metadata => AgentMetadata, + inherited_streams => OldStreams + }), + AgentState = agent_transition_to_initial_waiting_replaying( + Data0, Agent, AgentMetadata, OldStreams + ), + Data1 = set_agent_state(Data0, Agent, AgentState), + %% If client reconnected gracefully then it either had already sent all the final progresses + %% for the revoked streams (so `OldRevokedStreams` should be empty) or it had not started + %% to replay them (if we revoked streams after it desided to reconnect). So we can safely + %% unassign them. + %% + %% If client reconnects after a crash, then we wouldn't be here (the agent identity will be new). + Data2 = unassign_streams(Data1, OldRevokedStreams), + Data2. + %%-------------------------------------------------------------------- %% Disconnect agent gracefully diff --git a/apps/emqx_ds_shared_sub/test/emqx_ds_shared_sub_SUITE.erl b/apps/emqx_ds_shared_sub/test/emqx_ds_shared_sub_SUITE.erl index defc90c78..4c2e9a239 100644 --- a/apps/emqx_ds_shared_sub/test/emqx_ds_shared_sub_SUITE.erl +++ b/apps/emqx_ds_shared_sub/test/emqx_ds_shared_sub_SUITE.erl @@ -223,15 +223,8 @@ t_intensive_reassign(_Config) -> {Missing, Duplicate} = verify_received_pubs(Pubs, 2 * NPubs, ClientByBid), - ?assertEqual( - [], - Missing - ), - - ?assertEqual( - [], - Duplicate - ), + ?assertEqual([], Missing), + ?assertEqual([], Duplicate), ok = emqtt:disconnect(ConnShared1), ok = emqtt:disconnect(ConnShared2), @@ -276,15 +269,54 @@ t_unsubscribe(_Config) -> {Missing, Duplicate} = verify_received_pubs(Pubs, 2 * NPubs, ClientByBid), - ?assertEqual( - [], - Missing - ), + ?assertEqual([], Missing), + ?assertEqual([], Duplicate), - ?assertEqual( - [], - Duplicate - ), + ok = emqtt:disconnect(ConnShared1), + ok = emqtt:disconnect(ConnShared2), + ok = emqtt:disconnect(ConnPub). + +t_quick_resubscribe(_Config) -> + ConnPub = emqtt_connect_pub(<<"client_pub">>), + + ConnShared1 = emqtt_connect_sub(<<"client_shared1">>), + {ok, _, _} = emqtt:subscribe(ConnShared1, <<"$share/gr10/topic10/#">>, 1), + + ct:sleep(1000), + + NPubs = 10_000, + + Topics = [<<"topic10/1">>, <<"topic10/2">>, <<"topic10/3">>], + ok = publish_n(ConnPub, Topics, 1, NPubs), + + Self = self(), + _ = spawn_link(fun() -> + ok = publish_n(ConnPub, Topics, NPubs + 1, 2 * NPubs), + Self ! publish_done + end), + + ConnShared2 = emqtt_connect_sub(<<"client_shared2">>), + {ok, _, _} = emqtt:subscribe(ConnShared2, <<"$share/gr10/topic10/#">>, 1), + {ok, _, _} = emqtt:unsubscribe(ConnShared1, <<"$share/gr10/topic10/#">>), + {ok, _, _} = emqtt:subscribe(ConnShared1, <<"$share/gr10/topic10/#">>, 1), + + receive + publish_done -> ok + end, + + Pubs = drain_publishes(), + + ClientByBid = fun(Pid) -> + case Pid of + ConnShared1 -> <<"client_shared1">>; + ConnShared2 -> <<"client_shared2">> + end + end, + + {Missing, Duplicate} = verify_received_pubs(Pubs, 2 * NPubs, ClientByBid), + + ?assertEqual([], Missing), + ?assertEqual([], Duplicate), ok = emqtt:disconnect(ConnShared1), ok = emqtt:disconnect(ConnShared2), From 7d004b37da6ce8549d61707d557fba7374cd8320 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Wed, 3 Jul 2024 22:48:04 +0300 Subject: [PATCH 095/131] feat(queue): implement stream finalization --- .../src/emqx_ds_shared_sub_leader.erl | 237 +++++++++++++----- ...mqx_ds_shared_sub_leader_rank_progress.erl | 115 +++++++++ 2 files changed, 287 insertions(+), 65 deletions(-) create mode 100644 apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader_rank_progress.erl diff --git a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl index 8f6b7c683..196d667c6 100644 --- a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl +++ b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl @@ -58,14 +58,18 @@ %% TODO https://emqx.atlassian.net/browse/EMQX-12575 %% Implement some stats to assign evenly? stream_progresses := #{ - emqx_ds:stream() => emqx_ds:iterator() + emqx_ds:stream() => #{ + iterator => emqx_ds:iterator(), + rank => emqx_ds:stream_rank() + } }, agents := #{ emqx_ds_shared_sub_proto:agent() => agent_state() }, stream_owners := #{ emqx_ds:stream() => emqx_ds_shared_sub_proto:agent() - } + }, + rank_progress := emqx_ds_shared_sub_leader_rank_progress:t() }. -export_type([ @@ -139,7 +143,8 @@ init([#{topic_filter := #share{group = Group, topic = Topic}} = _Options]) -> start_time => now_ms() - ?START_TIME_THRESHOLD, stream_progresses => #{}, stream_owners => #{}, - agents => #{} + agents => #{}, + rank_progress => emqx_ds_shared_sub_leader_rank_progress:init() }, {ok, ?leader_waiting_registration, Data}. @@ -254,37 +259,87 @@ terminate(_Reason, _State, #{topic := Topic, router_id := RouterId} = _Data) -> %% * Revoke streams from agents having too many streams %% * Assign streams to agents having too few streams -renew_streams(#{start_time := StartTime, stream_progresses := Progresses, topic := Topic} = Data0) -> +renew_streams( + #{ + start_time := StartTime, + stream_progresses := Progresses, + topic := Topic, + rank_progress := RankProgress0 + } = Data0 +) -> TopicFilter = emqx_topic:words(Topic), - {_, Streams} = lists:unzip( - emqx_ds:get_streams(?PERSISTENT_MESSAGE_DB, TopicFilter, StartTime) + StreamsWRanks = emqx_ds:get_streams(?PERSISTENT_MESSAGE_DB, TopicFilter, StartTime), + + %% Discard streams that are already replayed and init new + {NewStreamsWRanks, RankProgress1} = emqx_ds_shared_sub_leader_rank_progress:add_streams( + StreamsWRanks, RankProgress0 ), - %% TODO https://emqx.atlassian.net/browse/EMQX-12572 - %% Handle stream removal - NewProgresses = lists:foldl( - fun(Stream, ProgressesAcc) -> - case ProgressesAcc of - #{Stream := _} -> - ProgressesAcc; + {NewProgresses, VanishedProgresses} = update_progresses( + Progresses, NewStreamsWRanks, TopicFilter, StartTime + ), + Data1 = removed_vanished_streams(Data0, VanishedProgresses), + Data2 = Data1#{stream_progresses => NewProgresses, rank_progress => RankProgress1}, + Data3 = revoke_streams(Data2), + Data4 = assign_streams(Data3), + ?SLOG(info, #{ + msg => leader_renew_streams, + topic_filter => TopicFilter, + new_streams => length(NewStreamsWRanks) + }), + Data4. + +update_progresses(Progresses, NewStreamsWRanks, TopicFilter, StartTime) -> + lists:foldl( + fun({Rank, Stream}, {NewProgressesAcc, OldProgressesAcc}) -> + case OldProgressesAcc of + #{Stream := StreamData} -> + { + NewProgressesAcc#{Stream => StreamData}, + maps:remove(Stream, OldProgressesAcc) + }; _ -> {ok, It} = emqx_ds:make_iterator( ?PERSISTENT_MESSAGE_DB, Stream, TopicFilter, StartTime ), - ProgressesAcc#{Stream => It} + {NewProgressesAcc#{Stream => #{iterator => It, rank => Rank}}, OldProgressesAcc} end end, - Progresses, - Streams + {#{}, Progresses}, + NewStreamsWRanks + ). + +%% We just remove disappeared streams from anywhere. +%% +%% If streams disappear from DS during leader being in replaying state +%% this is an abnormal situation (we should receive `end_of_stream` first), +%% but clients clients are unlikely to report any progress on them. +%% +%% If streams disappear after long leader sleep, it is a normal situation. +%% This removal will be a part of initialization before any agents connect. +removed_vanished_streams(Data0, VanishedProgresses) -> + VanishedStreams = maps:keys(VanishedProgresses), + Data1 = lists:foldl( + fun(Stream, #{stream_owners := StreamOwners0} = DataAcc) -> + case StreamOwners0 of + #{Stream := Agent} -> + #{streams := Streams0, revoked_streams := RevokedStreams0} = + AgentState0 = get_agent_state(Data0, Agent), + Streams1 = Streams0 -- [Stream], + RevokedStreams1 = RevokedStreams0 -- [Stream], + AgentState1 = AgentState0#{ + streams => Streams1, + revoked_streams => RevokedStreams1 + }, + set_agent_state(DataAcc, Agent, AgentState1); + _ -> + DataAcc + end + end, + Data0, + VanishedStreams ), - Data1 = Data0#{stream_progresses => NewProgresses}, - ?SLOG(info, #{ - msg => leader_renew_streams, - topic_filter => TopicFilter, - streams => length(Streams) - }), - Data2 = revoke_streams(Data1), - Data3 = assign_streams(Data2), - Data3. + Data2 = unassign_streams(Data1, VanishedStreams), + Data2. %% We revoke streams from agents that have too many streams (> desired_stream_count_per_agent). %% We revoke only from replaying agents. @@ -528,15 +583,19 @@ update_agent_stream_states(Data0, Agent, AgentStreamProgresses, Version) -> Data0; {?waiting_replaying, AgentVersion} -> %% Agent finished updating, now replaying - Data1 = update_stream_progresses(Data0, Agent, AgentStreamProgresses), - AgentState1 = update_agent_timeout(AgentState0), - AgentState2 = agent_transition_to_replaying(Agent, AgentState1), - set_agent_state(Data1, Agent, AgentState2); + {Data1, AgentState1} = update_stream_progresses( + Data0, Agent, AgentState0, AgentStreamProgresses + ), + AgentState2 = update_agent_timeout(AgentState1), + AgentState3 = agent_transition_to_replaying(Agent, AgentState2), + set_agent_state(Data1, Agent, AgentState3); {?replaying, AgentVersion} -> %% Common case, agent is replaying - Data1 = update_stream_progresses(Data0, Agent, AgentStreamProgresses), - AgentState1 = update_agent_timeout(AgentState0), - set_agent_state(Data1, Agent, AgentState1); + {Data1, AgentState1} = update_stream_progresses( + Data0, Agent, AgentState0, AgentStreamProgresses + ), + AgentState2 = update_agent_timeout(AgentState1), + set_agent_state(Data1, Agent, AgentState2); {OtherState, OtherVersion} -> ?tp(warning, unexpected_update, #{ agent => Agent, @@ -549,24 +608,63 @@ update_agent_stream_states(Data0, Agent, AgentStreamProgresses, Version) -> end. update_stream_progresses( - #{stream_progresses := StreamProgresses0, stream_owners := StreamOwners} = Data, + #{stream_progresses := StreamProgresses0, stream_owners := StreamOwners} = Data0, Agent, + AgentState0, ReceivedStreamProgresses ) -> - StreamProgresses1 = lists:foldl( - fun(#{stream := Stream, iterator := It}, ProgressesAcc) -> + {StreamProgresses1, ReplayedStreams} = lists:foldl( + fun(#{stream := Stream, iterator := It}, {ProgressesAcc, ReplayedStreamsAcc}) -> case StreamOwners of #{Stream := Agent} -> - ProgressesAcc#{Stream => It}; + StreamData0 = maps:get(Stream, ProgressesAcc), + case It of + end_of_stream -> + Rank = maps:get(rank, StreamData0), + {maps:remove(Stream, ProgressesAcc), ReplayedStreamsAcc#{Stream => Rank}}; + _ -> + StreamData1 = StreamData0#{iterator => It}, + {ProgressesAcc#{Stream => StreamData1}, ReplayedStreamsAcc} + end; _ -> - ProgressesAcc + {ProgressesAcc, ReplayedStreamsAcc} end end, - StreamProgresses0, + {StreamProgresses0, #{}}, ReceivedStreamProgresses ), - Data#{ - stream_progresses => StreamProgresses1 + Data1 = update_rank_progress(Data0, ReplayedStreams), + Data2 = Data1#{stream_progresses => StreamProgresses1}, + AgentState1 = filter_replayed_streams(AgentState0, ReplayedStreams), + {Data2, AgentState1}. + +update_rank_progress(#{rank_progress := RankProgress0} = Data, ReplayedStreams) -> + RankProgress1 = maps:fold( + fun(Stream, Rank, RankProgressAcc) -> + emqx_ds_shared_sub_leader_rank_progress:set_replayed({Rank, Stream}, RankProgressAcc) + end, + RankProgress0, + ReplayedStreams + ), + Data#{rank_progress => RankProgress1}. + +%% No need to revoke fully replayed streams. We do not assign them anymore. +%% The agent's session also will drop replayed streams itself. +filter_replayed_streams( + #{streams := Streams0, revoked_streams := RevokedStreams0} = AgentState0, + ReplayedStreams +) -> + Streams1 = lists:filter( + fun(Stream) -> not maps:is_key(Stream, ReplayedStreams) end, + Streams0 + ), + RevokedStreams1 = lists:filter( + fun(Stream) -> not maps:is_key(Stream, ReplayedStreams) end, + RevokedStreams0 + ), + AgentState0#{ + streams => Streams1, + revoked_streams => RevokedStreams1 }. clean_revoked_streams( @@ -613,41 +711,49 @@ update_agent_stream_states(Data0, Agent, AgentStreamProgresses, VersionOld, Vers case {State, VersionOld, VersionNew} of {?waiting_updating, AgentPrevVersion, AgentVersion} -> %% Client started updating - Data1 = update_stream_progresses(Data0, Agent, AgentStreamProgresses), - AgentState1 = update_agent_timeout(AgentState0), - {AgentState2, Data2} = clean_revoked_streams( - Data1, Agent, AgentState1, AgentStreamProgresses + {Data1, AgentState1} = update_stream_progresses( + Data0, Agent, AgentState0, AgentStreamProgresses ), - AgentState3 = - case AgentState2 of + AgentState2 = update_agent_timeout(AgentState1), + {AgentState3, Data2} = clean_revoked_streams( + Data1, Agent, AgentState2, AgentStreamProgresses + ), + AgentState4 = + case AgentState3 of #{revoked_streams := []} -> - agent_transition_to_waiting_replaying(Data1, Agent, AgentState2); + agent_transition_to_waiting_replaying(Data1, Agent, AgentState3); _ -> - agent_transition_to_updating(Agent, AgentState2) + agent_transition_to_updating(Agent, AgentState3) end, - set_agent_state(Data2, Agent, AgentState3); + set_agent_state(Data2, Agent, AgentState4); {?updating, AgentPrevVersion, AgentVersion} -> - Data1 = update_stream_progresses(Data0, Agent, AgentStreamProgresses), - AgentState1 = update_agent_timeout(AgentState0), - {AgentState2, Data2} = clean_revoked_streams( - Data1, Agent, AgentState1, AgentStreamProgresses + {Data1, AgentState1} = update_stream_progresses( + Data0, Agent, AgentState0, AgentStreamProgresses ), - AgentState3 = - case AgentState2 of + AgentState2 = update_agent_timeout(AgentState1), + {AgentState3, Data2} = clean_revoked_streams( + Data1, Agent, AgentState2, AgentStreamProgresses + ), + AgentState4 = + case AgentState3 of #{revoked_streams := []} -> - agent_transition_to_waiting_replaying(Data1, Agent, AgentState2); + agent_transition_to_waiting_replaying(Data1, Agent, AgentState3); _ -> - AgentState2 + AgentState3 end, - set_agent_state(Data2, Agent, AgentState3); + set_agent_state(Data2, Agent, AgentState4); {?waiting_replaying, _, AgentVersion} -> - Data1 = update_stream_progresses(Data0, Agent, AgentStreamProgresses), - AgentState1 = update_agent_timeout(AgentState0), - set_agent_state(Data1, Agent, AgentState1); + {Data1, AgentState1} = update_stream_progresses( + Data0, Agent, AgentState0, AgentStreamProgresses + ), + AgentState2 = update_agent_timeout(AgentState1), + set_agent_state(Data1, Agent, AgentState2); {?replaying, _, AgentVersion} -> - Data1 = update_stream_progresses(Data0, Agent, AgentStreamProgresses), - AgentState1 = update_agent_timeout(AgentState0), - set_agent_state(Data1, Agent, AgentState1); + {Data1, AgentState1} = update_stream_progresses( + Data0, Agent, AgentState0, AgentStreamProgresses + ), + AgentState2 = update_agent_timeout(AgentState1), + set_agent_state(Data1, Agent, AgentState2); {OtherState, OtherVersionOld, OtherVersionNew} -> ?tp(warning, unexpected_update, #{ agent => Agent, @@ -798,9 +904,10 @@ desired_stream_count_per_agent(#{stream_progresses := StreamProgresses}, AgentCo stream_progresses(#{stream_progresses := StreamProgresses} = _Data, Streams) -> lists:map( fun(Stream) -> + StreamData = maps:get(Stream, StreamProgresses), #{ stream => Stream, - iterator => maps:get(Stream, StreamProgresses) + iterator => maps:get(iterator, StreamData) } end, Streams diff --git a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader_rank_progress.erl b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader_rank_progress.erl new file mode 100644 index 000000000..689c4ba89 --- /dev/null +++ b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader_rank_progress.erl @@ -0,0 +1,115 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_ds_shared_sub_leader_rank_progress). + +-include_lib("emqx/include/logger.hrl"). + +-export([ + init/0, + set_replayed/2, + add_streams/2 +]). + +%% "shard" +-type rank_x() :: emqx_ds:rank_x(). + +%% "generation" +-type rank_y() :: emqx_ds:rank_y(). + +%% shard progress +-type x_progress() :: #{ + %% All streams with given rank_x and rank_y =< min_y are replayed. + min_y := rank_y(), + + ys := #{ + rank_y() => #{ + emqx_ds:stream() => _IdReplayed :: boolean() + } + } +}. + +-type t() :: #{ + rank_x() => x_progress() +}. + +-spec init() -> t(). +init() -> #{}. + +-spec set_replayed(emqx_ds:stream_rank(), t()) -> t(). +set_replayed({{RankX, RankY}, Stream}, State) -> + case State of + #{RankX := #{ys := #{RankY := #{Stream := false} = RankYStreams} = Ys0}} -> + Ys1 = Ys0#{RankY => RankYStreams#{Stream => true}}, + {MinY, Ys2} = update_min_y(maps:to_list(Ys1)), + State#{RankX => #{min_y => MinY, ys => Ys2}}; + _ -> + ?SLOG( + warning, + leader_rank_progress_double_or_invalid_update, + #{ + rank_x => RankX, + rank_y => RankY, + state => State + } + ), + State + end. + +-spec add_streams([{emqx_ds:stream_rank(), emqx_ds:stream()}], t()) -> false | {true, t()}. +add_streams(StreamsWithRanks, State) -> + SortedStreamsWithRanks = lists:sort( + fun({{_RankX1, RankY1}, _Stream1}, {{_RankX2, RankY2}, _Stream2}) -> + RankY1 =< RankY2 + end, + StreamsWithRanks + ), + lists:foldl( + fun({Rank, Stream} = StreamWithRank, {StreamAcc, StateAcc0}) -> + case add_stream({Rank, Stream}, StateAcc0) of + {true, StateAcc1} -> + {[StreamWithRank | StreamAcc], StateAcc1}; + false -> + {StreamAcc, StateAcc0} + end + end, + {[], State}, + SortedStreamsWithRanks + ). + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- + +add_stream({{RankX, RankY}, Stream}, State0) -> + case State0 of + #{RankX := #{min_y := MinY}} when RankY =< MinY -> + false; + #{RankX := #{ys := #{RankY := #{Stream := true}}}} -> + false; + _ -> + XProgress = maps:get(RankX, State0, #{min_y => RankY - 1, ys => #{}}), + Ys0 = maps:get(ys, XProgress), + RankYStreams0 = maps:get(RankY, Ys0, #{}), + RankYStreams1 = RankYStreams0#{Stream => false}, + Ys1 = Ys0#{RankY => RankYStreams1}, + State1 = State0#{RankX => XProgress#{ys => Ys1}}, + {true, State1} + end. + +update_min_y([{RankY, RankYStreams} | Rest] = Ys) -> + case {has_unreplayed_streams(RankYStreams), Rest} of + {true, _} -> + {RankY, maps:from_list(Ys)}; + {false, []} -> + {RankY - 1, #{}}; + {false, _} -> + update_min_y(Rest) + end. + +has_unreplayed_streams(RankYStreams) -> + lists:any( + fun(IsReplayed) -> not IsReplayed end, + maps:values(RankYStreams) + ). From 53d4cd3174abb3f0c176770fc00afae3a5c86fc2 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Wed, 3 Jul 2024 23:21:21 +0300 Subject: [PATCH 096/131] feat(queue): rename leader' stream_progresses to stream_states --- .../src/emqx_ds_shared_sub_leader.erl | 92 +++++++++++-------- 1 file changed, 53 insertions(+), 39 deletions(-) diff --git a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl index 196d667c6..f0d194dfc 100644 --- a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl +++ b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl @@ -48,28 +48,37 @@ revoked_streams := list(emqx_ds:stream()) }. +-type stream_state() :: #{ + iterator => emqx_ds:iterator(), + rank => emqx_ds:stream_rank() +}. + +%% TODO https://emqx.atlassian.net/browse/EMQX-12307 +%% Some data should be persisted -type data() :: #{ + %% + %% Persistent data + %% group := emqx_types:group(), topic := emqx_types:topic(), %% For ds router, not an actual session_id router_id := binary(), - %% TODO https://emqx.atlassian.net/browse/EMQX-12307 - %% Persist progress %% TODO https://emqx.atlassian.net/browse/EMQX-12575 %% Implement some stats to assign evenly? - stream_progresses := #{ - emqx_ds:stream() => #{ - iterator => emqx_ds:iterator(), - rank => emqx_ds:stream_rank() - } + stream_states := #{ + emqx_ds:stream() => stream_state() }, + rank_progress := emqx_ds_shared_sub_leader_rank_progress:t(), + + %% + %% Ephimeral data, should not be persisted + %% agents := #{ emqx_ds_shared_sub_proto:agent() => agent_state() }, stream_owners := #{ emqx_ds:stream() => emqx_ds_shared_sub_proto:agent() - }, - rank_progress := emqx_ds_shared_sub_leader_rank_progress:t() + } }. -export_type([ @@ -141,7 +150,7 @@ init([#{topic_filter := #share{group = Group, topic = Topic}} = _Options]) -> topic => Topic, router_id => gen_router_id(), start_time => now_ms() - ?START_TIME_THRESHOLD, - stream_progresses => #{}, + stream_states => #{}, stream_owners => #{}, agents => #{}, rank_progress => emqx_ds_shared_sub_leader_rank_progress:init() @@ -262,7 +271,7 @@ terminate(_Reason, _State, #{topic := Topic, router_id := RouterId} = _Data) -> renew_streams( #{ start_time := StartTime, - stream_progresses := Progresses, + stream_states := StreamStates, topic := Topic, rank_progress := RankProgress0 } = Data0 @@ -274,11 +283,11 @@ renew_streams( {NewStreamsWRanks, RankProgress1} = emqx_ds_shared_sub_leader_rank_progress:add_streams( StreamsWRanks, RankProgress0 ), - {NewProgresses, VanishedProgresses} = update_progresses( - Progresses, NewStreamsWRanks, TopicFilter, StartTime + {NewStreamStates, VanishedStreamStates} = update_progresses( + StreamStates, NewStreamsWRanks, TopicFilter, StartTime ), - Data1 = removed_vanished_streams(Data0, VanishedProgresses), - Data2 = Data1#{stream_progresses => NewProgresses, rank_progress => RankProgress1}, + Data1 = removed_vanished_streams(Data0, VanishedStreamStates), + Data2 = Data1#{stream_states => NewStreamStates, rank_progress => RankProgress1}, Data3 = revoke_streams(Data2), Data4 = assign_streams(Data3), ?SLOG(info, #{ @@ -288,23 +297,26 @@ renew_streams( }), Data4. -update_progresses(Progresses, NewStreamsWRanks, TopicFilter, StartTime) -> +update_progresses(StreamStates, NewStreamsWRanks, TopicFilter, StartTime) -> lists:foldl( - fun({Rank, Stream}, {NewProgressesAcc, OldProgressesAcc}) -> - case OldProgressesAcc of + fun({Rank, Stream}, {NewStreamStatesAcc, OldStreamStatesAcc}) -> + case OldStreamStatesAcc of #{Stream := StreamData} -> { - NewProgressesAcc#{Stream => StreamData}, - maps:remove(Stream, OldProgressesAcc) + NewStreamStatesAcc#{Stream => StreamData}, + maps:remove(Stream, OldStreamStatesAcc) }; _ -> {ok, It} = emqx_ds:make_iterator( ?PERSISTENT_MESSAGE_DB, Stream, TopicFilter, StartTime ), - {NewProgressesAcc#{Stream => #{iterator => It, rank => Rank}}, OldProgressesAcc} + { + NewStreamStatesAcc#{Stream => #{iterator => It, rank => Rank}}, + OldStreamStatesAcc + } end end, - {#{}, Progresses}, + {#{}, StreamStates}, NewStreamsWRanks ). @@ -316,8 +328,8 @@ update_progresses(Progresses, NewStreamsWRanks, TopicFilter, StartTime) -> %% %% If streams disappear after long leader sleep, it is a normal situation. %% This removal will be a part of initialization before any agents connect. -removed_vanished_streams(Data0, VanishedProgresses) -> - VanishedStreams = maps:keys(VanishedProgresses), +removed_vanished_streams(Data0, VanishedStreamStates) -> + VanishedStreams = maps:keys(VanishedStreamStates), Data1 = lists:foldl( fun(Stream, #{stream_owners := StreamOwners0} = DataAcc) -> case StreamOwners0 of @@ -608,33 +620,35 @@ update_agent_stream_states(Data0, Agent, AgentStreamProgresses, Version) -> end. update_stream_progresses( - #{stream_progresses := StreamProgresses0, stream_owners := StreamOwners} = Data0, + #{stream_states := StreamStates0, stream_owners := StreamOwners} = Data0, Agent, AgentState0, ReceivedStreamProgresses ) -> - {StreamProgresses1, ReplayedStreams} = lists:foldl( - fun(#{stream := Stream, iterator := It}, {ProgressesAcc, ReplayedStreamsAcc}) -> + {StreamStates1, ReplayedStreams} = lists:foldl( + fun(#{stream := Stream, iterator := It}, {StreamStatesAcc, ReplayedStreamsAcc}) -> case StreamOwners of #{Stream := Agent} -> - StreamData0 = maps:get(Stream, ProgressesAcc), + StreamData0 = maps:get(Stream, StreamStatesAcc), case It of end_of_stream -> Rank = maps:get(rank, StreamData0), - {maps:remove(Stream, ProgressesAcc), ReplayedStreamsAcc#{Stream => Rank}}; + {maps:remove(Stream, StreamStatesAcc), ReplayedStreamsAcc#{ + Stream => Rank + }}; _ -> StreamData1 = StreamData0#{iterator => It}, - {ProgressesAcc#{Stream => StreamData1}, ReplayedStreamsAcc} + {StreamStatesAcc#{Stream => StreamData1}, ReplayedStreamsAcc} end; _ -> - {ProgressesAcc, ReplayedStreamsAcc} + {StreamStatesAcc, ReplayedStreamsAcc} end end, - {StreamProgresses0, #{}}, + {StreamStates0, #{}}, ReceivedStreamProgresses ), Data1 = update_rank_progress(Data0, ReplayedStreams), - Data2 = Data1#{stream_progresses => StreamProgresses1}, + Data2 = Data1#{stream_states => StreamStates1}, AgentState1 = filter_replayed_streams(AgentState0, ReplayedStreams), {Data2, AgentState1}. @@ -864,8 +878,8 @@ gen_router_id() -> now_ms() -> erlang:system_time(millisecond). -unassigned_streams(#{stream_progresses := StreamProgresses, stream_owners := StreamOwners}) -> - Streams = maps:keys(StreamProgresses), +unassigned_streams(#{stream_states := StreamStates, stream_owners := StreamOwners}) -> + Streams = maps:keys(StreamStates), AssignedStreams = maps:keys(StreamOwners), Streams -- AssignedStreams. @@ -887,12 +901,12 @@ desired_stream_count_per_agent(#{agents := AgentStates} = Data) -> desired_stream_count_for_new_agent(#{agents := AgentStates} = Data) -> desired_stream_count_per_agent(Data, maps:size(AgentStates) + 1). -desired_stream_count_per_agent(#{stream_progresses := StreamProgresses}, AgentCount) -> +desired_stream_count_per_agent(#{stream_states := StreamStates}, AgentCount) -> case AgentCount of 0 -> 0; _ -> - StreamCount = maps:size(StreamProgresses), + StreamCount = maps:size(StreamStates), case StreamCount rem AgentCount of 0 -> StreamCount div AgentCount; @@ -901,10 +915,10 @@ desired_stream_count_per_agent(#{stream_progresses := StreamProgresses}, AgentCo end end. -stream_progresses(#{stream_progresses := StreamProgresses} = _Data, Streams) -> +stream_progresses(#{stream_states := StreamStates} = _Data, Streams) -> lists:map( fun(Stream) -> - StreamData = maps:get(Stream, StreamProgresses), + StreamData = maps:get(Stream, StreamStates), #{ stream => Stream, iterator => maps:get(iterator, StreamData) From 65ab81ff7487f8a1df61e58dd74c34f0b10fdcfa Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Thu, 4 Jul 2024 17:22:26 +0300 Subject: [PATCH 097/131] feat(queue): fix quick resubscription --- ...emqx_persistent_session_ds_shared_subs.erl | 68 +++++++++++++++---- .../src/emqx_ds_shared_sub_agent.erl | 7 ++ .../src/emqx_ds_shared_sub_group_sm.erl | 26 +++++-- .../src/emqx_ds_shared_sub_proto.erl | 22 ++++-- .../test/emqx_ds_shared_sub_SUITE.erl | 10 ++- 5 files changed, 106 insertions(+), 27 deletions(-) diff --git a/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl index 6709eb37a..7db86dfe0 100644 --- a/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl +++ b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl @@ -198,8 +198,16 @@ schedule_subscribe( ScheduledActions1 = ScheduledActions0#{ TopicFilter => ScheduledAction#{type => {?schedule_subscribe, SubOpts}} }, + ?tp(warning, shared_subs_schedule_subscribe_override, #{ + topic_filter => TopicFilter, + new_type => {?schedule_subscribe, SubOpts}, + old_action => format_schedule_action(ScheduledAction) + }), SharedSubS0#{scheduled_actions := ScheduledActions1}; _ -> + ?tp(warning, shared_subs_schedule_subscribe_new, #{ + topic_filter => TopicFilter, subopts => SubOpts + }), Agent1 = emqx_persistent_session_ds_shared_subs_agent:on_subscribe( Agent0, TopicFilter, SubOpts ), @@ -237,20 +245,30 @@ schedule_unsubscribe( S, #{scheduled_actions := ScheduledActions0} = SharedSubS0, UnsubscridedSubId, TopicFilter ) -> case ScheduledActions0 of - #{TopicFilter := ScheduledAction} -> + #{TopicFilter := ScheduledAction0} -> + ScheduledAction1 = ScheduledAction0#{type => ?schedule_unsubscribe}, ScheduledActions1 = ScheduledActions0#{ - TopicFilter => ScheduledAction#{type => ?schedule_unsubscribe} + TopicFilter => ScheduledAction1 }, + ?tp(warning, shared_subs_schedule_unsubscribe_override, #{ + topic_filter => TopicFilter, + new_type => ?schedule_unsubscribe, + old_action => format_schedule_action(ScheduledAction0) + }), SharedSubS0#{scheduled_actions := ScheduledActions1}; _ -> - StreamIdsToFinalize = stream_ids_by_sub_id(S, UnsubscridedSubId), + StreamKeys = stream_keys_by_sub_id(S, UnsubscridedSubId), ScheduledActions1 = ScheduledActions0#{ TopicFilter => #{ type => ?schedule_unsubscribe, - stream_keys_to_wait => StreamIdsToFinalize, + stream_keys_to_wait => StreamKeys, progresses => [] } }, + ?tp(warning, shared_subs_schedule_unsubscribe_new, #{ + topic_filter => TopicFilter, + stream_keys => emqx_ds_shared_sub_proto:format_stream_keys(StreamKeys) + }), SharedSubS0#{scheduled_actions := ScheduledActions1} end. @@ -400,28 +418,43 @@ run_scheduled_actions(S, Agent, ScheduledActions) -> run_scheduled_action( S, - Agent, - TopicFilter, + Agent0, + #share{group = Group} = TopicFilter, #{type := Type, stream_keys_to_wait := StreamKeysToWait0, progresses := Progresses0} = Action ) -> StreamKeysToWait1 = filter_unfinished_streams(S, StreamKeysToWait0), Progresses1 = stream_progresses(S, StreamKeysToWait0 -- StreamKeysToWait1) ++ Progresses0, case StreamKeysToWait1 of [] -> + ?tp(warning, shared_subs_schedule_action_complete, #{ + topic_filter => TopicFilter, + progresses => emqx_ds_shared_sub_proto:format_streams(Progresses1), + type => Type + }), + %% Regular progress won't se unsubscribed streams, so we need to + %% send the progress explicitly. + Agent1 = emqx_persistent_session_ds_shared_subs_agent:on_stream_progress( + Agent0, #{Group => Progresses1} + ), case Type of {?schedule_subscribe, SubOpts} -> {ok, emqx_persistent_session_ds_shared_subs_agent:on_subscribe( - Agent, TopicFilter, SubOpts + Agent1, TopicFilter, SubOpts )}; ?schedule_unsubscribe -> {ok, emqx_persistent_session_ds_shared_subs_agent:on_unsubscribe( - Agent, TopicFilter, Progresses1 + Agent1, TopicFilter, Progresses1 )} end; _ -> - {continue, Action#{stream_keys_to_wait => StreamKeysToWait1, progresses => Progresses1}} + Action1 = Action#{stream_keys_to_wait => StreamKeysToWait1, progresses => Progresses1}, + ?tp(warning, shared_subs_schedule_action_continue, #{ + topic_filter => TopicFilter, + new_action => format_schedule_action(Action1) + }), + {continue, Action1} end. filter_unfinished_streams(S, StreamKeysToWait) -> @@ -509,14 +542,14 @@ lookup(TopicFilter, S) -> undefined end. -stream_ids_by_sub_id(S, MatchSubId) -> +stream_keys_by_sub_id(S, MatchSubId) -> emqx_persistent_session_ds_state:fold_streams( - fun({SubId, _Stream} = StreamStateId, _SRS, StreamStateIds) -> + fun({SubId, _Stream} = StreamKey, _SRS, StreamKeys) -> case SubId of MatchSubId -> - [StreamStateId | StreamStateIds]; + [StreamKey | StreamKeys]; _ -> - StreamStateIds + StreamKeys end end, [], @@ -580,3 +613,12 @@ is_use_finished(_S, #srs{unsubscribed = Unsubscribed}) -> is_stream_fully_acked(S, SRS) -> emqx_persistent_session_ds_stream_scheduler:is_fully_acked(SRS, S). + +format_schedule_action(#{ + type := Type, progresses := Progresses, stream_keys_to_wait := StreamKeysToWait +}) -> + #{ + type => Type, + progresses => emqx_ds_shared_sub_proto:format_streams(Progresses), + stream_keys_to_wait => emqx_ds_shared_sub_proto:format_stream_keys(StreamKeysToWait) + }. diff --git a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_agent.erl b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_agent.erl index 70b203661..b896370f3 100644 --- a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_agent.erl +++ b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_agent.erl @@ -6,6 +6,7 @@ -include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("emqx/include/logger.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). -include("emqx_ds_shared_sub_proto.hrl"). @@ -41,6 +42,9 @@ open(TopicSubscriptions, Opts) -> State0 = init_state(Opts), State1 = lists:foldl( fun({ShareTopicFilter, #{}}, State) -> + ?tp(warning, ds_agent_open_subscription, #{ + topic_filter => ShareTopicFilter + }), add_group_subscription(State, ShareTopicFilter) end, State0, @@ -52,6 +56,9 @@ can_subscribe(_State, _TopicFilter, _SubOpts) -> ok. on_subscribe(State0, TopicFilter, _SubOpts) -> + ?tp(warning, ds_agent_on_subscribe, #{ + topic_filter => TopicFilter + }), add_group_subscription(State0, TopicFilter). on_unsubscribe(State, TopicFilter, GroupProgress) -> diff --git a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_group_sm.erl b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_group_sm.erl index aab47802b..f9a81bbb8 100644 --- a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_group_sm.erl +++ b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_group_sm.erl @@ -153,6 +153,10 @@ new(#{ agent => Agent, send_after => SendAfter }, + ?tp(warning, group_sm_new, #{ + agent => Agent, + topic_filter => ShareTopicFilter + }), transition(GSM0, ?connecting, #{}). -spec fetch_stream_events(group_sm()) -> {group_sm(), list(external_lease_event())}. @@ -191,6 +195,10 @@ handle_disconnect( %% Connecting state handle_connecting(#{agent := Agent, topic_filter := ShareTopicFilter} = GSM) -> + ?tp(warning, group_sm_enter_connecting, #{ + agent => Agent, + topic_filter => ShareTopicFilter + }), ok = emqx_ds_shared_sub_registry:lookup_leader(Agent, agent_metadata(GSM), ShareTopicFilter), ensure_state_timeout(GSM, find_leader_timeout, ?FIND_LEADER_TIMEOUT). @@ -215,6 +223,10 @@ handle_leader_lease_streams(GSM, _Leader, _StreamProgresses, _Version) -> GSM. handle_find_leader_timeout(#{agent := Agent, topic_filter := TopicFilter} = GSM0) -> + ?tp(warning, group_sm_find_leader_timeout, #{ + agent => Agent, + topic_filter => TopicFilter + }), ok = emqx_ds_shared_sub_registry:lookup_leader(Agent, agent_metadata(GSM0), TopicFilter), GSM1 = ensure_state_timeout(GSM0, find_leader_timeout, ?FIND_LEADER_TIMEOUT), GSM1. @@ -229,8 +241,8 @@ handle_replaying(GSM0) -> ), GSM2. -handle_renew_lease_timeout(GSM) -> - ?tp(debug, renew_lease_timeout, #{}), +handle_renew_lease_timeout(#{agent := Agent, topic_filter := TopicFilter} = GSM) -> + ?tp(warning, renew_lease_timeout, #{agent => Agent, topic_filter => TopicFilter}), transition(GSM, ?connecting, #{}). %%----------------------------------------------------------------------- @@ -326,12 +338,12 @@ handle_leader_update_streams( ) -> GSM; handle_leader_update_streams(GSM, VersionOld, VersionNew, _StreamProgresses) -> + %% Unexpected versions or state ?tp(warning, shared_sub_group_sm_unexpected_leader_update_streams, #{ gsm => GSM, version_old => VersionOld, version_new => VersionNew }), - %% Unexpected versions or state transition(GSM, ?connecting, #{}). handle_leader_renew_stream_lease( @@ -364,12 +376,12 @@ handle_leader_renew_stream_lease( ) -> GSM; handle_leader_renew_stream_lease(GSM, VersionOld, VersionNew) -> + %% Unexpected versions or state ?tp(warning, shared_sub_group_sm_unexpected_leader_renew_stream_lease, #{ gsm => GSM, version_old => VersionOld, version_new => VersionNew }), - %% Unexpected versions or state transition(GSM, ?connecting, #{}). -spec handle_stream_progress(group_sm(), list(emqx_ds_shared_sub_proto:agent_stream_progress())) -> @@ -410,7 +422,11 @@ handle_stream_progress( handle_stream_progress(#{state := ?disconnected} = GSM, _StreamProgresses) -> GSM. -handle_leader_invalidate(GSM) -> +handle_leader_invalidate(#{agent := Agent, topic_filter := TopicFilter} = GSM) -> + ?tp(warning, shared_sub_group_sm_leader_invalidate, #{ + agent => Agent, + topic_filter => TopicFilter + }), transition(GSM, ?connecting, #{}). %%----------------------------------------------------------------------- diff --git a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_proto.erl b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_proto.erl index 0b1770f3c..184e8d147 100644 --- a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_proto.erl +++ b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_proto.erl @@ -2,10 +2,6 @@ %% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- -%% TODO https://emqx.atlassian.net/browse/EMQX-12573 -%% This should be wrapped with a proto_v1 module. -%% For simplicity, send as simple OTP messages for now. - -module(emqx_ds_shared_sub_proto). -include("emqx_ds_shared_sub_proto.hrl"). @@ -27,6 +23,9 @@ -export([ format_streams/1, + format_stream/1, + format_stream_key/1, + format_stream_keys/1, agent/2 ]). @@ -254,12 +253,21 @@ format_streams(Streams) -> Streams ). +format_stream(#{stream := Stream, iterator := Iterator} = Value) -> + Value#{stream => format_opaque(Stream), iterator => format_opaque(Iterator)}. + +format_stream_key({SubId, Stream}) -> + {SubId, format_opaque(Stream)}. + +format_stream_keys(StreamKeys) -> + lists:map( + fun format_stream_key/1, + StreamKeys + ). + %%-------------------------------------------------------------------- %% Helpers %%-------------------------------------------------------------------- format_opaque(Opaque) -> erlang:phash2(Opaque). - -format_stream(#{stream := Stream, iterator := Iterator} = Value) -> - Value#{stream => format_opaque(Stream), iterator => format_opaque(Iterator)}. diff --git a/apps/emqx_ds_shared_sub/test/emqx_ds_shared_sub_SUITE.erl b/apps/emqx_ds_shared_sub/test/emqx_ds_shared_sub_SUITE.erl index 4c2e9a239..4733dc650 100644 --- a/apps/emqx_ds_shared_sub/test/emqx_ds_shared_sub_SUITE.erl +++ b/apps/emqx_ds_shared_sub/test/emqx_ds_shared_sub_SUITE.erl @@ -297,8 +297,14 @@ t_quick_resubscribe(_Config) -> ConnShared2 = emqtt_connect_sub(<<"client_shared2">>), {ok, _, _} = emqtt:subscribe(ConnShared2, <<"$share/gr10/topic10/#">>, 1), - {ok, _, _} = emqtt:unsubscribe(ConnShared1, <<"$share/gr10/topic10/#">>), - {ok, _, _} = emqtt:subscribe(ConnShared1, <<"$share/gr10/topic10/#">>, 1), + ok = lists:foreach( + fun(_) -> + {ok, _, _} = emqtt:unsubscribe(ConnShared1, <<"$share/gr10/topic10/#">>), + {ok, _, _} = emqtt:subscribe(ConnShared1, <<"$share/gr10/topic10/#">>, 1), + ct:sleep(5) + end, + lists:seq(1, 10) + ), receive publish_done -> ok From 91dd1183ad2dad3ac5f69411a8800ef31fa32e99 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Thu, 4 Jul 2024 17:51:42 +0300 Subject: [PATCH 098/131] feat(queue): fix dialyzer issues --- .../emqx_persistent_session_ds_shared_subs.erl | 5 ++++- .../src/emqx_ds_shared_sub_leader_rank_progress.erl | 13 +++++++++++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl index 7db86dfe0..eb45ef014 100644 --- a/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl +++ b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl @@ -107,7 +107,7 @@ open(S, Opts) -> Agent = emqx_persistent_session_ds_shared_subs_agent:open( SharedSubscriptions, agent_opts(Opts) ), - SharedSubS = #{agent => Agent}, + SharedSubS = #{agent => Agent, scheduled_actions => #{}}, {ok, S, SharedSubS}. %%-------------------------------------------------------------------- @@ -136,6 +136,7 @@ on_subscribe(undefined, TopicFilter, SubOpts, #{props := Props, s := S} = Sessio on_subscribe(Subscription, TopicFilter, SubOpts, Session) -> update_subscription(Subscription, TopicFilter, SubOpts, Session). +-dialyzer({nowarn_function, create_new_subscription/3}). create_new_subscription(TopicFilter, SubOpts, #{ s := S0, shared_sub_s := #{agent := Agent} = SharedSubS0, @@ -190,6 +191,7 @@ update_subscription(#{current_state := SStateId0, id := SubId} = Sub0, TopicFilt {ok, S, SharedSubS} end. +-dialyzer({nowarn_function, schedule_subscribe/3}). schedule_subscribe( #{agent := Agent0, scheduled_actions := ScheduledActions0} = SharedSubS0, TopicFilter, SubOpts ) -> @@ -605,6 +607,7 @@ to_agent_subscription(_S, Subscription) -> agent_opts(#{session_id := SessionId}) -> #{session_id => SessionId}. +-dialyzer({nowarn_function, now_ms/0}). now_ms() -> erlang:system_time(millisecond). diff --git a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader_rank_progress.erl b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader_rank_progress.erl index 689c4ba89..5cde51f16 100644 --- a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader_rank_progress.erl +++ b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader_rank_progress.erl @@ -34,6 +34,14 @@ rank_x() => x_progress() }. +-export_type([ + t/0 +]). + +%%-------------------------------------------------------------------- +%% API +%%-------------------------------------------------------------------- + -spec init() -> t(). init() -> #{}. @@ -47,8 +55,8 @@ set_replayed({{RankX, RankY}, Stream}, State) -> _ -> ?SLOG( warning, - leader_rank_progress_double_or_invalid_update, #{ + msg => leader_rank_progress_double_or_invalid_update, rank_x => RankX, rank_y => RankY, state => State @@ -57,7 +65,8 @@ set_replayed({{RankX, RankY}, Stream}, State) -> State end. --spec add_streams([{emqx_ds:stream_rank(), emqx_ds:stream()}], t()) -> false | {true, t()}. +-spec add_streams([{emqx_ds:stream_rank(), emqx_ds:stream()}], t()) -> + {[{emqx_ds:stream_rank(), emqx_ds:stream()}], t()}. add_streams(StreamsWithRanks, State) -> SortedStreamsWithRanks = lists:sort( fun({{_RankX1, RankY1}, _Stream1}, {{_RankX2, RankY2}, _Stream2}) -> From 1496f7f7788f5825a79a703ab17ccca633b2a3fc Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Thu, 4 Jul 2024 20:51:53 +0300 Subject: [PATCH 099/131] feat(queue): add leader_rank_progress test --- .../src/emqx_ds_shared_sub_leader.erl | 2 +- ...mqx_ds_shared_sub_leader_rank_progress.erl | 51 ++++++++++++++++++- 2 files changed, 50 insertions(+), 3 deletions(-) diff --git a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl index f0d194dfc..510d6a45f 100644 --- a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl +++ b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl @@ -71,7 +71,7 @@ rank_progress := emqx_ds_shared_sub_leader_rank_progress:t(), %% - %% Ephimeral data, should not be persisted + %% Ephemeral data, should not be persisted %% agents := #{ emqx_ds_shared_sub_proto:agent() => agent_state() diff --git a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader_rank_progress.erl b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader_rank_progress.erl index 5cde51f16..fa611463d 100644 --- a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader_rank_progress.erl +++ b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader_rank_progress.erl @@ -9,7 +9,8 @@ -export([ init/0, set_replayed/2, - add_streams/2 + add_streams/2, + replayed_up_to/2 ]). %% "shard" @@ -87,6 +88,15 @@ add_streams(StreamsWithRanks, State) -> SortedStreamsWithRanks ). +-spec replayed_up_to(emqx_ds:rank_x(), t()) -> emqx_ds:rank_y(). +replayed_up_to(RankX, State) -> + case State of + #{RankX := #{min_y := MinY}} -> + MinY; + _ -> + undefined + end. + %%-------------------------------------------------------------------- %% Internal functions %%-------------------------------------------------------------------- @@ -110,7 +120,7 @@ add_stream({{RankX, RankY}, Stream}, State0) -> update_min_y([{RankY, RankYStreams} | Rest] = Ys) -> case {has_unreplayed_streams(RankYStreams), Rest} of {true, _} -> - {RankY, maps:from_list(Ys)}; + {RankY - 1, maps:from_list(Ys)}; {false, []} -> {RankY - 1, #{}}; {false, _} -> @@ -122,3 +132,40 @@ has_unreplayed_streams(RankYStreams) -> fun(IsReplayed) -> not IsReplayed end, maps:values(RankYStreams) ). + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). + +add_streams_set_replayed_test() -> + State0 = init(), + {_, State1} = add_streams( + [ + {{shard1, 1}, s111}, + {{shard1, 1}, s112}, + {{shard1, 2}, s121}, + {{shard1, 2}, s122}, + {{shard1, 3}, s131}, + {{shard1, 4}, s141}, + + {{shard3, 5}, s51} + ], + State0 + ), + ?assertEqual(0, replayed_up_to(shard1, State1)), + + State2 = set_replayed({{shard1, 1}, s111}, State1), + State3 = set_replayed({{shard1, 3}, s131}, State2), + ?assertEqual(0, replayed_up_to(shard1, State3)), + State4 = set_replayed({{shard1, 1}, s112}, State3), + ?assertEqual(1, replayed_up_to(shard1, State4)), + + State5 = set_replayed({{shard1, 2}, s121}, State4), + State6 = set_replayed({{shard1, 2}, s122}, State5), + + ?assertEqual(3, replayed_up_to(shard1, State6)), + + State7 = set_replayed({{shard1, 4}, s141}, State6), + ?assertEqual(3, replayed_up_to(shard1, State7)). + +%% -ifdef(TEST) end +-endif. From 649cf880426cf6b20cb352580f06a44d188519f1 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Thu, 4 Jul 2024 21:24:21 +0300 Subject: [PATCH 100/131] feat(queue): kick agents that do not return to the replaying state for long --- .../src/emqx_ds_shared_sub_leader.erl | 59 ++++++++++++++----- 1 file changed, 45 insertions(+), 14 deletions(-) diff --git a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl index 510d6a45f..de277ece8 100644 --- a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl +++ b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl @@ -38,7 +38,7 @@ -define(updating, updating). -type agent_state() :: #{ - %% Our view of group gm's status + %% Our view of group sm's status %% it lags the actual state state := ?waiting_replaying | ?replaying | ?waiting_updating | ?updating, prev_version := emqx_maybe:t(emqx_ds_shared_sub_proto:version()), @@ -109,6 +109,7 @@ -define(DROP_TIMEOUT_INTERVAL, 1000). -define(AGENT_TIMEOUT, 5000). +-define(MAX_NOT_REPLAYING, 5000). -define(START_TIME_THRESHOLD, 5000). @@ -535,13 +536,24 @@ disconnect_agent(Data0, Agent, AgentStreamProgresses, Version) -> %% Drop agents that stopped reporting progress drop_timeout_agents(#{agents := Agents} = Data) -> - Now = now_ms(), + Now = now_ms_monotonic(), lists:foldl( - fun({Agent, #{update_deadline := Deadline} = _AgentState}, DataAcc) -> - case Deadline < Now of + fun( + {Agent, + #{update_deadline := UpdateDeadline, not_replaying_deadline := NoReplayingDeadline} = + _AgentState}, + DataAcc + ) -> + case + (UpdateDeadline < Now) orelse + (is_integer(NoReplayingDeadline) andalso NoReplayingDeadline < Now) + of true -> ?SLOG(info, #{ msg => leader_agent_timeout, + now => Now, + update_deadline => UpdateDeadline, + not_replaying_deadline => NoReplayingDeadline, agent => Agent }), drop_invalidate_agent(DataAcc, Agent); @@ -805,11 +817,12 @@ agent_transition_to_waiting_updating( prev_version => Version, version => NewVersion }, + AgentState2 = renew_no_replaying_deadline(AgentState1), StreamProgresses = stream_progresses(Data, Streams), ok = emqx_ds_shared_sub_proto:leader_update_streams( Agent, Group, Version, NewVersion, StreamProgresses ), - AgentState1. + AgentState2. agent_transition_to_waiting_replaying( #{group := Group} = _Data, Agent, #{state := OldState, version := Version} = AgentState0 @@ -820,10 +833,11 @@ agent_transition_to_waiting_replaying( new_state => ?waiting_replaying }), ok = emqx_ds_shared_sub_proto:leader_renew_stream_lease(Agent, Group, Version), - AgentState0#{ + AgentState1 = AgentState0#{ state => ?waiting_replaying, revoked_streams => [] - }. + }, + renew_no_replaying_deadline(AgentState1). agent_transition_to_initial_waiting_replaying( #{group := Group} = Data, Agent, AgentMetadata, InitialStreams @@ -839,15 +853,16 @@ agent_transition_to_initial_waiting_replaying( ok = emqx_ds_shared_sub_proto:leader_lease_streams( Agent, Group, Leader, StreamProgresses, Version ), - #{ + AgentState = #{ metadata => AgentMetadata, state => ?waiting_replaying, version => Version, prev_version => undefined, streams => InitialStreams, revoked_streams => [], - update_deadline => now_ms() + ?AGENT_TIMEOUT - }. + update_deadline => now_ms_monotonic() + ?AGENT_TIMEOUT + }, + renew_no_replaying_deadline(AgentState). agent_transition_to_replaying(Agent, #{state := ?waiting_replaying} = AgentState) -> ?tp(warning, shared_sub_leader_agent_state_transition, #{ @@ -857,16 +872,18 @@ agent_transition_to_replaying(Agent, #{state := ?waiting_replaying} = AgentState }), AgentState#{ state => ?replaying, - prev_version => undefined + prev_version => undefined, + not_replaying_deadline => undefined }. -agent_transition_to_updating(Agent, #{state := ?waiting_updating} = AgentState) -> +agent_transition_to_updating(Agent, #{state := ?waiting_updating} = AgentState0) -> ?tp(warning, shared_sub_leader_agent_state_transition, #{ agent => Agent, old_state => ?waiting_updating, new_state => ?updating }), - AgentState#{state => ?updating}. + AgentState1 = AgentState0#{state => ?updating}, + renew_no_replaying_deadline(AgentState1). %%-------------------------------------------------------------------- %% Helper functions @@ -878,6 +895,20 @@ gen_router_id() -> now_ms() -> erlang:system_time(millisecond). +now_ms_monotonic() -> + erlang:monotonic_time(millisecond). + +renew_no_replaying_deadline(#{not_replaying_deadline := undefined} = AgentState) -> + AgentState#{ + not_replaying_deadline => now_ms_monotonic() + ?MAX_NOT_REPLAYING + }; +renew_no_replaying_deadline(#{not_replaying_deadline := _Deadline} = AgentState) -> + AgentState; +renew_no_replaying_deadline(#{} = AgentState) -> + AgentState#{ + not_replaying_deadline => now_ms_monotonic() + ?MAX_NOT_REPLAYING + }. + unassigned_streams(#{stream_states := StreamStates, stream_owners := StreamOwners}) -> Streams = maps:keys(StreamStates), AssignedStreams = maps:keys(StreamOwners), @@ -960,7 +991,7 @@ set_agent_state(#{agents := Agents} = Data, Agent, AgentState) -> update_agent_timeout(AgentState) -> AgentState#{ - update_deadline => now_ms() + ?AGENT_TIMEOUT + update_deadline => now_ms_monotonic() + ?AGENT_TIMEOUT }. get_agent_state(#{agents := Agents} = _Data, Agent) -> From b74189570dfc2c0153dc3528f47fd4d5f14970d2 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Thu, 4 Jul 2024 21:33:22 +0300 Subject: [PATCH 101/131] feat(queue): do not use ee app from emqx app --- ...emqx_persistent_session_ds_shared_subs.erl | 42 ++++++++++++++++--- 1 file changed, 37 insertions(+), 5 deletions(-) diff --git a/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl index eb45ef014..0bdbff30a 100644 --- a/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl +++ b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl @@ -55,10 +55,17 @@ -type scheduled_action_type() :: {?schedule_subscribe, emqx_types:subopts()} | ?schedule_unsubscribe. + +-type agent_stream_progress() :: #{ + stream := emqx_ds:stream(), + iterator := emqx_ds:iterator(), + use_finished := boolean() +}. + -type scheduled_action() :: #{ type := scheduled_action_type(), stream_keys_to_wait := [stream_key()], - progresses := [emqx_ds_shared_sub_proto:agent_stream_progress()] + progresses := [agent_stream_progress()] }. -type t() :: #{ @@ -269,7 +276,7 @@ schedule_unsubscribe( }, ?tp(warning, shared_subs_schedule_unsubscribe_new, #{ topic_filter => TopicFilter, - stream_keys => emqx_ds_shared_sub_proto:format_stream_keys(StreamKeys) + stream_keys => format_stream_keys(StreamKeys) }), SharedSubS0#{scheduled_actions := ScheduledActions1} end. @@ -430,7 +437,7 @@ run_scheduled_action( [] -> ?tp(warning, shared_subs_schedule_action_complete, #{ topic_filter => TopicFilter, - progresses => emqx_ds_shared_sub_proto:format_streams(Progresses1), + progresses => format_streams(Progresses1), type => Type }), %% Regular progress won't se unsubscribed streams, so we need to @@ -617,11 +624,36 @@ is_use_finished(_S, #srs{unsubscribed = Unsubscribed}) -> is_stream_fully_acked(S, SRS) -> emqx_persistent_session_ds_stream_scheduler:is_fully_acked(SRS, S). +%%-------------------------------------------------------------------- +%% Formatters +%%-------------------------------------------------------------------- + format_schedule_action(#{ type := Type, progresses := Progresses, stream_keys_to_wait := StreamKeysToWait }) -> #{ type => Type, - progresses => emqx_ds_shared_sub_proto:format_streams(Progresses), - stream_keys_to_wait => emqx_ds_shared_sub_proto:format_stream_keys(StreamKeysToWait) + progresses => format_streams(Progresses), + stream_keys_to_wait => format_stream_keys(StreamKeysToWait) }. + +format_streams(Streams) -> + lists:map( + fun format_stream/1, + Streams + ). + +format_stream(#{stream := Stream, iterator := Iterator} = Value) -> + Value#{stream => format_opaque(Stream), iterator => format_opaque(Iterator)}. + +format_stream_key({SubId, Stream}) -> + {SubId, format_opaque(Stream)}. + +format_stream_keys(StreamKeys) -> + lists:map( + fun format_stream_key/1, + StreamKeys + ). + +format_opaque(Opaque) -> + erlang:phash2(Opaque). From 077ee3853081cd1339bd7e4813977142ee147f3f Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Fri, 5 Jul 2024 20:13:38 +0300 Subject: [PATCH 102/131] feat(queue): add config --- apps/emqx/test/emqx_cth_suite.erl | 2 + .../src/emqx_ds_shared_sub_app.erl | 2 + .../src/emqx_ds_shared_sub_config.erl | 84 +++++++++++++++++++ .../src/emqx_ds_shared_sub_config.hrl | 5 ++ .../src/emqx_ds_shared_sub_group_sm.erl | 45 +++++----- .../src/emqx_ds_shared_sub_leader.erl | 37 ++++---- .../src/emqx_ds_shared_sub_schema.erl | 57 +++++++++++++ .../test/emqx_ds_shared_sub_config_SUITE.erl | 62 ++++++++++++++ .../src/emqx_enterprise_schema.erl | 3 +- 9 files changed, 256 insertions(+), 41 deletions(-) create mode 100644 apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_config.erl create mode 100644 apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_config.hrl create mode 100644 apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_schema.erl create mode 100644 apps/emqx_ds_shared_sub/test/emqx_ds_shared_sub_config_SUITE.erl diff --git a/apps/emqx/test/emqx_cth_suite.erl b/apps/emqx/test/emqx_cth_suite.erl index 8e7c84580..c0e3430db 100644 --- a/apps/emqx/test/emqx_cth_suite.erl +++ b/apps/emqx/test/emqx_cth_suite.erl @@ -391,6 +391,8 @@ default_appspec(emqx_schema_validation, _SuiteOpts) -> #{schema_mod => emqx_schema_validation_schema, config => #{}}; default_appspec(emqx_message_transformation, _SuiteOpts) -> #{schema_mod => emqx_message_transformation_schema, config => #{}}; +default_appspec(emqx_ds_shared_sub, _SuiteOpts) -> + #{schema_mod => emqx_ds_shared_sub_schema, config => #{}}; default_appspec(_, _) -> #{}. diff --git a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_app.erl b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_app.erl index 5c2d8d964..80e728a80 100644 --- a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_app.erl +++ b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_app.erl @@ -15,9 +15,11 @@ -spec start(application:start_type(), term()) -> {ok, pid()}. start(_Type, _Args) -> + ok = emqx_ds_shared_sub_config:load(), {ok, Sup} = emqx_ds_shared_sub_sup:start_link(), {ok, Sup}. -spec stop(term()) -> ok. stop(_State) -> + ok = emqx_ds_shared_sub_config:unload(), ok. diff --git a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_config.erl b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_config.erl new file mode 100644 index 000000000..454e2b6e8 --- /dev/null +++ b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_config.erl @@ -0,0 +1,84 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_ds_shared_sub_config). + +-behaviour(emqx_config_handler). +-behaviour(emqx_config_backup). + +-type update_request() :: emqx_config:config(). + +%% callbacks for emqx_config_handler +-export([ + pre_config_update/3, + post_config_update/5 +]). + +%% callbacks for emqx_config_backup +-export([ + import_config/1 +]). + +%% API +-export([ + load/0, + unload/0, + get/1 +]). + +%%-------------------------------------------------------------------- +%% API +%%-------------------------------------------------------------------- + +-spec load() -> ok. +load() -> + emqx_conf:add_handler([durable_queues], ?MODULE). + +-spec unload() -> ok. +unload() -> + ok = emqx_conf:remove_handler([durable_queues]). + +-spec get(atom() | [atom()]) -> term(). +get(Name) when is_atom(Name) -> + emqx_config:get([durable_queues, Name]); +get(Name) when is_list(Name) -> + emqx_config:get([durable_queues | Name]). + +%%-------------------------------------------------------------------- +%% emqx_config_handler callbacks +%%-------------------------------------------------------------------- + +-spec pre_config_update(list(atom()), update_request(), emqx_config:raw_config()) -> + {ok, emqx_config:update_request()}. +pre_config_update([durable_queues | _], NewConfig, _OldConfig) -> + {ok, NewConfig}. + +-spec post_config_update( + list(atom()), + update_request(), + emqx_config:config(), + emqx_config:config(), + emqx_config:app_envs() +) -> + ok. +post_config_update([durable_queues | _], _Req, _NewConfig, _OldConfig, _AppEnvs) -> + ok. + +%%---------------------------------------------------------------------------------------- +%% Data backup +%%---------------------------------------------------------------------------------------- + +import_config(#{<<"durable_queues">> := DQConf}) -> + OldDQConf = emqx:get_raw_config([durable_queues], #{}), + NewDQConf = maps:merge(OldDQConf, DQConf), + case emqx_conf:update([durable_queues], NewDQConf, #{override_to => cluster}) of + {ok, #{raw_config := NewRawConf}} -> + Changed = maps:get(changed, emqx_utils_maps:diff_maps(NewRawConf, DQConf)), + ChangedPaths = [[durable_queues, K] || K <- maps:keys(Changed)], + {ok, #{root_key => durable_queues, changed => ChangedPaths}}; + Error -> + {error, #{root_key => durable_queues, reason => Error}} + end; +import_config(_) -> + {ok, #{root_key => durable_queues, changed => []}}. diff --git a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_config.hrl b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_config.hrl new file mode 100644 index 000000000..592a60643 --- /dev/null +++ b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_config.hrl @@ -0,0 +1,5 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-define(dq_config(Path), emqx_ds_shared_sub_config:get(Path)). diff --git a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_group_sm.erl b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_group_sm.erl index f9a81bbb8..81bca367a 100644 --- a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_group_sm.erl +++ b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_group_sm.erl @@ -10,6 +10,7 @@ -module(emqx_ds_shared_sub_group_sm). -include_lib("emqx/include/logger.hrl"). +-include("emqx_ds_shared_sub_config.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). -export([ @@ -118,16 +119,6 @@ state_timers => #{timer_name() => timer()} }. -%%----------------------------------------------------------------------- -%% Constants -%%----------------------------------------------------------------------- - -%% TODO https://emqx.atlassian.net/browse/EMQX-12574 -%% Move to settings --define(FIND_LEADER_TIMEOUT, 1000). --define(RENEW_LEASE_TIMEOUT, 5000). --define(MIN_UPDATE_STREAM_STATE_INTERVAL, 500). - %%----------------------------------------------------------------------- %% API %%----------------------------------------------------------------------- @@ -200,7 +191,7 @@ handle_connecting(#{agent := Agent, topic_filter := ShareTopicFilter} = GSM) -> topic_filter => ShareTopicFilter }), ok = emqx_ds_shared_sub_registry:lookup_leader(Agent, agent_metadata(GSM), ShareTopicFilter), - ensure_state_timeout(GSM, find_leader_timeout, ?FIND_LEADER_TIMEOUT). + ensure_state_timeout(GSM, find_leader_timeout, ?dq_config(session_find_leader_timeout_ms)). handle_leader_lease_streams( #{state := ?connecting, topic_filter := TopicFilter} = GSM0, Leader, StreamProgresses, Version @@ -228,16 +219,20 @@ handle_find_leader_timeout(#{agent := Agent, topic_filter := TopicFilter} = GSM0 topic_filter => TopicFilter }), ok = emqx_ds_shared_sub_registry:lookup_leader(Agent, agent_metadata(GSM0), TopicFilter), - GSM1 = ensure_state_timeout(GSM0, find_leader_timeout, ?FIND_LEADER_TIMEOUT), + GSM1 = ensure_state_timeout( + GSM0, find_leader_timeout, ?dq_config(session_find_leader_timeout_ms) + ), GSM1. %%----------------------------------------------------------------------- %% Replaying state handle_replaying(GSM0) -> - GSM1 = ensure_state_timeout(GSM0, renew_lease_timeout, ?RENEW_LEASE_TIMEOUT), + GSM1 = ensure_state_timeout( + GSM0, renew_lease_timeout, ?dq_config(session_renew_lease_timeout_ms) + ), GSM2 = ensure_state_timeout( - GSM1, update_stream_state_timeout, ?MIN_UPDATE_STREAM_STATE_INTERVAL + GSM1, update_stream_state_timeout, ?dq_config(session_min_update_stream_state_interval_ms) ), GSM2. @@ -249,9 +244,11 @@ handle_renew_lease_timeout(#{agent := Agent, topic_filter := TopicFilter} = GSM) %% Updating state handle_updating(GSM0) -> - GSM1 = ensure_state_timeout(GSM0, renew_lease_timeout, ?RENEW_LEASE_TIMEOUT), + GSM1 = ensure_state_timeout( + GSM0, renew_lease_timeout, ?dq_config(session_renew_lease_timeout_ms) + ), GSM2 = ensure_state_timeout( - GSM1, update_stream_state_timeout, ?MIN_UPDATE_STREAM_STATE_INTERVAL + GSM1, update_stream_state_timeout, ?dq_config(session_min_update_stream_state_interval_ms) ), GSM2. @@ -332,7 +329,7 @@ handle_leader_update_streams( VersionNew, _StreamProgresses ) -> - ensure_state_timeout(GSM, renew_lease_timeout, ?RENEW_LEASE_TIMEOUT); + ensure_state_timeout(GSM, renew_lease_timeout, ?dq_config(session_renew_lease_timeout_ms)); handle_leader_update_streams( #{state := ?disconnected} = GSM, _VersionOld, _VersionNew, _StreamProgresses ) -> @@ -349,7 +346,7 @@ handle_leader_update_streams(GSM, VersionOld, VersionNew, _StreamProgresses) -> handle_leader_renew_stream_lease( #{state := ?replaying, state_data := #{version := Version}} = GSM, Version ) -> - ensure_state_timeout(GSM, renew_lease_timeout, ?RENEW_LEASE_TIMEOUT); + ensure_state_timeout(GSM, renew_lease_timeout, ?dq_config(session_renew_lease_timeout_ms)); handle_leader_renew_stream_lease( #{state := ?updating, state_data := #{version := Version} = StateData} = GSM, Version ) -> @@ -364,13 +361,13 @@ handle_leader_renew_stream_lease(GSM, _Version) -> handle_leader_renew_stream_lease( #{state := ?replaying, state_data := #{version := Version}} = GSM, VersionOld, VersionNew ) when VersionOld =:= Version orelse VersionNew =:= Version -> - ensure_state_timeout(GSM, renew_lease_timeout, ?RENEW_LEASE_TIMEOUT); + ensure_state_timeout(GSM, renew_lease_timeout, ?dq_config(session_renew_lease_timeout_ms)); handle_leader_renew_stream_lease( #{state := ?updating, state_data := #{version := VersionNew, prev_version := VersionOld}} = GSM, VersionOld, VersionNew ) -> - ensure_state_timeout(GSM, renew_lease_timeout, ?RENEW_LEASE_TIMEOUT); + ensure_state_timeout(GSM, renew_lease_timeout, ?dq_config(session_renew_lease_timeout_ms)); handle_leader_renew_stream_lease( #{state := ?disconnected} = GSM, _VersionOld, _VersionNew ) -> @@ -402,7 +399,9 @@ handle_stream_progress( ok = emqx_ds_shared_sub_proto:agent_update_stream_states( Leader, Agent, StreamProgresses, Version ), - ensure_state_timeout(GSM, update_stream_state_timeout, ?MIN_UPDATE_STREAM_STATE_INTERVAL); + ensure_state_timeout( + GSM, update_stream_state_timeout, ?dq_config(session_min_update_stream_state_interval_ms) + ); handle_stream_progress( #{ state := ?updating, @@ -418,7 +417,9 @@ handle_stream_progress( ok = emqx_ds_shared_sub_proto:agent_update_stream_states( Leader, Agent, StreamProgresses, PrevVersion, Version ), - ensure_state_timeout(GSM, update_stream_state_timeout, ?MIN_UPDATE_STREAM_STATE_INTERVAL); + ensure_state_timeout( + GSM, update_stream_state_timeout, ?dq_config(session_min_update_stream_state_interval_ms) + ); handle_stream_progress(#{state := ?disconnected} = GSM, _StreamProgresses) -> GSM. diff --git a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl index de277ece8..143eed1fe 100644 --- a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl +++ b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl @@ -7,6 +7,7 @@ -behaviour(gen_statem). -include("emqx_ds_shared_sub_proto.hrl"). +-include("emqx_ds_shared_sub_config.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("emqx/include/logger.hrl"). @@ -102,15 +103,6 @@ %% Constants -%% TODO https://emqx.atlassian.net/browse/EMQX-12574 -%% Move to settings --define(RENEW_LEASE_INTERVAL, 1000). --define(RENEW_STREAMS_INTERVAL, 1000). --define(DROP_TIMEOUT_INTERVAL, 1000). - --define(AGENT_TIMEOUT, 5000). --define(MAX_NOT_REPLAYING, 5000). - -define(START_TIME_THRESHOLD, 5000). %%-------------------------------------------------------------------- @@ -176,8 +168,8 @@ handle_event(enter, _OldState, ?leader_active, #{topic := Topic, router_id := Ro ok = emqx_persistent_session_ds_router:do_add_route(Topic, RouterId), {keep_state_and_data, [ {{timeout, #renew_streams{}}, 0, #renew_streams{}}, - {{timeout, #renew_leases{}}, ?RENEW_LEASE_INTERVAL, #renew_leases{}}, - {{timeout, #drop_timeout{}}, ?DROP_TIMEOUT_INTERVAL, #drop_timeout{}} + {{timeout, #renew_leases{}}, ?dq_config(leader_renew_lease_interval_ms), #renew_leases{}}, + {{timeout, #drop_timeout{}}, ?dq_config(leader_drop_timeout_interval_ms), #drop_timeout{}} ]}; %%-------------------------------------------------------------------- %% timers @@ -185,17 +177,24 @@ handle_event(enter, _OldState, ?leader_active, #{topic := Topic, router_id := Ro handle_event({timeout, #renew_streams{}}, #renew_streams{}, ?leader_active, Data0) -> % ?tp(warning, shared_sub_leader_timeout, #{timeout => renew_streams}), Data1 = renew_streams(Data0), - {keep_state, Data1, {{timeout, #renew_streams{}}, ?RENEW_STREAMS_INTERVAL, #renew_streams{}}}; + {keep_state, Data1, + { + {timeout, #renew_streams{}}, + ?dq_config(leader_renew_streams_interval_ms), + #renew_streams{} + }}; %% renew_leases timer handle_event({timeout, #renew_leases{}}, #renew_leases{}, ?leader_active, Data0) -> % ?tp(warning, shared_sub_leader_timeout, #{timeout => renew_leases}), Data1 = renew_leases(Data0), - {keep_state, Data1, {{timeout, #renew_leases{}}, ?RENEW_LEASE_INTERVAL, #renew_leases{}}}; + {keep_state, Data1, + {{timeout, #renew_leases{}}, ?dq_config(leader_renew_lease_interval_ms), #renew_leases{}}}; %% drop_timeout timer handle_event({timeout, #drop_timeout{}}, #drop_timeout{}, ?leader_active, Data0) -> % ?tp(warning, shared_sub_leader_timeout, #{timeout => drop_timeout}), Data1 = drop_timeout_agents(Data0), - {keep_state, Data1, {{timeout, #drop_timeout{}}, ?DROP_TIMEOUT_INTERVAL, #drop_timeout{}}}; + {keep_state, Data1, + {{timeout, #drop_timeout{}}, ?dq_config(leader_drop_timeout_interval_ms), #drop_timeout{}}}; %%-------------------------------------------------------------------- %% agent events handle_event( @@ -860,7 +859,7 @@ agent_transition_to_initial_waiting_replaying( prev_version => undefined, streams => InitialStreams, revoked_streams => [], - update_deadline => now_ms_monotonic() + ?AGENT_TIMEOUT + update_deadline => now_ms_monotonic() + ?dq_config(leader_session_update_timeout_ms) }, renew_no_replaying_deadline(AgentState). @@ -900,13 +899,15 @@ now_ms_monotonic() -> renew_no_replaying_deadline(#{not_replaying_deadline := undefined} = AgentState) -> AgentState#{ - not_replaying_deadline => now_ms_monotonic() + ?MAX_NOT_REPLAYING + not_replaying_deadline => now_ms_monotonic() + + ?dq_config(leader_session_not_replaying_timeout_ms) }; renew_no_replaying_deadline(#{not_replaying_deadline := _Deadline} = AgentState) -> AgentState; renew_no_replaying_deadline(#{} = AgentState) -> AgentState#{ - not_replaying_deadline => now_ms_monotonic() + ?MAX_NOT_REPLAYING + not_replaying_deadline => now_ms_monotonic() + + ?dq_config(leader_session_not_replaying_timeout_ms) }. unassigned_streams(#{stream_states := StreamStates, stream_owners := StreamOwners}) -> @@ -991,7 +992,7 @@ set_agent_state(#{agents := Agents} = Data, Agent, AgentState) -> update_agent_timeout(AgentState) -> AgentState#{ - update_deadline => now_ms_monotonic() + ?AGENT_TIMEOUT + update_deadline => now_ms_monotonic() + ?dq_config(leader_session_update_timeout_ms) }. get_agent_state(#{agents := Agents} = _Data, Agent) -> diff --git a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_schema.erl b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_schema.erl new file mode 100644 index 000000000..198554d8a --- /dev/null +++ b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_schema.erl @@ -0,0 +1,57 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_ds_shared_sub_schema). + +-include_lib("hocon/include/hoconsc.hrl"). + +-export([ + namespace/0, + roots/0, + fields/1, + desc/1 +]). + +namespace() -> emqx_shared_subs. + +roots() -> + [ + durable_queues + ]. + +fields(durable_queues) -> + [ + {enable, + ?HOCON( + boolean(), + #{ + required => false, + default => true, + desc => ?DESC(durable_queues) + } + )}, + duration(session_find_leader_timeout_ms, 1000), + duration(session_renew_lease_timeout_ms, 5000), + duration(session_min_update_stream_state_interval_ms, 500), + + duration(leader_renew_lease_interval_ms, 1000), + duration(leader_renew_streams_interval_ms, 1000), + duration(leader_drop_timeout_interval_ms, 1000), + duration(leader_session_update_timeout_ms, 5000), + duration(leader_session_not_replaying_timeout_ms, 5000) + ]. + +duration(MsFieldName, Default) -> + {MsFieldName, + ?HOCON( + emqx_schema:timeout_duration_ms(), + #{ + required => false, + default => Default, + desc => ?DESC(MsFieldName), + importance => ?IMPORTANCE_HIDDEN + } + )}. + +desc(durable_queues) -> "Settings for durable queues". diff --git a/apps/emqx_ds_shared_sub/test/emqx_ds_shared_sub_config_SUITE.erl b/apps/emqx_ds_shared_sub/test/emqx_ds_shared_sub_config_SUITE.erl new file mode 100644 index 000000000..a3d58ebf9 --- /dev/null +++ b/apps/emqx_ds_shared_sub/test/emqx_ds_shared_sub_config_SUITE.erl @@ -0,0 +1,62 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_ds_shared_sub_config_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +-include_lib("emqx/include/asserts.hrl"). + +all() -> + emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + Apps = emqx_cth_suite:start( + [ + emqx_conf, + {emqx, #{ + config => #{ + <<"durable_sessions">> => #{ + <<"enable">> => true, + <<"renew_streams_interval">> => "100ms" + }, + <<"durable_storage">> => #{ + <<"messages">> => #{ + <<"backend">> => <<"builtin_raft">> + } + } + } + }}, + {emqx_ds_shared_sub, #{ + config => #{ + <<"durable_queues">> => #{ + <<"enable">> => true, + <<"session_find_leader_timeout_ms">> => "1200ms" + } + } + }} + ], + #{work_dir => ?config(priv_dir, Config)} + ), + [{apps, Apps} | Config]. + +end_per_suite(Config) -> + ok = emqx_cth_suite:stop(?config(apps, Config)), + ok. + +t_update_config(_Config) -> + ?assertEqual( + 1200, + emqx_ds_shared_sub_config:get(session_find_leader_timeout_ms) + ), + + {ok, _} = emqx_conf:update([durable_queues], #{session_find_leader_timeout_ms => 2000}, #{}), + ?assertEqual( + 2000, + emqx_ds_shared_sub_config:get(session_find_leader_timeout_ms) + ). diff --git a/apps/emqx_enterprise/src/emqx_enterprise_schema.erl b/apps/emqx_enterprise/src/emqx_enterprise_schema.erl index eeafe340a..eb43f67af 100644 --- a/apps/emqx_enterprise/src/emqx_enterprise_schema.erl +++ b/apps/emqx_enterprise/src/emqx_enterprise_schema.erl @@ -18,7 +18,8 @@ emqx_schema_registry_schema, emqx_schema_validation_schema, emqx_message_transformation_schema, - emqx_ft_schema + emqx_ft_schema, + emqx_ds_shared_sub_schema ]). %% Callback to upgrade config after loaded from config file but before validation. From 7daab1ab2325f394b20e8fcdc50539ab6dcb395e Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Mon, 8 Jul 2024 19:57:36 +0300 Subject: [PATCH 103/131] feat(queue): move replay progress to a separate data structure --- apps/emqx/src/emqx_persistent_session_ds.erl | 7 +- ...emqx_persistent_session_ds_shared_subs.erl | 184 ++++++++++++++---- .../src/emqx_ds_shared_sub_group_sm.erl | 30 +-- .../src/emqx_ds_shared_sub_leader.erl | 26 ++- .../src/emqx_ds_shared_sub_proto.erl | 56 ++++-- .../src/proto/emqx_ds_shared_sub_proto_v1.erl | 4 +- .../test/emqx_ds_shared_sub_SUITE.erl | 45 +++++ 7 files changed, 267 insertions(+), 85 deletions(-) diff --git a/apps/emqx/src/emqx_persistent_session_ds.erl b/apps/emqx/src/emqx_persistent_session_ds.erl index 517681f9a..124b1919a 100644 --- a/apps/emqx/src/emqx_persistent_session_ds.erl +++ b/apps/emqx/src/emqx_persistent_session_ds.erl @@ -993,11 +993,12 @@ do_ensure_all_iterators_closed(_DSSessionID) -> fetch_new_messages(Session0 = #{s := S0, shared_sub_s := SharedSubS0}, ClientInfo) -> {S1, SharedSubS1} = emqx_persistent_session_ds_shared_subs:on_streams_replay(S0, SharedSubS0), - LFS = maps:get(last_fetched_stream, Session0, beginning), + Session1 = Session0#{s => S1, shared_sub_s => SharedSubS1}, + LFS = maps:get(last_fetched_stream, Session1, beginning), ItStream = emqx_persistent_session_ds_stream_scheduler:iter_next_streams(LFS, S1), BatchSize = get_config(ClientInfo, [batch_size]), - Session1 = fetch_new_messages(ItStream, BatchSize, Session0, ClientInfo), - Session1#{shared_sub_s => SharedSubS1}. + Session2 = fetch_new_messages(ItStream, BatchSize, Session1, ClientInfo), + Session2#{shared_sub_s => SharedSubS1}. fetch_new_messages(ItStream0, BatchSize, Session0, ClientInfo) -> #{inflight := Inflight} = Session0, diff --git a/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl index 0bdbff30a..bb4c62726 100644 --- a/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl +++ b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl @@ -48,6 +48,8 @@ to_map/2 ]). +-define(EPOCH_BITS, 15). + -define(schedule_subscribe, schedule_subscribe). -define(schedule_unsubscribe, schedule_unsubscribe). @@ -58,10 +60,22 @@ -type agent_stream_progress() :: #{ stream := emqx_ds:stream(), - iterator := emqx_ds:iterator(), + progress := progress(), use_finished := boolean() }. +-type progress() :: + #{ + acked := true, + iterator := emqx_ds:iterator() + } + | #{ + acked := false, + iterator := emqx_ds:iterator(), + qos1_acked := boolean(), + qos2_acked := boolean() + }. + -type scheduled_action() :: #{ type := scheduled_action_type(), stream_keys_to_wait := [stream_key()], @@ -82,6 +96,11 @@ -define(rank_x, rank_shared). -define(rank_y, 0). +-export_type([ + progress/0, + agent_stream_progress/0 +]). + %%-------------------------------------------------------------------- %% API %%-------------------------------------------------------------------- @@ -290,7 +309,9 @@ renew_streams(S0, #{agent := Agent0, scheduled_actions := ScheduledActions} = Sh {StreamLeaseEvents, Agent1} = emqx_persistent_session_ds_shared_subs_agent:renew_streams( Agent0 ), - ?tp(info, shared_subs_new_stream_lease_events, #{stream_lease_events => StreamLeaseEvents}), + ?tp(warning, shared_subs_new_stream_lease_events, #{ + stream_lease_events => format_lease_events(StreamLeaseEvents) + }), S1 = lists:foldl( fun (#{type := lease} = Event, S) -> accept_stream(Event, S, ScheduledActions); @@ -317,8 +338,11 @@ accept_stream(#{topic_filter := TopicFilter} = Event, S, ScheduledActions) -> accept_stream(Event, S) end. +%% TODO: +%% handle unacked iterator accept_stream( - #{topic_filter := TopicFilter, stream := Stream, iterator := Iterator}, S0 + #{topic_filter := TopicFilter, stream := Stream, progress := #{iterator := Iterator}} = _Event, + S0 ) -> case emqx_persistent_session_ds_state:get_subscription(TopicFilter, S0) of undefined -> @@ -326,8 +350,17 @@ accept_stream( S0; #{id := SubId, current_state := SStateId} -> Key = {SubId, Stream}, - case emqx_persistent_session_ds_state:get_stream(Key, S0) of - undefined -> + NeedCreateStream = + case emqx_persistent_session_ds_state:get_stream(Key, S0) of + undefined -> + true; + #srs{unsubscribed = true} -> + true; + _SRS -> + false + end, + case NeedCreateStream of + true -> NewSRS = #srs{ rank_x = ?rank_x, @@ -338,7 +371,7 @@ accept_stream( }, S1 = emqx_persistent_session_ds_state:put_stream(Key, NewSRS, S0), S1; - _SRS -> + false -> S0 end end. @@ -371,22 +404,30 @@ revoke_stream( emqx_persistent_session_ds_state:t(), t() ) -> {emqx_persistent_session_ds_state:t(), t()}. -on_streams_replay(S, #{agent := Agent0, scheduled_actions := ScheduledActions0} = SharedSubS0) -> - Progresses = stream_progresses(S), +on_streams_replay(S0, SharedSubS0) -> + {S1, #{agent := Agent0, scheduled_actions := ScheduledActions0} = SharedSubS1} = + renew_streams(S0, SharedSubS0), + + Progresses = all_stream_progresses(S1, Agent0), Agent1 = emqx_persistent_session_ds_shared_subs_agent:on_stream_progress( Agent0, Progresses ), - {Agent2, ScheduledActions1} = run_scheduled_actions(S, Agent1, ScheduledActions0), - SharedSubS1 = SharedSubS0#{ + {Agent2, ScheduledActions1} = run_scheduled_actions(S1, Agent1, ScheduledActions0), + SharedSubS2 = SharedSubS1#{ agent => Agent2, scheduled_actions => ScheduledActions1 }, - {S, SharedSubS1}. + {S1, SharedSubS2}. %%-------------------------------------------------------------------- %% on_streams_replay internal functions -stream_progresses(S) -> +all_stream_progresses(S, Agent) -> + all_stream_progresses(S, Agent, _NeedUnacked = false). + +all_stream_progresses(S, _Agent, NeedUnacked) -> + CommQos1 = emqx_persistent_session_ds_state:get_seqno(?committed(?QOS_1), S), + CommQos2 = emqx_persistent_session_ds_state:get_seqno(?committed(?QOS_2), S), fold_shared_stream_states( fun( #share{group = Group}, @@ -394,9 +435,12 @@ stream_progresses(S) -> SRS, ProgressesAcc0 ) -> - case is_stream_fully_acked(S, SRS) of + case + is_stream_started(CommQos1, CommQos2, SRS) and + (NeedUnacked or is_stream_fully_acked(CommQos1, CommQos2, SRS)) + of true -> - StreamProgress = stream_progress(S, Stream, SRS), + StreamProgress = stream_progress(CommQos1, CommQos2, Stream, SRS), maps:update_with( Group, fun(Progresses) -> [StreamProgress | Progresses] end, @@ -437,7 +481,7 @@ run_scheduled_action( [] -> ?tp(warning, shared_subs_schedule_action_complete, #{ topic_filter => TopicFilter, - progresses => format_streams(Progresses1), + progresses => format_stream_progresses(Progresses1), type => Type }), %% Regular progress won't se unsubscribed streams, so we need to @@ -467,6 +511,8 @@ run_scheduled_action( end. filter_unfinished_streams(S, StreamKeysToWait) -> + CommQos1 = emqx_persistent_session_ds_state:get_seqno(?committed(?QOS_1), S), + CommQos2 = emqx_persistent_session_ds_state:get_seqno(?committed(?QOS_2), S), lists:filter( fun(Key) -> case emqx_persistent_session_ds_state:get_stream(Key, S) of @@ -475,21 +521,19 @@ filter_unfinished_streams(S, StreamKeysToWait) -> %% in completed state before deletion true; SRS -> - not is_stream_fully_acked(S, SRS) + not is_stream_fully_acked(CommQos1, CommQos2, SRS) end end, StreamKeysToWait ). stream_progresses(S, StreamKeys) -> + CommQos1 = emqx_persistent_session_ds_state:get_seqno(?committed(?QOS_1), S), + CommQos2 = emqx_persistent_session_ds_state:get_seqno(?committed(?QOS_2), S), lists:map( fun({_SubId, Stream} = Key) -> - #srs{it_end = ItEnd} = SRS = emqx_persistent_session_ds_state:get_stream(Key, S), - #{ - stream => Stream, - iterator => ItEnd, - use_finished => is_use_finished(S, SRS) - } + SRS = emqx_persistent_session_ds_state:get_stream(Key, S), + stream_progress(CommQos1, CommQos2, Stream, SRS) end, StreamKeys ). @@ -499,7 +543,7 @@ stream_progresses(S, StreamKeys) -> on_disconnect(S0, #{agent := Agent0} = SharedSubS0) -> S1 = revoke_all_streams(S0), - Progresses = stream_progresses(S1), + Progresses = all_stream_progresses(S1, Agent0), Agent1 = emqx_persistent_session_ds_shared_subs_agent:on_disconnect(Agent0, Progresses), SharedSubS1 = SharedSubS0#{agent => Agent1, scheduled_actions => #{}}, {S1, SharedSubS1}. @@ -565,12 +609,41 @@ stream_keys_by_sub_id(S, MatchSubId) -> S ). -stream_progress(S, Stream, #srs{it_end = EndIt} = SRS) -> - #{ - stream => Stream, - iterator => EndIt, - use_finished => is_use_finished(S, SRS) - }. +stream_progress( + CommQos1, + CommQos2, + Stream, + #srs{ + it_end = EndIt, + it_begin = BeginIt, + first_seqno_qos1 = StartQos1, + first_seqno_qos2 = StartQos2 + } = SRS +) -> + Qos1Acked = seqno_diff(?QOS_1, CommQos1, StartQos1), + Qos2Acked = seqno_diff(?QOS_2, CommQos2, StartQos2), + case is_stream_fully_acked(CommQos1, CommQos2, SRS) of + true -> + #{ + stream => Stream, + progress => #{ + acked => true, + iterator => EndIt + }, + use_finished => is_use_finished(SRS) + }; + false -> + #{ + stream => Stream, + progress => #{ + acked => false, + iterator => BeginIt, + qos1_acked => Qos1Acked, + qos2_acked => Qos2Acked + }, + use_finished => is_use_finished(SRS) + } + end. fold_shared_subs(Fun, Acc, S) -> emqx_persistent_session_ds_state:fold_subscriptions( @@ -618,11 +691,30 @@ agent_opts(#{session_id := SessionId}) -> now_ms() -> erlang:system_time(millisecond). -is_use_finished(_S, #srs{unsubscribed = Unsubscribed}) -> +is_use_finished(#srs{unsubscribed = Unsubscribed}) -> Unsubscribed. -is_stream_fully_acked(S, SRS) -> - emqx_persistent_session_ds_stream_scheduler:is_fully_acked(SRS, S). +is_stream_started(CommQos1, CommQos2, #srs{first_seqno_qos1 = Q1, last_seqno_qos1 = Q2}) -> + (CommQos1 >= Q1) or (CommQos2 >= Q2). + +is_stream_fully_acked(_, _, #srs{ + first_seqno_qos1 = Q1, last_seqno_qos1 = Q1, first_seqno_qos2 = Q2, last_seqno_qos2 = Q2 +}) -> + %% Streams where the last chunk doesn't contain any QoS1 and 2 + %% messages are considered fully acked: + true; +is_stream_fully_acked(Comm1, Comm2, #srs{last_seqno_qos1 = S1, last_seqno_qos2 = S2}) -> + (Comm1 >= S1) andalso (Comm2 >= S2). + +-dialyzer({nowarn_function, seqno_diff/3}). +seqno_diff(?QOS_1, A, B) -> + %% For QoS1 messages we skip a seqno every time the epoch changes, + %% we need to substract that from the diff: + EpochA = A bsr ?EPOCH_BITS, + EpochB = B bsr ?EPOCH_BITS, + A - B - (EpochA - EpochB); +seqno_diff(?QOS_2, A, B) -> + A - B. %%-------------------------------------------------------------------- %% Formatters @@ -633,21 +725,24 @@ format_schedule_action(#{ }) -> #{ type => Type, - progresses => format_streams(Progresses), + progresses => format_stream_progresses(Progresses), stream_keys_to_wait => format_stream_keys(StreamKeysToWait) }. -format_streams(Streams) -> +format_stream_progresses(Streams) -> lists:map( - fun format_stream/1, + fun format_stream_progress/1, Streams ). -format_stream(#{stream := Stream, iterator := Iterator} = Value) -> - Value#{stream => format_opaque(Stream), iterator => format_opaque(Iterator)}. +format_stream_progress(#{stream := Stream, progress := Progress} = Value) -> + Value#{stream => format_opaque(Stream), progress => format_progress(Progress)}. -format_stream_key({SubId, Stream}) -> - {SubId, format_opaque(Stream)}. +format_progress(#{iterator := Iterator} = Progress) -> + Progress#{iterator => format_opaque(Iterator)}. + +format_stream_key(beginning) -> beginning; +format_stream_key({SubId, Stream}) -> {SubId, format_opaque(Stream)}. format_stream_keys(StreamKeys) -> lists:map( @@ -655,5 +750,16 @@ format_stream_keys(StreamKeys) -> StreamKeys ). +format_lease_events(Events) -> + lists:map( + fun format_lease_event/1, + Events + ). + +format_lease_event(#{stream := Stream, progress := Progress} = Event) -> + Event#{stream => format_opaque(Stream), progress => format_progress(Progress)}; +format_lease_event(#{stream := Stream} = Event) -> + Event#{stream => format_opaque(Stream)}. + format_opaque(Opaque) -> erlang:phash2(Opaque). diff --git a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_group_sm.erl b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_group_sm.erl index 81bca367a..7d260dc0b 100644 --- a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_group_sm.erl +++ b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_group_sm.erl @@ -45,11 +45,13 @@ send_after := fun((non_neg_integer(), term()) -> reference()) }. +-type progress() :: emqx_persistent_session_ds_shared_subs:progress(). + -type stream_lease_event() :: #{ type => lease, stream => emqx_ds:stream(), - iterator => emqx_ds:iterator() + progress => progress() } | #{ type => revoke, @@ -60,7 +62,7 @@ #{ type => lease, stream => emqx_ds:stream(), - iterator => emqx_ds:iterator(), + progress => progress(), topic_filter => emqx_persistent_session_ds:share_topic_filter() } | #{ @@ -81,13 +83,13 @@ -type connecting_data() :: #{}. -type replaying_data() :: #{ leader => emqx_ds_shared_sub_proto:leader(), - streams => #{emqx_ds:stream() => emqx_ds:iterator()}, + streams => #{emqx_ds:stream() => progress()}, version => emqx_ds_shared_sub_proto:version(), prev_version => undefined }. -type updating_data() :: #{ leader => emqx_ds_shared_sub_proto:leader(), - streams => #{emqx_ds:stream() => emqx_ds:iterator()}, + streams => #{emqx_ds:stream() => progress()}, version => emqx_ds_shared_sub_proto:version(), prev_version => emqx_ds_shared_sub_proto:version() }. @@ -275,18 +277,18 @@ handle_leader_update_streams( id => Id, version_old => VersionOld, version_new => VersionNew, - stream_progresses => emqx_ds_shared_sub_proto:format_streams(StreamProgresses) + stream_progresses => emqx_ds_shared_sub_proto:format_stream_progresses(StreamProgresses) }), {AddEvents, Streams1} = lists:foldl( - fun(#{stream := Stream, iterator := It}, {AddEventAcc, StreamsAcc}) -> + fun(#{stream := Stream, progress := Progress}, {AddEventAcc, StreamsAcc}) -> case maps:is_key(Stream, StreamsAcc) of true -> %% We prefer our own progress {AddEventAcc, StreamsAcc}; false -> { - [#{type => lease, stream => Stream, iterator => It} | AddEventAcc], - StreamsAcc#{Stream => It} + [#{type => lease, stream => Stream, progress => Progress} | AddEventAcc], + StreamsAcc#{Stream => Progress} } end end, @@ -310,6 +312,10 @@ handle_leader_update_streams( maps:keys(Streams1) ), StreamLeaseEvents = AddEvents ++ RevokeEvents, + ?tp(warning, shared_sub_group_sm_leader_update_streams, #{ + id => Id, + stream_lease_events => emqx_ds_shared_sub_proto:format_lease_events(StreamLeaseEvents) + }), transition( GSM, ?updating, @@ -540,11 +546,11 @@ run_enter_callback(#{state := ?disconnected} = GSM) -> progresses_to_lease_events(StreamProgresses) -> lists:map( - fun(#{stream := Stream, iterator := It}) -> + fun(#{stream := Stream, progress := Progress}) -> #{ type => lease, stream => Stream, - iterator => It + progress => Progress } end, StreamProgresses @@ -552,8 +558,8 @@ progresses_to_lease_events(StreamProgresses) -> progresses_to_map(StreamProgresses) -> lists:foldl( - fun(#{stream := Stream, iterator := It}, Acc) -> - Acc#{Stream => It} + fun(#{stream := Stream, progress := Progress}, Acc) -> + Acc#{Stream => Progress} end, #{}, StreamProgresses diff --git a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl index 143eed1fe..976ce2437 100644 --- a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl +++ b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl @@ -49,8 +49,10 @@ revoked_streams := list(emqx_ds:stream()) }. +-type progress() :: emqx_persistent_session_ds_shared_subs:progress(). + -type stream_state() :: #{ - iterator => emqx_ds:iterator(), + progress => progress(), rank => emqx_ds:stream_rank() }. @@ -84,7 +86,8 @@ -export_type([ options/0, - data/0 + data/0, + progress/0 ]). %% States @@ -310,8 +313,12 @@ update_progresses(StreamStates, NewStreamsWRanks, TopicFilter, StartTime) -> {ok, It} = emqx_ds:make_iterator( ?PERSISTENT_MESSAGE_DB, Stream, TopicFilter, StartTime ), + Progress = #{ + iterator => It, + acked => true + }, { - NewStreamStatesAcc#{Stream => #{iterator => It, rank => Rank}}, + NewStreamStatesAcc#{Stream => #{progress => Progress, rank => Rank}}, OldStreamStatesAcc } end @@ -637,18 +644,18 @@ update_stream_progresses( ReceivedStreamProgresses ) -> {StreamStates1, ReplayedStreams} = lists:foldl( - fun(#{stream := Stream, iterator := It}, {StreamStatesAcc, ReplayedStreamsAcc}) -> + fun(#{stream := Stream, progress := Progress}, {StreamStatesAcc, ReplayedStreamsAcc}) -> case StreamOwners of #{Stream := Agent} -> StreamData0 = maps:get(Stream, StreamStatesAcc), - case It of - end_of_stream -> + case Progress of + #{iterator := end_of_stream} -> Rank = maps:get(rank, StreamData0), {maps:remove(Stream, StreamStatesAcc), ReplayedStreamsAcc#{ Stream => Rank }}; _ -> - StreamData1 = StreamData0#{iterator => It}, + StreamData1 = StreamData0#{progress => Progress}, {StreamStatesAcc#{Stream => StreamData1}, ReplayedStreamsAcc} end; _ -> @@ -701,6 +708,9 @@ clean_revoked_streams( ( #{ stream := Stream, + progress := #{ + acked := true + }, use_finished := true } ) -> @@ -953,7 +963,7 @@ stream_progresses(#{stream_states := StreamStates} = _Data, Streams) -> StreamData = maps:get(Stream, StreamStates), #{ stream => Stream, - iterator => maps:get(iterator, StreamData) + progress => maps:get(progress, StreamData) } end, Streams diff --git a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_proto.erl b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_proto.erl index 184e8d147..e74fae19c 100644 --- a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_proto.erl +++ b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_proto.erl @@ -22,10 +22,12 @@ ]). -export([ - format_streams/1, - format_stream/1, + format_stream_progresses/1, + format_stream_progress/1, format_stream_key/1, format_stream_keys/1, + format_lease_event/1, + format_lease_events/1, agent/2 ]). @@ -38,23 +40,19 @@ id := emqx_persistent_session_ds:id() }. --type stream_progress() :: #{ +-type leader_stream_progress() :: #{ stream := emqx_ds:stream(), - iterator := emqx_ds:iterator() + progress := emqx_persistent_session_ds_shared_subs:progress() }. --type agent_stream_progress() :: #{ - stream := emqx_ds:stream(), - iterator := emqx_ds:iterator(), - use_finished := boolean() -}. +-type agent_stream_progress() :: emqx_persistent_session_ds_shared_subs:agent_stream_progress(). -export_type([ agent/0, leader/0, group/0, version/0, - stream_progress/0, + leader_stream_progress/0, agent_stream_progress/0, agent_metadata/0 ]). @@ -91,7 +89,7 @@ agent_update_stream_states(ToLeader, FromAgent, StreamProgresses, Version) when type => agent_update_stream_states, to_leader => ToLeader, from_agent => FromAgent, - stream_progresses => format_streams(StreamProgresses), + stream_progresses => format_stream_progresses(StreamProgresses), version => Version }), _ = erlang:send(ToLeader, ?agent_update_stream_states(FromAgent, StreamProgresses, Version)), @@ -111,7 +109,7 @@ agent_update_stream_states(ToLeader, FromAgent, StreamProgresses, VersionOld, Ve type => agent_update_stream_states, to_leader => ToLeader, from_agent => FromAgent, - stream_progresses => format_streams(StreamProgresses), + stream_progresses => format_stream_progresses(StreamProgresses), version_old => VersionOld, version_new => VersionNew }), @@ -131,7 +129,7 @@ agent_disconnect(ToLeader, FromAgent, StreamProgresses, Version) when type => agent_disconnect, to_leader => ToLeader, from_agent => FromAgent, - stream_progresses => format_streams(StreamProgresses), + stream_progresses => format_stream_progresses(StreamProgresses), version => Version }), _ = erlang:send(ToLeader, ?agent_disconnect(FromAgent, StreamProgresses, Version)), @@ -143,14 +141,15 @@ agent_disconnect(ToLeader, FromAgent, StreamProgresses, Version) -> %% leader -> agent messages --spec leader_lease_streams(agent(), group(), leader(), list(stream_progress()), version()) -> ok. +-spec leader_lease_streams(agent(), group(), leader(), list(leader_stream_progress()), version()) -> + ok. leader_lease_streams(ToAgent, OfGroup, Leader, Streams, Version) when ?is_local_agent(ToAgent) -> ?tp(warning, shared_sub_proto_msg, #{ type => leader_lease_streams, to_agent => ToAgent, of_group => OfGroup, leader => Leader, - streams => format_streams(Streams), + streams => format_stream_progresses(Streams), version => Version }), _ = emqx_persistent_session_ds_shared_subs_agent:send( @@ -200,7 +199,8 @@ leader_renew_stream_lease(ToAgent, OfGroup, VersionOld, VersionNew) -> ?agent_node(ToAgent), ToAgent, OfGroup, VersionOld, VersionNew ). --spec leader_update_streams(agent(), group(), version(), version(), list(stream_progress())) -> ok. +-spec leader_update_streams(agent(), group(), version(), version(), list(leader_stream_progress())) -> + ok. leader_update_streams(ToAgent, OfGroup, VersionOld, VersionNew, StreamsNew) when ?is_local_agent(ToAgent) -> @@ -210,7 +210,7 @@ leader_update_streams(ToAgent, OfGroup, VersionOld, VersionNew, StreamsNew) when of_group => OfGroup, version_old => VersionOld, version_new => VersionNew, - streams_new => format_streams(StreamsNew) + streams_new => format_stream_progresses(StreamsNew) }), _ = emqx_persistent_session_ds_shared_subs_agent:send( ?agent_pid(ToAgent), @@ -247,14 +247,17 @@ agent(Id, Pid) -> _ = Id, ?agent(Id, Pid). -format_streams(Streams) -> +format_stream_progresses(Streams) -> lists:map( - fun format_stream/1, + fun format_stream_progress/1, Streams ). -format_stream(#{stream := Stream, iterator := Iterator} = Value) -> - Value#{stream => format_opaque(Stream), iterator => format_opaque(Iterator)}. +format_stream_progress(#{stream := Stream, progress := Progress} = Value) -> + Value#{stream => format_opaque(Stream), progress => format_progress(Progress)}. + +format_progress(#{iterator := Iterator} = Progress) -> + Progress#{iterator => format_opaque(Iterator)}. format_stream_key({SubId, Stream}) -> {SubId, format_opaque(Stream)}. @@ -265,6 +268,17 @@ format_stream_keys(StreamKeys) -> StreamKeys ). +format_lease_events(Events) -> + lists:map( + fun format_lease_event/1, + Events + ). + +format_lease_event(#{stream := Stream, progress := Progress} = Event) -> + Event#{stream => format_opaque(Stream), progress => format_progress(Progress)}; +format_lease_event(#{stream := Stream} = Event) -> + Event#{stream => format_opaque(Stream)}. + %%-------------------------------------------------------------------- %% Helpers %%-------------------------------------------------------------------- diff --git a/apps/emqx_ds_shared_sub/src/proto/emqx_ds_shared_sub_proto_v1.erl b/apps/emqx_ds_shared_sub/src/proto/emqx_ds_shared_sub_proto_v1.erl index 2dfc8be65..52f64937d 100644 --- a/apps/emqx_ds_shared_sub/src/proto/emqx_ds_shared_sub_proto_v1.erl +++ b/apps/emqx_ds_shared_sub/src/proto/emqx_ds_shared_sub_proto_v1.erl @@ -82,7 +82,7 @@ agent_disconnect(Node, ToLeader, FromAgent, StreamProgresses, Version) -> emqx_ds_shared_sub_proto:agent(), emqx_ds_shared_sub_proto:group(), emqx_ds_shared_sub_proto:leader(), - list(emqx_ds_shared_sub_proto:stream_progress()), + list(emqx_ds_shared_sub_proto:leader_stream_progress()), emqx_ds_shared_sub_proto:version() ) -> ok. leader_lease_streams(Node, ToAgent, OfGroup, Leader, Streams, Version) -> @@ -117,7 +117,7 @@ leader_renew_stream_lease(Node, ToAgent, OfGroup, VersionOld, VersionNew) -> emqx_ds_shared_sub_proto:group(), emqx_ds_shared_sub_proto:version(), emqx_ds_shared_sub_proto:version(), - list(emqx_ds_shared_sub_proto:stream_progress()) + list(emqx_ds_shared_sub_proto:leader_stream_progress()) ) -> ok. leader_update_streams(Node, ToAgent, OfGroup, VersionOld, VersionNew, StreamsNew) -> erpc:cast(Node, emqx_ds_shared_sub_proto, leader_update_streams, [ diff --git a/apps/emqx_ds_shared_sub/test/emqx_ds_shared_sub_SUITE.erl b/apps/emqx_ds_shared_sub/test/emqx_ds_shared_sub_SUITE.erl index 4733dc650..0f665b5a3 100644 --- a/apps/emqx_ds_shared_sub/test/emqx_ds_shared_sub_SUITE.erl +++ b/apps/emqx_ds_shared_sub/test/emqx_ds_shared_sub_SUITE.erl @@ -183,6 +183,51 @@ t_graceful_disconnect(_Config) -> ok = emqtt:disconnect(ConnShared2), ok = emqtt:disconnect(ConnPub). +t_disconnect_no_double_replay(_Config) -> + ConnPub = emqtt_connect_pub(<<"client_pub">>), + + ConnShared1 = emqtt_connect_sub(<<"client_shared1">>), + {ok, _, _} = emqtt:subscribe(ConnShared1, <<"$share/gr9/topic9/#">>, 1), + + ConnShared2 = emqtt_connect_sub(<<"client_shared2">>), + {ok, _, _} = emqtt:subscribe(ConnShared2, <<"$share/gr9/topic9/#">>, 1), + + ct:sleep(1000), + + NPubs = 10_000, + + Topics = [<<"topic9/1">>, <<"topic9/2">>, <<"topic9/3">>], + ok = publish_n(ConnPub, Topics, 1, NPubs), + + Self = self(), + _ = spawn_link(fun() -> + ok = publish_n(ConnPub, Topics, NPubs + 1, 2 * NPubs), + Self ! publish_done + end), + + ok = emqtt:disconnect(ConnShared2), + + receive + publish_done -> ok + end, + + Pubs = drain_publishes(), + + ClientByBid = fun(Pid) -> + case Pid of + ConnShared1 -> <<"client_shared1">>; + ConnShared2 -> <<"client_shared2">> + end + end, + + {Missing, Duplicate} = verify_received_pubs(Pubs, 2 * NPubs, ClientByBid), + + ?assertEqual([], Missing), + ?assertEqual([], Duplicate), + + ok = emqtt:disconnect(ConnShared1), + ok = emqtt:disconnect(ConnPub). + t_intensive_reassign(_Config) -> ConnPub = emqtt_connect_pub(<<"client_pub">>), From c569625dd17ba6a61a5530a0aed9f1699d0e0b29 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Mon, 8 Jul 2024 21:55:25 +0300 Subject: [PATCH 104/131] feat(queue): handle partially unacked ranges --- ...emqx_persistent_session_ds_shared_subs.erl | 55 ++++++- .../test/emqx_ds_shared_sub_SUITE.erl | 136 +++++++++++------- 2 files changed, 136 insertions(+), 55 deletions(-) diff --git a/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl index bb4c62726..7535e1a61 100644 --- a/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl +++ b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl @@ -31,6 +31,8 @@ -include("emqx_mqtt.hrl"). -include("logger.hrl"). -include("session_internals.hrl"). + +-include_lib("emqx/include/emqx_persistent_message.hrl"). -include_lib("snabbkaffe/include/trace.hrl"). -export([ @@ -338,10 +340,8 @@ accept_stream(#{topic_filter := TopicFilter} = Event, S, ScheduledActions) -> accept_stream(Event, S) end. -%% TODO: -%% handle unacked iterator accept_stream( - #{topic_filter := TopicFilter, stream := Stream, progress := #{iterator := Iterator}} = _Event, + #{topic_filter := TopicFilter, stream := Stream, progress := Progress} = _Event, S0 ) -> case emqx_persistent_session_ds_state:get_subscription(TopicFilter, S0) of @@ -361,6 +361,7 @@ accept_stream( end, case NeedCreateStream of true -> + Iterator = rewind_iterator(Progress), NewSRS = #srs{ rank_x = ?rank_x, @@ -376,6 +377,52 @@ accept_stream( end end. +%% Skip acked messages. +%% This may be a bit inefficient, and it is unclear how to handle errors. +%% +%% A better variant would be to wrap the iterator on `emqx_ds` level in a new one, +%% that will skip acked messages internally in `emqx_ds:next` function. +%% Unluckily, emqx_ds does not have a wrapping structure around iterators of +%% the underlying levels, so we cannot wrap it without a risk of confusion. + +rewind_iterator(#{iterator := Iterator, acked := true}) -> + Iterator; +rewind_iterator(#{iterator := Iterator0, acked := false, qos1_acked := 0, qos2_acked := 0}) -> + Iterator0; +%% This should not happen, means the DS is consistent +rewind_iterator(#{iterator := Iterator0, acked := false, qos1_acked := Q1, qos2_acked := Q2}) when + Q1 < 0 orelse Q2 < 0 +-> + Iterator0; +rewind_iterator( + #{iterator := Iterator0, acked := false, qos1_acked := Q1Old, qos2_acked := Q2Old} = Progress +) -> + case emqx_ds:next(?PERSISTENT_MESSAGE_DB, Iterator0, Q1Old + Q2Old) of + {ok, Iterator1, Messages} -> + {Q1New, Q2New} = update_qos_acked(Q1Old, Q2Old, Messages), + rewind_iterator(Progress#{ + iterator => Iterator1, qos1_acked => Q1New, qos2_acked => Q2New + }); + {ok, end_of_stream} -> + end_of_stream; + {error, _, _} -> + %% What to do here? + %% In the wrapping variant we do not have this problem. + Iterator0 + end. + +update_qos_acked(Q1, Q2, []) -> + {Q1, Q2}; +update_qos_acked(Q1, Q2, [{_Key, Message} | Messages]) -> + case emqx_message:qos(Message) of + ?QOS_1 -> + update_qos_acked(Q1 - 1, Q2, Messages); + ?QOS_2 -> + update_qos_acked(Q1, Q2 - 1, Messages); + _ -> + update_qos_acked(Q1, Q2, Messages) + end. + revoke_stream( #{topic_filter := TopicFilter, stream := Stream}, S0 ) -> @@ -543,7 +590,7 @@ stream_progresses(S, StreamKeys) -> on_disconnect(S0, #{agent := Agent0} = SharedSubS0) -> S1 = revoke_all_streams(S0), - Progresses = all_stream_progresses(S1, Agent0), + Progresses = all_stream_progresses(S1, Agent0, _NeedUnacked = true), Agent1 = emqx_persistent_session_ds_shared_subs_agent:on_disconnect(Agent0, Progresses), SharedSubS1 = SharedSubS0#{agent => Agent1, scheduled_actions => #{}}, {S1, SharedSubS1}. diff --git a/apps/emqx_ds_shared_sub/test/emqx_ds_shared_sub_SUITE.erl b/apps/emqx_ds_shared_sub/test/emqx_ds_shared_sub_SUITE.erl index 0f665b5a3..dfc2203c4 100644 --- a/apps/emqx_ds_shared_sub/test/emqx_ds_shared_sub_SUITE.erl +++ b/apps/emqx_ds_shared_sub/test/emqx_ds_shared_sub_SUITE.erl @@ -183,51 +183,6 @@ t_graceful_disconnect(_Config) -> ok = emqtt:disconnect(ConnShared2), ok = emqtt:disconnect(ConnPub). -t_disconnect_no_double_replay(_Config) -> - ConnPub = emqtt_connect_pub(<<"client_pub">>), - - ConnShared1 = emqtt_connect_sub(<<"client_shared1">>), - {ok, _, _} = emqtt:subscribe(ConnShared1, <<"$share/gr9/topic9/#">>, 1), - - ConnShared2 = emqtt_connect_sub(<<"client_shared2">>), - {ok, _, _} = emqtt:subscribe(ConnShared2, <<"$share/gr9/topic9/#">>, 1), - - ct:sleep(1000), - - NPubs = 10_000, - - Topics = [<<"topic9/1">>, <<"topic9/2">>, <<"topic9/3">>], - ok = publish_n(ConnPub, Topics, 1, NPubs), - - Self = self(), - _ = spawn_link(fun() -> - ok = publish_n(ConnPub, Topics, NPubs + 1, 2 * NPubs), - Self ! publish_done - end), - - ok = emqtt:disconnect(ConnShared2), - - receive - publish_done -> ok - end, - - Pubs = drain_publishes(), - - ClientByBid = fun(Pid) -> - case Pid of - ConnShared1 -> <<"client_shared1">>; - ConnShared2 -> <<"client_shared2">> - end - end, - - {Missing, Duplicate} = verify_received_pubs(Pubs, 2 * NPubs, ClientByBid), - - ?assertEqual([], Missing), - ?assertEqual([], Duplicate), - - ok = emqtt:disconnect(ConnShared1), - ok = emqtt:disconnect(ConnPub). - t_intensive_reassign(_Config) -> ConnPub = emqtt_connect_pub(<<"client_pub">>), @@ -373,6 +328,80 @@ t_quick_resubscribe(_Config) -> ok = emqtt:disconnect(ConnShared2), ok = emqtt:disconnect(ConnPub). +t_disconnect_no_double_replay1(_Config) -> + ConnPub = emqtt_connect_pub(<<"client_pub">>), + + ConnShared1 = emqtt_connect_sub(<<"client_shared1">>), + {ok, _, _} = emqtt:subscribe(ConnShared1, <<"$share/gr11/topic11/#">>, 1), + + ConnShared2 = emqtt_connect_sub(<<"client_shared2">>), + {ok, _, _} = emqtt:subscribe(ConnShared2, <<"$share/gr11/topic11/#">>, 1), + + ct:sleep(1000), + + NPubs = 10_000, + + Topics = [<<"topic11/1">>, <<"topic11/2">>, <<"topic11/3">>], + ok = publish_n(ConnPub, Topics, 1, NPubs), + + Self = self(), + _ = spawn_link(fun() -> + ok = publish_n(ConnPub, Topics, NPubs + 1, 2 * NPubs), + Self ! publish_done + end), + + ok = emqtt:disconnect(ConnShared2), + + receive + publish_done -> ok + end, + + Pubs = drain_publishes(), + + ClientByBid = fun(Pid) -> + case Pid of + ConnShared1 -> <<"client_shared1">>; + ConnShared2 -> <<"client_shared2">> + end + end, + + {Missing, Duplicate} = verify_received_pubs(Pubs, 2 * NPubs, ClientByBid), + + ?assertEqual([], Missing), + ?assertEqual([], Duplicate), + + ok = emqtt:disconnect(ConnShared1), + ok = emqtt:disconnect(ConnPub). + +t_disconnect_no_double_replay2(_Config) -> + ConnPub = emqtt_connect_pub(<<"client_pub">>), + + ConnShared1 = emqtt_connect_sub(<<"client_shared1">>, [{auto_ack, false}]), + {ok, _, _} = emqtt:subscribe(ConnShared1, <<"$share/gr12/topic12/#">>, 1), + + ct:sleep(1000), + + ok = publish_n(ConnPub, [<<"topic12/1">>], 1, 20), + + receive + {publish, #{payload := <<"1">>, packet_id := PacketId1}} -> + ok = emqtt:puback(ConnShared1, PacketId1) + after 5000 -> + ct:fail("No publish received") + end, + + ok = emqtt:disconnect(ConnShared1), + + ConnShared12 = emqtt_connect_sub(<<"client_shared12">>), + {ok, _, _} = emqtt:subscribe(ConnShared12, <<"$share/gr12/topic12/#">>, 1), + + ?assertNotReceive( + {publish, #{payload := <<"1">>}}, + 3000 + ), + + ok = emqtt:disconnect(ConnShared12). + t_lease_reconnect(_Config) -> ConnPub = emqtt_connect_pub(<<"client_pub">>), @@ -432,12 +461,17 @@ t_renew_lease_timeout(_Config) -> %%-------------------------------------------------------------------- emqtt_connect_sub(ClientId) -> - {ok, C} = emqtt:start_link([ - {clientid, ClientId}, - {clean_start, true}, - {proto_ver, v5}, - {properties, #{'Session-Expiry-Interval' => 7_200}} - ]), + emqtt_connect_sub(ClientId, []). + +emqtt_connect_sub(ClientId, Options) -> + {ok, C} = emqtt:start_link( + [ + {clientid, ClientId}, + {clean_start, true}, + {proto_ver, v5}, + {properties, #{'Session-Expiry-Interval' => 7_200}} + ] ++ Options + ), {ok, _} = emqtt:connect(C), C. From 143086b0ef72ef3e4d23b580c29b7cf7a8f6764f Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Tue, 9 Jul 2024 13:50:39 +0300 Subject: [PATCH 105/131] feat(queue): replace invalid rewing algorithm with skipping iterator --- ...emqx_persistent_session_ds_shared_subs.erl | 68 +++------------ apps/emqx_durable_storage/src/emqx_ds.erl | 4 + .../src/emqx_ds_skipping_iterator.erl | 87 +++++++++++++++++++ .../src/emqx_ds_skipping_iterator.hrl | 32 +++++++ 4 files changed, 137 insertions(+), 54 deletions(-) create mode 100644 apps/emqx_durable_storage/src/emqx_ds_skipping_iterator.erl create mode 100644 apps/emqx_durable_storage/src/emqx_ds_skipping_iterator.hrl diff --git a/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl index 7535e1a61..a4cc97c87 100644 --- a/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl +++ b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl @@ -341,7 +341,11 @@ accept_stream(#{topic_filter := TopicFilter} = Event, S, ScheduledActions) -> end. accept_stream( - #{topic_filter := TopicFilter, stream := Stream, progress := Progress} = _Event, + #{ + topic_filter := TopicFilter, + stream := Stream, + progress := #{iterator := Iterator} = _Progress + } = _Event, S0 ) -> case emqx_persistent_session_ds_state:get_subscription(TopicFilter, S0) of @@ -361,7 +365,6 @@ accept_stream( end, case NeedCreateStream of true -> - Iterator = rewind_iterator(Progress), NewSRS = #srs{ rank_x = ?rank_x, @@ -377,52 +380,6 @@ accept_stream( end end. -%% Skip acked messages. -%% This may be a bit inefficient, and it is unclear how to handle errors. -%% -%% A better variant would be to wrap the iterator on `emqx_ds` level in a new one, -%% that will skip acked messages internally in `emqx_ds:next` function. -%% Unluckily, emqx_ds does not have a wrapping structure around iterators of -%% the underlying levels, so we cannot wrap it without a risk of confusion. - -rewind_iterator(#{iterator := Iterator, acked := true}) -> - Iterator; -rewind_iterator(#{iterator := Iterator0, acked := false, qos1_acked := 0, qos2_acked := 0}) -> - Iterator0; -%% This should not happen, means the DS is consistent -rewind_iterator(#{iterator := Iterator0, acked := false, qos1_acked := Q1, qos2_acked := Q2}) when - Q1 < 0 orelse Q2 < 0 --> - Iterator0; -rewind_iterator( - #{iterator := Iterator0, acked := false, qos1_acked := Q1Old, qos2_acked := Q2Old} = Progress -) -> - case emqx_ds:next(?PERSISTENT_MESSAGE_DB, Iterator0, Q1Old + Q2Old) of - {ok, Iterator1, Messages} -> - {Q1New, Q2New} = update_qos_acked(Q1Old, Q2Old, Messages), - rewind_iterator(Progress#{ - iterator => Iterator1, qos1_acked => Q1New, qos2_acked => Q2New - }); - {ok, end_of_stream} -> - end_of_stream; - {error, _, _} -> - %% What to do here? - %% In the wrapping variant we do not have this problem. - Iterator0 - end. - -update_qos_acked(Q1, Q2, []) -> - {Q1, Q2}; -update_qos_acked(Q1, Q2, [{_Key, Message} | Messages]) -> - case emqx_message:qos(Message) of - ?QOS_1 -> - update_qos_acked(Q1 - 1, Q2, Messages); - ?QOS_2 -> - update_qos_acked(Q1, Q2 - 1, Messages); - _ -> - update_qos_acked(Q1, Q2, Messages) - end. - revoke_stream( #{topic_filter := TopicFilter, stream := Stream}, S0 ) -> @@ -667,8 +624,8 @@ stream_progress( first_seqno_qos2 = StartQos2 } = SRS ) -> - Qos1Acked = seqno_diff(?QOS_1, CommQos1, StartQos1), - Qos2Acked = seqno_diff(?QOS_2, CommQos2, StartQos2), + Qos1Acked = n_acked(?QOS_1, CommQos1, StartQos1), + Qos2Acked = n_acked(?QOS_2, CommQos2, StartQos2), case is_stream_fully_acked(CommQos1, CommQos2, SRS) of true -> #{ @@ -683,10 +640,10 @@ stream_progress( #{ stream => Stream, progress => #{ - acked => false, - iterator => BeginIt, - qos1_acked => Qos1Acked, - qos2_acked => Qos2Acked + acked => true, + iterator => emqx_ds_skipping_iterator:update_or_new( + BeginIt, Qos1Acked, Qos2Acked + ) }, use_finished => is_use_finished(SRS) } @@ -753,6 +710,9 @@ is_stream_fully_acked(_, _, #srs{ is_stream_fully_acked(Comm1, Comm2, #srs{last_seqno_qos1 = S1, last_seqno_qos2 = S2}) -> (Comm1 >= S1) andalso (Comm2 >= S2). +n_acked(Qos, A, B) -> + max(seqno_diff(Qos, A, B), 0). + -dialyzer({nowarn_function, seqno_diff/3}). seqno_diff(?QOS_1, A, B) -> %% For QoS1 messages we skip a seqno every time the epoch changes, diff --git a/apps/emqx_durable_storage/src/emqx_ds.erl b/apps/emqx_durable_storage/src/emqx_ds.erl index 69de92325..6aaba205d 100644 --- a/apps/emqx_durable_storage/src/emqx_ds.erl +++ b/apps/emqx_durable_storage/src/emqx_ds.erl @@ -401,10 +401,14 @@ make_iterator(DB, Stream, TopicFilter, StartTime) -> -spec update_iterator(db(), iterator(), message_key()) -> make_iterator_result(). +update_iterator(DB, ?skipping_iterator_match = OldIter, DSKey) -> + emqx_ds_skipping_iterator:update_iterator(DB, OldIter, DSKey); update_iterator(DB, OldIter, DSKey) -> ?module(DB):update_iterator(DB, OldIter, DSKey). -spec next(db(), iterator(), pos_integer()) -> next_result(). +next(DB, ?skipping_iterator_match = Iter, BatchSize) -> + emqx_ds_skipping_iterator:next(DB, Iter, BatchSize); next(DB, Iter, BatchSize) -> ?module(DB):next(DB, Iter, BatchSize). diff --git a/apps/emqx_durable_storage/src/emqx_ds_skipping_iterator.erl b/apps/emqx_durable_storage/src/emqx_ds_skipping_iterator.erl new file mode 100644 index 000000000..67d871e8a --- /dev/null +++ b/apps/emqx_durable_storage/src/emqx_ds_skipping_iterator.erl @@ -0,0 +1,87 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023-2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_ds_skipping_iterator). + +-include("emqx_ds_skipping_iterator.hrl"). +-include("emqx/include/emqx_mqtt.hrl"). + +-type t() :: ?skipping_iterator(emqx_ds:iterator(), non_neg_integer(), non_neg_integer()). + +-export([ + update_or_new/3, + update_iterator/3, + next/3 +]). + +%%-------------------------------------------------------------------- +%% API +%%-------------------------------------------------------------------- + +-spec update_or_new(t() | emqx_ds:iterator(), non_neg_integer(), non_neg_integer()) -> t(). +update_or_new(?skipping_iterator_match(Iterator, Q1Skip0, Q2Skip0), Q1Skip, Q2Skip) when + Q1Skip >= 0 andalso Q2Skip >= 0 +-> + ?skipping_iterator(Iterator, Q1Skip0 + Q1Skip, Q2Skip0 + Q2Skip); +update_or_new(Iterator, Q1Skip, Q2Skip) when Q1Skip >= 0 andalso Q2Skip >= 0 -> + ?skipping_iterator(Iterator, Q1Skip, Q2Skip). + +-spec next(emqx_ds:db(), t(), pos_integer()) -> emqx_ds:next_result(t()). +next(DB, ?skipping_iterator_match(Iterator0, Q1Skip0, Q2Skip0), Count) -> + case emqx_ds:next(DB, Iterator0, Count) of + {error, _, _} = Error -> + Error; + {ok, end_of_stream} -> + {ok, end_of_stream}; + {ok, Iterator1, Messages0} -> + {Messages1, Q1Skip1, Q2Skip1} = skip(Messages0, Q1Skip0, Q2Skip0), + case {Q1Skip1, Q2Skip1} of + {0, 0} -> {ok, Iterator1, Messages1}; + _ -> {ok, ?skipping_iterator(Iterator1, Q1Skip1, Q2Skip1)} + end + end. + +-spec update_iterator(emqx_ds:db(), emqx_ds:iterator(), emqx_ds:message_key()) -> + emqx_ds:make_iterator_result(). +update_iterator(DB, ?skipping_iterator_match(Iterator0, Q1Skip0, Q2Skip0), Key) -> + case emqx_ds:update_iterator(DB, Iterator0, Key) of + {error, _, _} = Error -> Error; + {ok, Iterator1} -> {ok, ?skipping_iterator(Iterator1, Q1Skip0, Q2Skip0)} + end. + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- + +skip(Messages, Q1Skip, Q2Skip) -> + skip(Messages, Q1Skip, Q2Skip, []). + +skip([], Q1Skip, Q2Skip, Agg) -> + {lists:reverse(Agg), Q1Skip, Q2Skip}; +skip([{Key, Message} | Messages], Q1Skip, Q2Skip, Agg) -> + Qos = emqx_message:qos(Message), + skip({Key, Message}, Qos, Messages, Q1Skip, Q2Skip, Agg). + +skip(_KeyMessage, ?QOS_0, Messages, Q1Skip, Q2Skip, Agg) -> + skip(Messages, Q1Skip, Q2Skip, Agg); +skip(_KeyMessage, ?QOS_1, Messages, Q1Skip, Q2Skip, Agg) when Q1Skip > 0 -> + skip(Messages, Q1Skip - 1, Q2Skip, Agg); +skip(KeyMessage, ?QOS_1, Messages, 0, Q2Skip, Agg) -> + skip(Messages, 0, Q2Skip, [KeyMessage | Agg]); +skip(_KeyMessage, ?QOS_2, Messages, Q1Skip, Q2Skip, Agg) when Q2Skip > 0 -> + skip(Messages, Q1Skip, Q2Skip - 1, Agg); +skip(KeyMessage, ?QOS_2, Messages, Q1Skip, 0, Agg) -> + skip(Messages, Q1Skip, 0, [KeyMessage | Agg]). diff --git a/apps/emqx_durable_storage/src/emqx_ds_skipping_iterator.hrl b/apps/emqx_durable_storage/src/emqx_ds_skipping_iterator.hrl new file mode 100644 index 000000000..2c0999fcc --- /dev/null +++ b/apps/emqx_durable_storage/src/emqx_ds_skipping_iterator.hrl @@ -0,0 +1,32 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023-2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-define(tag, 1). +-define(it, 2). +-define(qos1_skip, 3). +-define(qos2_skip, 4). + +-define(IT, -1000). + +-define(skipping_iterator_match, #{?tag := ?IT}). + +-define(skipping_iterator_match(Iterator, Q1Skip, Q2Skip), #{ + ?tag := ?IT, ?it := Iterator, ?qos1_skip := Q1Skip, ?qos2_skip := Q2Skip +}). + +-define(skipping_iterator(Iterator, Q1Skip, Q2Skip), #{ + ?tag => ?IT, ?it => Iterator, ?qos1_skip => Q1Skip, ?qos2_skip => Q2Skip +}). From 9e5e7a23c5cfb20af003b1319753335acdec790f Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Tue, 9 Jul 2024 14:14:22 +0300 Subject: [PATCH 106/131] feat(queue): remove unnecessary acked flag --- ...emqx_persistent_session_ds_shared_subs.erl | 45 +++++++------------ .../src/emqx_ds_shared_sub_leader.erl | 6 +-- 2 files changed, 17 insertions(+), 34 deletions(-) diff --git a/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl index a4cc97c87..6cf5cc40b 100644 --- a/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl +++ b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl @@ -68,14 +68,7 @@ -type progress() :: #{ - acked := true, iterator := emqx_ds:iterator() - } - | #{ - acked := false, - iterator := emqx_ds:iterator(), - qos1_acked := boolean(), - qos2_acked := boolean() }. -type scheduled_action() :: #{ @@ -626,28 +619,22 @@ stream_progress( ) -> Qos1Acked = n_acked(?QOS_1, CommQos1, StartQos1), Qos2Acked = n_acked(?QOS_2, CommQos2, StartQos2), - case is_stream_fully_acked(CommQos1, CommQos2, SRS) of - true -> - #{ - stream => Stream, - progress => #{ - acked => true, - iterator => EndIt - }, - use_finished => is_use_finished(SRS) - }; - false -> - #{ - stream => Stream, - progress => #{ - acked => true, - iterator => emqx_ds_skipping_iterator:update_or_new( - BeginIt, Qos1Acked, Qos2Acked - ) - }, - use_finished => is_use_finished(SRS) - } - end. + Iterator = + case is_stream_fully_acked(CommQos1, CommQos2, SRS) of + true -> + EndIt; + false -> + emqx_ds_skipping_iterator:update_or_new( + BeginIt, Qos1Acked, Qos2Acked + ) + end, + #{ + stream => Stream, + progress => #{ + iterator => Iterator + }, + use_finished => is_use_finished(SRS) + }. fold_shared_subs(Fun, Acc, S) -> emqx_persistent_session_ds_state:fold_subscriptions( diff --git a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl index 976ce2437..e98c74b27 100644 --- a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl +++ b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl @@ -314,8 +314,7 @@ update_progresses(StreamStates, NewStreamsWRanks, TopicFilter, StartTime) -> ?PERSISTENT_MESSAGE_DB, Stream, TopicFilter, StartTime ), Progress = #{ - iterator => It, - acked => true + iterator => It }, { NewStreamStatesAcc#{Stream => #{progress => Progress, rank => Rank}}, @@ -708,9 +707,6 @@ clean_revoked_streams( ( #{ stream := Stream, - progress := #{ - acked := true - }, use_finished := true } ) -> From a676ede6b8971978d75370fc62b52d107e8b0255 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Tue, 9 Jul 2024 17:53:37 +0300 Subject: [PATCH 107/131] feat(queue): improve logging --- .../emqx_persistent_session_ds_shared_subs.erl | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl index 6cf5cc40b..1f4d6f6e9 100644 --- a/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl +++ b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl @@ -304,9 +304,10 @@ renew_streams(S0, #{agent := Agent0, scheduled_actions := ScheduledActions} = Sh {StreamLeaseEvents, Agent1} = emqx_persistent_session_ds_shared_subs_agent:renew_streams( Agent0 ), - ?tp(warning, shared_subs_new_stream_lease_events, #{ - stream_lease_events => format_lease_events(StreamLeaseEvents) - }), + StreamLeaseEvents =/= [] andalso + ?tp(warning, shared_subs_new_stream_lease_events, #{ + stream_lease_events => format_lease_events(StreamLeaseEvents) + }), S1 = lists:foldl( fun (#{type := lease} = Event, S) -> accept_stream(Event, S, ScheduledActions); From 7e23f8d19fb167c9a0eca3f96c7940f511e61058 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Tue, 9 Jul 2024 17:53:59 +0300 Subject: [PATCH 108/131] feat(queue): fix include --- apps/emqx_durable_storage/src/emqx_ds_skipping_iterator.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_durable_storage/src/emqx_ds_skipping_iterator.erl b/apps/emqx_durable_storage/src/emqx_ds_skipping_iterator.erl index 67d871e8a..d8833f65e 100644 --- a/apps/emqx_durable_storage/src/emqx_ds_skipping_iterator.erl +++ b/apps/emqx_durable_storage/src/emqx_ds_skipping_iterator.erl @@ -17,7 +17,7 @@ -module(emqx_ds_skipping_iterator). -include("emqx_ds_skipping_iterator.hrl"). --include("emqx/include/emqx_mqtt.hrl"). +-include_lib("emqx/include/emqx_mqtt.hrl"). -type t() :: ?skipping_iterator(emqx_ds:iterator(), non_neg_integer(), non_neg_integer()). From f21356946089addbfd965cf39be3e3ecbd967568 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Wed, 10 Jul 2024 19:02:15 +0300 Subject: [PATCH 109/131] feat(queue): clarify naming; identify shared subs by full topic filter --- ...emqx_persistent_session_ds_shared_subs.erl | 137 ++++++++-------- ...ersistent_session_ds_shared_subs_agent.erl | 44 ++--- .../src/emqx_ds_shared_sub_agent.erl | 152 ++++++++++++------ .../src/emqx_ds_shared_sub_group_sm.erl | 61 +++---- .../src/emqx_ds_shared_sub_leader.erl | 60 +++---- .../src/emqx_ds_shared_sub_proto.erl | 14 +- .../src/emqx_ds_shared_sub_proto.hrl | 50 +++--- .../src/emqx_ds_shared_sub_registry.erl | 25 +-- .../src/proto/emqx_ds_shared_sub_proto_v1.erl | 4 +- 9 files changed, 303 insertions(+), 244 deletions(-) diff --git a/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl index 1f4d6f6e9..506114f35 100644 --- a/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl +++ b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl @@ -119,8 +119,8 @@ new(Opts) -> {ok, emqx_persistent_session_ds_state:t(), t()}. open(S, Opts) -> SharedSubscriptions = fold_shared_subs( - fun(#share{} = TopicFilter, Sub, Acc) -> - [{TopicFilter, to_agent_subscription(S, Sub)} | Acc] + fun(#share{} = ShareTopicFilter, Sub, Acc) -> + [{ShareTopicFilter, to_agent_subscription(S, Sub)} | Acc] end, [], S @@ -139,33 +139,33 @@ open(S, Opts) -> emqx_types:subopts(), emqx_persistent_session_ds:session() ) -> {ok, emqx_persistent_session_ds_state:t(), t()} | {error, emqx_types:reason_code()}. -on_subscribe(TopicFilter, SubOpts, #{s := S} = Session) -> - Subscription = emqx_persistent_session_ds_state:get_subscription(TopicFilter, S), - on_subscribe(Subscription, TopicFilter, SubOpts, Session). +on_subscribe(#share{} = ShareTopicFilter, SubOpts, #{s := S} = Session) -> + Subscription = emqx_persistent_session_ds_state:get_subscription(ShareTopicFilter, S), + on_subscribe(Subscription, ShareTopicFilter, SubOpts, Session). %%-------------------------------------------------------------------- %% on_subscribe internal functions -on_subscribe(undefined, TopicFilter, SubOpts, #{props := Props, s := S} = Session) -> +on_subscribe(undefined, ShareTopicFilter, SubOpts, #{props := Props, s := S} = Session) -> #{max_subscriptions := MaxSubscriptions} = Props, case emqx_persistent_session_ds_state:n_subscriptions(S) < MaxSubscriptions of true -> - create_new_subscription(TopicFilter, SubOpts, Session); + create_new_subscription(ShareTopicFilter, SubOpts, Session); false -> {error, ?RC_QUOTA_EXCEEDED} end; -on_subscribe(Subscription, TopicFilter, SubOpts, Session) -> - update_subscription(Subscription, TopicFilter, SubOpts, Session). +on_subscribe(Subscription, ShareTopicFilter, SubOpts, Session) -> + update_subscription(Subscription, ShareTopicFilter, SubOpts, Session). -dialyzer({nowarn_function, create_new_subscription/3}). -create_new_subscription(TopicFilter, SubOpts, #{ +create_new_subscription(ShareTopicFilter, SubOpts, #{ s := S0, shared_sub_s := #{agent := Agent} = SharedSubS0, props := Props }) -> case emqx_persistent_session_ds_shared_subs_agent:can_subscribe( - Agent, TopicFilter, SubOpts + Agent, ShareTopicFilter, SubOpts ) of ok -> @@ -184,17 +184,19 @@ create_new_subscription(TopicFilter, SubOpts, #{ start_time => now_ms() }, S = emqx_persistent_session_ds_state:put_subscription( - TopicFilter, Subscription, S3 + ShareTopicFilter, Subscription, S3 ), - SharedSubS = schedule_subscribe(SharedSubS0, TopicFilter, SubOpts), + SharedSubS = schedule_subscribe(SharedSubS0, ShareTopicFilter, SubOpts), {ok, S, SharedSubS}; {error, _} = Error -> Error end. -update_subscription(#{current_state := SStateId0, id := SubId} = Sub0, TopicFilter, SubOpts, #{ - s := S0, shared_sub_s := SharedSubS, props := Props -}) -> +update_subscription( + #{current_state := SStateId0, id := SubId} = Sub0, ShareTopicFilter, SubOpts, #{ + s := S0, shared_sub_s := SharedSubS, props := Props + } +) -> #{upgrade_qos := UpgradeQoS} = Props, SState = #{parent_subscription => SubId, upgrade_qos => UpgradeQoS, subopts => SubOpts}, case emqx_persistent_session_ds_state:get_subscription_state(SStateId0, S0) of @@ -208,31 +210,33 @@ update_subscription(#{current_state := SStateId0, id := SubId} = Sub0, TopicFilt SStateId, SState, S1 ), Sub = Sub0#{current_state => SStateId}, - S = emqx_persistent_session_ds_state:put_subscription(TopicFilter, Sub, S2), + S = emqx_persistent_session_ds_state:put_subscription(ShareTopicFilter, Sub, S2), {ok, S, SharedSubS} end. -dialyzer({nowarn_function, schedule_subscribe/3}). schedule_subscribe( - #{agent := Agent0, scheduled_actions := ScheduledActions0} = SharedSubS0, TopicFilter, SubOpts + #{agent := Agent0, scheduled_actions := ScheduledActions0} = SharedSubS0, + ShareTopicFilter, + SubOpts ) -> case ScheduledActions0 of - #{TopicFilter := ScheduledAction} -> + #{ShareTopicFilter := ScheduledAction} -> ScheduledActions1 = ScheduledActions0#{ - TopicFilter => ScheduledAction#{type => {?schedule_subscribe, SubOpts}} + ShareTopicFilter => ScheduledAction#{type => {?schedule_subscribe, SubOpts}} }, ?tp(warning, shared_subs_schedule_subscribe_override, #{ - topic_filter => TopicFilter, + share_topic_filter => ShareTopicFilter, new_type => {?schedule_subscribe, SubOpts}, old_action => format_schedule_action(ScheduledAction) }), SharedSubS0#{scheduled_actions := ScheduledActions1}; _ -> ?tp(warning, shared_subs_schedule_subscribe_new, #{ - topic_filter => TopicFilter, subopts => SubOpts + share_topic_filter => ShareTopicFilter, subopts => SubOpts }), Agent1 = emqx_persistent_session_ds_shared_subs_agent:on_subscribe( - Agent0, TopicFilter, SubOpts + Agent0, ShareTopicFilter, SubOpts ), SharedSubS0#{agent => Agent1} end. @@ -242,22 +246,22 @@ schedule_subscribe( -spec on_unsubscribe( emqx_persistent_session_ds:id(), - emqx_persistent_session_ds:topic_filter(), + share_topic_filter(), emqx_persistent_session_ds_state:t(), t() ) -> {ok, emqx_persistent_session_ds_state:t(), t(), emqx_persistent_session_ds:subscription()} | {error, emqx_types:reason_code()}. -on_unsubscribe(SessionId, TopicFilter, S0, SharedSubS0) -> - case lookup(TopicFilter, S0) of +on_unsubscribe(SessionId, ShareTopicFilter, S0, SharedSubS0) -> + case lookup(ShareTopicFilter, S0) of undefined -> {error, ?RC_NO_SUBSCRIPTION_EXISTED}; #{id := SubId} = Subscription -> ?tp(persistent_session_ds_subscription_delete, #{ - session_id => SessionId, topic_filter => TopicFilter + session_id => SessionId, share_topic_filter => ShareTopicFilter }), - S = emqx_persistent_session_ds_state:del_subscription(TopicFilter, S0), - SharedSubS = schedule_unsubscribe(S, SharedSubS0, SubId, TopicFilter), + S = emqx_persistent_session_ds_state:del_subscription(ShareTopicFilter, S0), + SharedSubS = schedule_unsubscribe(S, SharedSubS0, SubId, ShareTopicFilter), {ok, S, SharedSubS, Subscription} end. @@ -265,16 +269,16 @@ on_unsubscribe(SessionId, TopicFilter, S0, SharedSubS0) -> %% on_unsubscribe internal functions schedule_unsubscribe( - S, #{scheduled_actions := ScheduledActions0} = SharedSubS0, UnsubscridedSubId, TopicFilter + S, #{scheduled_actions := ScheduledActions0} = SharedSubS0, UnsubscridedSubId, ShareTopicFilter ) -> case ScheduledActions0 of - #{TopicFilter := ScheduledAction0} -> + #{ShareTopicFilter := ScheduledAction0} -> ScheduledAction1 = ScheduledAction0#{type => ?schedule_unsubscribe}, ScheduledActions1 = ScheduledActions0#{ - TopicFilter => ScheduledAction1 + ShareTopicFilter => ScheduledAction1 }, ?tp(warning, shared_subs_schedule_unsubscribe_override, #{ - topic_filter => TopicFilter, + share_topic_filter => ShareTopicFilter, new_type => ?schedule_unsubscribe, old_action => format_schedule_action(ScheduledAction0) }), @@ -282,14 +286,14 @@ schedule_unsubscribe( _ -> StreamKeys = stream_keys_by_sub_id(S, UnsubscridedSubId), ScheduledActions1 = ScheduledActions0#{ - TopicFilter => #{ + ShareTopicFilter => #{ type => ?schedule_unsubscribe, stream_keys_to_wait => StreamKeys, progresses => [] } }, ?tp(warning, shared_subs_schedule_unsubscribe_new, #{ - topic_filter => TopicFilter, + share_topic_filter => ShareTopicFilter, stream_keys => format_stream_keys(StreamKeys) }), SharedSubS0#{scheduled_actions := ScheduledActions1} @@ -322,13 +326,13 @@ renew_streams(S0, #{agent := Agent0, scheduled_actions := ScheduledActions} = Sh %%-------------------------------------------------------------------- %% renew_streams internal functions -accept_stream(#{topic_filter := TopicFilter} = Event, S, ScheduledActions) -> +accept_stream(#{share_topic_filter := ShareTopicFilter} = Event, S, ScheduledActions) -> %% If we have a pending action (subscribe or unsubscribe) for this topic filter, %% we should not accept a stream and start replaying it. We won't use it anyway: %% * if subscribe is pending, we will reset agent obtain a new lease %% * if unsubscribe is pending, we will drop connection case ScheduledActions of - #{TopicFilter := _Action} -> + #{ShareTopicFilter := _Action} -> S; _ -> accept_stream(Event, S) @@ -336,13 +340,13 @@ accept_stream(#{topic_filter := TopicFilter} = Event, S, ScheduledActions) -> accept_stream( #{ - topic_filter := TopicFilter, + share_topic_filter := ShareTopicFilter, stream := Stream, progress := #{iterator := Iterator} = _Progress } = _Event, S0 ) -> - case emqx_persistent_session_ds_state:get_subscription(TopicFilter, S0) of + case emqx_persistent_session_ds_state:get_subscription(ShareTopicFilter, S0) of undefined -> %% We unsubscribed S0; @@ -375,9 +379,9 @@ accept_stream( end. revoke_stream( - #{topic_filter := TopicFilter, stream := Stream}, S0 + #{share_topic_filter := ShareTopicFilter, stream := Stream}, S0 ) -> - case emqx_persistent_session_ds_state:get_subscription(TopicFilter, S0) of + case emqx_persistent_session_ds_state:get_subscription(ShareTopicFilter, S0) of undefined -> %% This should not happen. %% Agent should have received unsubscribe callback @@ -427,12 +431,7 @@ all_stream_progresses(S, _Agent, NeedUnacked) -> CommQos1 = emqx_persistent_session_ds_state:get_seqno(?committed(?QOS_1), S), CommQos2 = emqx_persistent_session_ds_state:get_seqno(?committed(?QOS_2), S), fold_shared_stream_states( - fun( - #share{group = Group}, - Stream, - SRS, - ProgressesAcc0 - ) -> + fun(ShareTopicFilter, Stream, SRS, ProgressesAcc0) -> case is_stream_started(CommQos1, CommQos2, SRS) and (NeedUnacked or is_stream_fully_acked(CommQos1, CommQos2, SRS)) @@ -440,7 +439,7 @@ all_stream_progresses(S, _Agent, NeedUnacked) -> true -> StreamProgress = stream_progress(CommQos1, CommQos2, Stream, SRS), maps:update_with( - Group, + ShareTopicFilter, fun(Progresses) -> [StreamProgress | Progresses] end, [StreamProgress], ProgressesAcc0 @@ -455,12 +454,12 @@ all_stream_progresses(S, _Agent, NeedUnacked) -> run_scheduled_actions(S, Agent, ScheduledActions) -> maps:fold( - fun(TopicFilter, Action0, {AgentAcc0, ScheduledActionsAcc}) -> - case run_scheduled_action(S, AgentAcc0, TopicFilter, Action0) of + fun(ShareTopicFilter, Action0, {AgentAcc0, ScheduledActionsAcc}) -> + case run_scheduled_action(S, AgentAcc0, ShareTopicFilter, Action0) of {ok, AgentAcc1} -> - {AgentAcc1, maps:remove(TopicFilter, ScheduledActionsAcc)}; + {AgentAcc1, maps:remove(ShareTopicFilter, ScheduledActionsAcc)}; {continue, Action1} -> - {AgentAcc0, ScheduledActionsAcc#{TopicFilter => Action1}} + {AgentAcc0, ScheduledActionsAcc#{ShareTopicFilter => Action1}} end end, {Agent, ScheduledActions}, @@ -470,7 +469,7 @@ run_scheduled_actions(S, Agent, ScheduledActions) -> run_scheduled_action( S, Agent0, - #share{group = Group} = TopicFilter, + ShareTopicFilter, #{type := Type, stream_keys_to_wait := StreamKeysToWait0, progresses := Progresses0} = Action ) -> StreamKeysToWait1 = filter_unfinished_streams(S, StreamKeysToWait0), @@ -478,31 +477,31 @@ run_scheduled_action( case StreamKeysToWait1 of [] -> ?tp(warning, shared_subs_schedule_action_complete, #{ - topic_filter => TopicFilter, + share_topic_filter => ShareTopicFilter, progresses => format_stream_progresses(Progresses1), type => Type }), %% Regular progress won't se unsubscribed streams, so we need to %% send the progress explicitly. Agent1 = emqx_persistent_session_ds_shared_subs_agent:on_stream_progress( - Agent0, #{Group => Progresses1} + Agent0, #{ShareTopicFilter => Progresses1} ), case Type of {?schedule_subscribe, SubOpts} -> {ok, emqx_persistent_session_ds_shared_subs_agent:on_subscribe( - Agent1, TopicFilter, SubOpts + Agent1, ShareTopicFilter, SubOpts )}; ?schedule_unsubscribe -> {ok, emqx_persistent_session_ds_shared_subs_agent:on_unsubscribe( - Agent1, TopicFilter, Progresses1 + Agent1, ShareTopicFilter, Progresses1 )} end; _ -> Action1 = Action#{stream_keys_to_wait => StreamKeysToWait1, progresses => Progresses1}, ?tp(warning, shared_subs_schedule_action_continue, #{ - topic_filter => TopicFilter, + share_topic_filter => ShareTopicFilter, new_action => format_schedule_action(Action1) }), {continue, Action1} @@ -551,8 +550,8 @@ on_disconnect(S0, #{agent := Agent0} = SharedSubS0) -> revoke_all_streams(S0) -> fold_shared_stream_states( - fun(TopicFilter, Stream, _SRS, S) -> - revoke_stream(#{topic_filter => TopicFilter, stream => Stream}, S) + fun(ShareTopicFilter, Stream, _SRS, S) -> + revoke_stream(#{share_topic_filter => ShareTopicFilter, stream => Stream}, S) end, S0, S0 @@ -580,8 +579,8 @@ to_map(_S, _SharedSubS) -> %% Generic helpers %%-------------------------------------------------------------------- -lookup(TopicFilter, S) -> - case emqx_persistent_session_ds_state:get_subscription(TopicFilter, S) of +lookup(ShareTopicFilter, S) -> + case emqx_persistent_session_ds_state:get_subscription(ShareTopicFilter, S) of Sub = #{current_state := SStateId} -> case emqx_persistent_session_ds_state:get_subscription_state(SStateId, S) of #{subopts := SubOpts} -> @@ -640,7 +639,7 @@ stream_progress( fold_shared_subs(Fun, Acc, S) -> emqx_persistent_session_ds_state:fold_subscriptions( fun - (#share{} = TopicFilter, Sub, Acc0) -> Fun(TopicFilter, Sub, Acc0); + (#share{} = ShareTopicFilter, Sub, Acc0) -> Fun(ShareTopicFilter, Sub, Acc0); (_, _Sub, Acc0) -> Acc0 end, Acc, @@ -650,10 +649,10 @@ fold_shared_subs(Fun, Acc, S) -> fold_shared_stream_states(Fun, Acc, S) -> %% TODO %% Optimize or cache - TopicFilters = fold_shared_subs( + ShareTopicFilters = fold_shared_subs( fun - (#share{} = TopicFilter, #{id := Id} = _Sub, Acc0) -> - Acc0#{Id => TopicFilter}; + (#share{} = ShareTopicFilter, #{id := Id} = _Sub, Acc0) -> + Acc0#{Id => ShareTopicFilter}; (_, _, Acc0) -> Acc0 end, @@ -662,9 +661,9 @@ fold_shared_stream_states(Fun, Acc, S) -> ), emqx_persistent_session_ds_state:fold_streams( fun({SubId, Stream}, SRS, Acc0) -> - case TopicFilters of - #{SubId := TopicFilter} -> - Fun(TopicFilter, Stream, SRS, Acc0); + case ShareTopicFilters of + #{SubId := ShareTopicFilter} -> + Fun(ShareTopicFilter, Stream, SRS, Acc0); _ -> Acc0 end diff --git a/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs_agent.erl b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs_agent.erl index b49ceabcf..022963ad9 100644 --- a/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs_agent.erl +++ b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs_agent.erl @@ -15,7 +15,7 @@ }. -type t() :: term(). --type topic_filter() :: emqx_persistent_session_ds:share_topic_filter(). +-type share_topic_filter() :: emqx_persistent_session_ds:share_topic_filter(). -type opts() :: #{ session_id := session_id() @@ -28,21 +28,21 @@ -type stream_lease() :: #{ type => lease, %% Used as "external" subscription_id - topic_filter := topic_filter(), + share_topic_filter := share_topic_filter(), stream := emqx_ds:stream(), iterator := emqx_ds:iterator() }. -type stream_revoke() :: #{ type => revoke, - topic_filter := topic_filter(), + share_topic_filter := share_topic_filter(), stream := emqx_ds:stream() }. -type stream_lease_event() :: stream_lease() | stream_revoke(). -type stream_progress() :: #{ - topic_filter := topic_filter(), + share_topic_filter := share_topic_filter(), stream := emqx_ds:stream(), iterator := emqx_ds:iterator(), use_finished := boolean() @@ -80,13 +80,13 @@ %%-------------------------------------------------------------------- -callback new(opts()) -> t(). --callback open([{topic_filter(), subscription()}], opts()) -> t(). --callback can_subscribe(t(), topic_filter(), emqx_types:subopts()) -> ok | {error, term()}. --callback on_subscribe(t(), topic_filter(), emqx_types:subopts()) -> t(). --callback on_unsubscribe(t(), topic_filter(), [stream_progress()]) -> t(). --callback on_disconnect(t(), #{emqx_types:group() => [stream_progress()]}) -> t(). +-callback open([{share_topic_filter(), subscription()}], opts()) -> t(). +-callback can_subscribe(t(), share_topic_filter(), emqx_types:subopts()) -> ok | {error, term()}. +-callback on_subscribe(t(), share_topic_filter(), emqx_types:subopts()) -> t(). +-callback on_unsubscribe(t(), share_topic_filter(), [stream_progress()]) -> t(). +-callback on_disconnect(t(), #{share_topic_filter() => [stream_progress()]}) -> t(). -callback renew_streams(t()) -> {[stream_lease_event()], t()}. --callback on_stream_progress(t(), #{emqx_types:group() => [stream_progress()]}) -> t(). +-callback on_stream_progress(t(), #{share_topic_filter() => [stream_progress()]}) -> t(). -callback on_info(t(), term()) -> t(). %%-------------------------------------------------------------------- @@ -97,23 +97,23 @@ new(Opts) -> ?shared_subs_agent:new(Opts). --spec open([{topic_filter(), subscription()}], opts()) -> t(). +-spec open([{share_topic_filter(), subscription()}], opts()) -> t(). open(Topics, Opts) -> ?shared_subs_agent:open(Topics, Opts). --spec can_subscribe(t(), topic_filter(), emqx_types:subopts()) -> ok | {error, term()}. -can_subscribe(Agent, TopicFilter, SubOpts) -> - ?shared_subs_agent:can_subscribe(Agent, TopicFilter, SubOpts). +-spec can_subscribe(t(), share_topic_filter(), emqx_types:subopts()) -> ok | {error, term()}. +can_subscribe(Agent, ShareTopicFilter, SubOpts) -> + ?shared_subs_agent:can_subscribe(Agent, ShareTopicFilter, SubOpts). --spec on_subscribe(t(), topic_filter(), emqx_types:subopts()) -> t(). -on_subscribe(Agent, TopicFilter, SubOpts) -> - ?shared_subs_agent:on_subscribe(Agent, TopicFilter, SubOpts). +-spec on_subscribe(t(), share_topic_filter(), emqx_types:subopts()) -> t(). +on_subscribe(Agent, ShareTopicFilter, SubOpts) -> + ?shared_subs_agent:on_subscribe(Agent, ShareTopicFilter, SubOpts). --spec on_unsubscribe(t(), topic_filter(), [stream_progress()]) -> t(). -on_unsubscribe(Agent, TopicFilter, StreamProgresses) -> - ?shared_subs_agent:on_unsubscribe(Agent, TopicFilter, StreamProgresses). +-spec on_unsubscribe(t(), share_topic_filter(), [stream_progress()]) -> t(). +on_unsubscribe(Agent, ShareTopicFilter, StreamProgresses) -> + ?shared_subs_agent:on_unsubscribe(Agent, ShareTopicFilter, StreamProgresses). --spec on_disconnect(t(), #{emqx_types:group() => [stream_progress()]}) -> t(). +-spec on_disconnect(t(), #{share_topic_filter() => [stream_progress()]}) -> t(). on_disconnect(Agent, StreamProgresses) -> ?shared_subs_agent:on_disconnect(Agent, StreamProgresses). @@ -121,7 +121,7 @@ on_disconnect(Agent, StreamProgresses) -> renew_streams(Agent) -> ?shared_subs_agent:renew_streams(Agent). --spec on_stream_progress(t(), #{emqx_types:group() => [stream_progress()]}) -> t(). +-spec on_stream_progress(t(), #{share_topic_filter() => [stream_progress()]}) -> t(). on_stream_progress(Agent, StreamProgress) -> ?shared_subs_agent:on_stream_progress(Agent, StreamProgress). diff --git a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_agent.erl b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_agent.erl index b896370f3..005307ca2 100644 --- a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_agent.erl +++ b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_agent.erl @@ -26,18 +26,66 @@ -behaviour(emqx_persistent_session_ds_shared_subs_agent). +-type share_topic_filter() :: emqx_persistent_session_ds:share_topic_filter(). +-type group_id() :: share_topic_filter(). + +-type progress() :: emqx_persistent_session_ds_shared_subs:progress(). +-type external_lease_event() :: + #{ + type => lease, + stream => emqx_ds:stream(), + progress => progress(), + share_topic_filter => emqx_persistent_session_ds:share_topic_filter() + } + | #{ + type => revoke, + stream => emqx_ds:stream(), + share_topic_filter => emqx_persistent_session_ds:share_topic_filter() + }. + +-type options() :: #{ + session_id := emqx_persistent_session_ds:id() +}. + +-type t() :: #{ + groups := #{ + group_id() => emqx_ds_shared_sub_group_sm:t() + }, + session_id := emqx_persistent_session_ds:id() +}. + +%% Techinically, group_id and share_topic_filter are the same. +%% However, we speak in the terms of share_topic_filter in the API, +%% which is known to the shared subscription handler of persistent session. +%% +%% And we speak in the terms of group_id internally: +%% * we keep group_sm's in the state by group_id +%% * we use group_id to address group_sm's, e.g. when sending messages to them +%% from leader or from themselves. +-define(group_id(ShareTopicFilter), ShareTopicFilter). +-define(share_topic_filter(GroupId), GroupId). + -record(message_to_group_sm, { - group :: emqx_types:group(), + group_id :: group_id(), message :: term() }). +-export_type([ + t/0, + group_id/0, + options/0, + external_lease_event/0 +]). + %%-------------------------------------------------------------------- %% API %%-------------------------------------------------------------------- +-spec new(options()) -> t(). new(Opts) -> init_state(Opts). +-spec open([{share_topic_filter(), emqx_types:subopts()}], options()) -> t(). open(TopicSubscriptions, Opts) -> State0 = init_state(Opts), State1 = lists:foldl( @@ -45,32 +93,41 @@ open(TopicSubscriptions, Opts) -> ?tp(warning, ds_agent_open_subscription, #{ topic_filter => ShareTopicFilter }), - add_group_subscription(State, ShareTopicFilter) + add_shared_subscription(State, ShareTopicFilter) end, State0, TopicSubscriptions ), State1. -can_subscribe(_State, _TopicFilter, _SubOpts) -> +-spec can_subscribe(t(), share_topic_filter(), emqx_types:subopts()) -> ok. +can_subscribe(_State, _ShareTopicFilter, _SubOpts) -> ok. -on_subscribe(State0, TopicFilter, _SubOpts) -> +-spec on_subscribe(t(), share_topic_filter(), emqx_types:subopts()) -> t(). +on_subscribe(State0, ShareTopicFilter, _SubOpts) -> ?tp(warning, ds_agent_on_subscribe, #{ - topic_filter => TopicFilter + share_topic_filter => ShareTopicFilter }), - add_group_subscription(State0, TopicFilter). + add_shared_subscription(State0, ShareTopicFilter). -on_unsubscribe(State, TopicFilter, GroupProgress) -> - delete_group_subscription(State, TopicFilter, GroupProgress). +-spec on_unsubscribe(t(), share_topic_filter(), [ + emqx_persistent_session_ds_shared_subs:agent_stream_progress() +]) -> t(). +on_unsubscribe(State, ShareTopicFilter, GroupProgress) -> + delete_shared_subscription(State, ShareTopicFilter, GroupProgress). +-spec renew_streams(t()) -> {[emqx_persistent_session_ds_shared_subs:agent_stream_event()], t()}. renew_streams(#{} = State) -> fetch_stream_events(State). +-spec on_stream_progress(t(), #{ + share_topic_filter() => [emqx_persistent_session_ds_shared_subs:agent_stream_progress()] +}) -> t(). on_stream_progress(State, StreamProgresses) -> maps:fold( - fun(Group, GroupProgresses, StateAcc) -> - with_group_sm(StateAcc, Group, fun(GSM) -> + fun(ShareTopicFilter, GroupProgresses, StateAcc) -> + with_group_sm(StateAcc, ?group_id(ShareTopicFilter), fun(GSM) -> emqx_ds_shared_sub_group_sm:handle_stream_progress(GSM, GroupProgresses) end) end, @@ -78,72 +135,74 @@ on_stream_progress(State, StreamProgresses) -> StreamProgresses ). +-spec on_disconnect(t(), [emqx_persistent_session_ds_shared_subs:agent_stream_progress()]) -> t(). on_disconnect(#{groups := Groups0} = State, StreamProgresses) -> ok = maps:foreach( - fun(Group, GroupSM0) -> - GroupProgresses = maps:get(Group, StreamProgresses, []), + fun(GroupId, GroupSM0) -> + GroupProgresses = maps:get(?share_topic_filter(GroupId), StreamProgresses, []), emqx_ds_shared_sub_group_sm:handle_disconnect(GroupSM0, GroupProgresses) end, Groups0 ), State#{groups => #{}}. -on_info(State, ?leader_lease_streams_match(Group, Leader, StreamProgresses, Version)) -> +-spec on_info(t(), term()) -> t(). +on_info(State, ?leader_lease_streams_match(GroupId, Leader, StreamProgresses, Version)) -> ?SLOG(info, #{ msg => leader_lease_streams, - group => Group, + group_id => GroupId, streams => StreamProgresses, version => Version, leader => Leader }), - with_group_sm(State, Group, fun(GSM) -> + with_group_sm(State, GroupId, fun(GSM) -> emqx_ds_shared_sub_group_sm:handle_leader_lease_streams( GSM, Leader, StreamProgresses, Version ) end); -on_info(State, ?leader_renew_stream_lease_match(Group, Version)) -> +on_info(State, ?leader_renew_stream_lease_match(GroupId, Version)) -> ?SLOG(info, #{ msg => leader_renew_stream_lease, - group => Group, + group_id => GroupId, version => Version }), - with_group_sm(State, Group, fun(GSM) -> + with_group_sm(State, GroupId, fun(GSM) -> emqx_ds_shared_sub_group_sm:handle_leader_renew_stream_lease(GSM, Version) end); -on_info(State, ?leader_renew_stream_lease_match(Group, VersionOld, VersionNew)) -> +on_info(State, ?leader_renew_stream_lease_match(GroupId, VersionOld, VersionNew)) -> ?SLOG(info, #{ msg => leader_renew_stream_lease, - group => Group, + group_id => GroupId, version_old => VersionOld, version_new => VersionNew }), - with_group_sm(State, Group, fun(GSM) -> + with_group_sm(State, GroupId, fun(GSM) -> emqx_ds_shared_sub_group_sm:handle_leader_renew_stream_lease(GSM, VersionOld, VersionNew) end); -on_info(State, ?leader_update_streams_match(Group, VersionOld, VersionNew, StreamsNew)) -> +on_info(State, ?leader_update_streams_match(GroupId, VersionOld, VersionNew, StreamsNew)) -> ?SLOG(info, #{ msg => leader_update_streams, - group => Group, + group_id => GroupId, version_old => VersionOld, version_new => VersionNew, streams_new => StreamsNew }), - with_group_sm(State, Group, fun(GSM) -> + with_group_sm(State, GroupId, fun(GSM) -> emqx_ds_shared_sub_group_sm:handle_leader_update_streams( GSM, VersionOld, VersionNew, StreamsNew ) end); -on_info(State, ?leader_invalidate_match(Group)) -> +on_info(State, ?leader_invalidate_match(GroupId)) -> ?SLOG(info, #{ msg => leader_invalidate, - group => Group + group_id => GroupId }), - with_group_sm(State, Group, fun(GSM) -> + with_group_sm(State, GroupId, fun(GSM) -> emqx_ds_shared_sub_group_sm:handle_leader_invalidate(GSM) end); %% Generic messages sent by group_sm's to themselves (timeouts). -on_info(State, #message_to_group_sm{group = Group, message = Message}) -> - with_group_sm(State, Group, fun(GSM) -> +on_info(State, #message_to_group_sm{group_id = GroupId, message = Message}) -> + with_group_sm(State, GroupId, fun(GSM) -> emqx_ds_shared_sub_group_sm:handle_info(GSM, Message) end). @@ -158,29 +217,30 @@ init_state(Opts) -> groups => #{} }. -delete_group_subscription(State, #share{group = Group}, GroupProgress) -> +delete_shared_subscription(State, ShareTopicFilter, GroupProgress) -> + GroupId = ?group_id(ShareTopicFilter), case State of - #{groups := #{Group := GSM} = Groups} -> + #{groups := #{GroupId := GSM} = Groups} -> _ = emqx_ds_shared_sub_group_sm:handle_disconnect(GSM, GroupProgress), - State#{groups => maps:remove(Group, Groups)}; + State#{groups => maps:remove(GroupId, Groups)}; _ -> State end. -add_group_subscription( +add_shared_subscription( #{session_id := SessionId, groups := Groups0} = State0, ShareTopicFilter ) -> ?SLOG(info, #{ - msg => agent_add_group_subscription, - topic_filter => ShareTopicFilter + msg => agent_add_shared_subscription, + share_topic_filter => ShareTopicFilter }), - #share{group = Group} = ShareTopicFilter, + GroupId = ?group_id(ShareTopicFilter), Groups1 = Groups0#{ - Group => emqx_ds_shared_sub_group_sm:new(#{ + GroupId => emqx_ds_shared_sub_group_sm:new(#{ session_id => SessionId, - topic_filter => ShareTopicFilter, + share_topic_filter => ShareTopicFilter, agent => this_agent(SessionId), - send_after => send_to_subscription_after(Group) + send_after => send_to_subscription_after(GroupId) }) }, State1 = State0#{groups => Groups1}, @@ -188,9 +248,9 @@ add_group_subscription( fetch_stream_events(#{groups := Groups0} = State0) -> {Groups1, Events} = maps:fold( - fun(Group, GroupSM0, {GroupsAcc, EventsAcc}) -> + fun(GroupId, GroupSM0, {GroupsAcc, EventsAcc}) -> {GroupSM1, Events} = emqx_ds_shared_sub_group_sm:fetch_stream_events(GroupSM0), - {GroupsAcc#{Group => GroupSM1}, [Events | EventsAcc]} + {GroupsAcc#{GroupId => GroupSM1}, [Events | EventsAcc]} end, {#{}, []}, Groups0 @@ -201,20 +261,20 @@ fetch_stream_events(#{groups := Groups0} = State0) -> this_agent(Id) -> emqx_ds_shared_sub_proto:agent(Id, self()). -send_to_subscription_after(Group) -> +send_to_subscription_after(GroupId) -> fun(Time, Msg) -> emqx_persistent_session_ds_shared_subs_agent:send_after( Time, self(), - #message_to_group_sm{group = Group, message = Msg} + #message_to_group_sm{group_id = GroupId, message = Msg} ) end. -with_group_sm(State, Group, Fun) -> +with_group_sm(State, GroupId, Fun) -> case State of - #{groups := #{Group := GSM0} = Groups} -> + #{groups := #{GroupId := GSM0} = Groups} -> #{} = GSM1 = Fun(GSM0), - State#{groups => Groups#{Group => GSM1}}; + State#{groups => Groups#{GroupId => GSM1}}; _ -> %% TODO %% Error? diff --git a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_group_sm.erl b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_group_sm.erl index 7d260dc0b..2b37328a2 100644 --- a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_group_sm.erl +++ b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_group_sm.erl @@ -41,7 +41,7 @@ -type options() :: #{ session_id := emqx_persistent_session_ds:id(), agent := emqx_ds_shared_sub_proto:agent(), - topic_filter := emqx_persistent_session_ds:share_topic_filter(), + share_topic_filter := emqx_persistent_session_ds:share_topic_filter(), send_after := fun((non_neg_integer(), term()) -> reference()) }. @@ -58,19 +58,6 @@ stream => emqx_ds:stream() }. --type external_lease_event() :: - #{ - type => lease, - stream => emqx_ds:stream(), - progress => progress(), - topic_filter => emqx_persistent_session_ds:share_topic_filter() - } - | #{ - type => revoke, - stream => emqx_ds:stream(), - topic_filter => emqx_persistent_session_ds:share_topic_filter() - }. - %% GroupSM States -define(connecting, connecting). @@ -111,7 +98,7 @@ -type timer() :: #timer{}. -type group_sm() :: #{ - topic_filter => emqx_persistent_session_ds:share_topic_filter(), + share_topic_filter => emqx_persistent_session_ds:share_topic_filter(), agent => emqx_ds_shared_sub_proto:agent(), send_after => fun((non_neg_integer(), term()) -> reference()), stream_lease_events => list(stream_lease_event()), @@ -129,7 +116,7 @@ new(#{ session_id := SessionId, agent := Agent, - topic_filter := ShareTopicFilter, + share_topic_filter := ShareTopicFilter, send_after := SendAfter }) -> ?SLOG( @@ -137,32 +124,33 @@ new(#{ #{ msg => group_sm_new, agent => Agent, - topic_filter => ShareTopicFilter + share_topic_filter => ShareTopicFilter } ), GSM0 = #{ id => SessionId, - topic_filter => ShareTopicFilter, + share_topic_filter => ShareTopicFilter, agent => Agent, send_after => SendAfter }, ?tp(warning, group_sm_new, #{ agent => Agent, - topic_filter => ShareTopicFilter + share_topic_filter => ShareTopicFilter }), transition(GSM0, ?connecting, #{}). --spec fetch_stream_events(group_sm()) -> {group_sm(), list(external_lease_event())}. +-spec fetch_stream_events(group_sm()) -> + {group_sm(), [emqx_ds_shared_sub_agent:external_lease_event()]}. fetch_stream_events( #{ state := _State, - topic_filter := TopicFilter, + share_topic_filter := ShareTopicFilter, stream_lease_events := Events0 } = GSM ) -> Events1 = lists:map( fun(Event) -> - Event#{topic_filter => TopicFilter} + Event#{share_topic_filter => ShareTopicFilter} end, Events0 ), @@ -187,18 +175,21 @@ handle_disconnect( %%----------------------------------------------------------------------- %% Connecting state -handle_connecting(#{agent := Agent, topic_filter := ShareTopicFilter} = GSM) -> +handle_connecting(#{agent := Agent, share_topic_filter := ShareTopicFilter} = GSM) -> ?tp(warning, group_sm_enter_connecting, #{ agent => Agent, - topic_filter => ShareTopicFilter + share_topic_filter => ShareTopicFilter }), ok = emqx_ds_shared_sub_registry:lookup_leader(Agent, agent_metadata(GSM), ShareTopicFilter), ensure_state_timeout(GSM, find_leader_timeout, ?dq_config(session_find_leader_timeout_ms)). handle_leader_lease_streams( - #{state := ?connecting, topic_filter := TopicFilter} = GSM0, Leader, StreamProgresses, Version + #{state := ?connecting, share_topic_filter := ShareTopicFilter} = GSM0, + Leader, + StreamProgresses, + Version ) -> - ?tp(debug, leader_lease_streams, #{topic_filter => TopicFilter}), + ?tp(debug, leader_lease_streams, #{share_topic_filter => ShareTopicFilter}), Streams = progresses_to_map(StreamProgresses), StreamLeaseEvents = progresses_to_lease_events(StreamProgresses), transition( @@ -215,12 +206,12 @@ handle_leader_lease_streams( handle_leader_lease_streams(GSM, _Leader, _StreamProgresses, _Version) -> GSM. -handle_find_leader_timeout(#{agent := Agent, topic_filter := TopicFilter} = GSM0) -> +handle_find_leader_timeout(#{agent := Agent, share_topic_filter := ShareTopicFilter} = GSM0) -> ?tp(warning, group_sm_find_leader_timeout, #{ agent => Agent, - topic_filter => TopicFilter + share_topic_filter => ShareTopicFilter }), - ok = emqx_ds_shared_sub_registry:lookup_leader(Agent, agent_metadata(GSM0), TopicFilter), + ok = emqx_ds_shared_sub_registry:lookup_leader(Agent, agent_metadata(GSM0), ShareTopicFilter), GSM1 = ensure_state_timeout( GSM0, find_leader_timeout, ?dq_config(session_find_leader_timeout_ms) ), @@ -238,8 +229,8 @@ handle_replaying(GSM0) -> ), GSM2. -handle_renew_lease_timeout(#{agent := Agent, topic_filter := TopicFilter} = GSM) -> - ?tp(warning, renew_lease_timeout, #{agent => Agent, topic_filter => TopicFilter}), +handle_renew_lease_timeout(#{agent := Agent, share_topic_filter := ShareTopicFilter} = GSM) -> + ?tp(warning, renew_lease_timeout, #{agent => Agent, share_topic_filter => ShareTopicFilter}), transition(GSM, ?connecting, #{}). %%----------------------------------------------------------------------- @@ -429,10 +420,10 @@ handle_stream_progress( handle_stream_progress(#{state := ?disconnected} = GSM, _StreamProgresses) -> GSM. -handle_leader_invalidate(#{agent := Agent, topic_filter := TopicFilter} = GSM) -> +handle_leader_invalidate(#{agent := Agent, share_topic_filter := ShareTopicFilter} = GSM) -> ?tp(warning, shared_sub_group_sm_leader_invalidate, #{ agent => Agent, - topic_filter => TopicFilter + share_topic_filter => ShareTopicFilter }), transition(GSM, ?connecting, #{}). @@ -441,11 +432,11 @@ handle_leader_invalidate(#{agent := Agent, topic_filter := TopicFilter} = GSM) - %%----------------------------------------------------------------------- handle_state_timeout( - #{state := ?connecting, topic_filter := TopicFilter} = GSM, + #{state := ?connecting, share_topic_filter := ShareTopicFilter} = GSM, find_leader_timeout, _Message ) -> - ?tp(debug, find_leader_timeout, #{topic_filter => TopicFilter}), + ?tp(debug, find_leader_timeout, #{share_topic_filter => ShareTopicFilter}), handle_find_leader_timeout(GSM); handle_state_timeout( #{state := ?replaying} = GSM, diff --git a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl index e98c74b27..3a2081f1b 100644 --- a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl +++ b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl @@ -27,8 +27,12 @@ terminate/3 ]). +-type share_topic_filter() :: emqx_persistent_session_ds:share_topic_filter(). + +-type group_id() :: share_topic_filter(). + -type options() :: #{ - topic_filter := emqx_persistent_session_ds:share_topic_filter() + share_topic_filter := share_topic_filter() }. %% Agent states @@ -39,7 +43,7 @@ -define(updating, updating). -type agent_state() :: #{ - %% Our view of group sm's status + %% Our view of group_id sm's status %% it lags the actual state state := ?waiting_replaying | ?replaying | ?waiting_updating | ?updating, prev_version := emqx_maybe:t(emqx_ds_shared_sub_proto:version()), @@ -62,7 +66,7 @@ %% %% Persistent data %% - group := emqx_types:group(), + group_id := group_id(), topic := emqx_types:topic(), %% For ds router, not an actual session_id router_id := binary(), @@ -119,9 +123,9 @@ register(Pid, Fun) -> %% Internal API %%-------------------------------------------------------------------- -child_spec(#{topic_filter := TopicFilter} = Options) -> +child_spec(#{share_topic_filter := ShareTopicFilter} = Options) -> #{ - id => id(TopicFilter), + id => id(ShareTopicFilter), start => {?MODULE, start_link, [Options]}, restart => temporary, shutdown => 5000, @@ -131,8 +135,8 @@ child_spec(#{topic_filter := TopicFilter} = Options) -> start_link(Options) -> gen_statem:start_link(?MODULE, [Options], []). -id(#share{group = Group} = _TopicFilter) -> - {?MODULE, Group}. +id(ShareTopicFilter) -> + {?MODULE, ShareTopicFilter}. %%-------------------------------------------------------------------- %% gen_statem callbacks @@ -140,9 +144,9 @@ id(#share{group = Group} = _TopicFilter) -> callback_mode() -> [handle_event_function, state_enter]. -init([#{topic_filter := #share{group = Group, topic = Topic}} = _Options]) -> +init([#{share_topic_filter := #share{topic = Topic} = ShareTopicFilter} = _Options]) -> Data = #{ - group => Group, + group_id => ShareTopicFilter, topic => Topic, router_id => gen_router_id(), start_time => now_ms() - ?START_TIME_THRESHOLD, @@ -463,14 +467,14 @@ select_streams_for_assign(Data0, _Agent, AssignCount) -> %% Handle a newly connected agent connect_agent( - #{group := Group, agents := Agents} = Data, + #{group_id := GroupId, agents := Agents} = Data, Agent, AgentMetadata ) -> ?SLOG(info, #{ msg => leader_agent_connected, agent => Agent, - group => Group + group_id => GroupId }), case Agents of #{Agent := AgentState} -> @@ -583,22 +587,22 @@ renew_leases(#{agents := AgentStates} = Data) -> ), Data. -renew_lease(#{group := Group}, Agent, #{state := ?replaying, version := Version}) -> - ok = emqx_ds_shared_sub_proto:leader_renew_stream_lease(Agent, Group, Version); -renew_lease(#{group := Group}, Agent, #{state := ?waiting_replaying, version := Version}) -> - ok = emqx_ds_shared_sub_proto:leader_renew_stream_lease(Agent, Group, Version); -renew_lease(#{group := Group} = Data, Agent, #{ +renew_lease(#{group_id := GroupId}, Agent, #{state := ?replaying, version := Version}) -> + ok = emqx_ds_shared_sub_proto:leader_renew_stream_lease(Agent, GroupId, Version); +renew_lease(#{group_id := GroupId}, Agent, #{state := ?waiting_replaying, version := Version}) -> + ok = emqx_ds_shared_sub_proto:leader_renew_stream_lease(Agent, GroupId, Version); +renew_lease(#{group_id := GroupId} = Data, Agent, #{ streams := Streams, state := ?waiting_updating, version := Version, prev_version := PrevVersion }) -> StreamProgresses = stream_progresses(Data, Streams), ok = emqx_ds_shared_sub_proto:leader_update_streams( - Agent, Group, PrevVersion, Version, StreamProgresses + Agent, GroupId, PrevVersion, Version, StreamProgresses ), - ok = emqx_ds_shared_sub_proto:leader_renew_stream_lease(Agent, Group, PrevVersion, Version); -renew_lease(#{group := Group}, Agent, #{ + ok = emqx_ds_shared_sub_proto:leader_renew_stream_lease(Agent, GroupId, PrevVersion, Version); +renew_lease(#{group_id := GroupId}, Agent, #{ state := ?updating, version := Version, prev_version := PrevVersion }) -> - ok = emqx_ds_shared_sub_proto:leader_renew_stream_lease(Agent, Group, PrevVersion, Version). + ok = emqx_ds_shared_sub_proto:leader_renew_stream_lease(Agent, GroupId, PrevVersion, Version). %%-------------------------------------------------------------------- %% Handle stream progress updates from agent in replaying state @@ -802,7 +806,7 @@ update_agent_stream_states(Data0, Agent, AgentStreamProgresses, VersionOld, Vers %%-------------------------------------------------------------------- agent_transition_to_waiting_updating( - #{group := Group} = Data, + #{group_id := GroupId} = Data, Agent, #{state := OldState, version := Version, prev_version := undefined} = AgentState0, Streams, @@ -825,19 +829,19 @@ agent_transition_to_waiting_updating( AgentState2 = renew_no_replaying_deadline(AgentState1), StreamProgresses = stream_progresses(Data, Streams), ok = emqx_ds_shared_sub_proto:leader_update_streams( - Agent, Group, Version, NewVersion, StreamProgresses + Agent, GroupId, Version, NewVersion, StreamProgresses ), AgentState2. agent_transition_to_waiting_replaying( - #{group := Group} = _Data, Agent, #{state := OldState, version := Version} = AgentState0 + #{group_id := GroupId} = _Data, Agent, #{state := OldState, version := Version} = AgentState0 ) -> ?tp(warning, shared_sub_leader_agent_state_transition, #{ agent => Agent, old_state => OldState, new_state => ?waiting_replaying }), - ok = emqx_ds_shared_sub_proto:leader_renew_stream_lease(Agent, Group, Version), + ok = emqx_ds_shared_sub_proto:leader_renew_stream_lease(Agent, GroupId, Version), AgentState1 = AgentState0#{ state => ?waiting_replaying, revoked_streams => [] @@ -845,7 +849,7 @@ agent_transition_to_waiting_replaying( renew_no_replaying_deadline(AgentState1). agent_transition_to_initial_waiting_replaying( - #{group := Group} = Data, Agent, AgentMetadata, InitialStreams + #{group_id := GroupId} = Data, Agent, AgentMetadata, InitialStreams ) -> ?tp(warning, shared_sub_leader_agent_state_transition, #{ agent => Agent, @@ -856,7 +860,7 @@ agent_transition_to_initial_waiting_replaying( StreamProgresses = stream_progresses(Data, InitialStreams), Leader = this_leader(Data), ok = emqx_ds_shared_sub_proto:leader_lease_streams( - Agent, Group, Leader, StreamProgresses, Version + Agent, GroupId, Leader, StreamProgresses, Version ), AgentState = #{ metadata => AgentMetadata, @@ -1015,8 +1019,8 @@ drop_agent(#{agents := Agents} = Data0, Agent) -> ?tp(warning, shared_sub_leader_drop_agent, #{agent => Agent}), Data1#{agents => maps:remove(Agent, Agents)}. -invalidate_agent(#{group := Group}, Agent) -> - ok = emqx_ds_shared_sub_proto:leader_invalidate(Agent, Group). +invalidate_agent(#{group_id := GroupId}, Agent) -> + ok = emqx_ds_shared_sub_proto:leader_invalidate(Agent, GroupId). drop_invalidate_agent(Data0, Agent) -> Data1 = drop_agent(Data0, Agent), diff --git a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_proto.erl b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_proto.erl index e74fae19c..383f66ff2 100644 --- a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_proto.erl +++ b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_proto.erl @@ -33,7 +33,7 @@ -type agent() :: ?agent(emqx_persistent_session_ds:id(), pid()). -type leader() :: pid(). --type topic_filter() :: emqx_persistent_session_ds:share_topic_filter(). +-type share_topic_filter() :: emqx_persistent_session_ds:share_topic_filter(). -type group() :: emqx_types:group(). -type version() :: non_neg_integer(). -type agent_metadata() :: #{ @@ -63,8 +63,8 @@ %% agent -> leader messages --spec agent_connect_leader(leader(), agent(), agent_metadata(), topic_filter()) -> ok. -agent_connect_leader(ToLeader, FromAgent, AgentMetadata, TopicFilter) when +-spec agent_connect_leader(leader(), agent(), agent_metadata(), share_topic_filter()) -> ok. +agent_connect_leader(ToLeader, FromAgent, AgentMetadata, ShareTopicFilter) when ?is_local_leader(ToLeader) -> ?tp(warning, shared_sub_proto_msg, #{ @@ -72,13 +72,13 @@ agent_connect_leader(ToLeader, FromAgent, AgentMetadata, TopicFilter) when to_leader => ToLeader, from_agent => FromAgent, agent_metadata => AgentMetadata, - topic_filter => TopicFilter + share_topic_filter => ShareTopicFilter }), - _ = erlang:send(ToLeader, ?agent_connect_leader(FromAgent, AgentMetadata, TopicFilter)), + _ = erlang:send(ToLeader, ?agent_connect_leader(FromAgent, AgentMetadata, ShareTopicFilter)), ok; -agent_connect_leader(ToLeader, FromAgent, AgentMetadata, TopicFilter) -> +agent_connect_leader(ToLeader, FromAgent, AgentMetadata, ShareTopicFilter) -> emqx_ds_shared_sub_proto_v1:agent_connect_leader( - ?leader_node(ToLeader), ToLeader, FromAgent, AgentMetadata, TopicFilter + ?leader_node(ToLeader), ToLeader, FromAgent, AgentMetadata, ShareTopicFilter ). -spec agent_update_stream_states(leader(), agent(), list(agent_stream_progress()), version()) -> ok. diff --git a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_proto.hrl b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_proto.hrl index f8158c918..bf54b2930 100644 --- a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_proto.hrl +++ b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_proto.hrl @@ -21,16 +21,16 @@ %% Agent messages sent to the leader. %% Leader talks to many agents, `agent` field is used to identify the sender. --define(agent_connect_leader(Agent, AgentMetadata, TopicFilter), #{ +-define(agent_connect_leader(Agent, AgentMetadata, ShareTopicFilter), #{ type => ?agent_connect_leader_msg, - topic_filter => TopicFilter, + share_topic_filter => ShareTopicFilter, agent_metadata => AgentMetadata, agent => Agent }). --define(agent_connect_leader_match(Agent, AgentMetadata, TopicFilter), #{ +-define(agent_connect_leader_match(Agent, AgentMetadata, ShareTopicFilter), #{ type := ?agent_connect_leader_msg, - topic_filter := TopicFilter, + share_topic_filter := ShareTopicFilter, agent_metadata := AgentMetadata, agent := Agent }). @@ -81,77 +81,77 @@ %% leader messages, sent from the leader to the agent %% Agent may have several shared subscriptions, so may talk to several leaders -%% `group` field is used to identify the leader. +%% `group_id` field is used to identify the leader. -define(leader_lease_streams_msg, leader_lease_streams). -define(leader_renew_stream_lease_msg, leader_renew_stream_lease). --define(leader_lease_streams(Group, Leader, Streams, Version), #{ +-define(leader_lease_streams(GrouId, Leader, Streams, Version), #{ type => ?leader_lease_streams_msg, streams => Streams, version => Version, leader => Leader, - group => Group + group_id => GrouId }). --define(leader_lease_streams_match(Group, Leader, Streams, Version), #{ +-define(leader_lease_streams_match(GroupId, Leader, Streams, Version), #{ type := ?leader_lease_streams_msg, streams := Streams, version := Version, leader := Leader, - group := Group + group_id := GroupId }). --define(leader_renew_stream_lease(Group, Version), #{ +-define(leader_renew_stream_lease(GroupId, Version), #{ type => ?leader_renew_stream_lease_msg, version => Version, - group => Group + group_id => GroupId }). --define(leader_renew_stream_lease_match(Group, Version), #{ +-define(leader_renew_stream_lease_match(GroupId, Version), #{ type := ?leader_renew_stream_lease_msg, version := Version, - group := Group + group_id := GroupId }). --define(leader_renew_stream_lease(Group, VersionOld, VersionNew), #{ +-define(leader_renew_stream_lease(GroupId, VersionOld, VersionNew), #{ type => ?leader_renew_stream_lease_msg, version_old => VersionOld, version_new => VersionNew, - group => Group + group_id => GroupId }). --define(leader_renew_stream_lease_match(Group, VersionOld, VersionNew), #{ +-define(leader_renew_stream_lease_match(GroupId, VersionOld, VersionNew), #{ type := ?leader_renew_stream_lease_msg, version_old := VersionOld, version_new := VersionNew, - group := Group + group_id := GroupId }). --define(leader_update_streams(Group, VersionOld, VersionNew, StreamsNew), #{ +-define(leader_update_streams(GroupId, VersionOld, VersionNew, StreamsNew), #{ type => leader_update_streams, version_old => VersionOld, version_new => VersionNew, streams_new => StreamsNew, - group => Group + group_id => GroupId }). --define(leader_update_streams_match(Group, VersionOld, VersionNew, StreamsNew), #{ +-define(leader_update_streams_match(GroupId, VersionOld, VersionNew, StreamsNew), #{ type := leader_update_streams, version_old := VersionOld, version_new := VersionNew, streams_new := StreamsNew, - group := Group + group_id := GroupId }). --define(leader_invalidate(Group), #{ +-define(leader_invalidate(GroupId), #{ type => leader_invalidate, - group => Group + group_id => GroupId }). --define(leader_invalidate_match(Group), #{ +-define(leader_invalidate_match(GroupId), #{ type := leader_invalidate, - group := Group + group_id := GroupId }). %% Helpers diff --git a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_registry.erl b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_registry.erl index bc732249a..eae212458 100644 --- a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_registry.erl +++ b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_registry.erl @@ -26,7 +26,7 @@ -record(lookup_leader, { agent :: emqx_ds_shared_sub_proto:agent(), agent_metadata :: emqx_ds_shared_sub_proto:agent_metadata(), - topic_filter :: emqx_persistent_session_ds:share_topic_filter() + share_topic_filter :: emqx_persistent_session_ds:share_topic_filter() }). -define(gproc_id(ID), {n, l, ID}). @@ -40,9 +40,9 @@ emqx_ds_shared_sub_proto:agent_metadata(), emqx_persistent_session_ds:share_topic_filter() ) -> ok. -lookup_leader(Agent, AgentMetadata, TopicFilter) -> +lookup_leader(Agent, AgentMetadata, ShareTopicFilter) -> gen_server:cast(?MODULE, #lookup_leader{ - agent = Agent, agent_metadata = AgentMetadata, topic_filter = TopicFilter + agent = Agent, agent_metadata = AgentMetadata, share_topic_filter = ShareTopicFilter }). %%-------------------------------------------------------------------- @@ -72,9 +72,14 @@ handle_call(_Request, _From, State) -> {reply, {error, unknown_request}, State}. handle_cast( - #lookup_leader{agent = Agent, agent_metadata = AgentMetadata, topic_filter = TopicFilter}, State + #lookup_leader{ + agent = Agent, + agent_metadata = AgentMetadata, + share_topic_filter = ShareTopicFilter + }, + State ) -> - State1 = do_lookup_leader(Agent, AgentMetadata, TopicFilter, State), + State1 = do_lookup_leader(Agent, AgentMetadata, ShareTopicFilter, State), {noreply, State1}. handle_info(_Info, State) -> @@ -87,15 +92,15 @@ terminate(_Reason, _State) -> %% Internal functions %%-------------------------------------------------------------------- -do_lookup_leader(Agent, AgentMetadata, TopicFilter, State) -> +do_lookup_leader(Agent, AgentMetadata, ShareTopicFilter, State) -> %% TODO https://emqx.atlassian.net/browse/EMQX-12309 %% Cluster-wide unique leader election should be implemented - Id = emqx_ds_shared_sub_leader:id(TopicFilter), + Id = emqx_ds_shared_sub_leader:id(ShareTopicFilter), LeaderPid = case gproc:where(?gproc_id(Id)) of undefined -> {ok, Pid} = emqx_ds_shared_sub_leader_sup:start_leader(#{ - topic_filter => TopicFilter + share_topic_filter => ShareTopicFilter }), {ok, NewLeaderPid} = emqx_ds_shared_sub_leader:register( Pid, @@ -111,10 +116,10 @@ do_lookup_leader(Agent, AgentMetadata, TopicFilter, State) -> ?SLOG(info, #{ msg => lookup_leader, agent => Agent, - topic_filter => TopicFilter, + share_topic_filter => ShareTopicFilter, leader => LeaderPid }), ok = emqx_ds_shared_sub_proto:agent_connect_leader( - LeaderPid, Agent, AgentMetadata, TopicFilter + LeaderPid, Agent, AgentMetadata, ShareTopicFilter ), State. diff --git a/apps/emqx_ds_shared_sub/src/proto/emqx_ds_shared_sub_proto_v1.erl b/apps/emqx_ds_shared_sub/src/proto/emqx_ds_shared_sub_proto_v1.erl index 52f64937d..17ceb4876 100644 --- a/apps/emqx_ds_shared_sub/src/proto/emqx_ds_shared_sub_proto_v1.erl +++ b/apps/emqx_ds_shared_sub/src/proto/emqx_ds_shared_sub_proto_v1.erl @@ -33,9 +33,9 @@ introduced_in() -> emqx_ds_shared_sub_proto:agent_metadata(), emqx_persistent_session_ds:share_topic_filter() ) -> ok. -agent_connect_leader(Node, ToLeader, FromAgent, AgentMetadata, TopicFilter) -> +agent_connect_leader(Node, ToLeader, FromAgent, AgentMetadata, ShareTopicFilter) -> erpc:cast(Node, emqx_ds_shared_sub_proto, agent_connect_leader, [ - ToLeader, FromAgent, AgentMetadata, TopicFilter + ToLeader, FromAgent, AgentMetadata, ShareTopicFilter ]). -spec agent_update_stream_states( From 8705956cdcafc8576b4c1c7b6295a269ef97f2c7 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Wed, 10 Jul 2024 19:31:40 +0300 Subject: [PATCH 110/131] feat(queue): update docs --- apps/emqx_ds_shared_sub/README.md | 1 - .../src/emqx_ds_shared_sub_agent.erl | 23 +++++++++++++------ 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/apps/emqx_ds_shared_sub/README.md b/apps/emqx_ds_shared_sub/README.md index 456d5fe52..6ff57b84e 100644 --- a/apps/emqx_ds_shared_sub/README.md +++ b/apps/emqx_ds_shared_sub/README.md @@ -13,7 +13,6 @@ On the code level, the application is organized in the following way: * The nesting reflects nesting/ownership of entity states. * The bold arrow represent the [most complex interaction](https://github.com/emqx/eip/blob/main/active/0028-durable-shared-subscriptions.md#shared-subscription-session-handler), between session-side group subscription state machine (**GroupSM**) and the shared subscription leader (**Leader**). - # Contributing Please see our [contributing.md](../../CONTRIBUTING.md). diff --git a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_agent.erl b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_agent.erl index 005307ca2..fea711d0f 100644 --- a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_agent.erl +++ b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_agent.erl @@ -54,14 +54,23 @@ session_id := emqx_persistent_session_ds:id() }. -%% Techinically, group_id and share_topic_filter are the same. -%% However, we speak in the terms of share_topic_filter in the API, -%% which is known to the shared subscription handler of persistent session. +%% We speak in the terms of share_topic_filter in the module API +%% which is consumed by persistent session. %% -%% And we speak in the terms of group_id internally: -%% * we keep group_sm's in the state by group_id -%% * we use group_id to address group_sm's, e.g. when sending messages to them -%% from leader or from themselves. +%% We speak in the terms of group_id internally: +%% * to identfy shared subscription's group_sm in the state; +%% * to addres agent's group_sm while communicating with leader. +%% * to identify the leader itself. +%% +%% share_topic_filter should be uniquely determined by group_id. See MQTT 5.0 spec: +%% +%% > Note that "$share/consumer1//finance" and "$share/consumer1/sport/tennis/+" +%% > are distinct shared subscriptions, even though they have the same ShareName. +%% > While they might be related in some way, no specific relationship between them +%% > is implied by them having the same ShareName. +%% +%% So we just use the full share_topic_filter record as group_id. + -define(group_id(ShareTopicFilter), ShareTopicFilter). -define(share_topic_filter(GroupId), GroupId). From a97a0d64006a3ecadd285d1365c032b2aeca4676 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Wed, 10 Jul 2024 20:25:53 +0300 Subject: [PATCH 111/131] feat(queue): fix dialyzer issues --- ...x_persistent_session_ds_shared_subs_agent.erl | 4 ++-- .../src/emqx_ds_shared_sub_agent.erl | 3 ++- .../src/emqx_ds_shared_sub_group_sm.erl | 16 ++++++++-------- apps/emqx_durable_storage/src/emqx_ds.erl | 2 +- 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs_agent.erl b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs_agent.erl index 022963ad9..dff66de0f 100644 --- a/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs_agent.erl +++ b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs_agent.erl @@ -52,7 +52,7 @@ t/0, subscription/0, session_id/0, - stream_lease/0, + stream_lease_event/0, opts/0 ]). @@ -84,7 +84,7 @@ -callback can_subscribe(t(), share_topic_filter(), emqx_types:subopts()) -> ok | {error, term()}. -callback on_subscribe(t(), share_topic_filter(), emqx_types:subopts()) -> t(). -callback on_unsubscribe(t(), share_topic_filter(), [stream_progress()]) -> t(). --callback on_disconnect(t(), #{share_topic_filter() => [stream_progress()]}) -> t(). +-callback on_disconnect(t(), [stream_progress()]) -> t(). -callback renew_streams(t()) -> {[stream_lease_event()], t()}. -callback on_stream_progress(t(), #{share_topic_filter() => [stream_progress()]}) -> t(). -callback on_info(t(), term()) -> t(). diff --git a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_agent.erl b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_agent.erl index fea711d0f..5b71a93e5 100644 --- a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_agent.erl +++ b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_agent.erl @@ -126,7 +126,8 @@ on_subscribe(State0, ShareTopicFilter, _SubOpts) -> on_unsubscribe(State, ShareTopicFilter, GroupProgress) -> delete_shared_subscription(State, ShareTopicFilter, GroupProgress). --spec renew_streams(t()) -> {[emqx_persistent_session_ds_shared_subs:agent_stream_event()], t()}. +-spec renew_streams(t()) -> + {[emqx_persistent_session_ds_shared_subs_agent:stream_lease_event()], t()}. renew_streams(#{} = State) -> fetch_stream_events(State). diff --git a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_group_sm.erl b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_group_sm.erl index 2b37328a2..a648bbaef 100644 --- a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_group_sm.erl +++ b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_group_sm.erl @@ -33,7 +33,7 @@ ]). -export_type([ - group_sm/0, + t/0, options/0, state/0 ]). @@ -97,7 +97,7 @@ -type timer_name() :: atom(). -type timer() :: #timer{}. --type group_sm() :: #{ +-type t() :: #{ share_topic_filter => emqx_persistent_session_ds:share_topic_filter(), agent => emqx_ds_shared_sub_proto:agent(), send_after => fun((non_neg_integer(), term()) -> reference()), @@ -112,7 +112,7 @@ %% API %%----------------------------------------------------------------------- --spec new(options()) -> group_sm(). +-spec new(options()) -> t(). new(#{ session_id := SessionId, agent := Agent, @@ -139,8 +139,8 @@ new(#{ }), transition(GSM0, ?connecting, #{}). --spec fetch_stream_events(group_sm()) -> - {group_sm(), [emqx_ds_shared_sub_agent:external_lease_event()]}. +-spec fetch_stream_events(t()) -> + {t(), [emqx_ds_shared_sub_agent:external_lease_event()]}. fetch_stream_events( #{ state := _State, @@ -156,7 +156,7 @@ fetch_stream_events( ), {GSM#{stream_lease_events => []}, Events1}. --spec handle_disconnect(group_sm(), emqx_ds_shared_sub_proto:agent_stream_progress()) -> group_sm(). +-spec handle_disconnect(t(), emqx_ds_shared_sub_proto:agent_stream_progress()) -> t(). handle_disconnect(#{state := ?connecting} = GSM, _StreamProgresses) -> transition(GSM, ?disconnected, #{}); handle_disconnect( @@ -378,8 +378,8 @@ handle_leader_renew_stream_lease(GSM, VersionOld, VersionNew) -> }), transition(GSM, ?connecting, #{}). --spec handle_stream_progress(group_sm(), list(emqx_ds_shared_sub_proto:agent_stream_progress())) -> - group_sm(). +-spec handle_stream_progress(t(), list(emqx_ds_shared_sub_proto:agent_stream_progress())) -> + t(). handle_stream_progress(#{state := ?connecting} = GSM, _StreamProgresses) -> GSM; handle_stream_progress( diff --git a/apps/emqx_durable_storage/src/emqx_ds.erl b/apps/emqx_durable_storage/src/emqx_ds.erl index 6aaba205d..38d63e41f 100644 --- a/apps/emqx_durable_storage/src/emqx_ds.erl +++ b/apps/emqx_durable_storage/src/emqx_ds.erl @@ -132,7 +132,7 @@ %% TODO: Not implemented -type iterator_id() :: term(). --opaque iterator() :: ds_specific_iterator(). +-type iterator() :: ds_specific_iterator(). -opaque delete_iterator() :: ds_specific_delete_iterator(). From b8e8f7c8e06911d0990e0e6ec90894f1aed2b199 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Wed, 10 Jul 2024 22:23:50 +0300 Subject: [PATCH 112/131] feat(queue): add pre_renew_streams callback --- apps/emqx/src/emqx_persistent_session_ds.erl | 2 +- .../emqx_persistent_session_ds_shared_subs.erl | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/apps/emqx/src/emqx_persistent_session_ds.erl b/apps/emqx/src/emqx_persistent_session_ds.erl index 124b1919a..0984f9de8 100644 --- a/apps/emqx/src/emqx_persistent_session_ds.erl +++ b/apps/emqx/src/emqx_persistent_session_ds.erl @@ -624,7 +624,7 @@ handle_timeout(ClientInfo, ?TIMER_GET_STREAMS, Session0 = #{s := S0, shared_sub_ %% `gc` and `renew_streams` methods may drop unsubscribed streams. %% Shared subscription handler must have a chance to see unsubscribed streams %% in the fully replayed state. - {S1, SharedSubS1} = emqx_persistent_session_ds_shared_subs:on_streams_replay(S0, SharedSubS0), + {S1, SharedSubS1} = emqx_persistent_session_ds_shared_subs:pre_renew_streams(S0, SharedSubS0), S2 = emqx_persistent_session_ds_subs:gc(S1), S3 = emqx_persistent_session_ds_stream_scheduler:renew_streams(S2), {S, SharedSubS} = emqx_persistent_session_ds_shared_subs:renew_streams(S3, SharedSubS1), diff --git a/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl index 506114f35..1b7a1b420 100644 --- a/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl +++ b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl @@ -46,6 +46,7 @@ on_streams_replay/2, on_info/3, + pre_renew_streams/2, renew_streams/2, to_map/2 ]). @@ -299,6 +300,14 @@ schedule_unsubscribe( SharedSubS0#{scheduled_actions := ScheduledActions1} end. +%%-------------------------------------------------------------------- +%% pre_renew_streams + +-spec pre_renew_streams(emqx_persistent_session_ds_state:t(), t()) -> + {emqx_persistent_session_ds_state:t(), t()}. +pre_renew_streams(S, SharedSubS) -> + on_streams_replay(S, SharedSubS). + %%-------------------------------------------------------------------- %% renew_streams From 9307a82004dd5728b2d2476ef164a667b2deaa20 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Wed, 10 Jul 2024 22:25:13 +0300 Subject: [PATCH 113/131] feat(queue): rearrange leader's code --- .../src/emqx_ds_shared_sub_leader.erl | 179 +++++++++--------- 1 file changed, 85 insertions(+), 94 deletions(-) diff --git a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl index 3a2081f1b..24b78155f 100644 --- a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl +++ b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl @@ -198,7 +198,6 @@ handle_event({timeout, #renew_leases{}}, #renew_leases{}, ?leader_active, Data0) {{timeout, #renew_leases{}}, ?dq_config(leader_renew_lease_interval_ms), #renew_leases{}}}; %% drop_timeout timer handle_event({timeout, #drop_timeout{}}, #drop_timeout{}, ?leader_active, Data0) -> - % ?tp(warning, shared_sub_leader_timeout, #{timeout => drop_timeout}), Data1 = drop_timeout_agents(Data0), {keep_state, Data1, {{timeout, #drop_timeout{}}, ?dq_config(leader_drop_timeout_interval_ms), #drop_timeout{}}}; @@ -207,7 +206,6 @@ handle_event({timeout, #drop_timeout{}}, #drop_timeout{}, ?leader_active, Data0) handle_event( info, ?agent_connect_leader_match(Agent, AgentMetadata, _TopicFilter), ?leader_active, Data0 ) -> - % ?tp(warning, shared_sub_leader_connect_agent, #{agent => Agent}), Data1 = connect_agent(Data0, Agent, AgentMetadata), {keep_state, Data1}; handle_event( @@ -216,7 +214,6 @@ handle_event( ?leader_active, Data0 ) -> - % ?tp(warning, shared_sub_leader_update_stream_states, #{agent => Agent, version => Version}), Data1 = with_agent(Data0, Agent, fun() -> update_agent_stream_states(Data0, Agent, StreamProgresses, Version) end), @@ -227,9 +224,6 @@ handle_event( ?leader_active, Data0 ) -> - % ?tp(warning, shared_sub_leader_update_stream_states, #{ - % agent => Agent, version_old => VersionOld, version_new => VersionNew - % }), Data1 = with_agent(Data0, Agent, fun() -> update_agent_stream_states(Data0, Agent, StreamProgresses, VersionOld, VersionNew) end), @@ -240,9 +234,6 @@ handle_event( ?leader_active, Data0 ) -> - % ?tp(warning, shared_sub_leader_disconnect, #{ - % agent => Agent, version => Version - % }), Data1 = with_agent(Data0, Agent, fun() -> disconnect_agent(Data0, Agent, StreamProgresses, Version) end), @@ -463,6 +454,69 @@ select_streams_for_assign(Data0, _Agent, AssignCount) -> UnassignedStreams = unassigned_streams(Data0), lists:sublist(shuffle(UnassignedStreams), AssignCount). +%%-------------------------------------------------------------------- +%% renew_leases - send lease confirmations to agents + +renew_leases(#{agents := AgentStates} = Data) -> + ?tp(warning, shared_sub_leader_renew_leases, #{agents => maps:keys(AgentStates)}), + ok = lists:foreach( + fun({Agent, AgentState}) -> + renew_lease(Data, Agent, AgentState) + end, + maps:to_list(AgentStates) + ), + Data. + +renew_lease(#{group_id := GroupId}, Agent, #{state := ?replaying, version := Version}) -> + ok = emqx_ds_shared_sub_proto:leader_renew_stream_lease(Agent, GroupId, Version); +renew_lease(#{group_id := GroupId}, Agent, #{state := ?waiting_replaying, version := Version}) -> + ok = emqx_ds_shared_sub_proto:leader_renew_stream_lease(Agent, GroupId, Version); +renew_lease(#{group_id := GroupId} = Data, Agent, #{ + streams := Streams, state := ?waiting_updating, version := Version, prev_version := PrevVersion +}) -> + StreamProgresses = stream_progresses(Data, Streams), + ok = emqx_ds_shared_sub_proto:leader_update_streams( + Agent, GroupId, PrevVersion, Version, StreamProgresses + ), + ok = emqx_ds_shared_sub_proto:leader_renew_stream_lease(Agent, GroupId, PrevVersion, Version); +renew_lease(#{group_id := GroupId}, Agent, #{ + state := ?updating, version := Version, prev_version := PrevVersion +}) -> + ok = emqx_ds_shared_sub_proto:leader_renew_stream_lease(Agent, GroupId, PrevVersion, Version). + +%%-------------------------------------------------------------------- +%% Drop agents that stopped reporting progress + +drop_timeout_agents(#{agents := Agents} = Data) -> + Now = now_ms_monotonic(), + lists:foldl( + fun( + {Agent, + #{update_deadline := UpdateDeadline, not_replaying_deadline := NoReplayingDeadline} = + _AgentState}, + DataAcc + ) -> + case + (UpdateDeadline < Now) orelse + (is_integer(NoReplayingDeadline) andalso NoReplayingDeadline < Now) + of + true -> + ?SLOG(info, #{ + msg => leader_agent_timeout, + now => Now, + update_deadline => UpdateDeadline, + not_replaying_deadline => NoReplayingDeadline, + agent => Agent + }), + drop_invalidate_agent(DataAcc, Agent); + false -> + DataAcc + end + end, + Data, + maps:to_list(Agents) + ). + %%-------------------------------------------------------------------- %% Handle a newly connected agent @@ -519,91 +573,6 @@ reconnect_agent( Data2 = unassign_streams(Data1, OldRevokedStreams), Data2. -%%-------------------------------------------------------------------- -%% Disconnect agent gracefully - -disconnect_agent(Data0, Agent, AgentStreamProgresses, Version) -> - case get_agent_state(Data0, Agent) of - #{version := Version} -> - ?tp(warning, shared_sub_leader_disconnect_agent, #{ - agent => Agent, - version => Version - }), - Data1 = update_agent_stream_states(Data0, Agent, AgentStreamProgresses, Version), - Data2 = drop_agent(Data1, Agent), - Data2; - _ -> - ?tp(warning, shared_sub_leader_unexpected_disconnect, #{ - agent => Agent, - version => Version - }), - Data1 = drop_agent(Data0, Agent), - Data1 - end. - -%%-------------------------------------------------------------------- -%% Drop agents that stopped reporting progress - -drop_timeout_agents(#{agents := Agents} = Data) -> - Now = now_ms_monotonic(), - lists:foldl( - fun( - {Agent, - #{update_deadline := UpdateDeadline, not_replaying_deadline := NoReplayingDeadline} = - _AgentState}, - DataAcc - ) -> - case - (UpdateDeadline < Now) orelse - (is_integer(NoReplayingDeadline) andalso NoReplayingDeadline < Now) - of - true -> - ?SLOG(info, #{ - msg => leader_agent_timeout, - now => Now, - update_deadline => UpdateDeadline, - not_replaying_deadline => NoReplayingDeadline, - agent => Agent - }), - drop_invalidate_agent(DataAcc, Agent); - false -> - DataAcc - end - end, - Data, - maps:to_list(Agents) - ). - -%%-------------------------------------------------------------------- -%% Send lease confirmations to agents - -renew_leases(#{agents := AgentStates} = Data) -> - ?tp(warning, shared_sub_leader_renew_leases, #{agents => maps:keys(AgentStates)}), - ok = lists:foreach( - fun({Agent, AgentState}) -> - renew_lease(Data, Agent, AgentState) - end, - maps:to_list(AgentStates) - ), - Data. - -renew_lease(#{group_id := GroupId}, Agent, #{state := ?replaying, version := Version}) -> - ok = emqx_ds_shared_sub_proto:leader_renew_stream_lease(Agent, GroupId, Version); -renew_lease(#{group_id := GroupId}, Agent, #{state := ?waiting_replaying, version := Version}) -> - ok = emqx_ds_shared_sub_proto:leader_renew_stream_lease(Agent, GroupId, Version); -renew_lease(#{group_id := GroupId} = Data, Agent, #{ - streams := Streams, state := ?waiting_updating, version := Version, prev_version := PrevVersion -}) -> - StreamProgresses = stream_progresses(Data, Streams), - ok = emqx_ds_shared_sub_proto:leader_update_streams( - Agent, GroupId, PrevVersion, Version, StreamProgresses - ), - ok = emqx_ds_shared_sub_proto:leader_renew_stream_lease(Agent, GroupId, PrevVersion, Version); -renew_lease(#{group_id := GroupId}, Agent, #{ - state := ?updating, version := Version, prev_version := PrevVersion -}) -> - ok = emqx_ds_shared_sub_proto:leader_renew_stream_lease(Agent, GroupId, PrevVersion, Version). - %%-------------------------------------------------------------------- %% Handle stream progress updates from agent in replaying state @@ -801,6 +770,28 @@ update_agent_stream_states(Data0, Agent, AgentStreamProgresses, VersionOld, Vers drop_invalidate_agent(Data0, Agent) end. +%%-------------------------------------------------------------------- +%% Disconnect agent gracefully + +disconnect_agent(Data0, Agent, AgentStreamProgresses, Version) -> + case get_agent_state(Data0, Agent) of + #{version := Version} -> + ?tp(warning, shared_sub_leader_disconnect_agent, #{ + agent => Agent, + version => Version + }), + Data1 = update_agent_stream_states(Data0, Agent, AgentStreamProgresses, Version), + Data2 = drop_agent(Data1, Agent), + Data2; + _ -> + ?tp(warning, shared_sub_leader_unexpected_disconnect, #{ + agent => Agent, + version => Version + }), + Data1 = drop_agent(Data0, Agent), + Data1 + end. + %%-------------------------------------------------------------------- %% Agent state transitions %%-------------------------------------------------------------------- From bab526be242490d3cc98f82c2880e8e581b76deb Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Wed, 10 Jul 2024 23:05:40 +0300 Subject: [PATCH 114/131] feat(queue): self-revoke all shared streams on session open --- .../emqx_persistent_session_ds_shared_subs.erl | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl index 1b7a1b420..bbaf3fd10 100644 --- a/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl +++ b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl @@ -118,19 +118,20 @@ new(Opts) -> -spec open(emqx_persistent_session_ds_state:t(), opts()) -> {ok, emqx_persistent_session_ds_state:t(), t()}. -open(S, Opts) -> +open(S0, Opts) -> SharedSubscriptions = fold_shared_subs( fun(#share{} = ShareTopicFilter, Sub, Acc) -> - [{ShareTopicFilter, to_agent_subscription(S, Sub)} | Acc] + [{ShareTopicFilter, to_agent_subscription(S0, Sub)} | Acc] end, [], - S + S0 ), Agent = emqx_persistent_session_ds_shared_subs_agent:open( SharedSubscriptions, agent_opts(Opts) ), SharedSubS = #{agent => Agent, scheduled_actions => #{}}, - {ok, S, SharedSubS}. + S1 = revoke_all_streams(S0), + {ok, S1, SharedSubS}. %%-------------------------------------------------------------------- %% on_subscribe From 81f4103d60bcfac0a7bf7b0f008bbb82161a06a8 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Thu, 11 Jul 2024 00:53:35 +0300 Subject: [PATCH 115/131] feat(queue): avoid cyclic dependencies --- apps/emqx_durable_storage/src/emqx_ds_skipping_iterator.erl | 1 - apps/emqx_durable_storage/src/emqx_ds_skipping_iterator.hrl | 4 ++++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/apps/emqx_durable_storage/src/emqx_ds_skipping_iterator.erl b/apps/emqx_durable_storage/src/emqx_ds_skipping_iterator.erl index d8833f65e..b14bd26f8 100644 --- a/apps/emqx_durable_storage/src/emqx_ds_skipping_iterator.erl +++ b/apps/emqx_durable_storage/src/emqx_ds_skipping_iterator.erl @@ -17,7 +17,6 @@ -module(emqx_ds_skipping_iterator). -include("emqx_ds_skipping_iterator.hrl"). --include_lib("emqx/include/emqx_mqtt.hrl"). -type t() :: ?skipping_iterator(emqx_ds:iterator(), non_neg_integer(), non_neg_integer()). diff --git a/apps/emqx_durable_storage/src/emqx_ds_skipping_iterator.hrl b/apps/emqx_durable_storage/src/emqx_ds_skipping_iterator.hrl index 2c0999fcc..6ec8ba16c 100644 --- a/apps/emqx_durable_storage/src/emqx_ds_skipping_iterator.hrl +++ b/apps/emqx_durable_storage/src/emqx_ds_skipping_iterator.hrl @@ -30,3 +30,7 @@ -define(skipping_iterator(Iterator, Q1Skip, Q2Skip), #{ ?tag => ?IT, ?it => Iterator, ?qos1_skip => Q1Skip, ?qos2_skip => Q2Skip }). + +-define(QOS_0, 0). +-define(QOS_1, 1). +-define(QOS_2, 2). From cae27293a5758664932c5d8987e990d40dbc7ab0 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Thu, 11 Jul 2024 11:36:20 +0300 Subject: [PATCH 116/131] feat(queue): move route registration to sessions --- ...emqx_persistent_session_ds_shared_subs.erl | 11 ++++++++-- .../shared_subs_agent.hrl | 1 + .../src/emqx_ds_shared_sub_leader.erl | 20 ++++--------------- 3 files changed, 14 insertions(+), 18 deletions(-) diff --git a/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl index bbaf3fd10..3bf24407a 100644 --- a/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl +++ b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl @@ -160,7 +160,8 @@ on_subscribe(Subscription, ShareTopicFilter, SubOpts, Session) -> update_subscription(Subscription, ShareTopicFilter, SubOpts, Session). -dialyzer({nowarn_function, create_new_subscription/3}). -create_new_subscription(ShareTopicFilter, SubOpts, #{ +create_new_subscription(#share{topic = TopicFilter} = ShareTopicFilter, SubOpts, #{ + id := SessionId, s := S0, shared_sub_s := #{agent := Agent} = SharedSubS0, props := Props @@ -171,6 +172,9 @@ create_new_subscription(ShareTopicFilter, SubOpts, #{ ) of ok -> + ok = emqx_persistent_session_ds_router:do_add_route(TopicFilter, SessionId), + _ = emqx_external_broker:add_persistent_route(TopicFilter, SessionId), + #{upgrade_qos := UpgradeQoS} = Props, {SubId, S1} = emqx_persistent_session_ds_state:new_id(S0), {SStateId, S2} = emqx_persistent_session_ds_state:new_id(S1), @@ -188,6 +192,7 @@ create_new_subscription(ShareTopicFilter, SubOpts, #{ S = emqx_persistent_session_ds_state:put_subscription( ShareTopicFilter, Subscription, S3 ), + SharedSubS = schedule_subscribe(SharedSubS0, ShareTopicFilter, SubOpts), {ok, S, SharedSubS}; {error, _} = Error -> @@ -254,7 +259,7 @@ schedule_subscribe( ) -> {ok, emqx_persistent_session_ds_state:t(), t(), emqx_persistent_session_ds:subscription()} | {error, emqx_types:reason_code()}. -on_unsubscribe(SessionId, ShareTopicFilter, S0, SharedSubS0) -> +on_unsubscribe(SessionId, #share{topic = TopicFilter} = ShareTopicFilter, S0, SharedSubS0) -> case lookup(ShareTopicFilter, S0) of undefined -> {error, ?RC_NO_SUBSCRIPTION_EXISTED}; @@ -262,6 +267,8 @@ on_unsubscribe(SessionId, ShareTopicFilter, S0, SharedSubS0) -> ?tp(persistent_session_ds_subscription_delete, #{ session_id => SessionId, share_topic_filter => ShareTopicFilter }), + ok = emqx_persistent_session_ds_router:do_delete_route(TopicFilter, SessionId), + _ = emqx_external_broker:delete_persistent_route(TopicFilter, SessionId), S = emqx_persistent_session_ds_state:del_subscription(ShareTopicFilter, S0), SharedSubS = schedule_unsubscribe(S, SharedSubS0, SubId, ShareTopicFilter), {ok, S, SharedSubS, Subscription} diff --git a/apps/emqx/src/emqx_persistent_session_ds/shared_subs_agent.hrl b/apps/emqx/src/emqx_persistent_session_ds/shared_subs_agent.hrl index 4fcd43e8a..ea2d41def 100644 --- a/apps/emqx/src/emqx_persistent_session_ds/shared_subs_agent.hrl +++ b/apps/emqx/src/emqx_persistent_session_ds/shared_subs_agent.hrl @@ -21,6 +21,7 @@ %% Till full implementation we need to dispach to the null agent. %% It will report "not implemented" error for attempts to use shared subscriptions. -define(shared_subs_agent, emqx_persistent_session_ds_shared_subs_null_agent). +% -define(shared_subs_agent, emqx_ds_shared_sub_agent). %% end of -ifdef(TEST). -endif. diff --git a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl index 24b78155f..912253205 100644 --- a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl +++ b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl @@ -68,8 +68,6 @@ %% group_id := group_id(), topic := emqx_types:topic(), - %% For ds router, not an actual session_id - router_id := binary(), %% TODO https://emqx.atlassian.net/browse/EMQX-12575 %% Implement some stats to assign evenly? stream_states := #{ @@ -108,10 +106,6 @@ -record(renew_leases, {}). -record(drop_timeout, {}). -%% Constants - --define(START_TIME_THRESHOLD, 5000). - %%-------------------------------------------------------------------- %% API %%-------------------------------------------------------------------- @@ -148,8 +142,7 @@ init([#{share_topic_filter := #share{topic = Topic} = ShareTopicFilter} = _Optio Data = #{ group_id => ShareTopicFilter, topic => Topic, - router_id => gen_router_id(), - start_time => now_ms() - ?START_TIME_THRESHOLD, + start_time => now_ms(), stream_states => #{}, stream_owners => #{}, agents => #{}, @@ -170,9 +163,8 @@ handle_event({call, From}, #register{register_fun = Fun}, ?leader_waiting_regist end; %%-------------------------------------------------------------------- %% repalying state -handle_event(enter, _OldState, ?leader_active, #{topic := Topic, router_id := RouterId} = _Data) -> - ?tp(warning, shared_sub_leader_enter_actve, #{topic => Topic, router_id => RouterId}), - ok = emqx_persistent_session_ds_router:do_add_route(Topic, RouterId), +handle_event(enter, _OldState, ?leader_active, #{topic := Topic} = _Data) -> + ?tp(warning, shared_sub_leader_enter_actve, #{topic => Topic}), {keep_state_and_data, [ {{timeout, #renew_streams{}}, 0, #renew_streams{}}, {{timeout, #renew_leases{}}, ?dq_config(leader_renew_lease_interval_ms), #renew_leases{}}, @@ -251,8 +243,7 @@ handle_event(Event, Content, State, _Data) -> }), keep_state_and_data. -terminate(_Reason, _State, #{topic := Topic, router_id := RouterId} = _Data) -> - ok = emqx_persistent_session_ds_router:do_delete_route(Topic, RouterId), +terminate(_Reason, _State, _Data) -> ok. %%-------------------------------------------------------------------- @@ -889,9 +880,6 @@ agent_transition_to_updating(Agent, #{state := ?waiting_updating} = AgentState0) %% Helper functions %%-------------------------------------------------------------------- -gen_router_id() -> - emqx_guid:to_hexstr(emqx_guid:gen()). - now_ms() -> erlang:system_time(millisecond). From 9b30320ddb17465305a417c5fb7bf3cc37fef81f Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Thu, 11 Jul 2024 19:39:09 +0300 Subject: [PATCH 117/131] feat(queue): simplify progress report on disconnect --- apps/emqx/include/emqx.hrl | 13 ++- apps/emqx/src/emqx_persistent_session_ds.erl | 8 +- ...emqx_persistent_session_ds_shared_subs.erl | 80 +++++++++-------- .../test/emqx_ds_shared_sub_SUITE.erl | 17 ++-- apps/emqx_durable_storage/src/emqx_ds.erl | 6 +- .../src/emqx_ds_skipping_iterator.erl | 86 ------------------- .../src/emqx_ds_skipping_iterator.hrl | 36 -------- .../src/emqx_mgmt_api_subscriptions.erl | 25 ++++-- 8 files changed, 92 insertions(+), 179 deletions(-) delete mode 100644 apps/emqx_durable_storage/src/emqx_ds_skipping_iterator.erl delete mode 100644 apps/emqx_durable_storage/src/emqx_ds_skipping_iterator.hrl diff --git a/apps/emqx/include/emqx.hrl b/apps/emqx/include/emqx.hrl index 0e17f71f2..78cf3825e 100644 --- a/apps/emqx/include/emqx.hrl +++ b/apps/emqx/include/emqx.hrl @@ -65,9 +65,20 @@ %% Route %%-------------------------------------------------------------------- +-record(share_dest, { + session_id :: emqx_session:session_id(), + group :: emqx_types:group() +}). + -record(route, { topic :: binary(), - dest :: node() | {binary(), node()} | emqx_session:session_id() | emqx_external_broker:dest() + dest :: + node() + | {binary(), node()} + | emqx_session:session_id() + %% One session can also have multiple subscriptions to the same topic through different groups + | #share_dest{} + | emqx_external_broker:dest() }). %%-------------------------------------------------------------------- diff --git a/apps/emqx/src/emqx_persistent_session_ds.erl b/apps/emqx/src/emqx_persistent_session_ds.erl index 0984f9de8..bd763e62f 100644 --- a/apps/emqx/src/emqx_persistent_session_ds.erl +++ b/apps/emqx/src/emqx_persistent_session_ds.erl @@ -821,10 +821,12 @@ list_client_subscriptions(ClientId) -> {error, not_found} end. --spec get_client_subscription(emqx_types:clientid(), emqx_types:topic()) -> +-spec get_client_subscription(emqx_types:clientid(), topic_filter()) -> subscription() | undefined. -get_client_subscription(ClientId, Topic) -> - emqx_persistent_session_ds_subs:cold_get_subscription(ClientId, Topic). +get_client_subscription(ClientId, #share{} = ShareTopicFilter) -> + emqx_persistent_session_ds_subs:cold_get_subscription(ClientId, ShareTopicFilter); +get_client_subscription(ClientId, TopicFilter) -> + emqx_persistent_session_ds_subs:cold_get_subscription(ClientId, TopicFilter). %%-------------------------------------------------------------------- %% Session tables operations diff --git a/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl index 3bf24407a..634207d12 100644 --- a/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl +++ b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl @@ -29,10 +29,10 @@ -module(emqx_persistent_session_ds_shared_subs). -include("emqx_mqtt.hrl"). +-include("emqx.hrl"). -include("logger.hrl"). -include("session_internals.hrl"). --include_lib("emqx/include/emqx_persistent_message.hrl"). -include_lib("snabbkaffe/include/trace.hrl"). -export([ @@ -51,7 +51,10 @@ to_map/2 ]). --define(EPOCH_BITS, 15). +%% Management API: +-export([ + cold_get_subscription/2 +]). -define(schedule_subscribe, schedule_subscribe). -define(schedule_unsubscribe, schedule_unsubscribe). @@ -160,7 +163,7 @@ on_subscribe(Subscription, ShareTopicFilter, SubOpts, Session) -> update_subscription(Subscription, ShareTopicFilter, SubOpts, Session). -dialyzer({nowarn_function, create_new_subscription/3}). -create_new_subscription(#share{topic = TopicFilter} = ShareTopicFilter, SubOpts, #{ +create_new_subscription(#share{topic = TopicFilter, group = Group} = ShareTopicFilter, SubOpts, #{ id := SessionId, s := S0, shared_sub_s := #{agent := Agent} = SharedSubS0, @@ -172,9 +175,9 @@ create_new_subscription(#share{topic = TopicFilter} = ShareTopicFilter, SubOpts, ) of ok -> - ok = emqx_persistent_session_ds_router:do_add_route(TopicFilter, SessionId), - _ = emqx_external_broker:add_persistent_route(TopicFilter, SessionId), - + ok = emqx_persistent_session_ds_router:do_add_route(TopicFilter, #share_dest{ + session_id = SessionId, group = Group + }), #{upgrade_qos := UpgradeQoS} = Props, {SubId, S1} = emqx_persistent_session_ds_state:new_id(S0), {SStateId, S2} = emqx_persistent_session_ds_state:new_id(S1), @@ -259,7 +262,9 @@ schedule_subscribe( ) -> {ok, emqx_persistent_session_ds_state:t(), t(), emqx_persistent_session_ds:subscription()} | {error, emqx_types:reason_code()}. -on_unsubscribe(SessionId, #share{topic = TopicFilter} = ShareTopicFilter, S0, SharedSubS0) -> +on_unsubscribe( + SessionId, #share{topic = TopicFilter, group = Group} = ShareTopicFilter, S0, SharedSubS0 +) -> case lookup(ShareTopicFilter, S0) of undefined -> {error, ?RC_NO_SUBSCRIPTION_EXISTED}; @@ -267,8 +272,9 @@ on_unsubscribe(SessionId, #share{topic = TopicFilter} = ShareTopicFilter, S0, Sh ?tp(persistent_session_ds_subscription_delete, #{ session_id => SessionId, share_topic_filter => ShareTopicFilter }), - ok = emqx_persistent_session_ds_router:do_delete_route(TopicFilter, SessionId), - _ = emqx_external_broker:delete_persistent_route(TopicFilter, SessionId), + ok = emqx_persistent_session_ds_router:do_delete_route(TopicFilter, #share_dest{ + session_id = SessionId, group = Group + }), S = emqx_persistent_session_ds_state:del_subscription(ShareTopicFilter, S0), SharedSubS = schedule_unsubscribe(S, SharedSubS0, SubId, ShareTopicFilter), {ok, S, SharedSubS, Subscription} @@ -588,9 +594,32 @@ on_info(S, #{agent := Agent0} = SharedSubS0, Info) -> %% to_map -spec to_map(emqx_persistent_session_ds_state:t(), t()) -> map(). -to_map(_S, _SharedSubS) -> - %% TODO - #{}. +to_map(S, _SharedSubS) -> + fold_shared_subs( + fun(ShareTopicFilter, _, Acc) -> Acc#{ShareTopicFilter => lookup(ShareTopicFilter, S)} end, + #{}, + S + ). + +%%-------------------------------------------------------------------- +%% cold_get_subscription + +-spec cold_get_subscription(emqx_persistent_session_ds:id(), emqx_types:topic()) -> + emqx_persistent_session_ds:subscription() | undefined. +cold_get_subscription(SessionId, ShareTopicFilter) -> + case emqx_persistent_session_ds_state:cold_get_subscription(SessionId, ShareTopicFilter) of + [Sub = #{current_state := SStateId}] -> + case + emqx_persistent_session_ds_state:cold_get_subscription_state(SessionId, SStateId) + of + [#{subopts := Subopts}] -> + Sub#{subopts => Subopts}; + _ -> + undefined + end; + _ -> + undefined + end. %%-------------------------------------------------------------------- %% Generic helpers @@ -629,21 +658,13 @@ stream_progress( Stream, #srs{ it_end = EndIt, - it_begin = BeginIt, - first_seqno_qos1 = StartQos1, - first_seqno_qos2 = StartQos2 + it_begin = BeginIt } = SRS ) -> - Qos1Acked = n_acked(?QOS_1, CommQos1, StartQos1), - Qos2Acked = n_acked(?QOS_2, CommQos2, StartQos2), Iterator = case is_stream_fully_acked(CommQos1, CommQos2, SRS) of - true -> - EndIt; - false -> - emqx_ds_skipping_iterator:update_or_new( - BeginIt, Qos1Acked, Qos2Acked - ) + true -> EndIt; + false -> BeginIt end, #{ stream => Stream, @@ -714,19 +735,6 @@ is_stream_fully_acked(_, _, #srs{ is_stream_fully_acked(Comm1, Comm2, #srs{last_seqno_qos1 = S1, last_seqno_qos2 = S2}) -> (Comm1 >= S1) andalso (Comm2 >= S2). -n_acked(Qos, A, B) -> - max(seqno_diff(Qos, A, B), 0). - --dialyzer({nowarn_function, seqno_diff/3}). -seqno_diff(?QOS_1, A, B) -> - %% For QoS1 messages we skip a seqno every time the epoch changes, - %% we need to substract that from the diff: - EpochA = A bsr ?EPOCH_BITS, - EpochB = B bsr ?EPOCH_BITS, - A - B - (EpochA - EpochB); -seqno_diff(?QOS_2, A, B) -> - A - B. - %%-------------------------------------------------------------------- %% Formatters %%-------------------------------------------------------------------- diff --git a/apps/emqx_ds_shared_sub/test/emqx_ds_shared_sub_SUITE.erl b/apps/emqx_ds_shared_sub/test/emqx_ds_shared_sub_SUITE.erl index dfc2203c4..4f99a8455 100644 --- a/apps/emqx_ds_shared_sub/test/emqx_ds_shared_sub_SUITE.erl +++ b/apps/emqx_ds_shared_sub/test/emqx_ds_shared_sub_SUITE.erl @@ -365,10 +365,13 @@ t_disconnect_no_double_replay1(_Config) -> end end, - {Missing, Duplicate} = verify_received_pubs(Pubs, 2 * NPubs, ClientByBid), + {Missing, _Duplicate} = verify_received_pubs(Pubs, 2 * NPubs, ClientByBid), ?assertEqual([], Missing), - ?assertEqual([], Duplicate), + + %% We cannnot garantee that the message are not duplicated until we are able + %% to send progress of a partially replayed stream range to the leader. + % ?assertEqual([], Duplicate), ok = emqtt:disconnect(ConnShared1), ok = emqtt:disconnect(ConnPub). @@ -395,10 +398,12 @@ t_disconnect_no_double_replay2(_Config) -> ConnShared12 = emqtt_connect_sub(<<"client_shared12">>), {ok, _, _} = emqtt:subscribe(ConnShared12, <<"$share/gr12/topic12/#">>, 1), - ?assertNotReceive( - {publish, #{payload := <<"1">>}}, - 3000 - ), + %% We cannnot garantee that the message is not duplicated until we are able + %% to send progress of a partially replayed stream range to the leader. + % ?assertNotReceive( + % {publish, #{payload := <<"1">>}}, + % 3000 + % ), ok = emqtt:disconnect(ConnShared12). diff --git a/apps/emqx_durable_storage/src/emqx_ds.erl b/apps/emqx_durable_storage/src/emqx_ds.erl index 38d63e41f..69de92325 100644 --- a/apps/emqx_durable_storage/src/emqx_ds.erl +++ b/apps/emqx_durable_storage/src/emqx_ds.erl @@ -132,7 +132,7 @@ %% TODO: Not implemented -type iterator_id() :: term(). --type iterator() :: ds_specific_iterator(). +-opaque iterator() :: ds_specific_iterator(). -opaque delete_iterator() :: ds_specific_delete_iterator(). @@ -401,14 +401,10 @@ make_iterator(DB, Stream, TopicFilter, StartTime) -> -spec update_iterator(db(), iterator(), message_key()) -> make_iterator_result(). -update_iterator(DB, ?skipping_iterator_match = OldIter, DSKey) -> - emqx_ds_skipping_iterator:update_iterator(DB, OldIter, DSKey); update_iterator(DB, OldIter, DSKey) -> ?module(DB):update_iterator(DB, OldIter, DSKey). -spec next(db(), iterator(), pos_integer()) -> next_result(). -next(DB, ?skipping_iterator_match = Iter, BatchSize) -> - emqx_ds_skipping_iterator:next(DB, Iter, BatchSize); next(DB, Iter, BatchSize) -> ?module(DB):next(DB, Iter, BatchSize). diff --git a/apps/emqx_durable_storage/src/emqx_ds_skipping_iterator.erl b/apps/emqx_durable_storage/src/emqx_ds_skipping_iterator.erl deleted file mode 100644 index b14bd26f8..000000000 --- a/apps/emqx_durable_storage/src/emqx_ds_skipping_iterator.erl +++ /dev/null @@ -1,86 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2023-2024 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_ds_skipping_iterator). - --include("emqx_ds_skipping_iterator.hrl"). - --type t() :: ?skipping_iterator(emqx_ds:iterator(), non_neg_integer(), non_neg_integer()). - --export([ - update_or_new/3, - update_iterator/3, - next/3 -]). - -%%-------------------------------------------------------------------- -%% API -%%-------------------------------------------------------------------- - --spec update_or_new(t() | emqx_ds:iterator(), non_neg_integer(), non_neg_integer()) -> t(). -update_or_new(?skipping_iterator_match(Iterator, Q1Skip0, Q2Skip0), Q1Skip, Q2Skip) when - Q1Skip >= 0 andalso Q2Skip >= 0 --> - ?skipping_iterator(Iterator, Q1Skip0 + Q1Skip, Q2Skip0 + Q2Skip); -update_or_new(Iterator, Q1Skip, Q2Skip) when Q1Skip >= 0 andalso Q2Skip >= 0 -> - ?skipping_iterator(Iterator, Q1Skip, Q2Skip). - --spec next(emqx_ds:db(), t(), pos_integer()) -> emqx_ds:next_result(t()). -next(DB, ?skipping_iterator_match(Iterator0, Q1Skip0, Q2Skip0), Count) -> - case emqx_ds:next(DB, Iterator0, Count) of - {error, _, _} = Error -> - Error; - {ok, end_of_stream} -> - {ok, end_of_stream}; - {ok, Iterator1, Messages0} -> - {Messages1, Q1Skip1, Q2Skip1} = skip(Messages0, Q1Skip0, Q2Skip0), - case {Q1Skip1, Q2Skip1} of - {0, 0} -> {ok, Iterator1, Messages1}; - _ -> {ok, ?skipping_iterator(Iterator1, Q1Skip1, Q2Skip1)} - end - end. - --spec update_iterator(emqx_ds:db(), emqx_ds:iterator(), emqx_ds:message_key()) -> - emqx_ds:make_iterator_result(). -update_iterator(DB, ?skipping_iterator_match(Iterator0, Q1Skip0, Q2Skip0), Key) -> - case emqx_ds:update_iterator(DB, Iterator0, Key) of - {error, _, _} = Error -> Error; - {ok, Iterator1} -> {ok, ?skipping_iterator(Iterator1, Q1Skip0, Q2Skip0)} - end. - -%%-------------------------------------------------------------------- -%% Internal functions -%%-------------------------------------------------------------------- - -skip(Messages, Q1Skip, Q2Skip) -> - skip(Messages, Q1Skip, Q2Skip, []). - -skip([], Q1Skip, Q2Skip, Agg) -> - {lists:reverse(Agg), Q1Skip, Q2Skip}; -skip([{Key, Message} | Messages], Q1Skip, Q2Skip, Agg) -> - Qos = emqx_message:qos(Message), - skip({Key, Message}, Qos, Messages, Q1Skip, Q2Skip, Agg). - -skip(_KeyMessage, ?QOS_0, Messages, Q1Skip, Q2Skip, Agg) -> - skip(Messages, Q1Skip, Q2Skip, Agg); -skip(_KeyMessage, ?QOS_1, Messages, Q1Skip, Q2Skip, Agg) when Q1Skip > 0 -> - skip(Messages, Q1Skip - 1, Q2Skip, Agg); -skip(KeyMessage, ?QOS_1, Messages, 0, Q2Skip, Agg) -> - skip(Messages, 0, Q2Skip, [KeyMessage | Agg]); -skip(_KeyMessage, ?QOS_2, Messages, Q1Skip, Q2Skip, Agg) when Q2Skip > 0 -> - skip(Messages, Q1Skip, Q2Skip - 1, Agg); -skip(KeyMessage, ?QOS_2, Messages, Q1Skip, 0, Agg) -> - skip(Messages, Q1Skip, 0, [KeyMessage | Agg]). diff --git a/apps/emqx_durable_storage/src/emqx_ds_skipping_iterator.hrl b/apps/emqx_durable_storage/src/emqx_ds_skipping_iterator.hrl deleted file mode 100644 index 6ec8ba16c..000000000 --- a/apps/emqx_durable_storage/src/emqx_ds_skipping_iterator.hrl +++ /dev/null @@ -1,36 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2023-2024 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --define(tag, 1). --define(it, 2). --define(qos1_skip, 3). --define(qos2_skip, 4). - --define(IT, -1000). - --define(skipping_iterator_match, #{?tag := ?IT}). - --define(skipping_iterator_match(Iterator, Q1Skip, Q2Skip), #{ - ?tag := ?IT, ?it := Iterator, ?qos1_skip := Q1Skip, ?qos2_skip := Q2Skip -}). - --define(skipping_iterator(Iterator, Q1Skip, Q2Skip), #{ - ?tag => ?IT, ?it => Iterator, ?qos1_skip => Q1Skip, ?qos2_skip => Q2Skip -}). - --define(QOS_0, 0). --define(QOS_1, 1). --define(QOS_2, 2). diff --git a/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl b/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl index b9cefeb1f..c4aa55463 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl @@ -328,13 +328,13 @@ consume_n_matching(Map, Pred, N, S0, Acc) -> end end. -persistent_route_to_subscription(#route{topic = Topic, dest = SessionId}) -> - case emqx_persistent_session_ds:get_client_subscription(SessionId, Topic) of +persistent_route_to_subscription(#route{dest = Dest} = Route) -> + case get_client_subscription(Route) of #{subopts := SubOpts} -> #{qos := Qos, nl := Nl, rh := Rh, rap := Rap} = SubOpts, #{ - topic => Topic, - clientid => SessionId, + topic => format_topic(Route), + clientid => session_id(Dest), node => all, qos => Qos, @@ -345,13 +345,26 @@ persistent_route_to_subscription(#route{topic = Topic, dest = SessionId}) -> }; undefined -> #{ - topic => Topic, - clientid => SessionId, + topic => format_topic(Route), + clientid => session_id(Dest), node => all, durable => true } end. +get_client_subscription(#route{topic = Topic, dest = #share_dest{session_id = SessionId, group = Group}}) -> + emqx_persistent_session_ds:get_client_subscription(SessionId, #share{topic = Topic, group = Group}); +get_client_subscription(#route{topic = Topic, dest = SessionId}) -> + emqx_persistent_session_ds:get_client_subscription(SessionId, Topic). + +session_id(#share_dest{session_id = SessionId}) -> SessionId; +session_id(SessionId) -> SessionId. + +format_topic(#route{topic = Topic, dest = #share_dest{group = Group}}) -> + <<"$share/", Group/binary, "/", Topic/binary>>; +format_topic(#route{topic = Topic}) -> + Topic. + %% @private This function merges paginated results from two sources. %% %% Note: this implementation is far from ideal: `count' for the From f0dd1bc4f4b316f2aa4eb7e1deeb76b6c8c410f4 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Fri, 12 Jul 2024 17:22:11 +0300 Subject: [PATCH 118/131] feat(queue): add shared sub support to the management API --- ...shared_sub_mgmt_api_subscription_SUITE.erl | 96 +++++++++++++++++++ .../src/emqx_mgmt_api_subscriptions.erl | 80 ++++++++++------ .../test/emqx_mgmt_api_subscription_SUITE.erl | 3 +- 3 files changed, 151 insertions(+), 28 deletions(-) create mode 100644 apps/emqx_ds_shared_sub/test/emqx_ds_shared_sub_mgmt_api_subscription_SUITE.erl diff --git a/apps/emqx_ds_shared_sub/test/emqx_ds_shared_sub_mgmt_api_subscription_SUITE.erl b/apps/emqx_ds_shared_sub/test/emqx_ds_shared_sub_mgmt_api_subscription_SUITE.erl new file mode 100644 index 000000000..ce73aa59f --- /dev/null +++ b/apps/emqx_ds_shared_sub/test/emqx_ds_shared_sub_mgmt_api_subscription_SUITE.erl @@ -0,0 +1,96 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_ds_shared_sub_mgmt_api_subscription_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +-define(CLIENTID, <<"api_clientid">>). +-define(USERNAME, <<"api_username">>). + +all() -> emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + Apps = emqx_cth_suite:start( + [ + {emqx, + "durable_sessions {\n" + " enable = true\n" + " renew_streams_interval = 10ms\n" + "}"}, + {emqx_ds_shared_sub, #{ + config => #{ + <<"durable_queues">> => #{ + <<"enable">> => true, + <<"session_find_leader_timeout_ms">> => "1200ms" + } + } + }}, + emqx_management, + emqx_mgmt_api_test_util:emqx_dashboard() + ], + #{work_dir => emqx_cth_suite:work_dir(Config)} + ), + [{apps, Apps} | Config]. + +end_per_suite(Config) -> + ok = emqx_cth_suite:stop(?config(apps, Config)). + +init_per_testcase(_TC, Config) -> + ClientConfig = #{ + username => ?USERNAME, + clientid => ?CLIENTID, + proto_ver => v5, + clean_start => true, + properties => #{'Session-Expiry-Interval' => 300} + }, + + {ok, Client} = emqtt:start_link(ClientConfig), + {ok, _} = emqtt:connect(Client), + [{client_config, ClientConfig}, {client, Client} | Config]. + +end_per_testcase(_TC, Config) -> + Client = proplists:get_value(client, Config), + emqtt:disconnect(Client). + +t_list_with_shared_sub(_Config) -> + Client = proplists:get_value(client, _Config), + RealTopic = <<"t/+">>, + Topic = <<"$share/g1/", RealTopic/binary>>, + + {ok, _, _} = emqtt:subscribe(Client, Topic), + {ok, _, _} = emqtt:subscribe(Client, RealTopic), + + QS0 = [ + {"clientid", ?CLIENTID}, + {"match_topic", "t/#"} + ], + Headers = emqx_mgmt_api_test_util:auth_header_(), + + ?assertMatch( + #{<<"data">> := [#{<<"clientid">> := ?CLIENTID}, #{<<"clientid">> := ?CLIENTID}]}, + request_json(get, QS0, Headers) + ), + + QS1 = [ + {"clientid", ?CLIENTID}, + {"share_group", "g1"} + ], + + ?assertMatch( + #{<<"data">> := [#{<<"clientid">> := ?CLIENTID, <<"topic">> := <<"$share/g1/t/+">>}]}, + request_json(get, QS1, Headers) + ). + +request_json(Method, Query, Headers) when is_list(Query) -> + Qs = uri_string:compose_query(Query), + {ok, MatchRes} = emqx_mgmt_api_test_util:request_api(Method, path(), Qs, Headers), + emqx_utils_json:decode(MatchRes, [return_maps]). + +path() -> + emqx_mgmt_api_test_util:api_path(["subscriptions"]). diff --git a/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl b/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl index c4aa55463..b662061a6 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl @@ -242,20 +242,25 @@ do_subscriptions_query_persistent(#{<<"page">> := Page, <<"limit">> := Limit} = %% TODO: filtering by client ID can be implemented more efficiently: FilterTopic = maps:get(<<"topic">>, QString, '_'), Stream0 = emqx_persistent_session_ds_router:stream(FilterTopic), + SubPred = fun(Sub) -> - compare_optional(<<"topic">>, QString, topic, Sub) andalso + compare_optional(<<"topic">>, QString, '_real_topic', Sub) andalso compare_optional(<<"clientid">>, QString, clientid, Sub) andalso compare_optional(<<"qos">>, QString, qos, Sub) andalso - compare_match_topic_optional(<<"match_topic">>, QString, topic, Sub) + compare_optional(<<"share_group">>, QString, '_group', Sub) andalso + compare_match_topic_optional(<<"match_topic">>, QString, '_real_topic', Sub) end, NDropped = (Page - 1) * Limit, {_, Stream} = consume_n_matching( fun persistent_route_to_subscription/1, SubPred, NDropped, Stream0 ), - {Subscriptions, Stream1} = consume_n_matching( + {Subscriptions0, Stream1} = consume_n_matching( fun persistent_route_to_subscription/1, SubPred, Limit, Stream ), HasNext = Stream1 =/= [], + Subscriptions1 = lists:map( + fun remove_temp_match_fields/1, Subscriptions0 + ), Meta = case maps:is_key(<<"match_topic">>, QString) orelse maps:is_key(<<"qos">>, QString) of true -> @@ -276,7 +281,7 @@ do_subscriptions_query_persistent(#{<<"page">> := Page, <<"limit">> := Limit} = #{ meta => Meta, - data => Subscriptions + data => Subscriptions1 }. compare_optional(QField, Query, SField, Subscription) -> @@ -329,37 +334,58 @@ consume_n_matching(Map, Pred, N, S0, Acc) -> end. persistent_route_to_subscription(#route{dest = Dest} = Route) -> - case get_client_subscription(Route) of - #{subopts := SubOpts} -> - #{qos := Qos, nl := Nl, rh := Rh, rap := Rap} = SubOpts, - #{ - topic => format_topic(Route), - clientid => session_id(Dest), - node => all, + Sub = + case get_client_subscription(Route) of + #{subopts := SubOpts} -> + #{qos := Qos, nl := Nl, rh := Rh, rap := Rap} = SubOpts, + #{ + topic => format_topic(Route), + clientid => session_id(Dest), + node => all, - qos => Qos, - nl => Nl, - rh => Rh, - rap => Rap, - durable => true - }; - undefined -> - #{ - topic => format_topic(Route), - clientid => session_id(Dest), - node => all, - durable => true - } - end. + qos => Qos, + nl => Nl, + rh => Rh, + rap => Rap, + durable => true + }; + undefined -> + #{ + topic => format_topic(Route), + clientid => session_id(Dest), + node => all, + durable => true + } + end, + add_temp_match_fields(Route, Sub). -get_client_subscription(#route{topic = Topic, dest = #share_dest{session_id = SessionId, group = Group}}) -> - emqx_persistent_session_ds:get_client_subscription(SessionId, #share{topic = Topic, group = Group}); +get_client_subscription(#route{ + topic = Topic, dest = #share_dest{session_id = SessionId, group = Group} +}) -> + emqx_persistent_session_ds:get_client_subscription(SessionId, #share{ + topic = Topic, group = Group + }); get_client_subscription(#route{topic = Topic, dest = SessionId}) -> emqx_persistent_session_ds:get_client_subscription(SessionId, Topic). session_id(#share_dest{session_id = SessionId}) -> SessionId; session_id(SessionId) -> SessionId. +add_temp_match_fields(Route, Sub) -> + add_temp_match_fields(['_real_topic', '_group'], Route, Sub). + +add_temp_match_fields([], _Route, Sub) -> + Sub; +add_temp_match_fields(['_real_topic' | Rest], #route{topic = Topic} = Route, Sub) -> + add_temp_match_fields(Rest, Route, Sub#{'_real_topic' => Topic}); +add_temp_match_fields(['_group' | Rest], #route{dest = #share_dest{group = Group}} = Route, Sub) -> + add_temp_match_fields(Rest, Route, Sub#{'_group' => Group}); +add_temp_match_fields(['_group' | Rest], Route, Sub) -> + add_temp_match_fields(Rest, Route, Sub#{'_group' => undefined}). + +remove_temp_match_fields(Sub) -> + maps:without(['_real_topic', '_group'], Sub). + format_topic(#route{topic = Topic, dest = #share_dest{group = Group}}) -> <<"$share/", Group/binary, "/", Topic/binary>>; format_topic(#route{topic = Topic}) -> diff --git a/apps/emqx_management/test/emqx_mgmt_api_subscription_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_subscription_SUITE.erl index 9a55fa1a0..274e0c5dd 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_subscription_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_subscription_SUITE.erl @@ -47,7 +47,8 @@ groups() -> CommonTCs = AllTCs -- persistent_only_tcs(), [ {mem, CommonTCs}, - %% Shared subscriptions are currently not supported: + %% Persistent shared subscriptions are an EE app. + %% So they are tested outside emqx_management app which is CE. {persistent, (CommonTCs -- [t_list_with_shared_sub, t_subscription_api]) ++ persistent_only_tcs()} ]. From 23f0e88b458c412d72102fef9457175f7f2ef90d Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Wed, 17 Jul 2024 16:23:05 +0300 Subject: [PATCH 119/131] feat(queue): add integration with external broker --- apps/emqx/src/emqx_external_broker.erl | 10 +++++++++- .../emqx_persistent_session_ds_router.erl | 8 +++++--- .../emqx_persistent_session_ds_shared_subs.erl | 2 ++ .../emqx_persistent_session_ds/emqx_ps_ds_int.hrl | 2 +- apps/emqx_cluster_link/include/emqx_cluster_link.hrl | 3 +++ apps/emqx_cluster_link/src/emqx_cluster_link.erl | 12 ++++++++++++ .../src/emqx_cluster_link_router_bootstrap.erl | 12 +++++++++--- 7 files changed, 41 insertions(+), 8 deletions(-) diff --git a/apps/emqx/src/emqx_external_broker.erl b/apps/emqx/src/emqx_external_broker.erl index fe360a5b8..5fcee71f0 100644 --- a/apps/emqx/src/emqx_external_broker.erl +++ b/apps/emqx/src/emqx_external_broker.erl @@ -43,7 +43,9 @@ add_shared_route/2, delete_shared_route/2, add_persistent_route/2, - delete_persistent_route/2 + delete_persistent_route/2, + add_persistent_shared_route/3, + delete_persistent_shared_route/3 ]). -export_type([dest/0]). @@ -129,6 +131,12 @@ add_persistent_route(Topic, ID) -> delete_persistent_route(Topic, ID) -> ?safe_with_provider(?FUNCTION_NAME(Topic, ID), ok). +add_persistent_shared_route(Topic, Group, ID) -> + ?safe_with_provider(?FUNCTION_NAME(Topic, Group, ID), ok). + +delete_persistent_shared_route(Topic, Group, ID) -> + ?safe_with_provider(?FUNCTION_NAME(Topic, Group, ID), ok). + %%-------------------------------------------------------------------- %% Internal functions %%-------------------------------------------------------------------- diff --git a/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_router.erl b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_router.erl index b0ee14963..1b80a28d2 100644 --- a/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_router.erl +++ b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_router.erl @@ -17,7 +17,7 @@ -module(emqx_persistent_session_ds_router). -include("emqx.hrl"). --include("emqx_persistent_session_ds/emqx_ps_ds_int.hrl"). +-include("emqx_ps_ds_int.hrl"). -export([init_tables/0]). @@ -47,7 +47,7 @@ -endif. -type route() :: #ps_route{}. --type dest() :: emqx_persistent_session_ds:id(). +-type dest() :: emqx_persistent_session_ds:id() | #share_dest{}. -export_type([dest/0, route/0]). @@ -161,7 +161,7 @@ topics() -> print_routes(Topic) -> lists:foreach( fun(#ps_route{topic = To, dest = Dest}) -> - io:format("~ts -> ~ts~n", [To, Dest]) + io:format("~ts -> ~tp~n", [To, Dest]) end, match_routes(Topic) ). @@ -247,6 +247,8 @@ mk_filtertab_fold_fun(FoldFun) -> match_filters(Topic) -> emqx_topic_index:matches(Topic, ?PS_FILTERS_TAB, []). +get_dest_session_id(#share_dest{session_id = DSSessionId}) -> + DSSessionId; get_dest_session_id({_, DSSessionId}) -> DSSessionId; get_dest_session_id(DSSessionId) -> diff --git a/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl index 634207d12..11b89441d 100644 --- a/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl +++ b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl @@ -178,6 +178,7 @@ create_new_subscription(#share{topic = TopicFilter, group = Group} = ShareTopicF ok = emqx_persistent_session_ds_router:do_add_route(TopicFilter, #share_dest{ session_id = SessionId, group = Group }), + _ = emqx_external_broker:add_persistent_shared_route(TopicFilter, Group, SessionId), #{upgrade_qos := UpgradeQoS} = Props, {SubId, S1} = emqx_persistent_session_ds_state:new_id(S0), {SStateId, S2} = emqx_persistent_session_ds_state:new_id(S1), @@ -272,6 +273,7 @@ on_unsubscribe( ?tp(persistent_session_ds_subscription_delete, #{ session_id => SessionId, share_topic_filter => ShareTopicFilter }), + _ = emqx_external_broker:delete_persistent_shared_route(TopicFilter, Group, SessionId), ok = emqx_persistent_session_ds_router:do_delete_route(TopicFilter, #share_dest{ session_id = SessionId, group = Group }), diff --git a/apps/emqx/src/emqx_persistent_session_ds/emqx_ps_ds_int.hrl b/apps/emqx/src/emqx_persistent_session_ds/emqx_ps_ds_int.hrl index dc487376b..e533cfcb9 100644 --- a/apps/emqx/src/emqx_persistent_session_ds/emqx_ps_ds_int.hrl +++ b/apps/emqx/src/emqx_persistent_session_ds/emqx_ps_ds_int.hrl @@ -21,7 +21,7 @@ -record(ps_route, { topic :: binary(), - dest :: emqx_persistent_session_ds:id() | '_' + dest :: emqx_persistent_session_ds_router:dest() | '_' }). -record(ps_routeidx, { diff --git a/apps/emqx_cluster_link/include/emqx_cluster_link.hrl b/apps/emqx_cluster_link/include/emqx_cluster_link.hrl index 32c675d8d..8a0c374ed 100644 --- a/apps/emqx_cluster_link/include/emqx_cluster_link.hrl +++ b/apps/emqx_cluster_link/include/emqx_cluster_link.hrl @@ -21,3 +21,6 @@ -define(METRIC_NAME, cluster_link). -define(route_metric, 'routes'). +-define(PERSISTENT_SHARED_ROUTE_ID(Topic, Group, ID), + <<"$sp/", Group/binary, "/", ID/binary, "/", Topic/binary>> +). diff --git a/apps/emqx_cluster_link/src/emqx_cluster_link.erl b/apps/emqx_cluster_link/src/emqx_cluster_link.erl index 76228c052..d68ffb4be 100644 --- a/apps/emqx_cluster_link/src/emqx_cluster_link.erl +++ b/apps/emqx_cluster_link/src/emqx_cluster_link.erl @@ -16,6 +16,8 @@ delete_shared_route/2, add_persistent_route/2, delete_persistent_route/2, + add_persistent_shared_route/3, + delete_persistent_shared_route/3, forward/1 ]). @@ -71,6 +73,16 @@ add_persistent_route(Topic, ID) -> delete_persistent_route(Topic, ID) -> maybe_push_route_op(delete, Topic, ?PERSISTENT_ROUTE_ID(Topic, ID), push_persistent_route). +add_persistent_shared_route(Topic, Group, ID) -> + maybe_push_route_op( + add, Topic, ?PERSISTENT_SHARED_ROUTE_ID(Topic, Group, ID), push_persistent_route + ). + +delete_persistent_shared_route(Topic, Group, ID) -> + maybe_push_route_op( + delete, Topic, ?PERSISTENT_SHARED_ROUTE_ID(Topic, Group, ID), push_persistent_route + ). + forward(#delivery{message = #message{extra = #{link_origin := _}}}) -> %% Do not forward any external messages to other links. %% Only forward locally originated messages to all the relevant links, i.e. no gossip diff --git a/apps/emqx_cluster_link/src/emqx_cluster_link_router_bootstrap.erl b/apps/emqx_cluster_link/src/emqx_cluster_link_router_bootstrap.erl index 1670c2ab4..6656c8c89 100644 --- a/apps/emqx_cluster_link/src/emqx_cluster_link_router_bootstrap.erl +++ b/apps/emqx_cluster_link/src/emqx_cluster_link_router_bootstrap.erl @@ -3,6 +3,7 @@ %%-------------------------------------------------------------------- -module(emqx_cluster_link_router_bootstrap). +-include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/emqx_router.hrl"). -include_lib("emqx/include/emqx_shared_sub.hrl"). -include_lib("emqx/src/emqx_persistent_session_ds/emqx_ps_ds_int.hrl"). @@ -67,7 +68,7 @@ routes_by_topic(Topics, _IsPersistentRoute = true) -> lists:foldl( fun(T, Acc) -> Routes = emqx_persistent_session_ds_router:lookup_routes(T), - [encode_route(T, ?PERSISTENT_ROUTE_ID(T, D)) || #ps_route{dest = D} <- Routes] ++ Acc + [encode_route(T, ps_route_id(PSRoute)) || #ps_route{} = PSRoute <- Routes] ++ Acc end, [], Topics @@ -79,17 +80,22 @@ routes_by_wildcards(Wildcards, _IsPersistentRoute = false) -> Routes ++ SharedRoutes; routes_by_wildcards(Wildcards, _IsPersistentRoute = true) -> emqx_persistent_session_ds_router:foldl_routes( - fun(#ps_route{dest = D, topic = T}, Acc) -> + fun(#ps_route{topic = T} = PSRoute, Acc) -> case topic_intersect_any(T, Wildcards) of false -> Acc; Intersec -> - [encode_route(Intersec, ?PERSISTENT_ROUTE_ID(T, D)) | Acc] + [encode_route(Intersec, ps_route_id(PSRoute)) | Acc] end end, [] ). +ps_route_id(#ps_route{topic = T, dest = #share_dest{group = Group, session_id = SessionId}}) -> + ?PERSISTENT_SHARED_ROUTE_ID(T, Group, SessionId); +ps_route_id(#ps_route{topic = T, dest = SessionId}) -> + ?PERSISTENT_ROUTE_ID(T, SessionId). + select_routes_by_topics(Topics) -> [encode_route(Topic, Topic) || Topic <- Topics, emqx_broker:subscribers(Topic) =/= []]. From 303ff95e10a3a94be2fbe37aadaf9907ab5a40be Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Fri, 19 Jul 2024 17:40:22 +0300 Subject: [PATCH 120/131] feat(queue): add stub for CRUD API --- .../src/emqx_ds_shared_sub_api.erl | 218 ++++++++++++++++++ .../test/emqx_ds_shared_sub_api_SUITE.erl | 140 +++++++++++ rel/i18n/emqx_ds_shared_sub_api.hocon | 34 +++ 3 files changed, 392 insertions(+) create mode 100644 apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_api.erl create mode 100644 apps/emqx_ds_shared_sub/test/emqx_ds_shared_sub_api_SUITE.erl create mode 100644 rel/i18n/emqx_ds_shared_sub_api.hocon diff --git a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_api.erl b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_api.erl new file mode 100644 index 000000000..0a8d41116 --- /dev/null +++ b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_api.erl @@ -0,0 +1,218 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_ds_shared_sub_api). + +-behaviour(minirest_api). + +-include_lib("typerefl/include/types.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). +-include_lib("emqx/include/logger.hrl"). + +%% Swagger specs from hocon schema +-export([ + api_spec/0, + paths/0, + schema/1, + namespace/0 +]). + +-export([ + fields/1, + roots/0 +]). + +-define(TAGS, [<<"Durable Queues">>]). + +%% API callbacks +-export([ + '/durable_queues'/2, + '/durable_queues/:id'/2 +]). + +-import(hoconsc, [mk/2, ref/1, ref/2]). +-import(emqx_dashboard_swagger, [error_codes/2]). + +namespace() -> "durable_queues". + +api_spec() -> + emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}). + +paths() -> + [ + "/durable_queues", + "/durable_queues/:id" + ]. + +-define(NOT_FOUND, 'NOT_FOUND'). + +schema("/durable_queues") -> + #{ + 'operationId' => '/durable_queues', + get => #{ + tags => ?TAGS, + summary => <<"List declared durable queues">>, + description => ?DESC("durable_queues_get"), + responses => #{ + 200 => emqx_dashboard_swagger:schema_with_example( + durable_queues_get(), + durable_queues_get_example() + ) + } + } + }; +schema("/durable_queues/:id") -> + #{ + 'operationId' => '/durable_queues/:id', + get => #{ + tags => ?TAGS, + summary => <<"Get a declared durable queue">>, + description => ?DESC("durable_queue_get"), + parameters => [param_queue_id()], + responses => #{ + 200 => emqx_dashboard_swagger:schema_with_example( + durable_queue_get(), + durable_queue_get_example() + ), + 404 => error_codes([?NOT_FOUND], <<"Queue Not Found">>) + } + }, + delete => #{ + tags => ?TAGS, + summary => <<"Delete a declared durable queue">>, + description => ?DESC("durable_queue_delete"), + parameters => [param_queue_id()], + responses => #{ + 200 => <<"Queue deleted">>, + 404 => error_codes([?NOT_FOUND], <<"Queue Not Found">>) + } + }, + put => #{ + tags => ?TAGS, + summary => <<"Declare a durable queue">>, + description => ?DESC("durable_queues_put"), + parameters => [param_queue_id()], + 'requestBody' => durable_queue_put(), + responses => #{ + 200 => emqx_dashboard_swagger:schema_with_example( + durable_queue_get(), + durable_queue_get_example() + ) + } + } + }. + +'/durable_queues'(get, _Params) -> + {200, queue_list()}. + +'/durable_queues/:id'(get, Params) -> + case queue_get(Params) of + {ok, Queue} -> {200, Queue}; + not_found -> serialize_error(not_found) + end; +'/durable_queues/:id'(delete, Params) -> + case queue_delete(Params) of + ok -> {200, <<"Queue deleted">>}; + not_found -> serialize_error(not_found) + end; +'/durable_queues/:id'(put, Params) -> + {200, queue_put(Params)}. + +%%-------------------------------------------------------------------- +%% Actual handlers: stubs +%%-------------------------------------------------------------------- + +queue_list() -> + persistent_term:get({?MODULE, queues}, []). + +queue_get(#{bindings := #{id := ReqId}}) -> + case [Q || #{id := Id} = Q <- queue_list(), Id =:= ReqId] of + [Queue] -> {ok, Queue}; + [] -> not_found + end. + +queue_delete(#{bindings := #{id := ReqId}}) -> + Queues0 = queue_list(), + Queues1 = [Q || #{id := Id} = Q <- Queues0, Id =/= ReqId], + persistent_term:put({?MODULE, queues}, Queues1), + case Queues0 =:= Queues1 of + true -> not_found; + false -> ok + end. + +queue_put(#{bindings := #{id := ReqId}}) -> + Queues0 = queue_list(), + Queues1 = [Q || #{id := Id} = Q <- Queues0, Id =/= ReqId], + NewQueue = #{ + id => ReqId + }, + Queues2 = [NewQueue | Queues1], + persistent_term:put({?MODULE, queues}, Queues2), + NewQueue. + +%%-------------------------------------------------------------------- +%% Schemas +%%-------------------------------------------------------------------- + +param_queue_id() -> + { + id, + mk(binary(), #{ + in => path, + desc => ?DESC(param_queue_id), + required => true, + validator => fun validate_queue_id/1 + }) + }. + +validate_queue_id(Id) -> + case emqx_topic:words(Id) of + [Segment] when is_binary(Segment) -> true; + _ -> {error, <<"Invalid queue id">>} + end. + +durable_queues_get() -> + hoconsc:array(ref(durable_queue_get)). + +durable_queue_get() -> + ref(durable_queue_get). + +durable_queue_put() -> + map(). + +roots() -> []. + +fields(durable_queue_get) -> + [ + {id, mk(binary(), #{})} + ]. + +%%-------------------------------------------------------------------- +%% Examples +%%-------------------------------------------------------------------- + +durable_queue_get_example() -> + #{ + id => <<"queue1">> + }. + +durable_queues_get_example() -> + [ + #{ + id => <<"queue1">> + }, + #{ + id => <<"queue2">> + } + ]. + +%%-------------------------------------------------------------------- +%% Error codes +%%-------------------------------------------------------------------- + +serialize_error(not_found) -> + {404, #{ + code => <<"NOT_FOUND">>, + message => <<"Queue Not Found">> + }}. diff --git a/apps/emqx_ds_shared_sub/test/emqx_ds_shared_sub_api_SUITE.erl b/apps/emqx_ds_shared_sub/test/emqx_ds_shared_sub_api_SUITE.erl new file mode 100644 index 000000000..0969bdcd1 --- /dev/null +++ b/apps/emqx_ds_shared_sub/test/emqx_ds_shared_sub_api_SUITE.erl @@ -0,0 +1,140 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_ds_shared_sub_api_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +-import( + emqx_mgmt_api_test_util, + [ + request_api/2, + request/3, + uri/1 + ] +). + +all() -> + emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + Apps = emqx_cth_suite:start( + [ + {emqx, #{ + config => #{ + <<"durable_sessions">> => #{ + <<"enable">> => true, + <<"renew_streams_interval">> => "100ms" + }, + <<"durable_storage">> => #{ + <<"messages">> => #{ + <<"backend">> => <<"builtin_raft">> + } + } + } + }}, + emqx_ds_shared_sub, + emqx_management, + emqx_mgmt_api_test_util:emqx_dashboard() + ], + #{work_dir => ?config(priv_dir, Config)} + ), + [{apps, Apps} | Config]. + +end_per_suite(Config) -> + ok = emqx_cth_suite:stop(?config(apps, Config)), + ok. + +init_per_testcase(_TC, Config) -> + ok = snabbkaffe:start_trace(), + Config. + +end_per_testcase(_TC, _Config) -> + ok = snabbkaffe:stop(), + ok = terminate_leaders(), + ok. +%%-------------------------------------------------------------------- +%% Tests +%%-------------------------------------------------------------------- + +t_basic_crud(_Config) -> + ?assertMatch( + {ok, []}, + api_get(["durable_queues"]) + ), + + ?assertMatch( + {ok, 200, #{ + <<"id">> := <<"q1">> + }}, + api(put, ["durable_queues", "q1"], #{}) + ), + + ?assertMatch( + {error, {_, 404, _}}, + api_get(["durable_queues", "q2"]) + ), + + ?assertMatch( + {ok, 200, #{ + <<"id">> := <<"q2">> + }}, + api(put, ["durable_queues", "q2"], #{}) + ), + + ?assertMatch( + {ok, #{ + <<"id">> := <<"q2">> + }}, + api_get(["durable_queues", "q2"]) + ), + + ?assertMatch( + {ok, [#{<<"id">> := <<"q2">>}, #{<<"id">> := <<"q1">>}]}, + api_get(["durable_queues"]) + ), + + ?assertMatch( + {ok, 200, <<"Queue deleted">>}, + api(delete, ["durable_queues", "q2"], #{}) + ), + + ?assertMatch( + {ok, [#{<<"id">> := <<"q1">>}]}, + api_get(["durable_queues"]) + ). + +%%-------------------------------------------------------------------- +%% Helpers +%%-------------------------------------------------------------------- + +api_get(Path) -> + case request_api(get, uri(Path)) of + {ok, ResponseBody} -> + {ok, jiffy:decode(list_to_binary(ResponseBody), [return_maps])}; + {error, _} = Error -> + Error + end. + +api(Method, Path, Data) -> + case request(Method, uri(Path), Data) of + {ok, Code, ResponseBody} -> + Res = + case emqx_utils_json:safe_decode(ResponseBody, [return_maps]) of + {ok, Decoded} -> Decoded; + {error, _} -> ResponseBody + end, + {ok, Code, Res}; + {error, _} = Error -> + Error + end. + +terminate_leaders() -> + ok = supervisor:terminate_child(emqx_ds_shared_sub_sup, emqx_ds_shared_sub_leader_sup), + {ok, _} = supervisor:restart_child(emqx_ds_shared_sub_sup, emqx_ds_shared_sub_leader_sup), + ok. diff --git a/rel/i18n/emqx_ds_shared_sub_api.hocon b/rel/i18n/emqx_ds_shared_sub_api.hocon new file mode 100644 index 000000000..369aeb88e --- /dev/null +++ b/rel/i18n/emqx_ds_shared_sub_api.hocon @@ -0,0 +1,34 @@ +emqx_ds_shared_sub_api { + +param_queue_id.desc: +"""The ID of the durable queue.""" + +param_queue_id.label: +"""Queue ID""" + +durable_queues_get.desc: +"""Get the list of durable queues.""" + +durable_queues_get.label: +"""Durable Queues""" + +durable_queue_get.desc: +"""Get the information of a durable queue.""" + +durable_queue_get.label: +"""Durable Queue""" + +durable_queue_delete.desc: +"""Delete a durable queue.""" + +durable_queue_delete.label: +"""Delete Durable Queue""" + +durable_queues_put.desc: +"""Create a durable queue.""" + +durable_queues_put.label: +"""Create Durable Queue""" + + +} From e294d35703e47d67e30f016a444768e8f11617fb Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Fri, 19 Jul 2024 19:06:48 +0300 Subject: [PATCH 121/131] feat(queue): add schema descriptions --- .../src/emqx_ds_shared_sub_schema.erl | 2 +- rel/i18n/emqx_ds_shared_sub_schema.hocon | 57 +++++++++++++++++++ 2 files changed, 58 insertions(+), 1 deletion(-) create mode 100644 rel/i18n/emqx_ds_shared_sub_schema.hocon diff --git a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_schema.erl b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_schema.erl index 198554d8a..d60893678 100644 --- a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_schema.erl +++ b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_schema.erl @@ -28,7 +28,7 @@ fields(durable_queues) -> #{ required => false, default => true, - desc => ?DESC(durable_queues) + desc => ?DESC(enable) } )}, duration(session_find_leader_timeout_ms, 1000), diff --git a/rel/i18n/emqx_ds_shared_sub_schema.hocon b/rel/i18n/emqx_ds_shared_sub_schema.hocon new file mode 100644 index 000000000..5a95e9693 --- /dev/null +++ b/rel/i18n/emqx_ds_shared_sub_schema.hocon @@ -0,0 +1,57 @@ +emqx_ds_shared_sub_schema { + +session_find_leader_timeout_ms.desc: +"""The timeout in milliseconds for the session to find a leader. +If the session cannot find a leader within this time, the session will retry.""" + +session_find_leader_timeout_ms.label: +"""Session Find Leader Timeout""" + +session_renew_lease_timeout_ms.desc: +"""The timeout in milliseconds for the session to wait for the leader to renew the lease. +If the leader does not renew the lease within this time, the session will consider +the leader as lost and try to find a new leader.""" + +session_renew_lease_timeout_ms.label: +"""Session Renew Lease Timeout""" + +session_min_update_stream_state_interval_ms.desc: +"""The minimum interval in milliseconds for the session to update the stream state. +If session has no updates for the stream state within this time, the session will +send empty updates.""" + +session_min_update_stream_state_interval_ms.label: +"""Session Min Update Stream State Interval""" + +leader_renew_lease_interval_ms.desc: +"""The interval in milliseconds for the leader to renew the lease.""" + +leader_renew_lease_interval_ms.label: +"""Leader Renew Lease Interval""" + +leader_renew_streams_interval_ms.desc: +"""The interval in milliseconds for the leader to renew the streams.""" + +leader_renew_streams_interval_ms.label: +"""Leader Renew Streams Interval""" + +leader_drop_timeout_interval_ms.desc: +"""The interval in milliseconds for the leader to drop non-responsive sessions.""" + +leader_drop_timeout_interval_ms.label: +"""Leader Drop Timeout Interval""" + +leader_session_update_timeout_ms.desc: +"""The timeout in milliseconds for the leader to wait for the session to update the stream state. +If the session does not update the stream state within this time, the leader will drop the session.""" + +leader_session_update_timeout_ms.label: +"""Leader Session Update Timeout""" + +leader_session_not_replaying_timeout_ms.desc: +"""The timeout in milliseconds for the leader to wait for the session leave intermediate states.""" + +leader_session_not_replaying_timeout_ms.label: +"""Leader Session Not Replaying Timeout""" + +} From e408804efb7528595e15772c79a20566fba23993 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Fri, 19 Jul 2024 21:17:29 +0300 Subject: [PATCH 122/131] feat(queue): fix dialyzer issues --- apps/emqx/src/emqx_persistent_session_ds.erl | 4 ++-- .../emqx_persistent_session_ds_shared_subs.erl | 2 +- .../emqx_persistent_session_ds_state.erl | 4 +++- apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_agent.erl | 9 +++++++-- rel/i18n/emqx_ds_shared_sub_schema.hocon | 6 ++++++ 5 files changed, 19 insertions(+), 6 deletions(-) diff --git a/apps/emqx/src/emqx_persistent_session_ds.erl b/apps/emqx/src/emqx_persistent_session_ds.erl index bd763e62f..b86b44611 100644 --- a/apps/emqx/src/emqx_persistent_session_ds.erl +++ b/apps/emqx/src/emqx_persistent_session_ds.erl @@ -821,10 +821,10 @@ list_client_subscriptions(ClientId) -> {error, not_found} end. --spec get_client_subscription(emqx_types:clientid(), topic_filter()) -> +-spec get_client_subscription(emqx_types:clientid(), topic_filter() | share_topic_filter()) -> subscription() | undefined. get_client_subscription(ClientId, #share{} = ShareTopicFilter) -> - emqx_persistent_session_ds_subs:cold_get_subscription(ClientId, ShareTopicFilter); + emqx_persistent_session_ds_shared_subs:cold_get_subscription(ClientId, ShareTopicFilter); get_client_subscription(ClientId, TopicFilter) -> emqx_persistent_session_ds_subs:cold_get_subscription(ClientId, TopicFilter). diff --git a/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl index 11b89441d..5b54c6f73 100644 --- a/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl +++ b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl @@ -606,7 +606,7 @@ to_map(S, _SharedSubS) -> %%-------------------------------------------------------------------- %% cold_get_subscription --spec cold_get_subscription(emqx_persistent_session_ds:id(), emqx_types:topic()) -> +-spec cold_get_subscription(emqx_persistent_session_ds:id(), share_topic_filter()) -> emqx_persistent_session_ds:subscription() | undefined. cold_get_subscription(SessionId, ShareTopicFilter) -> case emqx_persistent_session_ds_state:cold_get_subscription(SessionId, ShareTopicFilter) of diff --git a/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_state.erl b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_state.erl index 1d60250ea..3d3840307 100644 --- a/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_state.erl +++ b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_state.erl @@ -399,7 +399,9 @@ new_id(Rec) -> get_subscription(TopicFilter, Rec) -> gen_get(?subscriptions, TopicFilter, Rec). --spec cold_get_subscription(emqx_persistent_session_ds:id(), emqx_types:topic()) -> +-spec cold_get_subscription( + emqx_persistent_session_ds:id(), emqx_types:topic() | emqx_types:share() +) -> [emqx_persistent_session_ds_subs:subscription()]. cold_get_subscription(SessionId, Topic) -> kv_pmap_read(?subscription_tab, SessionId, Topic). diff --git a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_agent.erl b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_agent.erl index 5b71a93e5..a90f1286d 100644 --- a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_agent.erl +++ b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_agent.erl @@ -9,6 +9,7 @@ -include_lib("snabbkaffe/include/snabbkaffe.hrl"). -include("emqx_ds_shared_sub_proto.hrl"). +-include("emqx_ds_shared_sub_config.hrl"). -export([ new/1, @@ -109,9 +110,13 @@ open(TopicSubscriptions, Opts) -> ), State1. --spec can_subscribe(t(), share_topic_filter(), emqx_types:subopts()) -> ok. +-spec can_subscribe(t(), share_topic_filter(), emqx_types:subopts()) -> + ok | {error, emqx_types:reason_code()}. can_subscribe(_State, _ShareTopicFilter, _SubOpts) -> - ok. + case ?dq_config(enable) of + true -> ok; + false -> {error, ?RC_SHARED_SUBSCRIPTIONS_NOT_SUPPORTED} + end. -spec on_subscribe(t(), share_topic_filter(), emqx_types:subopts()) -> t(). on_subscribe(State0, ShareTopicFilter, _SubOpts) -> diff --git a/rel/i18n/emqx_ds_shared_sub_schema.hocon b/rel/i18n/emqx_ds_shared_sub_schema.hocon index 5a95e9693..2ee28cc30 100644 --- a/rel/i18n/emqx_ds_shared_sub_schema.hocon +++ b/rel/i18n/emqx_ds_shared_sub_schema.hocon @@ -1,5 +1,11 @@ emqx_ds_shared_sub_schema { +enable.desc: +"""Enable the shared subscription feature.""" + +enable.label: +"""Enable Shared Subscription""" + session_find_leader_timeout_ms.desc: """The timeout in milliseconds for the session to find a leader. If the session cannot find a leader within this time, the session will retry.""" From 08f70e4a25ab260f7cd943c2acbd436f96ca9f77 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Tue, 30 Jul 2024 14:19:39 +0300 Subject: [PATCH 123/131] feat(queue): move ds shared sub dependent test to emqx_ds_shared_sub app --- ...shared_sub_mgmt_api_subscription_SUITE.erl | 29 +++++++++++++++++++ .../test/emqx_mgmt_api_subscription_SUITE.erl | 4 ++- 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/apps/emqx_ds_shared_sub/test/emqx_ds_shared_sub_mgmt_api_subscription_SUITE.erl b/apps/emqx_ds_shared_sub/test/emqx_ds_shared_sub_mgmt_api_subscription_SUITE.erl index ce73aa59f..fde9acbea 100644 --- a/apps/emqx_ds_shared_sub/test/emqx_ds_shared_sub_mgmt_api_subscription_SUITE.erl +++ b/apps/emqx_ds_shared_sub/test/emqx_ds_shared_sub_mgmt_api_subscription_SUITE.erl @@ -87,6 +87,35 @@ t_list_with_shared_sub(_Config) -> request_json(get, QS1, Headers) ). +t_list_with_invalid_match_topic(Config) -> + Client = proplists:get_value(client, Config), + RealTopic = <<"t/+">>, + Topic = <<"$share/g1/", RealTopic/binary>>, + + {ok, _, _} = emqtt:subscribe(Client, Topic), + {ok, _, _} = emqtt:subscribe(Client, RealTopic), + + QS = [ + {"clientid", ?CLIENTID}, + {"match_topic", "$share/g1/t/1"} + ], + Headers = emqx_mgmt_api_test_util:auth_header_(), + + ?assertMatch( + {error, + {{_, 400, _}, _, #{ + <<"message">> := <<"match_topic_invalid">>, + <<"code">> := <<"INVALID_PARAMETER">> + }}}, + begin + {error, {R, _H, Body}} = emqx_mgmt_api_test_util:request_api( + get, path(), uri_string:compose_query(QS), Headers, [], #{return_all => true} + ), + {error, {R, _H, emqx_utils_json:decode(Body, [return_maps])}} + end + ), + ok. + request_json(Method, Query, Headers) when is_list(Query) -> Qs = uri_string:compose_query(Query), {ok, MatchRes} = emqx_mgmt_api_test_util:request_api(Method, path(), Qs, Headers), diff --git a/apps/emqx_management/test/emqx_mgmt_api_subscription_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_subscription_SUITE.erl index 274e0c5dd..604239379 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_subscription_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_subscription_SUITE.erl @@ -50,7 +50,9 @@ groups() -> %% Persistent shared subscriptions are an EE app. %% So they are tested outside emqx_management app which is CE. {persistent, - (CommonTCs -- [t_list_with_shared_sub, t_subscription_api]) ++ persistent_only_tcs()} + (CommonTCs -- + [t_list_with_shared_sub, t_list_with_invalid_match_topic, t_subscription_api]) ++ + persistent_only_tcs()} ]. persistent_only_tcs() -> From 1c4402b12c66c68255cbaaeb095350b16f0865c5 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 30 Jul 2024 14:07:08 -0300 Subject: [PATCH 124/131] test(clients v2 api): attempt to reduce flakiness https://github.com/emqx/emqx/actions/runs/10161391242/job/28101183920#step:6:331 --- .../test/emqx_mgmt_api_clients_SUITE.erl | 27 ++++++++++++------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl index ef7b1fa54..4845dedde 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl @@ -550,13 +550,10 @@ t_persistent_sessions5(Config) -> lists:sort(lists:map(fun(#{<<"clientid">> := CId}) -> CId end, R3 ++ R4)) ), - lists:foreach(fun emqtt:stop/1, [C3, C4]), - lists:foreach( - fun(ClientId) -> - ok = erpc:call(N1, emqx_persistent_session_ds, destroy_session, [ClientId]) - end, - ClientIds - ), + lists:foreach(fun disconnect_and_destroy_session/1, [C3, C4]), + C1B = connect_client(#{port => Port1, clientid => ClientId1}), + C2B = connect_client(#{port => Port2, clientid => ClientId2}), + lists:foreach(fun disconnect_and_destroy_session/1, [C1B, C2B]), ok end, @@ -1623,8 +1620,7 @@ t_list_clients_v2(Config) -> port => Port2, clientid => ClientId6, expiry => 0, clean_start => true }), %% offline persistent clients - ok = emqtt:stop(C3), - ok = emqtt:stop(C4), + lists:foreach(fun stop_and_commit/1, [C3, C4]), %% one by one QueryParams1 = #{limit => "1"}, @@ -2143,3 +2139,16 @@ do_traverse_in_reverse_v2(QueryParams0, Config, [Cursor | Rest], DirectOrderClie disconnect_and_destroy_session(Client) -> ok = emqtt:disconnect(Client, ?RC_SUCCESS, #{'Session-Expiry-Interval' => 0}). + +%% To avoid a race condition where we try to delete the session while it's terminating and +%% committing. This shouldn't happen realistically, because we have a safe grace period +%% before attempting to GC a session. +%% Also, we need to wait until offline metadata is committed before checking the v2 client +%% list, to avoid flaky batch results. +stop_and_commit(Client) -> + {ok, {ok, _}} = + ?wait_async_action( + emqtt:stop(Client), + #{?snk_kind := persistent_session_ds_terminate} + ), + ok. From 3d1f0c756cd8a63412858c2b631b7879fba5ccc1 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Mon, 29 Jul 2024 15:29:01 +0800 Subject: [PATCH 125/131] feat: call plugin's app module `on_config_changed/2` callback assume the module: `[PluginName]_app` --- apps/emqx_plugins/src/emqx_plugins.erl | 30 ++++++++++++++++++++++++-- changes/ce/feat-13548.en.md | 6 ++++++ 2 files changed, 34 insertions(+), 2 deletions(-) create mode 100644 changes/ce/feat-13548.en.md diff --git a/apps/emqx_plugins/src/emqx_plugins.erl b/apps/emqx_plugins/src/emqx_plugins.erl index 253d1b4ef..56a88d7cb 100644 --- a/apps/emqx_plugins/src/emqx_plugins.erl +++ b/apps/emqx_plugins/src/emqx_plugins.erl @@ -356,13 +356,13 @@ get_config_bin(NameVsn) -> %% RPC call from Management API or CLI. %% The plugin config Json Map was valid by avro schema %% Or: if no and plugin config ALWAYS be valid before calling this function. -put_config(NameVsn, ConfigJsonMap, AvroValue) when not is_binary(NameVsn) -> +put_config(NameVsn, ConfigJsonMap, AvroValue) when (not is_binary(NameVsn)) -> put_config(bin(NameVsn), ConfigJsonMap, AvroValue); put_config(NameVsn, ConfigJsonMap, _AvroValue) -> HoconBin = hocon_pp:do(ConfigJsonMap, #{}), ok = backup_and_write_hocon_bin(NameVsn, HoconBin), - %% TODO: callback in plugin's on_config_changed (config update by mgmt API) %% TODO: callback in plugin's on_config_upgraded (config vsn upgrade v1 -> v2) + ok = maybe_call_on_config_changed(NameVsn, ConfigJsonMap), ok = persistent_term:put(?PLUGIN_PERSIS_CONFIG_KEY(NameVsn), ConfigJsonMap), ok. @@ -373,6 +373,32 @@ restart(NameVsn) -> {error, Reason} -> {error, Reason} end. +%% @doc Call plugin's callback on_config_changed/2 +maybe_call_on_config_changed(NameVsn, NewConf) -> + FuncName = on_config_changed, + maybe + {ok, PluginAppModule} ?= app_module_name(NameVsn), + true ?= erlang:function_exported(PluginAppModule, FuncName, 2), + {ok, OldConf} = get_config(NameVsn), + _ = erlang:apply(PluginAppModule, FuncName, [OldConf, NewConf]) + else + {error, Reason} -> + ?SLOG(info, #{msg => "failed_to_call_on_config_changed", reason => Reason}); + false -> + ?SLOG(info, #{msg => "on_config_changed_callback_not_exported"}); + _ -> + ok + end. + +app_module_name(NameVsn) -> + case read_plugin_info(NameVsn, #{}) of + {ok, #{<<"name">> := Name} = _PluginInfo} -> + emqx_utils:safe_to_existing_atom(<>); + {error, Reason} -> + ?SLOG(error, Reason#{msg => "failed_to_read_plugin_info"}), + {error, Reason} + end. + %% @doc List all installed plugins. %% Including the ones that are installed, but not enabled in config. -spec list() -> [plugin_info()]. diff --git a/changes/ce/feat-13548.en.md b/changes/ce/feat-13548.en.md new file mode 100644 index 000000000..75b56cd43 --- /dev/null +++ b/changes/ce/feat-13548.en.md @@ -0,0 +1,6 @@ +Optionally calls the `on_config_changed/2` callback function when the plugin configuration is updated via the REST API. + +This callback function is assumed to be exported by the `_app` module. +i.e: +Plugin NameVsn: `my_plugin-1.0.0` +This callback function is assumed to be `my_plugin_app:on_config_changed/2` From e6bfc14cc9f8f7c792dc55c96fe81e38f9acb237 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Wed, 31 Jul 2024 09:26:44 +0800 Subject: [PATCH 126/131] fix: try-catch optional `on_config_changed/2` plugin app callback --- apps/emqx_plugins/src/emqx_plugins.erl | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/apps/emqx_plugins/src/emqx_plugins.erl b/apps/emqx_plugins/src/emqx_plugins.erl index 56a88d7cb..94c6aa4e1 100644 --- a/apps/emqx_plugins/src/emqx_plugins.erl +++ b/apps/emqx_plugins/src/emqx_plugins.erl @@ -380,7 +380,18 @@ maybe_call_on_config_changed(NameVsn, NewConf) -> {ok, PluginAppModule} ?= app_module_name(NameVsn), true ?= erlang:function_exported(PluginAppModule, FuncName, 2), {ok, OldConf} = get_config(NameVsn), - _ = erlang:apply(PluginAppModule, FuncName, [OldConf, NewConf]) + try erlang:apply(PluginAppModule, FuncName, [OldConf, NewConf]) of + _ -> ok + catch + Class:CatchReason:Stacktrace -> + ?SLOG(error, #{ + msg => "failed_to_call_on_config_changed", + exception => Class, + reason => CatchReason, + stacktrace => Stacktrace + }), + ok + end else {error, Reason} -> ?SLOG(info, #{msg => "failed_to_call_on_config_changed", reason => Reason}); From d69342a2fca7abc20b34226a720eaef49f1f483c Mon Sep 17 00:00:00 2001 From: ieQu1 <99872536+ieQu1@users.noreply.github.com> Date: Wed, 31 Jul 2024 09:56:28 +0200 Subject: [PATCH 127/131] chore(emqx): Remove ra from the list of EMQX dependencies --- apps/emqx/rebar.config | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index 8a3e35537..b98728ed1 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -35,8 +35,7 @@ {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.3"}}}, {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}, {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}}, - {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.10"}}}, - {ra, "2.7.3"} + {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.10"}}} ]}. {plugins, [{rebar3_proper, "0.12.1"}, rebar3_path_deps]}. From a23b8266b1a55b861435d0b41a7e01bd774e3711 Mon Sep 17 00:00:00 2001 From: ieQu1 <99872536+ieQu1@users.noreply.github.com> Date: Wed, 31 Jul 2024 10:18:38 +0200 Subject: [PATCH 128/131] fix(sessds): Expose durable sessions in the config API --- apps/emqx_conf/src/emqx_conf_cli.erl | 2 -- 1 file changed, 2 deletions(-) diff --git a/apps/emqx_conf/src/emqx_conf_cli.erl b/apps/emqx_conf/src/emqx_conf_cli.erl index 3c0e98096..cb0c0b7d8 100644 --- a/apps/emqx_conf/src/emqx_conf_cli.erl +++ b/apps/emqx_conf/src/emqx_conf_cli.erl @@ -302,8 +302,6 @@ hidden_roots() -> <<"trace">>, <<"stats">>, <<"broker">>, - <<"persistent_session_store">>, - <<"durable_sessions">>, <<"plugins">>, <<"zones">> ]. From 569f48f5a1e0398584f43b925cda26d0e005d311 Mon Sep 17 00:00:00 2001 From: ieQu1 <99872536+ieQu1@users.noreply.github.com> Date: Wed, 31 Jul 2024 13:33:51 +0200 Subject: [PATCH 129/131] fix(mgmt): Return 404 for /ds/ API endpoints when DS is disabled --- apps/emqx_management/src/emqx_mgmt_api_ds.erl | 111 +++++++++++++----- 1 file changed, 80 insertions(+), 31 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt_api_ds.erl b/apps/emqx_management/src/emqx_mgmt_api_ds.erl index bc949cd8a..3eb80f50e 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_ds.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_ds.erl @@ -87,7 +87,8 @@ schema("/ds/sites") -> tags => ?TAGS, responses => #{ - 200 => mk(array(binary()), #{desc => <<"List sites">>}) + 200 => mk(array(binary()), #{desc => <<"List sites">>}), + 404 => disabled_schema() } } }; @@ -115,7 +116,8 @@ schema("/ds/storages") -> tags => ?TAGS, responses => #{ - 200 => mk(array(atom()), #{desc => <<"List durable storages">>}) + 200 => mk(array(atom()), #{desc => <<"List durable storages">>}), + 404 => disabled_schema() } } }; @@ -130,7 +132,8 @@ schema("/ds/storages/:ds") -> responses => #{ 200 => mk(ref(db), #{desc => <<"Get information about a durable storage">>}), - 400 => not_found(<<"Durable storage">>) + 400 => not_found(<<"Durable storage">>), + 404 => disabled_schema() } } }; @@ -148,7 +151,8 @@ schema("/ds/storages/:ds/replicas") -> 200 => mk(array(binary()), #{ desc => <<"List sites that contain replicas of the durable storage">> }), - 400 => not_found(<<"Durable storage">>) + 400 => not_found(<<"Durable storage">>), + 404 => disabled_schema() } }, put => @@ -159,7 +163,8 @@ schema("/ds/storages/:ds/replicas") -> responses => #{ 202 => mk(array(binary()), #{}), - 400 => bad_request() + 400 => bad_request(), + 404 => disabled_schema() }, 'requestBody' => mk(array(binary()), #{desc => <<"New list of sites">>}) } @@ -296,10 +301,15 @@ fields(db_site) -> %%================================================================================ list_sites(get, _Params) -> - {200, emqx_ds_replication_layer_meta:sites()}. + case is_enabled() of + true -> + {200, emqx_ds_replication_layer_meta:sites()}; + false -> + err_disabled() + end. get_site(get, #{bindings := #{site := Site}}) -> - case lists:member(Site, emqx_ds_replication_layer_meta:sites()) of + case is_enabled() andalso lists:member(Site, emqx_ds_replication_layer_meta:sites()) of false -> ?NOT_FOUND(<<"Site not found: ", Site/binary>>); true -> @@ -314,40 +324,70 @@ get_site(get, #{bindings := #{site := Site}}) -> end. list_dbs(get, _Params) -> - ?OK(dbs()). + case is_enabled() of + true -> + ?OK(dbs()); + false -> + err_disabled() + end. get_db(get, #{bindings := #{ds := DB}}) -> - ?OK(#{ - name => DB, - shards => list_shards(DB) - }). + case is_enabled() of + true -> + ?OK(#{ + name => DB, + shards => list_shards(DB) + }); + false -> + err_disabled() + end. db_replicas(get, #{bindings := #{ds := DB}}) -> - Replicas = emqx_ds_replication_layer_meta:db_sites(DB), - ?OK(Replicas); + case is_enabled() of + true -> + Replicas = emqx_ds_replication_layer_meta:db_sites(DB), + ?OK(Replicas); + false -> + err_disabled() + end; db_replicas(put, #{bindings := #{ds := DB}, body := Sites}) -> - case update_db_sites(DB, Sites, rest) of - {ok, _} -> - {202, <<"OK">>}; - {error, Description} -> - ?BAD_REQUEST(400, Description) + case is_enabled() of + true -> + case update_db_sites(DB, Sites, rest) of + {ok, _} -> + {202, <<"OK">>}; + {error, Description} -> + ?BAD_REQUEST(400, Description) + end; + false -> + err_disabled() end. db_replica(put, #{bindings := #{ds := DB, site := Site}}) -> - case join(DB, Site, rest) of - {ok, _} -> - {202, <<"OK">>}; - {error, Description} -> - ?BAD_REQUEST(400, Description) + case is_enabled() of + true -> + case join(DB, Site, rest) of + {ok, _} -> + {202, <<"OK">>}; + {error, Description} -> + ?BAD_REQUEST(400, Description) + end; + false -> + err_disabled() end; db_replica(delete, #{bindings := #{ds := DB, site := Site}}) -> - case leave(DB, Site, rest) of - {ok, Sites} when is_list(Sites) -> - {202, <<"OK">>}; - {ok, unchanged} -> - ?NOT_FOUND(<<"Site is not part of replica set">>); - {error, Description} -> - ?BAD_REQUEST(400, Description) + case is_enabled() of + true -> + case leave(DB, Site, rest) of + {ok, Sites} when is_list(Sites) -> + {202, <<"OK">>}; + {ok, unchanged} -> + ?NOT_FOUND(<<"Site is not part of replica set">>); + {error, Description} -> + ?BAD_REQUEST(400, Description) + end; + false -> + err_disabled() end. -spec update_db_sites(emqx_ds:db(), [emqx_ds_replication_layer_meta:site()], rest | cli) -> @@ -391,6 +431,9 @@ forget(Site, Via) -> %% site_info(Site) -> %% #{}. +disabled_schema() -> + emqx_dashboard_swagger:error_codes(['NOT_FOUND'], <<"Durable storage is disabled">>). + not_found(What) -> emqx_dashboard_swagger:error_codes(['NOT_FOUND'], <>). @@ -492,4 +535,10 @@ meta_result_to_binary({error, Err}) -> IOList = io_lib:format("Error: ~p", [Err]), {error, iolist_to_binary(IOList)}. +is_enabled() -> + emqx_persistent_message:is_persistence_enabled(). + +err_disabled() -> + ?NOT_FOUND(<<"Durable storage is disabled">>). + -endif. From 85cff5e7ebf7c25d2d4298995a5dd2c614cd333b Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Wed, 31 Jul 2024 09:14:29 -0300 Subject: [PATCH 130/131] fix: merge conflicts --- .../src/emqx_bridge_kafka_impl_producer.erl | 12 +++++++----- .../test/emqx_bridge_v2_kafka_producer_SUITE.erl | 15 +++++++++++++++ 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl index c886b8d58..1b18a1767 100644 --- a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl +++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl @@ -564,7 +564,7 @@ on_kafka_ack(_Partition, message_too_large, {ReplyFn, Args}) -> %% `emqx_resource_buffer_worker', we must avoid returning `disconnected' here. Otherwise, %% `emqx_resource_manager' will kill the wolff producers and messages might be lost. on_get_status( - _InstId, + ConnResId, #{client_id := ClientId} = State ) -> %% Note: we must avoid returning `?status_disconnected' here if the connector ever was @@ -574,7 +574,7 @@ on_get_status( %% held in wolff producer's replayq. case check_client_connectivity(ClientId) of ok -> - maybe_check_health_check_topic(State); + maybe_check_health_check_topic(ConnResId, State); {error, {find_client, _Error}} -> ?status_connecting; {error, {connectivity, Error}} -> @@ -648,21 +648,23 @@ check_client_connectivity(ClientId) -> {error, {find_client, Reason}} end. -maybe_check_health_check_topic(#{health_check_topic := Topic} = ConnectorState) when +maybe_check_health_check_topic(ConnResId, #{health_check_topic := Topic} = ConnectorState) when is_binary(Topic) -> #{client_id := ClientId} = ConnectorState, MaxPartitions = all_partitions, - try check_topic_and_leader_connections(ClientId, Topic, MaxPartitions) of + try check_topic_and_leader_connections(ConnResId, ClientId, Topic, MaxPartitions) of ok -> ?status_connected catch + throw:{unhealthy_target, Msg} -> + {?status_disconnected, ConnectorState, Msg}; throw:#{reason := {connection_down, _} = Reason} -> {?status_disconnected, ConnectorState, Reason}; throw:#{reason := Reason} -> {?status_connecting, ConnectorState, Reason} end; -maybe_check_health_check_topic(_) -> +maybe_check_health_check_topic(_ConnResId, _ConnState) -> %% Cannot infer further information. Maybe upgraded from older version. ?status_connected. diff --git a/apps/emqx_bridge_kafka/test/emqx_bridge_v2_kafka_producer_SUITE.erl b/apps/emqx_bridge_kafka/test/emqx_bridge_v2_kafka_producer_SUITE.erl index 5e9d53fc5..1db3c1725 100644 --- a/apps/emqx_bridge_kafka/test/emqx_bridge_v2_kafka_producer_SUITE.erl +++ b/apps/emqx_bridge_kafka/test/emqx_bridge_v2_kafka_producer_SUITE.erl @@ -740,6 +740,21 @@ t_connector_health_check_topic(_Config) -> emqx_bridge_v2_testlib:update_connector_api(Name, Type, ConnectorConfig1) ), + %% By providing an inexistent health check topic, we should detect it's + %% disconnected without the need for an action. + ConnectorConfig2 = connector_config(#{ + <<"bootstrap_hosts">> => iolist_to_binary(kafka_hosts_string()), + <<"health_check_topic">> => <<"i-dont-exist-999">> + }), + ?assertMatch( + {ok, + {{_, 200, _}, _, #{ + <<"status">> := <<"disconnected">>, + <<"status_reason">> := <<"Unknown topic or partition", _/binary>> + }}}, + emqx_bridge_v2_testlib:update_connector_api(Name, Type, ConnectorConfig2) + ), + ok end, [] From a6a9538e73ac059822050660e556b6f671b92167 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Tue, 30 Jul 2024 16:05:46 -0300 Subject: [PATCH 131/131] refactor: move JWT worker and helpers to separate app Some bridge applications might need to use JWTs before the `emqx_connector` is started, so we must move JWT table initialization to a separate dependency application. --- apps/emqx_bridge_gcp_pubsub/mix.exs | 1 + apps/emqx_bridge_gcp_pubsub/rebar.config | 1 + .../src/emqx_bridge_gcp_pubsub.app.src | 3 +- .../src/emqx_bridge_gcp_pubsub_client.erl | 2 +- .../emqx_bridge_gcp_pubsub_consumer_SUITE.erl | 2 +- apps/emqx_connector/mix.exs | 1 + apps/emqx_connector/rebar.config | 3 +- .../emqx_connector/src/emqx_connector_sup.erl | 14 +------ apps/emqx_connector_jwt/README.md | 3 ++ .../include/emqx_connector_jwt_tables.hrl} | 0 apps/emqx_connector_jwt/mix.exs | 34 ++++++++++++++++ apps/emqx_connector_jwt/rebar.config | 10 +++++ .../src/emqx_connector_jwt.app.src | 16 ++++++++ .../src/emqx_connector_jwt.erl | 12 +++--- .../src/emqx_connector_jwt_app.erl | 39 +++++++++++++++++++ .../src/emqx_connector_jwt_sup.erl | 2 +- .../src/emqx_connector_jwt_worker.erl | 0 .../test/emqx_connector_jwt_SUITE.erl | 11 +++--- .../test/emqx_connector_jwt_worker_SUITE.erl | 0 apps/emqx_machine/priv/reboot_lists.eterm | 1 + 20 files changed, 127 insertions(+), 28 deletions(-) create mode 100644 apps/emqx_connector_jwt/README.md rename apps/{emqx_connector/include/emqx_connector_tables.hrl => emqx_connector_jwt/include/emqx_connector_jwt_tables.hrl} (100%) create mode 100644 apps/emqx_connector_jwt/mix.exs create mode 100644 apps/emqx_connector_jwt/rebar.config create mode 100644 apps/emqx_connector_jwt/src/emqx_connector_jwt.app.src rename apps/{emqx_connector => emqx_connector_jwt}/src/emqx_connector_jwt.erl (92%) create mode 100644 apps/emqx_connector_jwt/src/emqx_connector_jwt_app.erl rename apps/{emqx_connector => emqx_connector_jwt}/src/emqx_connector_jwt_sup.erl (97%) rename apps/{emqx_connector => emqx_connector_jwt}/src/emqx_connector_jwt_worker.erl (100%) rename apps/{emqx_connector => emqx_connector_jwt}/test/emqx_connector_jwt_SUITE.erl (94%) rename apps/{emqx_connector => emqx_connector_jwt}/test/emqx_connector_jwt_worker_SUITE.erl (100%) diff --git a/apps/emqx_bridge_gcp_pubsub/mix.exs b/apps/emqx_bridge_gcp_pubsub/mix.exs index 3a9fae0a1..34e02ca9f 100644 --- a/apps/emqx_bridge_gcp_pubsub/mix.exs +++ b/apps/emqx_bridge_gcp_pubsub/mix.exs @@ -23,6 +23,7 @@ defmodule EMQXBridgeGcpPubsub.MixProject do def deps() do [ + {:emqx_connector_jwt, in_umbrella: true}, {:emqx_connector, in_umbrella: true, runtime: false}, {:emqx_resource, in_umbrella: true}, {:emqx_bridge, in_umbrella: true, runtime: false}, diff --git a/apps/emqx_bridge_gcp_pubsub/rebar.config b/apps/emqx_bridge_gcp_pubsub/rebar.config index a6a12b429..e5a65b745 100644 --- a/apps/emqx_bridge_gcp_pubsub/rebar.config +++ b/apps/emqx_bridge_gcp_pubsub/rebar.config @@ -9,6 +9,7 @@ debug_info ]}. {deps, [ + {emqx_connector_jwt, {path, "../../apps/emqx_connector_jwt"}}, {emqx_connector, {path, "../../apps/emqx_connector"}}, {emqx_resource, {path, "../../apps/emqx_resource"}}, {emqx_bridge, {path, "../../apps/emqx_bridge"}}, diff --git a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.app.src b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.app.src index eff7847f2..a39c4be99 100644 --- a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.app.src +++ b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.app.src @@ -6,7 +6,8 @@ kernel, stdlib, emqx_resource, - ehttpc + ehttpc, + emqx_connector_jwt ]}, {env, [ {emqx_action_info_modules, [ diff --git a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_client.erl b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_client.erl index 67218fcf0..ea6f67112 100644 --- a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_client.erl +++ b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_client.erl @@ -5,7 +5,7 @@ -module(emqx_bridge_gcp_pubsub_client). -include_lib("jose/include/jose_jwk.hrl"). --include_lib("emqx_connector/include/emqx_connector_tables.hrl"). +-include_lib("emqx_connector_jwt/include/emqx_connector_jwt_tables.hrl"). -include_lib("emqx_resource/include/emqx_resource.hrl"). -include_lib("typerefl/include/types.hrl"). -include_lib("emqx/include/logger.hrl"). diff --git a/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_consumer_SUITE.erl b/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_consumer_SUITE.erl index 0e6956d58..656413225 100644 --- a/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_consumer_SUITE.erl +++ b/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_consumer_SUITE.erl @@ -594,7 +594,7 @@ cluster(Config) -> Cluster = emqx_common_test_helpers:emqx_cluster( [core, core], [ - {apps, [emqx_conf, emqx_rule_engine, emqx_bridge]}, + {apps, [emqx_conf, emqx_rule_engine, emqx_bridge_gcp_pubsub, emqx_bridge]}, {listener_ports, []}, {priv_data_dir, PrivDataDir}, {load_schema, true}, diff --git a/apps/emqx_connector/mix.exs b/apps/emqx_connector/mix.exs index a641c27fe..a818d8072 100644 --- a/apps/emqx_connector/mix.exs +++ b/apps/emqx_connector/mix.exs @@ -33,6 +33,7 @@ defmodule EMQXConnector.MixProject do [ {:emqx, in_umbrella: true}, {:emqx_resource, in_umbrella: true}, + {:emqx_connector_jwt, in_umbrella: true}, UMP.common_dep(:jose), UMP.common_dep(:ecpool), {:eredis_cluster, github: "emqx/eredis_cluster", tag: "0.8.4"}, diff --git a/apps/emqx_connector/rebar.config b/apps/emqx_connector/rebar.config index 94da3c580..7e0ff4ea3 100644 --- a/apps/emqx_connector/rebar.config +++ b/apps/emqx_connector/rebar.config @@ -8,7 +8,8 @@ {deps, [ {emqx, {path, "../emqx"}}, {emqx_utils, {path, "../emqx_utils"}}, - {emqx_resource, {path, "../emqx_resource"}} + {emqx_resource, {path, "../emqx_resource"}}, + {emqx_connector_jwt, {path, "../emqx_connector_jwt"}} ]}. {shell, [ diff --git a/apps/emqx_connector/src/emqx_connector_sup.erl b/apps/emqx_connector/src/emqx_connector_sup.erl index 0d2f337a3..09cd8ea68 100644 --- a/apps/emqx_connector/src/emqx_connector_sup.erl +++ b/apps/emqx_connector/src/emqx_connector_sup.erl @@ -32,17 +32,5 @@ init([]) -> intensity => 5, period => 20 }, - ChildSpecs = [ - child_spec(emqx_connector_jwt_sup) - ], + ChildSpecs = [], {ok, {SupFlags, ChildSpecs}}. - -child_spec(Mod) -> - #{ - id => Mod, - start => {Mod, start_link, []}, - restart => permanent, - shutdown => 3000, - type => supervisor, - modules => [Mod] - }. diff --git a/apps/emqx_connector_jwt/README.md b/apps/emqx_connector_jwt/README.md new file mode 100644 index 000000000..868d7b08f --- /dev/null +++ b/apps/emqx_connector_jwt/README.md @@ -0,0 +1,3 @@ +# emqx_connector_jwt + +This is a small helper application for connectors, actions and sources to generate JWTs. diff --git a/apps/emqx_connector/include/emqx_connector_tables.hrl b/apps/emqx_connector_jwt/include/emqx_connector_jwt_tables.hrl similarity index 100% rename from apps/emqx_connector/include/emqx_connector_tables.hrl rename to apps/emqx_connector_jwt/include/emqx_connector_jwt_tables.hrl diff --git a/apps/emqx_connector_jwt/mix.exs b/apps/emqx_connector_jwt/mix.exs new file mode 100644 index 000000000..752275a56 --- /dev/null +++ b/apps/emqx_connector_jwt/mix.exs @@ -0,0 +1,34 @@ +defmodule EMQXConnectorJWT.MixProject do + use Mix.Project + alias EMQXUmbrella.MixProject, as: UMP + + def project do + [ + app: :emqx_connector_jwt, + version: "0.1.0", + build_path: "../../_build", + erlc_options: UMP.erlc_options(), + erlc_paths: UMP.erlc_paths(), + deps_path: "../../deps", + lockfile: "../../mix.lock", + elixir: "~> 1.14", + start_permanent: Mix.env() == :prod, + deps: deps() + ] + end + + # Run "mix help compile.app" to learn about applications + def application do + [ + extra_applications: UMP.extra_applications(), + mod: {:emqx_connector_jwt_app, []} + ] + end + + def deps() do + [ + {:emqx_resource, in_umbrella: true}, + UMP.common_dep(:jose), + ] + end +end diff --git a/apps/emqx_connector_jwt/rebar.config b/apps/emqx_connector_jwt/rebar.config new file mode 100644 index 000000000..252534c46 --- /dev/null +++ b/apps/emqx_connector_jwt/rebar.config @@ -0,0 +1,10 @@ +%% -*- mode: erlang -*- + +{erl_opts, [ + nowarn_unused_import, + debug_info +]}. + +{deps, [ + {emqx_resource, {path, "../emqx_resource"}} +]}. diff --git a/apps/emqx_connector_jwt/src/emqx_connector_jwt.app.src b/apps/emqx_connector_jwt/src/emqx_connector_jwt.app.src new file mode 100644 index 000000000..e284d7471 --- /dev/null +++ b/apps/emqx_connector_jwt/src/emqx_connector_jwt.app.src @@ -0,0 +1,16 @@ +%% -*- mode: erlang -*- +{application, emqx_connector_jwt, [ + {description, "EMQX JWT Connector Utility"}, + {vsn, "0.1.0"}, + {registered, []}, + {applications, [ + kernel, + stdlib + ]}, + {env, []}, + {modules, []}, + {mod, {emqx_connector_jwt_app, []}}, + + {licenses, ["Apache 2.0"]}, + {links, []} +]}. diff --git a/apps/emqx_connector/src/emqx_connector_jwt.erl b/apps/emqx_connector_jwt/src/emqx_connector_jwt.erl similarity index 92% rename from apps/emqx_connector/src/emqx_connector_jwt.erl rename to apps/emqx_connector_jwt/src/emqx_connector_jwt.erl index 60b35ddbb..933a259de 100644 --- a/apps/emqx_connector/src/emqx_connector_jwt.erl +++ b/apps/emqx_connector_jwt/src/emqx_connector_jwt.erl @@ -16,7 +16,7 @@ -module(emqx_connector_jwt). --include_lib("emqx_connector/include/emqx_connector_tables.hrl"). +-include("emqx_connector_jwt_tables.hrl"). -include_lib("emqx_resource/include/emqx_resource.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). -include_lib("jose/include/jose_jwt.hrl"). @@ -37,7 +37,7 @@ -type jwt_config() :: #{ expiration := duration(), resource_id := resource_id(), - table := ets:table(), + table => ets:table(), jwk := wrapped_jwk() | jwk(), iss := binary(), sub := binary(), @@ -82,7 +82,8 @@ delete_jwt(TId, ResourceId) -> %% one. -spec ensure_jwt(jwt_config()) -> jwt(). ensure_jwt(JWTConfig) -> - #{resource_id := ResourceId, table := Table} = JWTConfig, + #{resource_id := ResourceId} = JWTConfig, + Table = maps:get(table, JWTConfig, ?JWT_TABLE), case lookup_jwt(Table, ResourceId) of {error, not_found} -> JWT = do_generate_jwt(JWTConfig), @@ -132,8 +133,9 @@ do_generate_jwt(#{ JWT. -spec store_jwt(jwt_config(), jwt()) -> ok. -store_jwt(#{resource_id := ResourceId, table := TId}, JWT) -> - true = ets:insert(TId, {{ResourceId, jwt}, JWT}), +store_jwt(#{resource_id := ResourceId} = JWTConfig, JWT) -> + Table = maps:get(table, JWTConfig, ?JWT_TABLE), + true = ets:insert(Table, {{ResourceId, jwt}, JWT}), ?tp(emqx_connector_jwt_token_stored, #{resource_id => ResourceId}), ok. diff --git a/apps/emqx_connector_jwt/src/emqx_connector_jwt_app.erl b/apps/emqx_connector_jwt/src/emqx_connector_jwt_app.erl new file mode 100644 index 000000000..9c9c134de --- /dev/null +++ b/apps/emqx_connector_jwt/src/emqx_connector_jwt_app.erl @@ -0,0 +1,39 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_connector_jwt_app). + +-behaviour(application). + +%% `application' API +-export([start/2, stop/1]). + +%%------------------------------------------------------------------------------ +%% Type declarations +%%------------------------------------------------------------------------------ + +%%------------------------------------------------------------------------------ +%% `application' API +%%------------------------------------------------------------------------------ + +start(_StartType, _StartArgs) -> + emqx_connector_jwt_sup:start_link(). + +stop(_State) -> + ok. + +%%------------------------------------------------------------------------------ +%% Internal fns +%%------------------------------------------------------------------------------ diff --git a/apps/emqx_connector/src/emqx_connector_jwt_sup.erl b/apps/emqx_connector_jwt/src/emqx_connector_jwt_sup.erl similarity index 97% rename from apps/emqx_connector/src/emqx_connector_jwt_sup.erl rename to apps/emqx_connector_jwt/src/emqx_connector_jwt_sup.erl index d50be6395..4579b221c 100644 --- a/apps/emqx_connector/src/emqx_connector_jwt_sup.erl +++ b/apps/emqx_connector_jwt/src/emqx_connector_jwt_sup.erl @@ -18,7 +18,7 @@ -behaviour(supervisor). --include_lib("emqx_connector/include/emqx_connector_tables.hrl"). +-include("emqx_connector_jwt_tables.hrl"). -export([ start_link/0, diff --git a/apps/emqx_connector/src/emqx_connector_jwt_worker.erl b/apps/emqx_connector_jwt/src/emqx_connector_jwt_worker.erl similarity index 100% rename from apps/emqx_connector/src/emqx_connector_jwt_worker.erl rename to apps/emqx_connector_jwt/src/emqx_connector_jwt_worker.erl diff --git a/apps/emqx_connector/test/emqx_connector_jwt_SUITE.erl b/apps/emqx_connector_jwt/test/emqx_connector_jwt_SUITE.erl similarity index 94% rename from apps/emqx_connector/test/emqx_connector_jwt_SUITE.erl rename to apps/emqx_connector_jwt/test/emqx_connector_jwt_SUITE.erl index aef0e660c..a0416b9d5 100644 --- a/apps/emqx_connector/test/emqx_connector_jwt_SUITE.erl +++ b/apps/emqx_connector_jwt/test/emqx_connector_jwt_SUITE.erl @@ -20,7 +20,7 @@ -include_lib("common_test/include/ct.hrl"). -include_lib("jose/include/jose_jwt.hrl"). -include_lib("jose/include/jose_jws.hrl"). --include("emqx_connector_tables.hrl"). +-include("emqx_connector_jwt_tables.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). -compile([export_all, nowarn_export_all]). @@ -33,11 +33,12 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - emqx_common_test_helpers:start_apps([emqx_connector]), - Config. + Apps = emqx_cth_suite:start([emqx_connector_jwt], #{work_dir => emqx_cth_suite:work_dir(Config)}), + [{apps, Apps} | Config]. -end_per_suite(_Config) -> - emqx_common_test_helpers:stop_apps([emqx_connector]), +end_per_suite(Config) -> + Apps = ?config(apps, Config), + emqx_cth_suite:stop(Apps), ok. init_per_testcase(_TestCase, Config) -> diff --git a/apps/emqx_connector/test/emqx_connector_jwt_worker_SUITE.erl b/apps/emqx_connector_jwt/test/emqx_connector_jwt_worker_SUITE.erl similarity index 100% rename from apps/emqx_connector/test/emqx_connector_jwt_worker_SUITE.erl rename to apps/emqx_connector_jwt/test/emqx_connector_jwt_worker_SUITE.erl diff --git a/apps/emqx_machine/priv/reboot_lists.eterm b/apps/emqx_machine/priv/reboot_lists.eterm index 4830f51c4..251b9790a 100644 --- a/apps/emqx_machine/priv/reboot_lists.eterm +++ b/apps/emqx_machine/priv/reboot_lists.eterm @@ -45,6 +45,7 @@ emqx_ds_backends, emqx_http_lib, emqx_resource, + emqx_connector_jwt, emqx_connector, emqx_auth, emqx_auth_http,

#8w;M5(l%v&(!Z+)o@<^_Rg6%uSZ)Fjr)EH|>^I8A={3;LZ}Y%d_n zQODb7dE_zM)4uM+Lx{D#;CX)8#(_heYZmcm+Oki{5@nC27*daEKaGBxo$laK+|{ea z)i}OB|CB5r*8%yOeM+gHgi1~=|H_ewZLmMkXtj2JbEzlIPIu8p1nB9~&)WY;3fw8> zm0d<3wJaskj^ih-E5pMiopZEG=i~{0W@Sffxtnw9$a|U8i#}tSt*ajNkuObr6~UvvYBlRd^;H4xtQ;h4;hpJ#{W;U~c2zm%n@hK0 z6jS_(ZK*P?UKne+*gAJ|zrbmO(b|k{q5fl=oa!@<{p_I*^EkP1^ zKONDda*LIBwehnjZL4mcNXyHvQmLQ}aK=7i5ekjE?>iShlH;#6I6dotg$<7*@o`yG z6&~7}mBkg$IfS)Yhz3%dqPaH^79>AR*#1$?hQsKjO+}>c#^^6XSaHqCQByCjB8t>% z&goNCG+S9JU#!z?1D>o=vt*|jMxQQEvGM?{OTi76v~F#YY%R;Vbdz}#@5WO?MGFp@ zzC25AO-hfm!-ah0T{;dGs6L)AU^yA>MrfM68d{7~`=wbCqcs^FcIrXXL5zlWb9*Mq z;8Oh|mXqaQW@)DLK3U`(T6&C^{Nk52i`jrG)7O<7oZe%88HiI^IE4M-uX9rBvgSt) zQ@NvVJzZQbSDGD=f79uK*peM^FE4d*T1o*f zJeuWj&bq|YV|g;*o)*P^SG4UR{QI%t`uGlGE9$ro$8|T`oC>s7i<{LS7os$7 z_K8f;T00w5`<)d>qeo#PkcxVqH_Q@d6af;?;KMHQ61a_v0QH*DGo}^+oTcU_ z7y0fvIA#QsY#lL#KslAUFF6H2zYrF*jC9RRI1~X*keTF5TIx+o%b-avk#A3Rhz?mD z4%pjuQ8%e7y*mOjjR_gum0n5r41)}C5+T#1lYU{Lf63@>L`IcXWK=iiUAWe`(;?0B zmO+ou54ADp_$W>UnF;)^MTBSW>UsGy>CizFdLlS`U+*mm1LX16 z3;ijc;JJlM>bIdUk7MYzVCk=v=0`9<7K!*smJ(*QanimK*Zq7|p47-_WNX$=d3D(7 z){FXxXIwc0ah}6jf<%Z{ZglYL7{ImEg?5aI1O?{WKTIEmJ3;g4*_T*T7s905}2a;_UVGOax0NJ+)%zqAJ}G@AqKGE`Jl_T%Le%tc#y%`!J+ zhUik&?5i%&ONq{K$#*oAVe&FOI?Pg?4Ck)S<$Z7ucZhHhsxqIrK%zD{f%z`9&+t{oAY-=}> z<@3>a61XKuPxZEmaGHz=#x<3G)=SrhZl)62MpK;ZeH-Ie5oM#iH2dA44@XP=9Os*! zYNvf9xapE9?KqFn9eQ`1bbtC3y_@JXJ>P`trOx z4@sH!q1pXWh5c=65Z6=bI?mW${)Mn6kz(1r#Kdbz2-&APqgZg?rXH_}Sxx+9>|{1a ze#CjgOb)N>TkPT$m%FS;G49N~#xs+1oD%Y;x<(-l zEZx42i4Wv*^)*EY<+#i=*G~Xozl&lq-gI;oHEASh?ECu?hesHnp8^?{CmNyExgeay zRLX7SmPOF%B@q_dAYS_Lq}Y_v=)|(chm2u@=C=%u)6qy;!yuu0>LQyP?W^vBczhPG zy5&`)=LTbJx!03u3xQf`1EX$(#q?{jq%nt@1B)Am?Bg6%GEY(q66asYpz2oGm>1C% zmOGikOt90n?U~yBn%xj1k|v=R7x$xCrtUa*kapP+WH(9V^H|?da{u{kUVRBOn!UIX zzx{9mfH6xnKP?JuW3%=ex=xST%64RaXIhUy@N}QqL*f+C?00Ja)Dnxacb@E@ez{Ly zvly5}>B+^{<5(BXp7m!4C3G8o8xW7J%U@EWm-Foq@2GWu+5f2b_oSvXp;<0h_w;?W znAe-sD5mu+d~qM}Q?-wJe^~sm!x}FCa*~@Hq@)izVB<&64qV3MrKcwA8l_44V3Ve4 zf!{I2A4wN7oB%{FL%nl^#-pwD__ayU8QN6`-W092xzwpQ3eb$zTu>bfzKUkL^}xkm zEkB3g6qYJSJ$ig1@$?aiqAH zMjXjGy2<>#Mau-_XHXE)l z8+5o$OTA(&+IG{w-xjxuFm+cBYaklrepsl=-YZGVGFosSU&8*3vm*f6b^hF!?&Daq zTdhYt?5g*UylHL~MU!t)O)4S$Z1*|ssM0a~qBHB82E8&eHsC{^b=>0)dJ2rD9)puQ zey8T_ixaque}8EdpmLOH%|#nFPY_ro4Q!|0{~CNi@5Z-3n_PDRsv>1pYLvE3*6Z~| ztErD4&sRXf^5^^?L`Al7ucWa!&OSb4hq$YgGcDu2)t$??j~D;w>`H5YGxpamQxT8r z19)7yJZ)LRN52`nV@%wKGj?6x6neO>KMK-UZ;DPECZ6e=sqF;5M$}N74=BRvpaA0~JO($-?s) zepLgbMkOQPBlCOPb0o-kZb~Wr_Q|HZSBYW|D;GXNJ&R5nrQgrsOyzwL!yoG3hzZ1} z8{rc8=1C^G_LnhJKVz9rn=nkEVPVCw;yl=JoMEZ!L&O>bs@Y6yHfd>w{(khy(Y0Ox z8@hvF#$PL^O{q3>lCW+mYg!a^^$zu~_p-yhNFy8ek}^q+(n0KA__(IFI&m45SN?T> zvYcdDg*CBaPLpJ)Fp=Hg40O zGuBmZ!szc)!j4d?eBoj0XV>56Z@Asm^=>4gVFzko$9aWgcLJeyY~AK@Uah&*^c^15 zSw#Y2{Lf0ogCIo4e=y%*^}t3;3cX#)JfOljNSPZ*x<@63TWn-2`!lqDN2@15yaS<* z#(GB$r^`yEcL+NXj~!mj!8m+#He9cU#Bd=%Uljr$gmHdrUh8}emL0x5g}Q z%2-D!&70wvMuVr-tV>4zcENU;Z|pKn#rcr$ugu|PgjLhbcgd6rt#}LfB#~2Sbdhbe z5hrfX>x$BlSRB)IG6Zm$d*QTMhha6c6RTK4#Wv!NH{-npbF@u?kDc)<@nFVASK8^G zew6si(m>=vrq+FD_Aa=pUMb;gpCb_smXh&({+P?g_nhlwKxY#g!`3Pu=24rYQlF{JB8V<;70YhMjai+2|3KcQnS%eA>%A=#T%}Csv&6 z=#J^5$SGyunG3h*?xHsSR(}q2ieAz&IcdQrz@3=``{~ zuIRmW^}@)n2!PgR=DHoJC6DKkz2g!`ug!Bk)AOi6iQhx_>4=L~W&nJ#+)Gsaf0pUB zuZX`jBK~jz+(-9?g(2JW3OCok2Hiz?a`bqNJc!wmi_QVHOGX#%hlyVpcj=tu#?$_#K6PJ?%_mL%%)a^$PfVl7T-kqm~iu+CuM04`!^L;Z++kiam5h^)*M$2G? zGV|r841EbQIJWkCEo1r1s4AiNt$dC8DaKtFE|+~uwjBdci)91p z_uZeSfyl(4rP=*Vb;Xlmuj9TO3^(kx%gBfFl9+MwG$NQGgSMXT#y>pKqYCC zak)0FCpmIas6Kf(){_rZ7H!wYuZHCu$DyjG_1Yl?^yuXdCot|;Qhck|470VRk63DyDMr+y=uRh=6 ztx|rWMF7^Fu|&_QIRmzJuVthsQ`h>dw=uMzfuOzzFP=WtznOLFGXGhl1dG0tKLQ`Y=@C+^FcAzuT@ zgh+aT{OgyBaD%C+lIj1f(~IgyVM|oWuwJf!eKEgP*|Pc1E!#i{ICnF*Wqo67#vd!& z7=`LBt%6G(wN>mEw=K3>iyK0wk<7uAax*0^c1`q1g+a=ZKb%*x7N)XNLta?EB$+$d z6a5ZoK3^peg%$RHg=b1ajP%N0a=hhZ>d#>BnR8UzqFUYb>^+5b;*w2yRs9;S7}kyU zmZ&LQ{BKlzV&KsG&8+L)M4)X7PDK+|P$+`xQ^3oj3(739#JFV@4&vubEMP}dMRPEl zZt05pt7oHHzxi|JiUCPZLA5?+o2eT3z;^>jK?nS#weU zvvo%N^NMRtp=TSIU!!U!J1$`sb7|Y5o!#l~A=< zdE}{Ca$>FSE|8fM_eyn?*VW+lgsC%6Aee~x*ZwQ6R#ksr8dNnp^HD5mm$!NtgjE&m z5oj5gGt)Xd|97{L*X2kcnb%lx7fZnUjr#Hy?832*&jvM}S8l5sdRuu%)c!C3TzM)R z4uG8nDr0XqLoi>`3WTW<8X>Tf8Zji@(}KXYZn1OcBaKNBYX`2q0i4HKfbr?HeqWqS zSIbNK{X0chOEN{CY7)U-QedGeWNNm-ExXldr6ug65@KK1nK)kY9Kq^+LHwt$N z@K&AxTW^DTCsvjrz*^+6067?Y1iS-4NgI#tl35A1ev2Mfe_ns#G0^aFiJr71kyXFe zH8E}BXx;k|Z=T6xO}92Eec<1F!j4;I@}Wsz0or zVs#K{HG(l5Cp}wE-_6^xP` zffT~3$c?x-9#c*Q_u`%6gZ7KSP^572^R}68&f>C(gwaAC6AafBfKam*y-~h&ciZJwI~AB8^empC9f>wpi5mvRGIKhKkd!Z zH=(7J%r>C=N_WnuT)=k2#@oYoK6ZwQDfty^14ajJjMFa?f zP`UxW&o;j+Z!HdJ_(?t^SCd`>jaZJyVQZ^|3E16B=FJ*{6ntJ)Xgo44%PIdl^qN^) zS49H=uiZL_8a(jq>LtMI^DWrqEsBILF&bUUeruFW&+r~< zPLmHatfedl46oRq)iO}KNRhb>=}GD|X;D)BTP5_WoP(v63_~rzW#9L@ahU8{i;P-B z22w8eYpc1*EChukJRr(2P_ol~ht#PhKx^dt-YA83sG;??v#;^*aVN%;mgkaOZokj* zTXYg=urKbsR=!}3&{b7u?<4w9ryBsYJOsr7M-|ev3Os%Y*wmgQn54zu$iz{ElLnG$ ztxdzw&?K}Ni-*Oy$KAQ0CF-=azprxFbe$P>STmii6+I+G(RZjLz?4@yp9w(KO9lxdfm3>-IJFq?%a1!Y37A@m=?yNyO>kyxA<-!hw1e@w7+-*J#tad z2ZEEl#0e#U9CluS4l>7tpE!Is0=WxPK~MvJFOWnYE77ta=z`G;>)tP2u} z4h~Gz-f`nC4CXy6kW1f2+>Q4$(5_Q-&vw)7!=1pn>@t;VKIU}FAi$-j zp3g~?3r+`aqmkWgniz@iA>1UnDy=?%BAq^_@UimFG*vLNqDKJgy9FWLYH zUN>KCYk7Qi5R0T*rKH!YR%tvnLq6j{8R=_3^o%|8nlvs7CL3i?IKN`~rcBt>w?npj zK8BR%P)gFx=KRI`)8VvlG`{V7d)C(}q2)25PPL>PKeq~~XZCyDT$)I_woLIi4{&iX z5zk!E@zD@@l%3@LI>1~(wQz3+?P($&H0forPv@qn6Mzh@oEaZV~?Isea` zpMG2!e`Of%Af>HAq_kDr_H=sq$uU`?lH+99YYrcW#thETSQDDS4+M=z5p}tlk`Ua#}1#0=FMa7dxiW%w<|Y=x(8eFiqDzsLm(S~ zu}ObQ*DKX}WRQ5cbvu4F@E{F+ZR)29c&oK(;$F(_HYI#ct%%AAXURl(kX0fKL{oTq z2evJFJ;2CU8Em}?cCaIGK=pRWT5itn-yfd9$g?&;U=M3up?~yuEa%1&2H7=pQa%$+ zj&PoqD>Ws+gb>A2lHD;}jhxqU=cWv$=~1Q&}T%14|7m$7K1^ zE&B?%N3-TyuUpToSRx z$~iB3;fa=`H@&?dCJitf%vcI-TW{I9!RjT2_DkP21lX+NJp1*C@U@<0BsRJ8LmKX>?!L@^gbNfN3pJrY3Usr$455D}|GeV2_(TCsY#6v6Ox0 z7<~u#Snfe;QW)Wip62Nw(nYOmgoj4SR2NqucSftyC~2?t!Q(j+_Qqe0@yqtIz1q8> zJS!hQIa+rXd%R)t7@u{$7fEm4SnIuMZMqXZB>M@srJRO-{T6fV)ZMg&7(ZQ`qQUGW z7>Qvn{m|S*ob0DDnlJ=9KZFnUy1@=0#lnG5J7BX{w!G z1%bQG{W3BO#@Ulrn(k1hy6u7Y7phgnAGUuC`#?7HLQC`^z7>=F1i2UMG3dSUiHCbJ z6$UgDeG2C0zj_3`nqsbmZZK6Rny6wiTt9B{mJaVauQ}#=`4%tviYTGzO_WDrie0tVAl5Bo#3YUubZ7hjPj-3qhFMZG>Dpb!w;N_wSTVdOyTCn1xv7f-Cs zy;r|OO_$u^UfMsYq$u%VvQow(o9q|Ia)R8yqls57)OV6FTs5UK><1kr`FFDL{iSW1 zUEgh%sWa;tdrj)}JEXd@*Ez^uI|-tDEtLR!?Gaq1GG`ES)+ws8plnL21o~Hop}c!F zWM53S$!2GQmZXD0GKi3EbHgDvtkL1I+kzt0ORf74X1J^;B!dUx*tJNLT5qV=pI)jXS-e z`_{9};AISK;?`!R%6>E|lIpZ=yt*>JVWoRL#4{U#Q3x;I_0$;}{y>ded= z9`^|?Ua^{Ip}1sTXtQE9Nr1cZEw(JG{8AmXx$Ltl$1hRsoBa)4$GLpAHuWdj%Mxia zUIVedJA52Zkqtnu63qmxAC!vP8W<>lyS-Vd?OD$t=S}L*2$IBONltr!Y_}*`KaaDk zl9Ao6(uK4EuCKAg!60L;c!|A}M^3r>m2WKdwUCajGt)4Tc<>lr;E_osSx|TJ>C2hAu@ElXw=yS_;i@e*9G|{GNRlYULW#y^(p7y0kLA*1YXSiD+ClN4`%G|k9Pn}cXDvR+U-_zjaDP^u9k#eBQe(B+@UzI5 zTFdrhV_qRzrxzhub2B2;d#TgJS{=q*)B^Xo$o8g(lM^webv86{sjb-CwRuxUVK}>V zr)RP{1Ye*O+}sW>cxOr#)TR=9V$aL=wr50K&21E#D(Dx-hO*Rae8olwl6Ngwx@Tr+ zy`S@F!di1rNvXYl)--MZ<53&l;cXP@K33S&wjSe*pQ)MI9Aw~ywWEqB;~w@lN7j{+ zB>Zi`SydWZ!kp}1b1LY4S+{%PSzW>YYhk`JZ@JQ$>1qiNY|t z3XkhO2I+RRO^d14aLV~{dw!&|&-vsf1&aDeG~E-CilqF~H1fjT003?$Zd^BnyW*xS z((QB-_Ojyw{+xA)(t`Z0$pe&IgjJfhwOQ*(a>K^$Sx8&83MGfyACDCio9_7*M!Pn%BE zHym*$ENuZBvQb`%+(EVEPNiMPYa93&hkRMM%WZ$VKIQI?Hqzh%Pw0G#);Rrsl<)A% zFkfy>A3gz9t9E?~nT={40+4z$VV>m&A4}f5!wU)=7l%X!Z z;}2n{ZL57o%r@zH~P@}`mCg^@`Zo+Co~F(i0*bo$=Rw=4)R(HzBi)P#hl;XEF-tWhoub&oxWx?ewGjQLy6Ffw69g+ zvXTYwqDZ|CR^|Qqmlko{^1Ax`fDN0(Yc>l$j!J9yi)==7(Fhq&Yzdb8b*2nI~u< z#uesQY|G(&?)9*aZMNEAv7V}0{-J!{u;}jA+3ya#<_%VAGxeU03PH9AVgRrGo$g_< zd5SR6(`A%5lFoy}DDp`)t~EmsU|~}`nj&AtV&iX@sbR}8yym?9u)%_*|LdJ>$CXD4 z&pN##$P{B~d0vyF(}5DD*$z5w%{eF2lIApHe3Vm9-}5DTT$1CfmPs8J=m;}38h66Y zo$X(ru5rLtZTlN`JT32PD&ZbAN64CBtVhuQo!ZKo%^NGRZNkG+h4=X!*~m5fR?{p^ zz-{^50g7VU6@$L}(k(2ae443g`)zz+u%J+lNxkG+jliUF!IGTbblBb@;x zKc}7KkJx^euOm;c^9Fy<&wZ~?QDe+0K9yF(ZxzrQ)cNxSrx(&0TVU}8x^)_hlM9z9 zsSaM81W4{8Zj?Su2K@S$5Dvwrq|9CtuOua18wuQMxq09 zH90_s-Y+N5c8v2dq0)%ZuupC(D>0SlYNmIQm^^k>JanlcsT@jrMgt<>N^}UBd$N5T zUS+W@Oubbv7ca@NqYU5A{%+wiPf{BJ9daJhA4Lo4BU`Nrb*XGEi>ZCvtsm6^_&UKt zB%Q?|=qc@&xrhn9pXk;%77)E$(p8(;F>j-kW4ooaPCUD|>muMmw0+{4)jgyls0(Ev zzH=)oCi?3w3BMg|68K#w*PoL#`$b6C9pnL5-tFl7laMKGxBT`7=3+M$+utZs5Z_YA z@)dvXR7jfSaq)sJUhcL*7nREL>&;Su+ml-M?3b2pVctV`y2f{8uw+(}mRfY;QQJ0i zZl9E{hoAxiBGyh${&am-!i`b zlH~Ohkn^ij&`F8W>i829XCrN78yfi(UeuAvVXqHF#_N%~&RuWWItx_nQ?Lb*bfbp&hKJ)Il-G|L@}r@Xg+Jj{o|-fPU*t8(7Ni1kx$ z|8L9@;wEc#wUM4a*Q1g`#+usr8|=&OWxCS6ugT0KdpyI@w^GvlxrtfI(*2gWCw5>< zV>k!>dUd|ltVB@Wh(OgeMuo(4EbsBhM`9%s_oJ3K*ScE-CDW!Fb90M;Q}`Lcn+@_B zA1V0U7u3(QysXAo8=jx(>1CJ4PxHqzKn@n|Vl5k13ghBv-Awn}3=5JP1_ad~Y}DCV zAQ>D@G(6#QdV-pn8Z;*FY^sUVmJd~^vCC|#)or(YnQcBP2d_wM@MhgP`Ao;{ z4vo8FY5-)>t6K-tHJDHnJ#;CQl92C6HQZp2G5zCRV$I~~sHUp%Z{}NBC0o897t%1WZuJ8whULOY36%?7dfm$Pn%RNkIxg@t&V~L1J#l{~ zgazEBdHfD!SnY-rn`H z@I%Rm&~+bD35{D3aRS89y<7B4m*xla&r}9yLObIiQJUrke+Z;c`t&MCGy7N<3N{ z;LdXhKm5Mn81d_dtyEoOvNXC@<8zSN+oq--4pj2efBt&ivPihxU6yZG!7>QsPD5vB zx6iGzJ}AP@5NdlkBu2Tk?;qt-_zEBB%lCwT+M(_IqVmx5TZ>=W)t)acE|ete)c-~h z!6opW&LbT~*^d#zH-wvd8ku^SVM6HmGTnL2c4pLTy-@NX-#I^qU;)mNfohucMTJ3K z7~p|sVHFJt*SZ|Ex4GV4PxPHB={^bE!%dkr1C3e8NR}WA;LFHD8poU`5KbzfE}<`Z zH)fhQS zrS`}pbTt&9EgPAFf=P4nwb79!Zy?ZrjtsQtm%^W07<;db1~M8{Dpun;K@obwjC*WZX0wm-= zczzVE}C!U!vDycqxb_rpy09jg==&LD)j5fa-ZkKWzjA!6S5Ibk^d>gN~GG4j@H zddLCr&;Cfh&o#$y(D$Z)_S1)J7!pAhWfuF16Muy&t`K3nZ6LYNLyJs|)>E2F9ygtY}&b)*)W3d+tyE&ODMQ}+cmRdun~AC53oDd(>;{Oyoe z3ywV5EFO%z1vS2XlghIBrf^M%T{xd+$sue)k3keJ6c z=I32{%ifQtw(v7N3z}<7@IjA~_N-Qh@CNn|sgsojG`CM~O?_Br~RUQBbIt{*;Kyh;z zIA*-E7jJXd7L8wZLv#Zj-8qBq00&`o!_)St%fgp?0QR7@qv+}|EV@AC=$$mrh%7}} zq2w%+GlIzC0t%U;*j2a~xX84daKNBsZ;=tgO;C83VD`^*Af-`?&Ld7ekSjj}Rozr{weOR+m{9To zvA)~LO)y4ZpR9BsbOp+F3=!HtNC>gd%L_HR=c^y*ku^bcVr_l^p>ah%P=Fp&4$u=< zo7j5o6M#~8`Ci(2eu>MWgV1R}jz=dx#v;u3v(C4PQx-&@G$>6C%&Ci;rmbfqR`_A4$nlt5xaW(JYte60K zr0W1ZtLgPjU@~%zIY|3CNeBlKjDB&wSOO_FXPBv|;MgE8VAyDS;`g8YQu&NS5f5fM zYErstgdL3Q4erf#u;yFYaLBN%A+X^d7X=Socp$h>9-<~hOq~Ygkx@|)^)*n9f8F>ObCOX;IFRHdmWGJOaf zN&f{Xkv5=Wy?{*oOI-A?{BpIi4{?yxCs3d2=V%3W1$12p@ex=ZL=tLXC{Lpuup2S3 zK!9Da2j*R0d~->U>)fgYMMW{B=^=0v^8nU1K5^KK-l<(!hH%TDdCP$~2<;8YFx^5r zdH(aYva?`S!vQiRL0fM_x4Y^8`Sz1Xh*GXV_x;vzi`+IuO~IoF045JhnFrEmB+fse z@?w8kq}ucVB9FDI>U~;Y?n5{f@ZUod8Ge7Go2Gz_%_W)A$w;J@=QY`HQk`w?Tn*+kwuOP? z7e*2L2QL)Z9Yo)({)&Ty*%hkx&G(Cu#3 zQb@!y(hYr*d8KndpDJALI*ibJb>bFmyocr6fg9kv&7kKDs4wiZZH$JPucN5QC^2yF zfAjJj5vnVojCzNR%i$!OLCrZQmNcURJ<<8##}_3f0Fkc3`6^yz1hfcb`mQNaK<1QtDwf`6XIAJ9F}1RixIRF#x1H-{!F!EPch=u4rcUw0{a z$f&jJ)0z9WrR|J6pq`GJ$)3ShMX zeuZ*K3>pwp0i9c~Wd6lyto~jmLci2F2}56VO$b`MyPksDKP16?=si@k(T5lUf?R#* z7D4a%RPKQn0Js%>fz*qBW)k#plLbm49gEI3+Ukl zQ1t6Oo%uDrKvj#VX^)xPz11;RKs7k1YDixl>GJnyFxUEToHO|W^xV1Qp&O-Bz-oQ6 zAA}H;g+r{k`+9!9<>7F9E{-tv@{WX>+o zGMxPSlv71@liGvFku-tMsqdL#75}qZgvJ#Wh)O(xeGkb(G?Nrmyr-#Uq75Ie^cLFTt>FZ26PKd;e zdzsz6HfS2e=-+7&hkcU(*v)VHicDTZKw=1%7lAT6x8|Pr1eVfPaCJKs06kc)0wV%l zb396rXob&W0|Tb~>l&&&|Cw?_lqaw?wr#)G@vZ3*0)f2*>dq=lAZe8@E2TCCScSAf)2MC!(ihP1w91w=SHuz zw`~7*Aj!2P`{c_#BO)taIdM1P4diXL14KB9d03lit<2f zIFRkaWi__Cc&RCnmembmM4-76C4qsDT0*g zG;Bo(Aewk_bA{I!IYx|$aL7#K&LSw|?rg=aM3#V2gBH0$o0B(_;}2d=NP>!%2j@dlc^xm8!$5d1HThM6tjr%d|3TixvnX|#a zI1|LEUtAkGWf_7nwNiEy)5j^`(dD1}xXJ3dT>yeME>%U8s;u2By!=Rad3k%;f1G1M zSX&`FZ0uBqIW^`;C_JdN0n%YTXD>3e)dp0@bgW`XDsp`o9d{C9uh{zw3pSz&OEg_U zA#p$dNBIwuS_25JC*V-LJ!U=usw!3fM6kCkG&w|{WoB|6(Kj?su4cv*%y7oBe+?6s zA@9z{Jiy~<{^_Z@z_*R)YBpx95(V7B>|QnFY@5=NX03mCHYgMd{#Uo>| zFUGsxasG4c-N<`j#>+#7aw`o!d!+lm=Ra{22EgHYzGlOrSD}x`iQwN)zgJtL@#-hQ zgou0?`CQCnhV=^U23+&D8GJzrXe-?5-eksCvf?HY-Of<(s{%Zk?hsJVcVXmU9QT_t zph|9or0Y3k!pwRS9*-Xa-Qy~n?(*lD7UWXXrL6Kl0qJ_=_4GZly8m-tqv>HWtY--5 za-PG0+uFt-$UxA|`t*x1J*RAnUjIHctr%vLUz+{(%{Av*C1Pe)46bl&F2wM)+$)P+ z&DsX-u@E1%L5A77Wirclu?l#j&GyCCt;hedD5f@d-yr@FjdC#BT_jfbe~4)y11nJW zi|wHN<%&Qym(}jkZla{g3Ak8oFmlN%9r&}pMo)v!w&DQ!+f{AZSLwW>V>#TR1Op0< zC{Lt%Ho(-qQ^MvFv0jh?y#!2?ZiFl(c!t)4>pk0_+F@NynQ4fPXzX6s`d5(wX#j`h z%0}kh;p8)W3Mt=Fn$45v?)TIPEBMQZ)c3ZX>@{(=i3%1#dgc>eFv+wRw}0<}2XwdV-pKx|}RH6KL564v9{P+hN8Phsw?9(G%A- zSFiw(IE(4}r*o$mJD%_keb0z|wj<;kEi2z@ zF}`abdbReIu$3^PRm9e5A%}pBKy3pb_7dRQb1C_4*yfQB?ERvDN`Sg3*o=OxV(C^(!-FnpJ?dofHxsY0!H@gX*qkn26|PFL+uZ=k0YeZWkchX zemp!n1nSsu5M$|a>VI0Jes5L-I~W9He0?j77#+#{g^cWgtfSjtBoKTRymLLn*t!n% z3yZGFVpjx>OY+q{GG~l*2zQhpAlQL*)&X%1O7Ql&-LC@HI1@q@GFwvQsxl#FyML{4 zZ`;%BMATruK^;~>lBlT(`W@m(&u-wP;Ty zSIcfPqaF)l#Cr@a_X-~rZA2_l95_=*yryuqo9K)xv4;#A4e0bNp4pCRQhfqh7RnXhiYoPGdkHToD@H5ZlK3++!2tuQpmJwgooZqKB-2j07*I>NYXYuv5s z`4Q>5j;e>AsY8ypQ4czl(Km`nSEqQv-YG;iVxajOsTsD%mPX=NWSs}BC-NuwikK&TC;*~T$~Ah@6JAQ|52|zbe}PA01Oj@=tI%3&0AQ)f7T$=w$c~l}o^=R^ZWW zT`;hi*(w_{ZGq$d->z4%G!xWdK3NhqkEm?};fI>W6G+S79)mbuJtr{yo`805gG49B zX}K-|=!bUl^$P!G%07dEME)pt-;PI63LHm-Lz0o98=hc6-u)bc435-^=dYmrwRK^O z?|a~&yS7_#Wxs@tp=7~euVCr8XNFX|?JWXefJ7~{V%o!mmTLy5US2S8G+2bzvtjtB z0J5WBpevJo0lK^F78ho&9eG5WLo!63FnhTj$|cs@clgepMzT7?FmB`pWC@Crh9T2F z(mll=NcLiDUC}EVD3u3iuS==0*;)sl$4Y-wGFn@d90_><*8WW_1DuKSB+-6vn(jNkl1b_0`5aVh*_Y8$Eu_I zv%2$_93On(aa^+L7C>9SYVya{{JREx2G{Mk>;r|05bs66yX7HC$mK4W6f*>_A{8hE zTU8f^Z~}Z4X4W@{Rya^r|1b95J1VLy>l;=YY{0e~38JWo0g)geQ8K82l4BttL8U;l zfPiE{F%uLDkgUWiqJ$y`133v4AX!B~zybjQ$?v`uy62hq`DT6Jzu%gfwPsDzx9&Y> zpM7@z?M-UIv{!@f*1K*4;8?Cbfv$m(0Oir>8s&rjN-!w(b((tg0K`t5AYWixf`wpD z$tHvpi%Iyr75xKJuBk9qrW^dItFWPKW(Lr!E|;-@6hKAJid?6HZLJn&P3_R%L%Joj zv;zkZv+YUUz!uQkA>&Zu6auOf0N4rfugJ$a8<1pnh=BNy{xYP;rMh2D^7(H|xlNQ7 z=?&q4DO@s}06wurDurh*p0>#NOa50a zoe{@@v;_!z4}xKS9f5}(1{BP!4NR1HN-Ss*bTUoQdLL|n4-zfK19*lLU^O!B090+DNzcE%H;-&ya zshV?HB}^JU{Eb_b%fLixa|CkYl&1_45lVKf4`rnJ@+{RR{r=x4hsD)$s-)f+4BtCM z5+d+FG*njoYMt={VO4P&%-d6^ugOCqa87%6oP87vKDRu6uNCOnoY1X&URCs1y=}(@K!eDIPJ9BTn5I zl^q<^9lrf{6!z3}O6k_K*^AII8`{dA3jbio6`141K8nVL%H4=uOdm5 zazAv-pnTn!QoaQvV7n0sZp?V%b+}jjnl&3i07@<7Ud-7jtQ#BlYye8ua?qhr({Ql2 zKo>>KNz-zm);!U(>QZ#qL(h}$NRE(sU!&_SgEE}WUV<#dZJZ7Z$R*?r6!@;X&J0(0 z7RMP?nL$3VT;!k98`jH|SJXAj8iw+zhR5MTnFBRCGTKN0+#BLqPT0h`sE zBV>rZ@n&(4#K&aLiGp=z4Q{RGofHQm`4WUNkpIU1oJco@(vMul5oiI%nEvy96>=xP zb*2!Zxp`ypZ(x-*cl_N>B9NlFIKJIa!y!k`9e6Axw3$C~|f!TfdL zCs?QUzmcqRAU_oHgj3G)oFvNwB(hk*oGE6gtrWR$TTqW{Todj+Qow=)6_5!YiE#G~ zdP8x`95$UJb~A51DA(eS;xh^n9{WDfLsAS7U{_AU^>qd}AImfbjNxIJohbohTVvJx1TOM*#Bjzt z^g~0x@4Y#ub$m4d`9+-WdRx7Ub*s*i_xx?toewigMipndeD-4Pqfmd+H9H@o}J?j+;y+29v_MC@)X0$MT7-1D5|M}fMP?>5! zEey$?c%l0R3f;CT3&aLLTTn|?Ab$z;J!rGHi`(Md}_}6^rhd$!AFWQ1A!Kr z1*J?Lqkf)o5{|KLaguo_-y##XPR2Rtb<6iS#NYDClHwtI8;HvhLg{SK^Uzb!U1dk$ z{eKY-PtKj9<7+#lqAaA>rGyM9L!!R3tPlpuLjq(fAskTpYQp#=sqa-C{%^Keps2>C zQKLh9K@8kHN`RDH(4Q%wfBhHZMn58H0If6Dt@|af*#Umj1Gv7Au#QCSpnfgWHpxyx zOZs8fMEpO+Ph!;mBgaR|2q5`Mga2iQ(4N3s`z@U!NfpCQ$VTOXF1TA!o{G)HT~0SW{7fJt~GnGunMu~J7CzlSq|{t z-s37C!IB$I4}xKJZaaHCs}~OsLbx~^QuFg1I%DRbLhJcnBOWyAuIgN*Ea}`xEE3sz zj4Wv_4-{w{lz^sKcp1}1icG1>wK1`(NO<&>u3)b**by(SYppjk!^n;vAdM+=dhkMv`{j^+Bw3*V_7DCxgay2xN!avCn}RZPA}qtTWJsyAP~24>-LuXO+7aZ#a7xMTq_#{Ai*WX55|%E+-J9 zW~YkkxZAS=M!OOJbw_p;QbQC-vE=rAgc8O38CT_Q_#Ej3Nz#~sWOpB$s(n&Pg=yx{jx;5uOVVlX4>0*yPHN;sbK^Y25F&oC zJyc=qHmRlu5_EZ&+rX%dOW&~VTt^by?t$pef3nbU_KHfBi^1qBst1{EMSg7N+mY-L_AcLO=O4SFw@ zbmT{p&sWUY3c(0_VQJiLcBvT~TM5MU5Jv-&+$L|9mMYgOU!G+JQOTpj`;-Ib zLqdC=+Mkl#3W2Za#GkMW;gQLl5OcHw`Ec=2I-ql;dK)Z{|kn&ho-AKs@u?j3o5u<;+YagRbI=JFS3 zq3x9wPpv`K_v{F1eUuV1k$hsG;x_b+Anp^H%S1+*dO#VYXOVGRo=wk4EPp-H0}s(< zcNE`HiS41UcNh#*q1tm3A;Y4eS^u{TxC=C2{eqde^Y_pHq&iX6|!62GFU<> zkBERD*ecpl)FN2Xdgc$9iU_4w>TzPW&(~Lw;e|-|6cG3wh^w?hs78jPN*DqtugG1fl!HBzUs`5mdIKqhcGuJ@Wcmu+j61IMJ!wEokk zSe8QBh0?{dJT&x0jKb1Jua@}LYcE4REE)GwygcOP7au755= zqY~M2;d^i}O#c|u^Bww2=gJ2#gR$WKLywu*eTsbpx_n4Bi$&|~8yJ9S6({dkzHU5P z?KhmsX932ZZHaal_mkz5W;UVJSPK2uqZEupLqdbpUV6nl!L7fQHj0a#$x5Dpw4M9$ zWJuZ&VA{nczwx&jFsr8lq6`M-`EjFSS<5Jk<9NiPBl!c5c`1R3R-XuyqaVU)o`%{r zwnSY@EBw<=e&fe^Q?MHy^WW^s^+RIRNMn{7NoXn#Md9y;3~GwhdWo7@hyRADFkLjI zY?oN^bnH0O9T1i@rH$?qXGnpq-hR1{R2PPZ(hHUG7yAxP)nrZ(+C3@kBh{oXu9B6U znolTFiHnz_F;ZtwhTc`=f)z4;J;yWDntC_(|4y9u4k%15pEOsZ6g5*UzT?ezCmjkx@9gWEGJl|Fa9~ zsn|PjGlO@t$Nr08Vo1P8B+p)Xs-qrL0VTQ&8B1AE8`7)op_`szfR_e}@dSBMlsdZd|t^F<^v6zm&S?qJM7`aJnSmO^Go zV%89(vdc_8xjGP$`OheV^YJZJJ$Bk@l5G}9w#}yV(S2)luzFwN;v1dE5PNhW=Oq=76V3! zW0Xkl4jLjwSInEzK+GCUvt5-p0*w#@Vx-bb0!FEq)Ds|L<_NZ$kMR)O2@MYa&Hh>XkAobT~h1yc% ztR+#AiOnUm1XdOIwPVs7FGk8(0+a+5$8Pj0Z{4l7xU;5af;5pI_-C}Ej z9c;Fh>1ujYhu6WYYxQ!M_Yp%(CI6`{eoVQWhN&{o%V~LaY;lP+yg30WHN5m<ID5 zvtR_q1G?m5jw^>(PADae)5E}u?$!2 zf3lJU4}mNTcFVD~Z9tByJ44%fPb?Bw9o?MuBqG(qh&rXy_kLIl#kylET1eS%Z=sF0 zDfkjdlwfQZp;Q^746oH3{?5v>>QSN$;UyBYoW@?Xe@)wefhF#&%PUi7zNzQRJ`uU1Y3eF*t%BT}#WN3c*S)@%1Zfg4rZ%^@**ek!4dY{AHINca-{ z)4lms)up}{z)YAB<4gLK=e-lR+fKk*Q>bbF zAy#>e>l*^FxQz_4DB_@*ICL|V+4%8}C8Pa6-2di!mP|2eHIkSE8r+VKZS3fP`TryXYrryp@ZkZ96I<#M7L?z z-bTJmDX1Dn^okqFpMvmVEh_iQ6bVP>SVYPF7uY#-xKEPH!4Tp6uNpV&DDCCO%A(|(Xcs#TGgu4` zzWYZTvr@Hj<(sp&=IQnyHr%kkuqW<3NYx1>hk?Mjppp;`4ZI9e4Nn-$|I-}XsQV15g>|jIL-xfeBMPofLk2|aN5(1uB zjTqrK8v!$;-Z*o0_1)S%{6BkAT-%V$R$&9DPmGEtk}2?jqOHkhsLgV7H?G>18 z+B?LVBPQCVp>EgHJ^9PZQ*}Dm8)s7Ct-s`69S%O*XCk|PR!Y>hi()~w1Ko3g)aj0^ zN7@5U^mq7sAXCGO`x>LF?|o>Hn|0wa7tP8>O6CqbM_Af5TuKPvv6F_Dp6Rmg&Nq>k zYQn{@Bt1ih^%^Vdb^9h?(zhPZl!B7MxW4M;V*O{n+v}swT7J{|+DnN!mA5%Jht*^% z?~`A^mH(Y4h!@^r*FUF69Mn-f>c(=fR+x5TFM&S&{u%b7=alC@urX;cq|@rlv+Uxv zbSH2<&`jaq7o#e?1aPSGRVdh(#Vla8+;J21s%PIC%=I@vnd}_RDSBs5&%~v82VT-5 z$Q8p3uZY?d?5P&tnU$iki{>3$=IKmqPR+Z77baTBL$ve)CW^w`@Zjk+!CRY}blIT; zk&V0Q1*#Msl{6hU&DIz)A*znNvbKwn9@Z0tO{sAWVkq=a%3~p zYbncOn0aWm-P0-5qbMauwgEv+o)o{=)z?=!AuiK9>aZ={?h@L4XzHTsz)U>ShA38M zcuK3yJ=ZN^IpA>NZo)nSp?9?3=DsI0c@M?A$)6lhpo_}fW=I)X>n)L(3)o(dLt0qc zpH`>merlqi1CXuDZdi3Q+~zwZ>eKc}+xH7MN0Wc-0)Xc#y?)$1G9mw^p$k8u%Y)uAiWk_{+wRihYsax1p z&xbgR*+_T4z`GB}j$Sd<(KijSPaIsMP|`u0Oi9N%&V|{-Ofd#B)1V%mJct8^o70&t z?~Cd5M}a!A-GItyI}_&BdNQjn)bIQC4zWdq8D&51{ObniVoZaE?>3#_&A^j@0H|q`f}y^RREu$}GX$C2?BVn#%KNHjTl?}>LyGCd ziZII{gq)oo~u)ARd*Eohu8rYhB}Ti=`0|LwV# zh28MiDpSA$G$=?kNcXu&B8I-RIHu&T>C)&=gVS`TP4#~padP9+_OxJ6$S-j z)>dk^hd1Cn?X_+Ou{<@aTYx`QZ7$jh6z1@U#VLGd&_28LaTA)c#*{wiA;yatSsw_u z4Z_+^s!l8+JHS_&=9sY+{+zeskui3Af1J}r-K_Z+TI&$#fzcmJ{VJZolv#e7G^r8L zeA!fbyOf>#mqzWadeOw!tM7T_Upr}tShyx^$f^z)tu1`FQK%?;cX;+g{8F}!BZ6noJ3-DW7GC4^rm6PJy0c+iH>gEg2i4w_! zaBtcMN0#x+>d--Kt*5x#4-qK#pRB}nsugVpb*AQf}LP zsAuH_M0_n4GgakqUYj!(qrC-9&w386u7)`Kj=csh;#~jOAaP{1QFVYribuNhQEUsC zwr!41ZgCUL(WUPPeDsP|O-I*IZk>wbKJ=>NH}6bT z@M{#uyM2|#{$*gd8tIlWdn?2KnosS{-^F+pq4$nOc~10Ccyoc@CA+Ou$D}nvMr;Sb zS=&xKd$r(0Z0+NZtduqs=dB~1p_dg@Y2IbyyZ|me0VVV zap~@g@Fs$FF$U)5wfm+13+) zOM1u*Lxbn~PM2@Vrq_8l!WxJ#qVMqhz%#WeI1e_lb| z985P2SdTp$|M?G_8vzL)&d_zaS;<%o%*1FV<7;f}X69}czR4nA?wvH-JX|rc!Uyfv ze@8oRCM=)lvZyR<1a1;)SKT~;%Uux21(dwl%Xgaw@NZek9 zD_qgn!93&(fy6{R?!IrABpB1vUmTHg_GMY)-gt4#D!3D>fai-XdFINdG@S=_X+(q~ ze{lS05Soo^_7TaeKg=-8ymsYB5c`$&d8C|B;(d!>v?BkCuC1G$R^19~##!Y>$bR3< zGt`*UK9_R{VXHw);T4Ba+7_d-X2WGWm3r~9xZ{hi8nM5{PGeA)PJUnvWeIA#oF zw2@mHzZrJZ5q0+~13E)?oOcjox>7ag7l+WC*wKs&l^qxStQWfRePdD~V135-j1a%p z!upkUC(jDg^}=l7rsn7Oc6LCY??J=MrgA5HLe)yclfPI?Rv1TM1XkTqdgB*QMV>}6 zL^1EE<5f)zipnuy;3`+r@Q>0t_1UzNg@*fw`VWJ(B&OqU&aZ`6kHA<-+m07EUsuow zY*vGD)VU=v*EAh>{Lp9mHmEiZo=Cp?r=3yF_^*+b>b~y~dLibi3ma;1&OcNwT0DBy zVxVhJO4lgu0pli&yv>lC?}TWQ?POziL#n0S!0NLZo=AbkWBgAFVo!}FS)R@@TK4 zmCYtp`(iEH?y>sf7U;jUBD*T z_dWON^!=UviD#XB4Eyr$Niv8v=PfSM(v7Qc$$&NCZd&o>xuYI~2=jN@AAx^4XiH)1 z4J0345)LQ&4r8>D&cS`Cs1S4hNlnGu(G^2!RVqRG_Z~|*-AXwqZj6aCv_~x9KFtsU zS9zndVAq!MO8l^;-7P~Mlq>hmrZe8{v~uXW_9x~)Usd5fwM2y)==2L)!>)*a)<+#f zZ*75sGAVh^pA|2X#KM8rvTm^rb+8f0dMVPdFyd;}9zP+LVH;pd)AiI1?=;c4+`z1Z zSwVVMTbRD?+jG8ROoD?IEn~s3o5foW1p5{H^O6GoZ&?!a@r-uQU8e20ulT&#ZT_M9 z#P#?U_b{iJ3e5Vo*!k?w#l@R)i18KkZtWu3jQx2WB;+TPL4{NHGjl~H`jRotDbs}_ zsfYTzBH1LQm~&599gr)}ief&A?$MHrQLDld@}@bl#*vErswkTiPpAC7w8JrMM^jYa z2Ho2wwSbn{X<{tZwI``y;e;u3p&s{DozR6kg{$hQH&1&%a8&h{_UK}G8H+5m8qS(ZYi_eEpfJ~`&}v=PFMa1mns zA$E;6X$6rCFkYQY#3R9IUCg#uLp|oE)r@1Yj><2p&=ll}O@x2vC|J1oywh2cuCosPhvf8TcqdW@R}0ooXVth`#oaLOJligH z)CNTtMT|Gz`ywYQMY+F1361}WcE?Z;_MD+-HWRUs_@)jY5!L;PKd zA-&#DYPMIUnSHvGc%cl1l0_>UhU$sp4r+w>bX~3%;HhoaeJt$x4(D)vCU-xdUo&Hx zgGKW*Htr1R9_)bkSzhx5!_&rw7&gH)`dg@i6F-}E@zG|V8I5ij-3-~S&8)(Gv1QxC zNYy zlo$ld8xRiypYQSZQ9RF8ODuO(U&!us=KCiIQ=G#+)?@e?^dP$4X9)Mwc4EBL(yn(^ zH#g2wg#TUF?EbGcvmZg>wyxiMIvliYyc(T+O1T4(wRu^qkv$iA*Ai2*Lk|+2QB{~& z-8%8>7&fzrb6wfyt;j*Lf2zE zTTo%XP1Q>|&Q(rz26*)l|Gk{-8t8*PsX|L*xfYKlJxjv<=AT$Nyo)4QG&hG6*!e!| zGAfMDdX$*%exf&nll|=6HJwSgt9D9!>f)yejL5Li_Hf_H`LS~Z-lKQj;=sdyp_Rq!0g8rtrqwp~zu!gME#826#Z?rvrd zcO#YYD-kGFM?zFK$9nDW@*nn{(auSnjrd;Rn9q7Hi92T6DWmmqwh!F8gXLE4hWM+S zP$%rXJ3RXIl_?|C`Uwp+`WHC?Y4j_=YenJSg5`|}&pi_saZhkf9p(MX8J(S~ZW(b? zWRqi0I-@Hp8tt%XfmiS_(#2$qVsfIqI@B%Jy7n~itehJNPa*6WykTnlHZQvJE4RIJ z)020lvBpw1j|*)K4MHnV7WXaW2Ren6xa`qTaGRTmfnn{o3Z{16evSbh;CjbS6jW1O zZyA1s%Z#a9)~WKwV0t`NLOayqH@>=vioRbk0JuL_u<2QjBDO+QUAQO5D^h3$y<309 zgy9aeY+A1NxmZ`L9=+=P3@1bUTlAnlRv&}W#WG-y^A{A1pXN6&&$6**NGW)G!aZ6jW!Vl5y>#u$dsaCt?(DbJHtV1yn0Ei3HjKA8fd`Y*ry4veVZ<(jZKp` zDdQ=x+fZ*S>RcCIlvtGP}!2(-?%Ehdo z)K+VK#~j$`k2`zI3C*2q`1xZQs9B;|?<0r4y^Q;F9O z#A`Q19mjBJ4GS%=dZDbos=I~~ikqTWjf1cbGW<))3%6>K;E|8}xSdgjNk#1STZ306 zi%vQ7DQ6bCW}4Pk_;eZ5U>CL?$bG-6i8tal_ytibJnBUChOD+VzjC8Kt(@{#8_*td z1^0p&cp%Q=mR;BsRR@8h4Pp|vZ&|y0Zg=hw?@{HT0;llkzHzJrdZ~vg2x~l{>dB@P z)Z)-K!^wLh;N@2`IiDp8qsOj){Q;;K6BP8xq z-xJ#0G5X`aWYLlz_^nR@f*u{hj_M~qj~dOm{{fSGEA6CsdAaGA^5Os_)-L9yXBz5K zS6$&avoF29d(nLUv5{DV{POF;=&#U?8l}iBs}X{8U{@by$^Y7Dhy8^fS`d3jdkk^I z`UE?33=JUb{(hdMo#%(Z-?K*)0+uNX|&y zZu1{O_Ne*x^E(!*hjv3L|IG?Vk*p52z_ojZNWf`e74A!7XqW38a`KAib0LL4?9M$7P@5B6x7{}&3C_LXn>8t7T>t45C_esZZ^@>F=VaQ75 zqcNgWI>tS`s!OVl>OJqey3<6$W0+rjqfE(+LYdYwoe~YRJzvs$s9>nck7fYx$V=$tO|e zqnw^6AOC>boy-1FJ5DY=DBwHIAM9Gq+-&7zfiem)J<~HR662OQE?{HqSyez36_0Ly zoMId=xe(*w|F!86Lk4q{M;PrR21~tY{<=o>H){(9#YIPpQ|slLgg#Gz*%@CptM)tzN;anpWgVsQ`NxSa!)A1FoW0adm7)%h$%t^xcq?qpJ=Wo$d z=U@EPljrX)V{@pRfv~)y0U-@1iQD3&06Q0w^8x{2eco|9IXkDu&z~Nu#%pX#n9yum zj1D&ZCcIxOyZhRjQ?j*I;VnJ!5L9q9N{-BE#Mp-n;-c^HY^QzlV({@OcTAa7pI?dD zfEcht$wzb@3Z;!;TCNKv+~kg8U_TkG6EP5v(P4;Z{sTN-vs^+n-=gnNKi(aQ7_FHkp4=*~S{cq*XL$CHSv(J&Illo&lAzm6lYygV!p|w1B z_p?`2N)9d+lw9;*AL`j>KmEYF2~=2j6)COuit-uae6DCD~oj^WCyoeEEhx< zn+~K)G0TSJYQ;`$idRa|)!6hiM`27DI_#ti3?&Bj^$+4$Q__1&g#O$yu}iZry4RUi z#?Jy!Hg;iV!@u|3gqt0{1>!-9!aZWpWWcADmb}Y5I|BDmV&KpHFK7(lq80gqV@mW} zeJ@pH8`IKf=PNq$KZGbBqqdpZakA0iy^oynksGz@j+ehm08s5n?$ZKkI}WKlZbi#} zI#SSouAoDKNclyS5hZu?W)mB}%KS&rNhYojS>((O(S6~p6!}W!cTrMKN+qS|CUysT z8q?73JElX*U$Bc{uS8OqfPs86;*+NR)0t9tD-P|XQ9nw4Ec)7WTUzqlgfCJadtllJ zk8xGN-_=i39`jzXgD?1Q6){45N`6OqtwV-ApSTF zqhp%U9&;nZ-)pwFWPFt~V3)NEl%&vjQlJUN)fb?g$3S3(D49xM0~Rf$n9nh!cPfj!LRtXP(~=@KPP7n4Cb z^Mz3TMqf2yA@IEvyTwR}j=mEVM~uFekHUesS58?~E1wcVR=huOASiZV&AyAur`D8A z-oYU`lxo{r=NCVq&t)Mp#o9b63{#s;Sz+s;d}+JhFxENbWw$9W8~OK_ZOioW3&hkbY5rlaooE%c;;GBCND%lk4(QLB=tx8G(SLUKR){t2U#SZ_$*EgQ%k z`4g7&@n|5~h?No3jHl2bQ>XSZmf@~B*@{ig@g`A?I~@1polZ0Z(>Eh>pxU{nu%Vw~ zyBU&L*d&l+Yp2=U%%8v?hFh1rC244*bd&Gb*p$yg?Cii~yu@De6@JHra6C#rMz7I# z{}C0FOLuDRDCgaiSmmus3GMJARL-#yP7C)+I&!cbU>|`~l*{l_^sNjMk42u`Ceazi z7gHeJta=d05T&0?l0Jfy+1-3(GF#FkR3<=!_pFrrKXg+0iAQW&2t7FT|b@(p8z@lDn&`TmWECpI2^c7-p*=#p38e)e=_2t^bn|5{Cz z(t!hseqD3Wv8Scf`0zN(Q}~zf_Fk^9u~{QzfyOkEh5_El67vUWP#T*1N5M*wvJd!6 z>T9@1PnXtV#usyusf$PuyNuR))5p5Z`}I;UIDa`Ux4EgzQ$BHXZ_2JxVpF_rb#vWQF`Rq$}si?^gSF zVtb7Ico5dQ!Ga9gF#}gWOFfG!R0Wu{f7Q#<~HsGMt{3+%Am6o} zH13e{J^8dEKyjNr{j9_IGKdX+HS_bZE@`@rDwE058LCNTDT;_}JF^`I z5huG<=srls-SH3E=A8|fG$<$fnIN6ZPX7!gV8 zoujQM5y^*6KO(WVr1SKN_=TOYf}J!6;qR?~kq;RD`3-^9Ex9*9k;k$*pzsj(493w9 zs_@}0!hAY2qgjb(#>^>No!&vfvW`HTWU{@pYWSea*pm==r0evYJ?VZDK9L=C$I^~XMfV7G$ui)Z zGiwmpG#uSy-OuA(j-nV}+0bF`lCBsmnzTz~F@Y{5S<(6?()2OuUpg2n3Ktvl=7C@) z{WF0Zc@snWbot~`6#QlPUTC#+Z&;anKHn~RIO)b2!fzC^C>|k; zd|a4Jixl!B%LHi10+os6*E8)W?T;bdR{}YI#W`@Cx95wt4@76vNk z*Q?u8KflOKwkH&1pUjzLAhwekp?(~H?q|5qdA#|@0M_T_Ld`R-I z98AI2b3DgGb;hz&k+4Rwk+8U$UJwqD)_Gqwll2h~_CpO!a16L^M1ArPT~c6&U4saH zMIq*Len5dgFf!ThVF_LWqIeK=q1A5@=QJU?$?$^9`$*1to~2!+RZ23S>1MNs0zWQl z9N&w0$O1{=zOL3^ctUkwTS#7;BZr3jBY7w{l|b5YMr^kQq^L6B2pJ~#2Tc#cqxrJn zERG-8eVQ6v5Qrd!WekPIn~RNg(!$q7hG$7Jj5=`C_u0%flY<^R0@0)p9-d7jWC7zC z5P>q_O|5P{^80`aIK*49^v!;0QHqOf*+_DbLwD&%1n~tE9&DU>NrcyYtcE48-*2b7 zv6q#R4dVJNaV@6!&GC+UI>TxGoG+?R<-w2tR0Ho24W#(Aik75YM{&N;A8|by%m%9; zj)Cis#Mw|b0!#NOPvu9DzWX)^tGm{QeA#oM*HaLP)e7-&aSLZ7s)PQAGU+H{p`h)Q z+EMoHOQbUreq-R7 z$*MQn6t;5`7N)ojMBWN9qpn`~0_mynH-vBbj;JKHqf1BqWo?04Sr9WZN$VuAQyl-= zc2d-Yf3HT!pVzV4m|DK=5Y9BG9tazH&N{gwVGE|1Q?NupTT?N99nfXp*evevqhm)LejI&zIs>j5UsKue7>C}b@0O>by?=}=hNJnSEf4kNsu2nr zU|>HV+4Pd;9)xD?<1Vbds7TNUj=2LW&7??iVLP$dEiS!ZbXT_1w9mHQFS6mrj8%fOVEeuSV zfIu-^`Y|I*ed-m$st~xF`2D?#66W0W;QJ*Hh^(o=jTAYLJx<;3VM<(G=C(TU=-n7C z+tZ8EImE{7mB2|?egi7*PYMjO$? zo+Fqr`H-jwR}TX_#NI!Doq7oBP-BQ}ao=qjzSL?(-{_Nymbt(JWv&(bscRReeD9Hy zQ7|F;hOk)hjzDCg#8Vj&l&}?PCnU(t6tU9S54Z5X1`$51#YhOM1vt}WqO((idI(=A zB>OU3gUzip)^UfX7$uV|2AXx#b-?kLfa7^?P@|9Ze`t_5vgdpH={Cb8yX4<8`f2&# z`-aj#${}!0TIR^Rg%mH^@p39t~DLikc^)2Q3Cr z_+RdWSot22&{&DRmmT3`?ZWa#sC%L&ISNPQe)Ar+Uc%BE{>Gu9$t|n{4=VnyRY;9W z$x$FZiOvOR*-dNYd|S@i4P3DN2$Z}UmefnlbgWUrSAN50ze(q<8T`f}@Sk6IG|@jL z8RxLBUIR%ax%ack&D#wbhSNKSxv5|Ojg)bq)&H$jm%aZ&Fo7LAH$aPmIB~J`bu15+ zvl1wqm+?|HyqViJtr?LjULz8|M|XcRr|QAI)&NznjuYb>Gcm80H8KhpU*D8XjvP6$y2zX9c5 z@>R}9r?tef=?N`ik)qYq%<{WC0zX3x`M!(?6%5AoO`psfOI{D8Rt#tuvD$Uxb+o%6 z(8)o#X)8e8h7)ChP9<@>HTjdn*NU#0Yg>9jtIunu{Uk`s9&}TT6 zD$f?p&T^R#cB{ANLH*zRfQwDK4sFEDquulls3bmd>_XC-L3gJ2>Hl5lz=dw~%*W?3 zIP3%c;SWK5uh$seyo$Z<7;i#^7bjfkvOEmB>oConNmW5K|MvFQ} zxe4!cvRUF_h1D;@lUTli8tKW)E2MJbeM*||zW0jV3Z51ez`i6&mKYFQ;^HqPsoTI& zSj%u|TDt66EnERsJ;?xi2MGEW`<=I;%E@esbTg@wF-^jA=aZB%{&rQF-p)4$X}{$S zQkhVbeDNd07--_?Gqur={~r`9Z1S-uUVe#>D{~st*JNY>-+T32mMQf;|B4JVFH!7Y zW)Ci?%j16~Z#cYoer_8;YiLq@nRI4Pp~{_vIy^}KJsH^pt)a|AqQPCbi+k)!#gHgB z@?K#{GI_;I2HS$|SU(D6@?~Yitus3wCT|4F4tFEpoyyI^!DMkB`{n+;9?c+|PLf?k z(1?NoaDk4hRt5EDZc+ej{&Qz+c=BJ?swf%uFGsUn)?K%ga~z9 zl23~&MtL*>KolCx7e-n^OeW7Vl-e;z$M|(%rh$jX6ZcsX^l#v zamte|Ue0!GfmO7iJZVv@&Odc74yPJh!)rytLh)2dw~rE)-{-kqCKBj~FwJRs=fpyI zXZt5XO3spL|3mVwIU3t=FK8uLbT0ad9#yb_8)5fsEte{ZFPl@?v~ndQYHy4&@ac=G z%GW%=3flQv>eM_yDdkxoNr?;d%zYS!=nrnNJ zs~4+J%v)HoBZ?|g3I=!5jL~HSI!Y8CJ6*^^10`rRo?pb;mB2z04?KQ&>f&9(kCkZz zKug_J6-fu>C^}l~_pT}2S^>f8b*bXdKyO&WyZ!b7d4FDyrvh;=SCWZ8JulCyvGj0; zKvJiBu79F~M|pB68MA$qRF8M%Uv&C%U!=tY)j$}&+kg8FLAd@BKl#IfbrjvlIrk2| zJDmh(MZhd^ZV5A2+KO}Cb2xow380QX0h~<5ZBk~mtPCJ%ABSelR|cMiju(CgkfWbl z5KFw}_f_-7*gmlipOM8_+0Q>=b!}qUidi+oD3I!tMZ1 zsP;l26+^%3C7l4(!S0wfY4PhVdn`aS8%V&}2^#j&T70{Bp5*p7RRz21Uv)wf05RSh z%WpQNwpi}K*4IC^n%_Mc=gze7X@CSEgye9q@3-?u>;zq(J!$pEO(0XO7NHz5cNL1S zh7KX%Yp+nR*6R{Bax=zpNGUW4nO5w!{WC+s5rIle+MIoNAQnO9w&{ktMiwVAq+4?a zl~wBsiyJwyQw;iDU>psch~fD0v7PYKm%;6Od@bMa`X!}xr%B+vHn!ZG!4n8hu>IxI z?R2C?9|_yI%!nY|yeswmGDav%zfj=Ht2Ddw_d_2l0b7-@B-RFXvXdT|Yc#o+8V*Bo z-UNdE8VF-Sjj#OW_w9V#?*PvY!oFByUsQQRxHe=)k#74F5$-6~0-0MV`P$O1@;C4M z)LH-=xe_0VHv|tlo|=G+yQvBCqXKfG!LSh^Yty$loD&_X`_%Y6!fEQ*D`I(eH*2#k4oNeV82 z*Zzq$Slg8{Go&B@MBDy+kPEeUGp1UE^1I&{RC6Im>>PUg{c$Jk;x?2s_f~oEo33sl zJv)%8hOuLuE2V9n0N#Go_u}N)UvRWtPRi4ZyO$K;=?SR!_Huj0l8`Aits5qdAMW0z zkMvH6yMAQ-V;l5LDC}Dm!Iw>RhWYST$@$4M7XZ+f?%t?ok8%XVL=?A{ zYQ|SquBb?s>n;M!Gk;jiB$R|i?r-i>!Y(QZ28RKRR8$hcs+$8uS>7vK0L*hHw#=|3{dzxfD??`3d{X`L0)mw_ z`#k?T%rAS7UBU2dx@48@BF?DF&LIR_$y1zxK-|Pf0n6Uo{$ge1yS1eKtao391P@?P z2Nbpu48afrx~tw87s{)eJv4b%)eW$&uEZy9{(xoQNbaxl@ibNE$x_l&=kWt7%D&))gcuW*Ufs7|Eqq=$xx!nNB-Ir&n{?GvY(hZ>g) z;w7sY#23~Q=&{Z5w*l~3ojVL%wL0k17C$iU`|Y}antAs-&gJ|GIe?%JT&et~EHn`6 zR)0-Qc*riGSORUC|@m;pLyDgb6=K*PP+rq313(EM@ zpcvNc`}noE@KjTJ`K)+Bmq+Si2*#cVKM#0(r^RNYYFFZ9TCNVwV_FpYiO~w}Yk;&X zlUt&>yW>XT>W7AnW;~u?ANw_ShtO4X#>L5) zbcM7U26IHk;A=L4K?u9Tr9?P0q}pr;EO;z6lV!Y5u(oYRN<9nb9gFJ-Jj8v4!%v&0 z0g*^VAAis^xtnJfb7@16hM9wwdcxvu8SlTC^T(LPSMgJffp>^qX}Ne3Lc+&qlNcI%M2r;y?7a5 zo7JBtAKjKRVyNZZ`1E@Te}c6f^C^tsndq`s^nl?tuX5vRf{l3p7|6|K0>l3H+x4P& zaqoq?>THju-fV|Qt++xbVZ(ulJJa!#vNr1%@Y(m&bYkoy=Sz`R08R%s0Hxl!&p_zt z70KHdf1yzAvNOgVc1Q(x34iH=Rqf9jTFmg8((swq;sU-?M>X(9)=MVF;?YUm`)gg*d?mM2Z#bp!ba;)v-jzfuk;L!FppH3OQPYqPJYCw0|i-=21=;?Ig(C?xf=J6HQux!=_o zKguoK?)S_s)w<`~-iGY<<} z{<>hv6ND|8&KMlOCkM?8`F5R=f#-AHo-cX5~UtS9Af3{bvS z;ti!j{i_85IQ<0N9y4YK^x##glRd1>(eohs%80AXvE12ws^@}>PAexD9y>L-X}$S8 z!7XW0cG6ZmA%i9=2fgZ-8{rtvpvU|@we(i#CGm8XyV&eM`LBXdT0Ak-{4D?2plC${ zJ0LoWF?#*w7&cL<=H;f&awP!W+^79Bg*dlMA8n{thQfM9j)oI%bZKMmzh?^AieeOyDRs5RSq;#KcK%C<%<2U%uXO{eLt}_g{I$v4B8JVp_4f*s> zn7(~uJojSHL6HYXdP(h1q%@93Q;j#OFV}K3p9Sz&hmi&y7JV;ih0;E_Ro}C5*P+2> zFF=b2Med8d&Kjl#+(M?$5c1jstZvjc)GdBY454P9+2s1@wts~*_e&$12vxahU3-Ni z>=&$B46xCW!lzz-`1Xv6*W$gKRh{wi{l;S~v$|qvBMxL`bEr9H8i!@u@qac!S;-+iQ0~45+}M?R zX-EFxUpyeHrD|L^k$xk@=6qUt27i^XY$s_FxZdTDYiHC0n)%XP2A9eQXTC%V7aU5v z?1kS#@BWl}eB~zo7kcTsv7@BZ?D(ek?YAtwZ+lq1N9~@i7&p}9t@MznT|E)9NQnky zfeml6d8YRr9pm>4!We1gvBZt2FCUH3E|{zN}daj5iKr=8RmisnZFH zV@=gcmYBw4EOFV2Gx#Y%&$1SiUpgp@5WEz>*B2D3_*TFcoLjY--y$>?$nn!WSL`(Z z%jFHBbkV9(bXKa~u}^(R)F+oAT(0d?(NKo-72V9Fg{Q%M@q|IDnev-jgZ}HS1Yr|oN8=QbH$aQ2NQWG|0)bQ-^ghnKtOf%}O1MhcEU^#c3o_?0kcLQOJ+sO;+IwHB20Vr+s z(iP8&PtbmVXVPdkW0tX?t^WAR1+1-vV}RP#Ey6yTU!U%-lp9=OuAXR|F1hz|*zrDM zzxJV=Fp@UK`t6txxI;!213d$5VQI;0rSk5(TPLwE#-gr)= z*GcUDruSFo`iTnGIYIs#)_c;MmtPl&W*JFH7pU>yXE-W?A8&GD_%2e<5K%BaNy~Fq z39-=g<$3^oI;4MS@_+_L-g^8BAWSDrJ=qLVfoV*)LoY!Tzddgj)tMd2@7{YdM0EmX zd=;owu)yn=P;`xN@B$0d|7q{q!=YN&xUEz}o!WBQMK-(0ZJMc2WROaGkZI3k#7GF0 zj7m{csF7-?T*A17AsQOSZDPdCp2#j4_XrUsCc;RO&dApJ)~uO*I?p-(?{l7$zvl6* zwZ8Se-|cBg#HBH+*FmTVXODkWq8nt&Nl%E~ZW1 zZ%o_s^1crWE`W6~>ob9h6QOt|4%o@&5`$U0a%2&~pd%yiQgG~qy69xhK94lAt?&80 zl;i!$uk;T_-SgJp#a_uD!X7sV%k5=3u3iIUzXVJ;Nd2P;=qB~@jdyhtIx0Js3Njgg zbCORC-Tu-ZO(S2WgvF$R&rpckXYGK9L4ixq9lZ_{BhRW~!3--D+akzmJ7H8F#ugY@ zzq@k&9ylM;e2Ow#kG~jAQc;LK?;i{)i#CDJ#kHArG;9ncxyTn-eni!=6$oz+(KjT! z?OAX2{#sY}=FF9Lx~DbQ41z&+YAJTV*Eqn^b39@eI7aek`Hp#dB#5^Q5E(AkpN_@_X?p+#O>;4EpRdW;L#TdDdY~MN@x8 zfNYkTy)2ViF9uj|>mdeXfyxtgv5Kb^ggkcUX`lIY%?%1OJEJ^spp^NU*EuM!F!QJ) za&=$!>%R51s+>q~CO0t$I7;fD@o39lu|yVz0D@HO--zjy z9&-Qd4RbXSr>bzGG{0Y`BC7+g+fOFS_d5IgX8cGAe={4h1&=U<5DmnwE^+_7s zQz7o?XLWOi5iX@6swnb%*P28cUP6XK9%Y0vw{#Rn6F5hN4MEK$#ppAc7iY2-W9}Y!DJrmzWBCzX@VocTyrJN_e9|TT5T>c>Rd*vV96iJ(W@6=xfFM?@q@f09 zQtB*o$sS8lC#XuSa#OFy%c#(SkfobBuHuA4FQC{fxHikO9zBJevfdpY0#1*^BCRJ; z4;vd?X9IMANCvcZT(p>v z`x=A20`iHgC%TV{<-j*(bZsoAQd@XxlS9`7V*(55bKyCjhSn{dJG_bxV?ib@yWw)S z#+9KKe4QSt)c$)B{sfncVRqyi4*%SgAL+*kGvAFUt13TLRq*SFes-1KZ6Yx>G5a~9 zNI6AU(};}Qrd&cR#NAu!>pyoVkU(L(`u|PWy&=CEu4pI9(v4YX3q z4gmon0k*oeT#)pDZLxtg&z zml)Qj#`|YtSt}atYYH`Uf`yw}8j>EU^Ij>CYbZ4x#C7UlR@ZdAuCq~sbF@@1kbJ$e z(aFtxFPNa_9~2<%w7IIhexXt@XT--DD}RkO|0JnM(^tcd^N3M>NUS&C z7A&_sfl3WEZ?Vs7TR$%R2*&x8uz}1Udh;Yfq^!jQA`u796fMMnX`s~WhUo$lVx5Xn z!=KFAdr)2-P)2W_ufftC!TymkeQ5fApD3`4yRi;(=F?|xR%FDYy;UWsRu5mQ=#_`9T z$Z*$RlF8wNM91I?rmnXAtLGSjnIL^F=>Z40q3%6a6WC~kTqclHGX?vVxO2nCqgK-! z9|W~QW*2qGF5($c}rZa2O_+xV}jZktW4!0sgVkn7n4G`+9rG@_>1@O(`8-k{4oJ~_2vA5KLTScnGqcIb!@&a$a&zMgA=*|$OY9LJ8`A@F~%5`DB+>u21hRnzdhB4nGZ}vH)(D;q_$j4T|Uq1wpTbU7bxR+B^C<%cAG`b%H(4f(1rWW!hIBu^k+Cf;-s2%ruFuAIO(UZ3_ z#wft5)}UfKN#$`1r(W~9#Vv9fq$pCatVVnIWiTS26)3pS+c(p-4Af}k!Ff?={Ei;jFfX)W|JVDb8f>jEQTo|Ru3m$ zRXrVBXhkbt=q!H8e#ksecfFLKZZxp(Q@23Cl#mg(C2Q2~l5Q$A0Cp7)m5g?|kffKmCP7rJ)X#lDg67ZzKvKc5+v3hoA z%b77JNoaSlVukE700wukg8E@>&QRq^(gVDHm{A{UU?2R(0{6+)@DndqZw1qs05{Ic>vI%Ji;<-0c;%_V0Mq(~e zn0zJdv1^yG!Lw*<+}6P3)v#7d13sL46agC_Z31lHYUc=%>;Qx(C*^I*<=xSfn$Q8@ zfo}2C)}?TN1yaTzZMcXpeG3R0p8W_&;*vqnI(iEx*v88cMX2_q$#58=)k19~N!v(T zAE7P-`t1O+m;2wS#&(Qj z;h{J{PtDbpcDogHafil65lpOJpaREN3)16nIl{Bq_!?MS*hRLRVKW&@nYPY!?yR(8 zV;VSXqg`E6Ht%dAp{L)&p&eH>Y%5WL0HL zL1x;tmD(O4*`Ltsz$PDO=RlI3VN<}3ZffbF!7j(2n+W&;P&R{hS;4HVLKsxs>VSjf zatT-e>uV)pNxgSyCj?cqi@&;1daD|A$SmPrmz@Xy zyy9!d@1KoRF9SzR?cqc=ythhEJIU)t7@YdIHvjYMe;RzBFDEBojorC@pQOG2-L8Cl kec25AD<;xkYW=wjI^Sb_)|@REkOP0%T~0fTY&|3X0p(gcIRF3v literal 0 HcmV?d00001 From bceb5d43ed00b601154c35e04acd14790a3a0a38 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Wed, 26 Jun 2024 22:36:31 +0300 Subject: [PATCH 082/131] feat(queue): fix stream rebalancing issues, update tests --- .../src/emqx_ds_shared_sub_agent.erl | 2 +- .../src/emqx_ds_shared_sub_group_sm.erl | 61 ++++++-- .../src/emqx_ds_shared_sub_leader.erl | 141 +++++++++++++----- .../src/emqx_ds_shared_sub_proto.erl | 76 ++++++++++ .../test/emqx_ds_shared_sub_SUITE.erl | 35 ++--- 5 files changed, 239 insertions(+), 76 deletions(-) diff --git a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_agent.erl b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_agent.erl index 6e43e0a65..5e27f290a 100644 --- a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_agent.erl +++ b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_agent.erl @@ -190,7 +190,7 @@ send_to_subscription_after(Group) -> with_group_sm(State, Group, Fun) -> case State of #{groups := #{Group := GSM0} = Groups} -> - GSM1 = Fun(GSM0), + #{} = GSM1 = Fun(GSM0), State#{groups => Groups#{Group => GSM1}}; _ -> %% TODO diff --git a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_group_sm.erl b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_group_sm.erl index eb13e7147..1bf023e56 100644 --- a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_group_sm.erl +++ b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_group_sm.erl @@ -122,7 +122,8 @@ %% TODO https://emqx.atlassian.net/browse/EMQX-12574 %% Move to settings -define(FIND_LEADER_TIMEOUT, 1000). --define(RENEW_LEASE_TIMEOUT, 2000). +-define(RENEW_LEASE_TIMEOUT, 5000). +-define(MIN_UPDATE_STREAM_STATE_INTERVAL, 500). %%----------------------------------------------------------------------- %% API @@ -204,8 +205,12 @@ handle_find_leader_timeout(#{agent := Agent, topic_filter := TopicFilter} = GSM0 %%----------------------------------------------------------------------- %% Replaying state -handle_replaying(GSM) -> - ensure_state_timeout(GSM, renew_lease_timeout, ?RENEW_LEASE_TIMEOUT). +handle_replaying(GSM0) -> + GSM1 = ensure_state_timeout(GSM0, renew_lease_timeout, ?RENEW_LEASE_TIMEOUT), + GSM2 = ensure_state_timeout( + GSM1, update_stream_state_timeout, ?MIN_UPDATE_STREAM_STATE_INTERVAL + ), + GSM2. handle_renew_lease_timeout(GSM) -> ?tp(debug, renew_lease_timeout, #{}), @@ -214,8 +219,12 @@ handle_renew_lease_timeout(GSM) -> %%----------------------------------------------------------------------- %% Updating state -handle_updating(GSM) -> - ensure_state_timeout(GSM, renew_lease_timeout, ?RENEW_LEASE_TIMEOUT). +handle_updating(GSM0) -> + GSM1 = ensure_state_timeout(GSM0, renew_lease_timeout, ?RENEW_LEASE_TIMEOUT), + GSM2 = ensure_state_timeout( + GSM1, update_stream_state_timeout, ?MIN_UPDATE_STREAM_STATE_INTERVAL + ), + GSM2. %%----------------------------------------------------------------------- %% Common handlers @@ -223,7 +232,7 @@ handle_updating(GSM) -> handle_leader_update_streams( #{ state := ?replaying, - stream_data := #{streams := Streams0, version := VersionOld} = StateData + state_data := #{streams := Streams0, version := VersionOld} = StateData } = GSM, VersionOld, VersionNew, @@ -275,14 +284,19 @@ handle_leader_update_streams( handle_leader_update_streams( #{ state := ?updating, - stream_data := #{version := VersionNew} = _StreamData + state_data := #{version := VersionNew} = _StreamData } = GSM, _VersionOld, VersionNew, _StreamProgresses ) -> ensure_state_timeout(GSM, renew_lease_timeout, ?RENEW_LEASE_TIMEOUT); -handle_leader_update_streams(GSM, _VersionOld, _VersionNew, _StreamProgresses) -> +handle_leader_update_streams(GSM, VersionOld, VersionNew, _StreamProgresses) -> + ?tp(warning, shared_sub_group_sm_unexpected_leader_update_streams, #{ + gsm => GSM, + version_old => VersionOld, + version_new => VersionNew + }), %% Unexpected versions or state transition(GSM, ?connecting, #{}). @@ -311,7 +325,13 @@ handle_leader_renew_stream_lease( VersionNew ) -> ensure_state_timeout(GSM, renew_lease_timeout, ?RENEW_LEASE_TIMEOUT); -handle_leader_renew_stream_lease(GSM, _VersionOld, _VersionNew) -> +handle_leader_renew_stream_lease(GSM, VersionOld, VersionNew) -> + ?tp(warning, shared_sub_group_sm_unexpected_leader_renew_stream_lease, #{ + gsm => GSM, + version_old => VersionOld, + version_new => VersionNew + }), + %% Unexpected versions or state transition(GSM, ?connecting, #{}). handle_stream_progress(#{state := ?connecting} = GSM, _StreamProgresses) -> @@ -319,32 +339,34 @@ handle_stream_progress(#{state := ?connecting} = GSM, _StreamProgresses) -> handle_stream_progress( #{ state := ?replaying, + agent := Agent, state_data := #{ - agent := Agent, leader := Leader, version := Version } - } = _GSM, + } = GSM, StreamProgresses ) -> ok = emqx_ds_shared_sub_proto:agent_update_stream_states( Leader, Agent, StreamProgresses, Version - ); + ), + ensure_state_timeout(GSM, update_stream_state_timeout, ?MIN_UPDATE_STREAM_STATE_INTERVAL); handle_stream_progress( #{ state := ?updating, + agent := Agent, state_data := #{ - agent := Agent, leader := Leader, version := Version, prev_version := PrevVersion } - } = _GSM, + } = GSM, StreamProgresses ) -> ok = emqx_ds_shared_sub_proto:agent_update_stream_states( Leader, Agent, StreamProgresses, PrevVersion, Version - ). + ), + ensure_state_timeout(GSM, update_stream_state_timeout, ?MIN_UPDATE_STREAM_STATE_INTERVAL). handle_leader_invalidate(GSM) -> transition(GSM, ?connecting, #{}). @@ -365,7 +387,14 @@ handle_state_timeout( renew_lease_timeout, _Message ) -> - handle_renew_lease_timeout(GSM). + handle_renew_lease_timeout(GSM); +handle_state_timeout( + GSM, + update_stream_state_timeout, + _Message +) -> + ?tp(debug, update_stream_state_timeout, #{}), + handle_stream_progress(GSM, []). handle_info( #{state_timers := Timers} = GSM, #state_timeout{message = Message, name = Name, id = Id} = _Info diff --git a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl index 3f2a85424..64a74510a 100644 --- a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl +++ b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl @@ -75,7 +75,7 @@ %% States -define(leader_waiting_registration, leader_waiting_registration). --define(leader_replaying, leader_replaying). +-define(leader_active, leader_active). %% Events @@ -96,6 +96,8 @@ -define(AGENT_TIMEOUT, 5000). +-define(START_TIME_THRESHOLD, 5000). + %%-------------------------------------------------------------------- %% API %%-------------------------------------------------------------------- @@ -133,6 +135,7 @@ init([#{topic_filter := #share{group = Group, topic = Topic}} = _Options]) -> group => Group, topic => Topic, router_id => gen_router_id(), + start_time => now_ms() - ?START_TIME_THRESHOLD, stream_progresses => #{}, stream_owners => #{}, agents => #{} @@ -146,37 +149,50 @@ handle_event({call, From}, #register{register_fun = Fun}, ?leader_waiting_regist Self = self(), case Fun() of Self -> - {next_state, ?replaying, Data, {reply, From, {ok, Self}}}; + {next_state, ?leader_active, Data, {reply, From, {ok, Self}}}; OtherPid -> {stop_and_reply, normal, {reply, From, {ok, OtherPid}}} end; %%-------------------------------------------------------------------- %% repalying state -handle_event(enter, _OldState, ?leader_replaying, #{topic := Topic, router_id := RouterId} = _Data) -> +handle_event(enter, _OldState, ?leader_active, #{topic := Topic, router_id := RouterId} = _Data) -> + ?tp(warning, shared_sub_leader_enter_actve, #{topic => Topic, router_id => RouterId}), ok = emqx_persistent_session_ds_router:do_add_route(Topic, RouterId), {keep_state_and_data, [ - {state_timeout, ?RENEW_LEASE_INTERVAL, #renew_leases{}}, - {state_timeout, ?DROP_TIMEOUT_INTERVAL, #drop_timeout{}}, - {state_timeout, 0, #renew_streams{}} + {{timeout, #renew_streams{}}, 0, #renew_streams{}}, + {{timeout, #renew_leases{}}, ?RENEW_LEASE_INTERVAL, #renew_leases{}}, + {{timeout, #drop_timeout{}}, ?DROP_TIMEOUT_INTERVAL, #drop_timeout{}} ]}; -handle_event(state_timeout, #renew_streams{}, ?leader_replaying, Data0) -> +%%-------------------------------------------------------------------- +%% timers +%% renew_streams timer +handle_event({timeout, #renew_streams{}}, #renew_streams{}, ?leader_active, Data0) -> + % ?tp(warning, shared_sub_leader_timeout, #{timeout => renew_streams}), Data1 = renew_streams(Data0), - {keep_state, Data1, {state_timeout, ?RENEW_STREAMS_INTERVAL, #renew_streams{}}}; -handle_event(state_timeout, #renew_leases{}, ?leader_replaying, Data0) -> + {keep_state, Data1, {{timeout, #renew_streams{}}, ?RENEW_STREAMS_INTERVAL, #renew_streams{}}}; +%% renew_leases timer +handle_event({timeout, #renew_leases{}}, #renew_leases{}, ?leader_active, Data0) -> + % ?tp(warning, shared_sub_leader_timeout, #{timeout => renew_leases}), Data1 = renew_leases(Data0), - {keep_state, Data1, {state_timeout, ?RENEW_LEASE_INTERVAL, #renew_leases{}}}; -handle_event(state_timeout, #drop_timeout{}, ?leader_replaying, Data0) -> + {keep_state, Data1, {{timeout, #renew_leases{}}, ?RENEW_LEASE_INTERVAL, #renew_leases{}}}; +%% drop_timeout timer +handle_event({timeout, #drop_timeout{}}, #drop_timeout{}, ?leader_active, Data0) -> + % ?tp(warning, shared_sub_leader_timeout, #{timeout => drop_timeout}), Data1 = drop_timeout_agents(Data0), - {keep_state, Data1, {state_timeout, ?DROP_TIMEOUT_INTERVAL, #drop_timeout{}}}; -handle_event(info, ?agent_connect_leader_match(Agent, _TopicFilter), ?leader_replaying, Data0) -> + {keep_state, Data1, {{timeout, #drop_timeout{}}, ?DROP_TIMEOUT_INTERVAL, #drop_timeout{}}}; +%%-------------------------------------------------------------------- +%% agent events +handle_event(info, ?agent_connect_leader_match(Agent, _TopicFilter), ?leader_active, Data0) -> + % ?tp(warning, shared_sub_leader_connect_agent, #{agent => Agent}), Data1 = connect_agent(Data0, Agent), {keep_state, Data1}; handle_event( info, ?agent_update_stream_states_match(Agent, StreamProgresses, Version), - ?leader_replaying, + ?leader_active, Data0 ) -> + % ?tp(warning, shared_sub_leader_update_stream_states, #{agent => Agent, version => Version}), Data1 = with_agent(Data0, Agent, fun() -> update_agent_stream_states(Data0, Agent, StreamProgresses, Version) end), @@ -184,9 +200,12 @@ handle_event( handle_event( info, ?agent_update_stream_states_match(Agent, StreamProgresses, VersionOld, VersionNew), - ?leader_replaying, + ?leader_active, Data0 ) -> + % ?tp(warning, shared_sub_leader_update_stream_states, #{ + % agent => Agent, version_old => VersionOld, version_new => VersionNew + % }), Data1 = with_agent(Data0, Agent, fun() -> update_agent_stream_states(Data0, Agent, StreamProgresses, VersionOld, VersionNew) end), @@ -195,10 +214,11 @@ handle_event( %% fallback handle_event(enter, _OldState, _State, _Data) -> keep_state_and_data; -handle_event(Event, _Content, State, _Data) -> +handle_event(Event, Content, State, _Data) -> ?SLOG(warning, #{ msg => unexpected_event, event => Event, + content => Content, state => State }), keep_state_and_data. @@ -218,11 +238,10 @@ terminate(_Reason, _State, #{topic := Topic, router_id := RouterId} = _Data) -> %% * Revoke streams from agents having too many streams %% * Assign streams to agents having too few streams -renew_streams(#{stream_progresses := Progresses, topic := Topic} = Data0) -> +renew_streams(#{start_time := StartTime, stream_progresses := Progresses, topic := Topic} = Data0) -> TopicFilter = emqx_topic:words(Topic), - StartTime = now_ms(), {_, Streams} = lists:unzip( - emqx_ds:get_streams(?PERSISTENT_MESSAGE_DB, TopicFilter, now_ms()) + emqx_ds:get_streams(?PERSISTENT_MESSAGE_DB, TopicFilter, StartTime) ), %% TODO https://emqx.atlassian.net/browse/EMQX-12572 %% Handle stream removal @@ -274,6 +293,12 @@ revoke_excess_streams_from_agent(Data0, Agent, DesiredCount) -> false -> AgentState0; true -> + ?tp(warning, shared_sub_leader_revoke_streams, #{ + agent => Agent, + agent_stream_count => length(Streams0), + revoke_count => RevokeCount, + desired_count => DesiredCount + }), revoke_streams_from_agent(Data0, Agent, AgentState0, RevokeCount) end, set_agent_state(Data0, Agent, AgentState1). @@ -321,6 +346,12 @@ assign_lacking_streams(Data0, Agent, DesiredCount) -> false -> Data0; true -> + ?tp(warning, shared_sub_leader_assign_streams, #{ + agent => Agent, + agent_stream_count => length(Streams0), + assign_count => AssignCount, + desired_count => DesiredCount + }), assign_streams_to_agent(Data0, Agent, AssignCount) end. @@ -346,12 +377,15 @@ connect_agent( #{group := Group} = Data, Agent ) -> + %% TODO + %% implement graceful reconnection of the same agent ?SLOG(info, #{ msg => leader_agent_connected, agent => Agent, group => Group }), DesiredCount = desired_streams_per_agent(Data), + % DesiredCount = desired_streams_for_new_agent(Data), assign_initial_streams_to_agent(Data, Agent, DesiredCount). assign_initial_streams_to_agent(Data, Agent, AssignCount) -> @@ -388,6 +422,7 @@ drop_timeout_agents(#{agents := Agents} = Data) -> %% Send lease confirmations to agents renew_leases(#{agents := AgentStates} = Data) -> + ?tp(warning, shared_sub_leader_renew_leases, #{agents => maps:keys(AgentStates)}), ok = lists:foreach( fun({Agent, AgentState}) -> renew_lease(Data, Agent, AgentState) @@ -407,11 +442,11 @@ renew_lease(#{group := Group} = Data, Agent, #{ ok = emqx_ds_shared_sub_proto:leader_update_streams( Agent, Group, PrevVersion, Version, StreamProgresses ), - ok = emqx_ds_shared_sub_proto:leader_renew_stream_lease(Agent, Group, Version, PrevVersion); + ok = emqx_ds_shared_sub_proto:leader_renew_stream_lease(Agent, Group, PrevVersion, Version); renew_lease(#{group := Group}, Agent, #{ state := ?updating, version := Version, prev_version := PrevVersion }) -> - ok = emqx_ds_shared_sub_proto:leader_renew_stream_lease(Agent, Group, Version, PrevVersion). + ok = emqx_ds_shared_sub_proto:leader_renew_stream_lease(Agent, Group, PrevVersion, Version). %%-------------------------------------------------------------------- %% Handle stream progress updates from agent in replaying state @@ -427,7 +462,7 @@ update_agent_stream_states(Data0, Agent, AgentStreamProgresses, Version) -> %% Agent finished updating, now replaying Data1 = update_stream_progresses(Data0, Agent, AgentStreamProgresses), AgentState1 = update_agent_timeout(AgentState0), - AgentState2 = agent_transition_to_replaying(AgentState1), + AgentState2 = agent_transition_to_replaying(Agent, AgentState1), set_agent_state(Data1, Agent, AgentState2); {?replaying, AgentVersion} -> %% Common case, agent is replaying @@ -521,10 +556,10 @@ update_agent_stream_states(Data0, Agent, AgentStreamProgresses, VersionOld, Vers {AgentState2, Data2} = clean_revoked_streams(Data1, AgentState1, AgentStreamProgresses), AgentState3 = case AgentState2 of - #{revoke_streams := []} -> - agent_transition_to_waiting_replaying(AgentState2); + #{revoked_streams := []} -> + agent_transition_to_waiting_replaying(Data1, Agent, AgentState2); _ -> - agent_transition_to_updating(AgentState2) + agent_transition_to_updating(Agent, AgentState2) end, set_agent_state(Data2, Agent, AgentState3); {?updating, AgentPrevVersion, AgentVersion} -> @@ -533,8 +568,8 @@ update_agent_stream_states(Data0, Agent, AgentStreamProgresses, VersionOld, Vers {AgentState2, Data2} = clean_revoked_streams(Data1, AgentState1, AgentStreamProgresses), AgentState3 = case AgentState2 of - #{revoke_streams := []} -> - agent_transition_to_waiting_replaying(AgentState2); + #{revoked_streams := []} -> + agent_transition_to_waiting_replaying(Data1, Agent, AgentState2); _ -> AgentState2 end, @@ -566,10 +601,15 @@ update_agent_stream_states(Data0, Agent, AgentStreamProgresses, VersionOld, Vers agent_transition_to_waiting_updating( #{group := Group} = Data, Agent, - #{version := Version, prev_version := undefined} = AgentState0, + #{state := OldState, version := Version, prev_version := undefined} = AgentState0, Streams, RevokedStreams ) -> + ?tp(warning, shared_sub_leader_agent_state_transition, #{ + agent => Agent, + old_state => OldState, + new_state => ?waiting_updating + }), NewVersion = next_version(Version), AgentState1 = AgentState0#{ @@ -585,7 +625,15 @@ agent_transition_to_waiting_updating( ), AgentState1. -agent_transition_to_waiting_replaying(AgentState0) -> +agent_transition_to_waiting_replaying( + #{group := Group} = _Data, Agent, #{state := OldState, version := Version} = AgentState0 +) -> + ?tp(warning, shared_sub_leader_agent_state_transition, #{ + agent => Agent, + old_state => OldState, + new_state => ?waiting_replaying + }), + ok = emqx_ds_shared_sub_proto:leader_renew_stream_lease(Agent, Group, Version), AgentState0#{ state => ?waiting_replaying, revoked_streams => [] @@ -594,6 +642,11 @@ agent_transition_to_waiting_replaying(AgentState0) -> agent_transition_to_initial_waiting_replaying( #{group := Group} = Data, Agent, InitialStreams ) -> + ?tp(warning, shared_sub_leader_agent_state_transition, #{ + agent => Agent, + old_state => none, + new_state => ?waiting_replaying + }), Version = 0, StreamProgresses = stream_progresses(Data, InitialStreams), Leader = this_leader(Data), @@ -609,13 +662,23 @@ agent_transition_to_initial_waiting_replaying( update_deadline => now_ms() + ?AGENT_TIMEOUT }. -agent_transition_to_replaying(#{state := ?waiting_replaying} = AgentState) -> +agent_transition_to_replaying(Agent, #{state := ?waiting_replaying} = AgentState) -> + ?tp(warning, shared_sub_leader_agent_state_transition, #{ + agent => Agent, + old_state => ?waiting_replaying, + new_state => ?replaying + }), AgentState#{ state => ?replaying, prev_version => undefined }. -agent_transition_to_updating(#{state := ?waiting_updating} = AgentState) -> +agent_transition_to_updating(Agent, #{state := ?waiting_updating} = AgentState) -> + ?tp(warning, shared_sub_leader_agent_state_transition, #{ + agent => Agent, + old_state => ?waiting_updating, + new_state => ?updating + }), AgentState#{state => ?updating}. %%-------------------------------------------------------------------- @@ -645,14 +708,24 @@ replaying_agents(#{agents := AgentStates}) -> maps:to_list(AgentStates) ). -desired_streams_per_agent(#{agents := AgentStates, stream_progresses := StreamProgresses}) -> - AgentCount = maps:size(AgentStates), +desired_streams_per_agent(#{agents := AgentStates} = Data) -> + desired_streams_per_agent(Data, maps:size(AgentStates)). + +desired_streams_for_new_agent(#{agents := AgentStates} = Data) -> + desired_streams_per_agent(Data, maps:size(AgentStates) + 1). + +desired_streams_per_agent(#{stream_progresses := StreamProgresses}, AgentCount) -> case AgentCount of 0 -> 0; _ -> StreamCount = maps:size(StreamProgresses), - (StreamCount div AgentCount) + 1 + case StreamCount rem AgentCount of + 0 -> + StreamCount div AgentCount; + _ -> + 1 + StreamCount div AgentCount + end end. stream_progresses(#{stream_progresses := StreamProgresses} = _Data, Streams) -> diff --git a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_proto.erl b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_proto.erl index 7d81de083..01f63aaad 100644 --- a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_proto.erl +++ b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_proto.erl @@ -9,6 +9,7 @@ -module(emqx_ds_shared_sub_proto). -include("emqx_ds_shared_sub_proto.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). -export([ agent_connect_leader/3, @@ -47,15 +48,32 @@ stream_progress/0 ]). +%%-------------------------------------------------------------------- +%% API +%%-------------------------------------------------------------------- + %% agent -> leader messages -spec agent_connect_leader(leader(), agent(), topic_filter()) -> ok. agent_connect_leader(ToLeader, FromAgent, TopicFilter) -> + ?tp(warning, shared_sub_proto_msg, #{ + type => agent_connect_leader, + to_leader => ToLeader, + from_agent => FromAgent, + topic_filter => TopicFilter + }), _ = erlang:send(ToLeader, ?agent_connect_leader(FromAgent, TopicFilter)), ok. -spec agent_update_stream_states(leader(), agent(), list(agent_stream_progress()), version()) -> ok. agent_update_stream_states(ToLeader, FromAgent, StreamProgresses, Version) -> + ?tp(warning, shared_sub_proto_msg, #{ + type => agent_update_stream_states, + to_leader => ToLeader, + from_agent => FromAgent, + stream_progresses => format_streams(StreamProgresses), + version => Version + }), _ = erlang:send(ToLeader, ?agent_update_stream_states(FromAgent, StreamProgresses, Version)), ok. @@ -63,6 +81,14 @@ agent_update_stream_states(ToLeader, FromAgent, StreamProgresses, Version) -> leader(), agent(), list(agent_stream_progress()), version(), version() ) -> ok. agent_update_stream_states(ToLeader, FromAgent, StreamProgresses, VersionOld, VersionNew) -> + ?tp(warning, shared_sub_proto_msg, #{ + type => agent_update_stream_states, + to_leader => ToLeader, + from_agent => FromAgent, + stream_progresses => format_streams(StreamProgresses), + version_old => VersionOld, + version_new => VersionNew + }), _ = erlang:send( ToLeader, ?agent_update_stream_states(FromAgent, StreamProgresses, VersionOld, VersionNew) ), @@ -72,6 +98,14 @@ agent_update_stream_states(ToLeader, FromAgent, StreamProgresses, VersionOld, Ve -spec leader_lease_streams(agent(), group(), leader(), list(stream_progress()), version()) -> ok. leader_lease_streams(ToAgent, OfGroup, Leader, Streams, Version) -> + ?tp(warning, shared_sub_proto_msg, #{ + type => leader_lease_streams, + to_agent => ToAgent, + of_group => OfGroup, + leader => Leader, + streams => format_streams(Streams), + version => Version + }), _ = emqx_persistent_session_ds_shared_subs_agent:send( ToAgent, ?leader_lease_streams(OfGroup, Leader, Streams, Version) @@ -80,6 +114,12 @@ leader_lease_streams(ToAgent, OfGroup, Leader, Streams, Version) -> -spec leader_renew_stream_lease(agent(), group(), version()) -> ok. leader_renew_stream_lease(ToAgent, OfGroup, Version) -> + ?tp(warning, shared_sub_proto_msg, #{ + type => leader_renew_stream_lease, + to_agent => ToAgent, + of_group => OfGroup, + version => Version + }), _ = emqx_persistent_session_ds_shared_subs_agent:send( ToAgent, ?leader_renew_stream_lease(OfGroup, Version) @@ -88,6 +128,13 @@ leader_renew_stream_lease(ToAgent, OfGroup, Version) -> -spec leader_renew_stream_lease(agent(), group(), version(), version()) -> ok. leader_renew_stream_lease(ToAgent, OfGroup, VersionOld, VersionNew) -> + ?tp(warning, shared_sub_proto_msg, #{ + type => leader_renew_stream_lease, + to_agent => ToAgent, + of_group => OfGroup, + version_old => VersionOld, + version_new => VersionNew + }), _ = emqx_persistent_session_ds_shared_subs_agent:send( ToAgent, ?leader_renew_stream_lease(OfGroup, VersionOld, VersionNew) @@ -96,6 +143,14 @@ leader_renew_stream_lease(ToAgent, OfGroup, VersionOld, VersionNew) -> -spec leader_update_streams(agent(), group(), version(), version(), list(stream_progress())) -> ok. leader_update_streams(ToAgent, OfGroup, VersionOld, VersionNew, StreamsNew) -> + ?tp(warning, shared_sub_proto_msg, #{ + type => leader_update_streams, + to_agent => ToAgent, + of_group => OfGroup, + version_old => VersionOld, + version_new => VersionNew, + streams_new => format_streams(StreamsNew) + }), _ = emqx_persistent_session_ds_shared_subs_agent:send( ToAgent, ?leader_update_streams(OfGroup, VersionOld, VersionNew, StreamsNew) @@ -104,8 +159,29 @@ leader_update_streams(ToAgent, OfGroup, VersionOld, VersionNew, StreamsNew) -> -spec leader_invalidate(agent(), group()) -> ok. leader_invalidate(ToAgent, OfGroup) -> + ?tp(warning, shared_sub_proto_msg, #{ + type => leader_invalidate, + to_agent => ToAgent, + of_group => OfGroup + }), _ = emqx_persistent_session_ds_shared_subs_agent:send( ToAgent, ?leader_invalidate(OfGroup) ), ok. + +%%-------------------------------------------------------------------- +%% Helpers +%%-------------------------------------------------------------------- + +format_opaque(Opaque) -> + erlang:phash2(Opaque). + +format_streams(Streams) -> + lists:map( + fun format_stream/1, + Streams + ). + +format_stream(#{stream := Stream, iterator := Iterator} = Value) -> + Value#{stream => format_opaque(Stream), iterator => format_opaque(Iterator)}. diff --git a/apps/emqx_ds_shared_sub/test/emqx_ds_shared_sub_SUITE.erl b/apps/emqx_ds_shared_sub/test/emqx_ds_shared_sub_SUITE.erl index f18114918..6eafe0e4a 100644 --- a/apps/emqx_ds_shared_sub/test/emqx_ds_shared_sub_SUITE.erl +++ b/apps/emqx_ds_shared_sub/test/emqx_ds_shared_sub_SUITE.erl @@ -13,7 +13,8 @@ -include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("emqx/include/asserts.hrl"). -all() -> emqx_common_test_helpers:all(?MODULE). +all() -> + emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> Apps = emqx_cth_suite:start( @@ -51,17 +52,16 @@ end_per_testcase(_TC, _Config) -> ok. t_lease_initial(_Config) -> - ConnPub = emqtt_connect_pub(<<"client_pub">>), - - %% Need to pre-create some streams in "topic/#". - %% Leader is dummy by far and won't update streams after the first lease to the agent. - %% So there should be some streams already when the agent connects. - ok = init_streams(ConnPub, <<"topic1/1">>), - ConnShared = emqtt_connect_sub(<<"client_shared">>), {ok, _, _} = emqtt:subscribe(ConnShared, <<"$share/gr1/topic1/#">>, 1), - {ok, _} = emqtt:publish(ConnPub, <<"topic1/1">>, <<"hello2">>, 1), + ConnPub = emqtt_connect_pub(<<"client_pub">>), + + {ok, _} = emqtt:publish(ConnPub, <<"topic1/1">>, <<"hello1">>, 1), + ct:sleep(2_000), + {ok, _} = emqtt:publish(ConnPub, <<"topic1/2">>, <<"hello2">>, 1), + + ?assertReceive({publish, #{payload := <<"hello1">>}}, 10_000), ?assertReceive({publish, #{payload := <<"hello2">>}}, 10_000), ok = emqtt:disconnect(ConnShared), @@ -70,11 +70,6 @@ t_lease_initial(_Config) -> t_lease_reconnect(_Config) -> ConnPub = emqtt_connect_pub(<<"client_pub">>), - %% Need to pre-create some streams in "topic/#". - %% Leader is dummy by far and won't update streams after the first lease to the agent. - %% So there should be some streams already when the agent connects. - ok = init_streams(ConnPub, <<"topic2/2">>), - ConnShared = emqtt_connect_sub(<<"client_shared">>), %% Stop registry to simulate unability to find leader. @@ -93,7 +88,6 @@ t_lease_reconnect(_Config) -> 5_000 ), - ct:sleep(1_000), {ok, _} = emqtt:publish(ConnPub, <<"topic2/2">>, <<"hello2">>, 1), ?assertReceive({publish, #{payload := <<"hello2">>}}, 10_000), @@ -114,7 +108,7 @@ t_renew_lease_timeout(_Config) -> ?wait_async_action( ok = terminate_leaders(), #{?snk_kind := leader_lease_streams}, - 5_000 + 10_000 ), fun(Trace) -> ?strict_causality( @@ -131,15 +125,6 @@ t_renew_lease_timeout(_Config) -> %% Helper functions %%-------------------------------------------------------------------- -init_streams(ConnPub, Topic) -> - ConnRegular = emqtt_connect_sub(<<"client_regular">>), - {ok, _, _} = emqtt:subscribe(ConnRegular, Topic, 1), - {ok, _} = emqtt:publish(ConnPub, Topic, <<"hello1">>, 1), - - ?assertReceive({publish, #{payload := <<"hello1">>}}, 5_000), - - ok = emqtt:disconnect(ConnRegular). - emqtt_connect_sub(ClientId) -> {ok, C} = emqtt:start_link([ {client_id, ClientId}, From 8f0d807c00bbf90e1946e14d6eee266e4036295e Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Thu, 27 Jun 2024 17:18:59 +0300 Subject: [PATCH 083/131] feat(queue): add new test scenarios --- .../test/emqx_ds_shared_sub_SUITE.erl | 89 ++++++++++++++++++- 1 file changed, 87 insertions(+), 2 deletions(-) diff --git a/apps/emqx_ds_shared_sub/test/emqx_ds_shared_sub_SUITE.erl b/apps/emqx_ds_shared_sub/test/emqx_ds_shared_sub_SUITE.erl index 6eafe0e4a..8bb460967 100644 --- a/apps/emqx_ds_shared_sub/test/emqx_ds_shared_sub_SUITE.erl +++ b/apps/emqx_ds_shared_sub/test/emqx_ds_shared_sub_SUITE.erl @@ -67,6 +67,91 @@ t_lease_initial(_Config) -> ok = emqtt:disconnect(ConnShared), ok = emqtt:disconnect(ConnPub). +t_two_clients(_Config) -> + ConnShared1 = emqtt_connect_sub(<<"client_shared1">>), + {ok, _, _} = emqtt:subscribe(ConnShared1, <<"$share/gr4/topic4/#">>, 1), + + ConnShared2 = emqtt_connect_sub(<<"client_shared2">>), + {ok, _, _} = emqtt:subscribe(ConnShared2, <<"$share/gr4/topic4/#">>, 1), + + ConnPub = emqtt_connect_pub(<<"client_pub">>), + + {ok, _} = emqtt:publish(ConnPub, <<"topic4/1">>, <<"hello1">>, 1), + {ok, _} = emqtt:publish(ConnPub, <<"topic4/2">>, <<"hello2">>, 1), + ct:sleep(2_000), + {ok, _} = emqtt:publish(ConnPub, <<"topic4/1">>, <<"hello3">>, 1), + {ok, _} = emqtt:publish(ConnPub, <<"topic4/2">>, <<"hello4">>, 1), + + ?assertReceive({publish, #{payload := <<"hello1">>}}, 10_000), + ?assertReceive({publish, #{payload := <<"hello2">>}}, 10_000), + ?assertReceive({publish, #{payload := <<"hello3">>}}, 10_000), + ?assertReceive({publish, #{payload := <<"hello4">>}}, 10_000), + + ok = emqtt:disconnect(ConnShared1), + ok = emqtt:disconnect(ConnShared2), + ok = emqtt:disconnect(ConnPub). + +t_client_loss(_Config) -> + process_flag(trap_exit, true), + + ConnShared1 = emqtt_connect_sub(<<"client_shared1">>), + {ok, _, _} = emqtt:subscribe(ConnShared1, <<"$share/gr5/topic5/#">>, 1), + + ConnShared2 = emqtt_connect_sub(<<"client_shared2">>), + {ok, _, _} = emqtt:subscribe(ConnShared2, <<"$share/gr5/topic5/#">>, 1), + + ConnPub = emqtt_connect_pub(<<"client_pub">>), + + {ok, _} = emqtt:publish(ConnPub, <<"topic5/1">>, <<"hello1">>, 1), + {ok, _} = emqtt:publish(ConnPub, <<"topic5/2">>, <<"hello2">>, 1), + + exit(ConnShared1, kill), + + {ok, _} = emqtt:publish(ConnPub, <<"topic5/1">>, <<"hello3">>, 1), + {ok, _} = emqtt:publish(ConnPub, <<"topic5/2">>, <<"hello4">>, 1), + + ?assertReceive({publish, #{payload := <<"hello3">>}}, 10_000), + ?assertReceive({publish, #{payload := <<"hello4">>}}, 10_000), + + ok = emqtt:disconnect(ConnShared2), + ok = emqtt:disconnect(ConnPub). + +t_stream_revoke(_Config) -> + process_flag(trap_exit, true), + + ConnShared1 = emqtt_connect_sub(<<"client_shared1">>), + {ok, _, _} = emqtt:subscribe(ConnShared1, <<"$share/gr6/topic6/#">>, 1), + + ConnPub = emqtt_connect_pub(<<"client_pub">>), + + {ok, _} = emqtt:publish(ConnPub, <<"topic6/1">>, <<"hello1">>, 1), + {ok, _} = emqtt:publish(ConnPub, <<"topic6/2">>, <<"hello2">>, 1), + + ?assertReceive({publish, #{payload := <<"hello1">>}}, 10_000), + ?assertReceive({publish, #{payload := <<"hello2">>}}, 10_000), + + ConnShared2 = emqtt_connect_sub(<<"client_shared2">>), + + ?assertWaitEvent( + {ok, _, _} = emqtt:subscribe(ConnShared2, <<"$share/gr6/topic6/#">>, 1), + #{ + ?snk_kind := shared_sub_group_sm_leader_update_streams, + stream_progresses := [_ | _], + id := <<"client_shared2">> + }, + 5_000 + ), + + {ok, _} = emqtt:publish(ConnPub, <<"topic6/1">>, <<"hello3">>, 1), + {ok, _} = emqtt:publish(ConnPub, <<"topic6/2">>, <<"hello4">>, 1), + + ?assertReceive({publish, #{payload := <<"hello3">>}}, 10_000), + ?assertReceive({publish, #{payload := <<"hello4">>}}, 10_000), + + ok = emqtt:disconnect(ConnShared1), + ok = emqtt:disconnect(ConnShared2), + ok = emqtt:disconnect(ConnPub). + t_lease_reconnect(_Config) -> ConnPub = emqtt_connect_pub(<<"client_pub">>), @@ -127,7 +212,7 @@ t_renew_lease_timeout(_Config) -> emqtt_connect_sub(ClientId) -> {ok, C} = emqtt:start_link([ - {client_id, ClientId}, + {clientid, ClientId}, {clean_start, true}, {proto_ver, v5}, {properties, #{'Session-Expiry-Interval' => 7_200}} @@ -137,7 +222,7 @@ emqtt_connect_sub(ClientId) -> emqtt_connect_pub(ClientId) -> {ok, C} = emqtt:start_link([ - {client_id, ClientId}, + {clientid, ClientId}, {clean_start, true}, {proto_ver, v5} ]), From 61eda0ff31f9c1cc5d2ce96c68d75e030a17e5e4 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Thu, 27 Jun 2024 17:20:04 +0300 Subject: [PATCH 084/131] feat(queue): identify agents by SessionId in tests --- .../src/emqx_ds_shared_sub_agent.erl | 12 ++-- .../src/emqx_ds_shared_sub_group_sm.erl | 10 ++++ .../src/emqx_ds_shared_sub_leader.erl | 21 ++++--- .../src/emqx_ds_shared_sub_proto.erl | 55 +++++++++++++++---- 4 files changed, 69 insertions(+), 29 deletions(-) diff --git a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_agent.erl b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_agent.erl index 5e27f290a..11cb011d3 100644 --- a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_agent.erl +++ b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_agent.erl @@ -143,7 +143,7 @@ delete_group_subscription(State, _ShareTopicFilter) -> State. add_group_subscription( - #{groups := Groups0} = State0, ShareTopicFilter + #{session_id := SessionId, groups := Groups0} = State0, ShareTopicFilter ) -> ?SLOG(info, #{ msg => agent_add_group_subscription, @@ -152,8 +152,9 @@ add_group_subscription( #share{group = Group} = ShareTopicFilter, Groups1 = Groups0#{ Group => emqx_ds_shared_sub_group_sm:new(#{ + session_id => SessionId, topic_filter => ShareTopicFilter, - agent => this_agent(), + agent => this_agent(SessionId), send_after => send_to_subscription_after(Group) }) }, @@ -172,11 +173,8 @@ fetch_stream_events(#{groups := Groups0} = State0) -> State1 = State0#{groups => Groups1}, {lists:concat(Events), State1}. -%%-------------------------------------------------------------------- -%% Internal functions -%%-------------------------------------------------------------------- - -this_agent() -> self(). +this_agent(Id) -> + emqx_ds_shared_sub_proto:agent(Id, self()). send_to_subscription_after(Group) -> fun(Time, Msg) -> diff --git a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_group_sm.erl b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_group_sm.erl index 1bf023e56..a16b2bcf9 100644 --- a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_group_sm.erl +++ b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_group_sm.erl @@ -37,6 +37,7 @@ ]). -type options() :: #{ + session_id := emqx_persistent_session_ds:id(), agent := emqx_ds_shared_sub_proto:agent(), topic_filter := emqx_persistent_session_ds:share_topic_filter(), send_after := fun((non_neg_integer(), term()) -> reference()) @@ -131,6 +132,7 @@ -spec new(options()) -> group_sm(). new(#{ + session_id := SessionId, agent := Agent, topic_filter := ShareTopicFilter, send_after := SendAfter @@ -144,6 +146,7 @@ new(#{ } ), GSM0 = #{ + id => SessionId, topic_filter => ShareTopicFilter, agent => Agent, send_after => SendAfter @@ -231,6 +234,7 @@ handle_updating(GSM0) -> handle_leader_update_streams( #{ + id := Id, state := ?replaying, state_data := #{streams := Streams0, version := VersionOld} = StateData } = GSM, @@ -238,6 +242,12 @@ handle_leader_update_streams( VersionNew, StreamProgresses ) -> + ?tp(warning, shared_sub_group_sm_leader_update_streams, #{ + id => Id, + version_old => VersionOld, + version_new => VersionNew, + stream_progresses => emqx_ds_shared_sub_proto:format_streams(StreamProgresses) + }), {AddEvents, Streams1} = lists:foldl( fun(#{stream := Stream, iterator := It}, {AddEventAcc, StreamsAcc}) -> case maps:is_key(Stream, StreamsAcc) of diff --git a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl index 64a74510a..e1437dc36 100644 --- a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl +++ b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl @@ -270,12 +270,12 @@ renew_streams(#{start_time := StartTime, stream_progresses := Progresses, topic Data3 = assign_streams(Data2), Data3. -%% We revoke streams from agents that have too many streams (> desired_streams_per_agent). +%% We revoke streams from agents that have too many streams (> desired_stream_count_per_agent). %% We revoke only from replaying agents. %% After revoking, no unassigned streams appear. Streams will become unassigned %% only after agents report them as acked and unsubscribed. revoke_streams(Data0) -> - DesiredStreamsPerAgent = desired_streams_per_agent(Data0), + DesiredStreamsPerAgent = desired_stream_count_per_agent(Data0), Agents = replaying_agents(Data0), lists:foldl( fun(Agent, DataAcc) -> @@ -326,10 +326,10 @@ select_streams_for_revoke( %% * data locality (agents better preserve streams with data available on the agent's node) lists:sublist(shuffle(Streams), RevokeCount). -%% We assign streams to agents that have too few streams (< desired_streams_per_agent). +%% We assign streams to agents that have too few streams (< desired_stream_count_per_agent). %% We assign only to replaying agents. assign_streams(Data0) -> - DesiredStreamsPerAgent = desired_streams_per_agent(Data0), + DesiredStreamsPerAgent = desired_stream_count_per_agent(Data0), Agents = replaying_agents(Data0), lists:foldl( fun(Agent, DataAcc) -> @@ -384,8 +384,7 @@ connect_agent( agent => Agent, group => Group }), - DesiredCount = desired_streams_per_agent(Data), - % DesiredCount = desired_streams_for_new_agent(Data), + DesiredCount = desired_stream_count_for_new_agent(Data), assign_initial_streams_to_agent(Data, Agent, DesiredCount). assign_initial_streams_to_agent(Data, Agent, AssignCount) -> @@ -708,13 +707,13 @@ replaying_agents(#{agents := AgentStates}) -> maps:to_list(AgentStates) ). -desired_streams_per_agent(#{agents := AgentStates} = Data) -> - desired_streams_per_agent(Data, maps:size(AgentStates)). +desired_stream_count_per_agent(#{agents := AgentStates} = Data) -> + desired_stream_count_per_agent(Data, maps:size(AgentStates)). -desired_streams_for_new_agent(#{agents := AgentStates} = Data) -> - desired_streams_per_agent(Data, maps:size(AgentStates) + 1). +desired_stream_count_for_new_agent(#{agents := AgentStates} = Data) -> + desired_stream_count_per_agent(Data, maps:size(AgentStates) + 1). -desired_streams_per_agent(#{stream_progresses := StreamProgresses}, AgentCount) -> +desired_stream_count_per_agent(#{stream_progresses := StreamProgresses}, AgentCount) -> case AgentCount of 0 -> 0; diff --git a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_proto.erl b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_proto.erl index 01f63aaad..d7d85b8f2 100644 --- a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_proto.erl +++ b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_proto.erl @@ -23,7 +23,21 @@ leader_invalidate/2 ]). +-export([ + format_streams/1, + agent/2 +]). + +-ifdef(TEST). +-record(agent, { + pid :: pid(), + id :: term() +}). +-type agent() :: #agent{}. +-else. -type agent() :: pid(). +-endif. + -type leader() :: pid(). -type topic_filter() :: emqx_persistent_session_ds:share_topic_filter(). -type group() :: emqx_types:group(). @@ -107,7 +121,7 @@ leader_lease_streams(ToAgent, OfGroup, Leader, Streams, Version) -> version => Version }), _ = emqx_persistent_session_ds_shared_subs_agent:send( - ToAgent, + agent_pid(ToAgent), ?leader_lease_streams(OfGroup, Leader, Streams, Version) ), ok. @@ -121,7 +135,7 @@ leader_renew_stream_lease(ToAgent, OfGroup, Version) -> version => Version }), _ = emqx_persistent_session_ds_shared_subs_agent:send( - ToAgent, + agent_pid(ToAgent), ?leader_renew_stream_lease(OfGroup, Version) ), ok. @@ -136,7 +150,7 @@ leader_renew_stream_lease(ToAgent, OfGroup, VersionOld, VersionNew) -> version_new => VersionNew }), _ = emqx_persistent_session_ds_shared_subs_agent:send( - ToAgent, + agent_pid(ToAgent), ?leader_renew_stream_lease(OfGroup, VersionOld, VersionNew) ), ok. @@ -152,7 +166,7 @@ leader_update_streams(ToAgent, OfGroup, VersionOld, VersionNew, StreamsNew) -> streams_new => format_streams(StreamsNew) }), _ = emqx_persistent_session_ds_shared_subs_agent:send( - ToAgent, + agent_pid(ToAgent), ?leader_update_streams(OfGroup, VersionOld, VersionNew, StreamsNew) ), ok. @@ -165,11 +179,36 @@ leader_invalidate(ToAgent, OfGroup) -> of_group => OfGroup }), _ = emqx_persistent_session_ds_shared_subs_agent:send( - ToAgent, + agent_pid(ToAgent), ?leader_invalidate(OfGroup) ), ok. +%%-------------------------------------------------------------------- +%% Internal API +%%-------------------------------------------------------------------- + +-ifdef(TEST). +agent(Id, Pid) -> + #agent{id = Id, pid = Pid}. + +agent_pid(#agent{pid = Pid}) -> + Pid. + +-else. +agent(_Id, Pid) -> + Pid. + +agent_pid(Pid) -> + Pid. +-endif. + +format_streams(Streams) -> + lists:map( + fun format_stream/1, + Streams + ). + %%-------------------------------------------------------------------- %% Helpers %%-------------------------------------------------------------------- @@ -177,11 +216,5 @@ leader_invalidate(ToAgent, OfGroup) -> format_opaque(Opaque) -> erlang:phash2(Opaque). -format_streams(Streams) -> - lists:map( - fun format_stream/1, - Streams - ). - format_stream(#{stream := Stream, iterator := Iterator} = Value) -> Value#{stream => format_opaque(Stream), iterator => format_opaque(Iterator)}. From 49bff5c08a801a00422ce2dd5e421aebbf0b376e Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Thu, 27 Jun 2024 18:09:59 +0300 Subject: [PATCH 085/131] feat(queue): wrap remote calls in a proto --- .../src/emqx_ds_shared_sub_proto.erl | 106 +++++++++------- .../src/emqx_ds_shared_sub_proto.hrl | 31 ++++- .../src/proto/emqx_ds_shared_sub_proto_v1.erl | 116 ++++++++++++++++++ 3 files changed, 206 insertions(+), 47 deletions(-) create mode 100644 apps/emqx_ds_shared_sub/src/proto/emqx_ds_shared_sub_proto_v1.erl diff --git a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_proto.erl b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_proto.erl index d7d85b8f2..363a16e46 100644 --- a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_proto.erl +++ b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_proto.erl @@ -9,6 +9,7 @@ -module(emqx_ds_shared_sub_proto). -include("emqx_ds_shared_sub_proto.hrl"). + -include_lib("snabbkaffe/include/snabbkaffe.hrl"). -export([ @@ -28,16 +29,7 @@ agent/2 ]). --ifdef(TEST). --record(agent, { - pid :: pid(), - id :: term() -}). --type agent() :: #agent{}. --else. --type agent() :: pid(). --endif. - +-type agent() :: ?agent(emqx_persistent_session_ds:id(), pid()). -type leader() :: pid(). -type topic_filter() :: emqx_persistent_session_ds:share_topic_filter(). -type group() :: emqx_types:group(). @@ -69,7 +61,7 @@ %% agent -> leader messages -spec agent_connect_leader(leader(), agent(), topic_filter()) -> ok. -agent_connect_leader(ToLeader, FromAgent, TopicFilter) -> +agent_connect_leader(ToLeader, FromAgent, TopicFilter) when ?is_local_leader(ToLeader) -> ?tp(warning, shared_sub_proto_msg, #{ type => agent_connect_leader, to_leader => ToLeader, @@ -77,10 +69,16 @@ agent_connect_leader(ToLeader, FromAgent, TopicFilter) -> topic_filter => TopicFilter }), _ = erlang:send(ToLeader, ?agent_connect_leader(FromAgent, TopicFilter)), - ok. + ok; +agent_connect_leader(ToLeader, FromAgent, TopicFilter) -> + emqx_ds_shared_sub_proto_v1:agent_connect_leader( + ?leader_node(ToLeader), ToLeader, FromAgent, TopicFilter + ). -spec agent_update_stream_states(leader(), agent(), list(agent_stream_progress()), version()) -> ok. -agent_update_stream_states(ToLeader, FromAgent, StreamProgresses, Version) -> +agent_update_stream_states(ToLeader, FromAgent, StreamProgresses, Version) when + ?is_local_leader(ToLeader) +-> ?tp(warning, shared_sub_proto_msg, #{ type => agent_update_stream_states, to_leader => ToLeader, @@ -89,12 +87,18 @@ agent_update_stream_states(ToLeader, FromAgent, StreamProgresses, Version) -> version => Version }), _ = erlang:send(ToLeader, ?agent_update_stream_states(FromAgent, StreamProgresses, Version)), - ok. + ok; +agent_update_stream_states(ToLeader, FromAgent, StreamProgresses, Version) -> + emqx_ds_shared_sub_proto_v1:agent_update_stream_states( + ?leader_node(ToLeader), ToLeader, FromAgent, StreamProgresses, Version + ). -spec agent_update_stream_states( leader(), agent(), list(agent_stream_progress()), version(), version() ) -> ok. -agent_update_stream_states(ToLeader, FromAgent, StreamProgresses, VersionOld, VersionNew) -> +agent_update_stream_states(ToLeader, FromAgent, StreamProgresses, VersionOld, VersionNew) when + ?is_local_leader(ToLeader) +-> ?tp(warning, shared_sub_proto_msg, #{ type => agent_update_stream_states, to_leader => ToLeader, @@ -106,12 +110,16 @@ agent_update_stream_states(ToLeader, FromAgent, StreamProgresses, VersionOld, Ve _ = erlang:send( ToLeader, ?agent_update_stream_states(FromAgent, StreamProgresses, VersionOld, VersionNew) ), - ok. + ok; +agent_update_stream_states(ToLeader, FromAgent, StreamProgresses, VersionOld, VersionNew) -> + emqx_ds_shared_sub_proto_v1:agent_update_stream_states( + ?leader_node(ToLeader), ToLeader, FromAgent, StreamProgresses, VersionOld, VersionNew + ). %% leader -> agent messages -spec leader_lease_streams(agent(), group(), leader(), list(stream_progress()), version()) -> ok. -leader_lease_streams(ToAgent, OfGroup, Leader, Streams, Version) -> +leader_lease_streams(ToAgent, OfGroup, Leader, Streams, Version) when ?is_local_agent(ToAgent) -> ?tp(warning, shared_sub_proto_msg, #{ type => leader_lease_streams, to_agent => ToAgent, @@ -121,13 +129,17 @@ leader_lease_streams(ToAgent, OfGroup, Leader, Streams, Version) -> version => Version }), _ = emqx_persistent_session_ds_shared_subs_agent:send( - agent_pid(ToAgent), + ?agent_pid(ToAgent), ?leader_lease_streams(OfGroup, Leader, Streams, Version) ), - ok. + ok; +leader_lease_streams(ToAgent, OfGroup, Leader, Streams, Version) -> + emqx_ds_shared_sub_proto_v1:leader_lease_streams( + ?agent_node(ToAgent), ToAgent, OfGroup, Leader, Streams, Version + ). -spec leader_renew_stream_lease(agent(), group(), version()) -> ok. -leader_renew_stream_lease(ToAgent, OfGroup, Version) -> +leader_renew_stream_lease(ToAgent, OfGroup, Version) when ?is_local_agent(ToAgent) -> ?tp(warning, shared_sub_proto_msg, #{ type => leader_renew_stream_lease, to_agent => ToAgent, @@ -135,13 +147,17 @@ leader_renew_stream_lease(ToAgent, OfGroup, Version) -> version => Version }), _ = emqx_persistent_session_ds_shared_subs_agent:send( - agent_pid(ToAgent), + ?agent_pid(ToAgent), ?leader_renew_stream_lease(OfGroup, Version) ), - ok. + ok; +leader_renew_stream_lease(ToAgent, OfGroup, Version) -> + emqx_ds_shared_sub_proto_v1:leader_renew_stream_lease( + ?agent_node(ToAgent), ToAgent, OfGroup, Version + ). -spec leader_renew_stream_lease(agent(), group(), version(), version()) -> ok. -leader_renew_stream_lease(ToAgent, OfGroup, VersionOld, VersionNew) -> +leader_renew_stream_lease(ToAgent, OfGroup, VersionOld, VersionNew) when ?is_local_agent(ToAgent) -> ?tp(warning, shared_sub_proto_msg, #{ type => leader_renew_stream_lease, to_agent => ToAgent, @@ -150,13 +166,19 @@ leader_renew_stream_lease(ToAgent, OfGroup, VersionOld, VersionNew) -> version_new => VersionNew }), _ = emqx_persistent_session_ds_shared_subs_agent:send( - agent_pid(ToAgent), + ?agent_pid(ToAgent), ?leader_renew_stream_lease(OfGroup, VersionOld, VersionNew) ), - ok. + ok; +leader_renew_stream_lease(ToAgent, OfGroup, VersionOld, VersionNew) -> + emqx_ds_shared_sub_proto_v1:leader_renew_stream_lease( + ?agent_node(ToAgent), ToAgent, OfGroup, VersionOld, VersionNew + ). -spec leader_update_streams(agent(), group(), version(), version(), list(stream_progress())) -> ok. -leader_update_streams(ToAgent, OfGroup, VersionOld, VersionNew, StreamsNew) -> +leader_update_streams(ToAgent, OfGroup, VersionOld, VersionNew, StreamsNew) when + ?is_local_agent(ToAgent) +-> ?tp(warning, shared_sub_proto_msg, #{ type => leader_update_streams, to_agent => ToAgent, @@ -166,42 +188,38 @@ leader_update_streams(ToAgent, OfGroup, VersionOld, VersionNew, StreamsNew) -> streams_new => format_streams(StreamsNew) }), _ = emqx_persistent_session_ds_shared_subs_agent:send( - agent_pid(ToAgent), + ?agent_pid(ToAgent), ?leader_update_streams(OfGroup, VersionOld, VersionNew, StreamsNew) ), - ok. + ok; +leader_update_streams(ToAgent, OfGroup, VersionOld, VersionNew, StreamsNew) -> + emqx_ds_shared_sub_proto_v1:leader_update_streams( + ?agent_node(ToAgent), ToAgent, OfGroup, VersionOld, VersionNew, StreamsNew + ). -spec leader_invalidate(agent(), group()) -> ok. -leader_invalidate(ToAgent, OfGroup) -> +leader_invalidate(ToAgent, OfGroup) when ?is_local_agent(ToAgent) -> ?tp(warning, shared_sub_proto_msg, #{ type => leader_invalidate, to_agent => ToAgent, of_group => OfGroup }), _ = emqx_persistent_session_ds_shared_subs_agent:send( - agent_pid(ToAgent), + ?agent_pid(ToAgent), ?leader_invalidate(OfGroup) ), - ok. + ok; +leader_invalidate(ToAgent, OfGroup) -> + emqx_ds_shared_sub_proto_v1:leader_invalidate( + ?agent_node(ToAgent), ToAgent, OfGroup + ). %%-------------------------------------------------------------------- %% Internal API %%-------------------------------------------------------------------- --ifdef(TEST). -agent(Id, Pid) -> - #agent{id = Id, pid = Pid}. - -agent_pid(#agent{pid = Pid}) -> - Pid. - --else. agent(_Id, Pid) -> - Pid. - -agent_pid(Pid) -> - Pid. --endif. + ?agent(_Id, Pid). format_streams(Streams) -> lists:map( diff --git a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_proto.hrl b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_proto.hrl index 6689a0d3b..c9227ea2d 100644 --- a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_proto.hrl +++ b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_proto.hrl @@ -6,9 +6,6 @@ %% These messages are instantiated on the receiver's side, so they do not %% travel over the network. --ifndef(EMQX_DS_SHARED_SUB_PROTO_HRL). --define(EMQX_DS_SHARED_SUB_PROTO_HRL, true). - %% NOTE %% We do not need any kind of request/response identification, %% because the protocol is fully event-based. @@ -140,4 +137,32 @@ group := Group }). +%% Helpers +%% In test mode we extend agents with (session) Id to have more +%% readable traces. + +-ifdef(TEST). + +-define(agent(Id, Pid), {Id, Pid}). + +-define(agent_pid(Agent), element(2, Agent)). + +-define(agent_node(Agent), node(element(2, Agent))). + +%% -ifdef(TEST). +-else. + +-define(agent(Id, Pid), Pid). + +-define(agent_pid(Agent), Agent). + +-define(agent_node(Agent), node(Agent)). + +%% -ifdef(TEST). -endif. + +-define(is_local_agent(Agent), (?agent_node(Agent) =:= node())). + +-define(leader_node(Leader), node(Leader)). + +-define(is_local_leader(Leader), (?leader_node(Leader) =:= node())). diff --git a/apps/emqx_ds_shared_sub/src/proto/emqx_ds_shared_sub_proto_v1.erl b/apps/emqx_ds_shared_sub/src/proto/emqx_ds_shared_sub_proto_v1.erl new file mode 100644 index 000000000..b0a132ea5 --- /dev/null +++ b/apps/emqx_ds_shared_sub/src/proto/emqx_ds_shared_sub_proto_v1.erl @@ -0,0 +1,116 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_ds_shared_sub_proto_v1). + +-behaviour(emqx_bpapi). + +-include_lib("emqx/include/bpapi.hrl"). + +-export([ + introduced_in/0, + + agent_connect_leader/4, + agent_update_stream_states/5, + agent_update_stream_states/6, + + leader_lease_streams/6, + leader_renew_stream_lease/4, + leader_renew_stream_lease/5, + leader_update_streams/6, + leader_invalidate/3 +]). + +introduced_in() -> + "5.8.0". + +-spec agent_connect_leader( + node(), + emqx_ds_shared_sub_proto:leader(), + emqx_ds_shared_sub_proto:agent(), + emqx_ds_shared_sub_proto:topic_filter() +) -> ok. +agent_connect_leader(Node, ToLeader, FromAgent, TopicFilter) -> + erpc:cast(Node, emqx_ds_shared_sub_proto, agent_connect_leader, [ + ToLeader, FromAgent, TopicFilter + ]). + +-spec agent_update_stream_states( + node(), + emqx_ds_shared_sub_proto:leader(), + emqx_ds_shared_sub_proto:agent(), + list(emqx_ds_shared_sub_proto:agent_stream_progress()), + emqx_ds_shared_sub_proto:version() +) -> ok. +agent_update_stream_states(Node, ToLeader, FromAgent, StreamProgresses, Version) -> + erpc:cast(Node, emqx_ds_shared_sub_proto, agent_update_stream_states, [ + ToLeader, FromAgent, StreamProgresses, Version + ]). + +-spec agent_update_stream_states( + node(), + emqx_ds_shared_sub_proto:leader(), + emqx_ds_shared_sub_proto:agent(), + list(emqx_ds_shared_sub_proto:agent_stream_progress()), + emqx_ds_shared_sub_proto:version(), + emqx_ds_shared_sub_proto:version() +) -> ok. +agent_update_stream_states(Node, ToLeader, FromAgent, StreamProgresses, VersionOld, VersionNew) -> + erpc:cast(Node, emqx_ds_shared_sub_proto, agent_update_stream_states, [ + ToLeader, FromAgent, StreamProgresses, VersionOld, VersionNew + ]). + +%% leader -> agent messages + +-spec leader_lease_streams( + node(), + emqx_ds_shared_sub_proto:agent(), + emqx_ds_shared_sub_proto:group(), + emqx_ds_shared_sub_proto:leader(), + list(emqx_ds_shared_sub_proto:stream_progress()), + emqx_ds_shared_sub_proto:version() +) -> ok. +leader_lease_streams(Node, ToAgent, OfGroup, Leader, Streams, Version) -> + erpc:cast(Node, emqx_ds_shared_sub_proto, leader_lease_streams, [ + ToAgent, OfGroup, Leader, Streams, Version + ]). + +-spec leader_renew_stream_lease( + node(), + emqx_ds_shared_sub_proto:agent(), + emqx_ds_shared_sub_proto:group(), + emqx_ds_shared_sub_proto:version() +) -> ok. +leader_renew_stream_lease(Node, ToAgent, OfGroup, Version) -> + erpc:cast(Node, emqx_ds_shared_sub_proto, leader_renew_stream_lease, [ToAgent, OfGroup, Version]). + +-spec leader_renew_stream_lease( + node(), + emqx_ds_shared_sub_proto:agent(), + emqx_ds_shared_sub_proto:group(), + emqx_ds_shared_sub_proto:version(), + emqx_ds_shared_sub_proto:version() +) -> ok. +leader_renew_stream_lease(Node, ToAgent, OfGroup, VersionOld, VersionNew) -> + erpc:cast(Node, emqx_ds_shared_sub_proto, leader_renew_stream_lease, [ + ToAgent, OfGroup, VersionOld, VersionNew + ]). + +-spec leader_update_streams( + node(), + emqx_ds_shared_sub_proto:agent(), + emqx_ds_shared_sub_proto:group(), + emqx_ds_shared_sub_proto:version(), + emqx_ds_shared_sub_proto:version(), + list(emqx_ds_shared_sub_proto:stream_progress()) +) -> ok. +leader_update_streams(Node, ToAgent, OfGroup, VersionOld, VersionNew, StreamsNew) -> + erpc:cast(Node, emqx_ds_shared_sub_proto, leader_update_streams, [ + ToAgent, OfGroup, VersionOld, VersionNew, StreamsNew + ]). + +-spec leader_invalidate(node(), emqx_ds_shared_sub_proto:agent(), emqx_ds_shared_sub_proto:group()) -> + ok. +leader_invalidate(Node, ToAgent, OfGroup) -> + erpc:cast(Node, emqx_ds_shared_sub_proto, leader_invalidate, [ToAgent, OfGroup]). From 1d728a05b2eff5f2c8cae1257e6e084d14485335 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Thu, 27 Jun 2024 18:49:47 +0300 Subject: [PATCH 086/131] feat(queue): send metadata with agent when connecting to leader It will be used to attach agent taints to improve stream assignment. --- .../src/emqx_ds_shared_sub_group_sm.erl | 7 ++++-- .../src/emqx_ds_shared_sub_leader.erl | 19 ++++++++------ .../src/emqx_ds_shared_sub_proto.erl | 22 ++++++++++------ .../src/emqx_ds_shared_sub_proto.hrl | 6 +++-- .../src/emqx_ds_shared_sub_registry.erl | 25 +++++++++++++------ .../src/proto/emqx_ds_shared_sub_proto_v1.erl | 7 +++--- 6 files changed, 57 insertions(+), 29 deletions(-) diff --git a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_group_sm.erl b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_group_sm.erl index a16b2bcf9..cd029d8df 100644 --- a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_group_sm.erl +++ b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_group_sm.erl @@ -177,7 +177,7 @@ fetch_stream_events( %% Connecting state handle_connecting(#{agent := Agent, topic_filter := ShareTopicFilter} = GSM) -> - ok = emqx_ds_shared_sub_registry:lookup_leader(Agent, ShareTopicFilter), + ok = emqx_ds_shared_sub_registry:lookup_leader(Agent, agent_metadata(GSM), ShareTopicFilter), ensure_state_timeout(GSM, find_leader_timeout, ?FIND_LEADER_TIMEOUT). handle_leader_lease_streams( @@ -201,7 +201,7 @@ handle_leader_lease_streams(GSM, _Leader, _StreamProgresses, _Version) -> GSM. handle_find_leader_timeout(#{agent := Agent, topic_filter := TopicFilter} = GSM0) -> - ok = emqx_ds_shared_sub_registry:lookup_leader(Agent, TopicFilter), + ok = emqx_ds_shared_sub_registry:lookup_leader(Agent, agent_metadata(GSM0), TopicFilter), GSM1 = ensure_state_timeout(GSM0, find_leader_timeout, ?FIND_LEADER_TIMEOUT), GSM1. @@ -444,6 +444,9 @@ transition(GSM0, NewState, NewStateData, LeaseEvents) -> }, run_enter_callback(GSM2). +agent_metadata(#{id := Id} = _GSM) -> + #{id => Id}. + ensure_state_timeout(GSM0, Name, Delay) -> ensure_state_timeout(GSM0, Name, Delay, Name). diff --git a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl index e1437dc36..379c0278f 100644 --- a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl +++ b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl @@ -43,6 +43,7 @@ state := emqx_ds_shared_sub_agent:status(), prev_version := emqx_maybe:t(emqx_ds_shared_sub_proto:version()), version := emqx_ds_shared_sub_proto:version(), + agent_metadata := emqx_ds_shared_sub_proto:agent_metadata(), streams := list(emqx_ds:stream()), revoked_streams := list(emqx_ds:stream()) }. @@ -182,9 +183,11 @@ handle_event({timeout, #drop_timeout{}}, #drop_timeout{}, ?leader_active, Data0) {keep_state, Data1, {{timeout, #drop_timeout{}}, ?DROP_TIMEOUT_INTERVAL, #drop_timeout{}}}; %%-------------------------------------------------------------------- %% agent events -handle_event(info, ?agent_connect_leader_match(Agent, _TopicFilter), ?leader_active, Data0) -> +handle_event( + info, ?agent_connect_leader_match(Agent, AgentMetadata, _TopicFilter), ?leader_active, Data0 +) -> % ?tp(warning, shared_sub_leader_connect_agent, #{agent => Agent}), - Data1 = connect_agent(Data0, Agent), + Data1 = connect_agent(Data0, Agent, AgentMetadata), {keep_state, Data1}; handle_event( info, @@ -375,7 +378,8 @@ select_streams_for_assign(Data0, _Agent, AssignCount) -> connect_agent( #{group := Group} = Data, - Agent + Agent, + AgentMetadata ) -> %% TODO %% implement graceful reconnection of the same agent @@ -385,13 +389,13 @@ connect_agent( group => Group }), DesiredCount = desired_stream_count_for_new_agent(Data), - assign_initial_streams_to_agent(Data, Agent, DesiredCount). + assign_initial_streams_to_agent(Data, Agent, AgentMetadata, DesiredCount). -assign_initial_streams_to_agent(Data, Agent, AssignCount) -> +assign_initial_streams_to_agent(Data, Agent, AgentMetadata, AssignCount) -> InitialStreamsToAssign = select_streams_for_assign(Data, Agent, AssignCount), Data1 = set_stream_ownership_to_agent(Data, Agent, InitialStreamsToAssign), AgentState = agent_transition_to_initial_waiting_replaying( - Data1, Agent, InitialStreamsToAssign + Data1, Agent, AgentMetadata, InitialStreamsToAssign ), set_agent_state(Data1, Agent, AgentState). @@ -639,7 +643,7 @@ agent_transition_to_waiting_replaying( }. agent_transition_to_initial_waiting_replaying( - #{group := Group} = Data, Agent, InitialStreams + #{group := Group} = Data, Agent, AgentMetadata, InitialStreams ) -> ?tp(warning, shared_sub_leader_agent_state_transition, #{ agent => Agent, @@ -653,6 +657,7 @@ agent_transition_to_initial_waiting_replaying( Agent, Group, Leader, StreamProgresses, Version ), #{ + metadata => AgentMetadata, state => ?waiting_replaying, version => Version, prev_version => undefined, diff --git a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_proto.erl b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_proto.erl index 363a16e46..ec0a25f14 100644 --- a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_proto.erl +++ b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_proto.erl @@ -13,7 +13,7 @@ -include_lib("snabbkaffe/include/snabbkaffe.hrl"). -export([ - agent_connect_leader/3, + agent_connect_leader/4, agent_update_stream_states/4, agent_update_stream_states/5, @@ -34,6 +34,9 @@ -type topic_filter() :: emqx_persistent_session_ds:share_topic_filter(). -type group() :: emqx_types:group(). -type version() :: non_neg_integer(). +-type agent_metadata() :: #{ + id := emqx_persistent_session_ds:id() +}. -type stream_progress() :: #{ stream := emqx_ds:stream(), @@ -51,7 +54,9 @@ leader/0, group/0, version/0, - stream_progress/0 + stream_progress/0, + agent_stream_progress/0, + agent_metadata/0 ]). %%-------------------------------------------------------------------- @@ -60,19 +65,22 @@ %% agent -> leader messages --spec agent_connect_leader(leader(), agent(), topic_filter()) -> ok. -agent_connect_leader(ToLeader, FromAgent, TopicFilter) when ?is_local_leader(ToLeader) -> +-spec agent_connect_leader(leader(), agent(), agent_metadata(), topic_filter()) -> ok. +agent_connect_leader(ToLeader, FromAgent, AgentMetadata, TopicFilter) when + ?is_local_leader(ToLeader) +-> ?tp(warning, shared_sub_proto_msg, #{ type => agent_connect_leader, to_leader => ToLeader, from_agent => FromAgent, + agent_metadata => AgentMetadata, topic_filter => TopicFilter }), - _ = erlang:send(ToLeader, ?agent_connect_leader(FromAgent, TopicFilter)), + _ = erlang:send(ToLeader, ?agent_connect_leader(FromAgent, AgentMetadata, TopicFilter)), ok; -agent_connect_leader(ToLeader, FromAgent, TopicFilter) -> +agent_connect_leader(ToLeader, FromAgent, AgentMetadata, TopicFilter) -> emqx_ds_shared_sub_proto_v1:agent_connect_leader( - ?leader_node(ToLeader), ToLeader, FromAgent, TopicFilter + ?leader_node(ToLeader), ToLeader, FromAgent, AgentMetadata, TopicFilter ). -spec agent_update_stream_states(leader(), agent(), list(agent_stream_progress()), version()) -> ok. diff --git a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_proto.hrl b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_proto.hrl index c9227ea2d..a2cf284f3 100644 --- a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_proto.hrl +++ b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_proto.hrl @@ -20,15 +20,17 @@ %% Agent messages sent to the leader. %% Leader talks to many agents, `agent` field is used to identify the sender. --define(agent_connect_leader(Agent, TopicFilter), #{ +-define(agent_connect_leader(Agent, AgentMetadata, TopicFilter), #{ type => ?agent_connect_leader_msg, topic_filter => TopicFilter, + agent_metadata => AgentMetadata, agent => Agent }). --define(agent_connect_leader_match(Agent, TopicFilter), #{ +-define(agent_connect_leader_match(Agent, AgentMetadata, TopicFilter), #{ type := ?agent_connect_leader_msg, topic_filter := TopicFilter, + agent_metadata := AgentMetadata, agent := Agent }). diff --git a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_registry.erl b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_registry.erl index 9b4a6bd11..bc732249a 100644 --- a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_registry.erl +++ b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_registry.erl @@ -20,11 +20,12 @@ ]). -export([ - lookup_leader/2 + lookup_leader/3 ]). -record(lookup_leader, { agent :: emqx_ds_shared_sub_proto:agent(), + agent_metadata :: emqx_ds_shared_sub_proto:agent_metadata(), topic_filter :: emqx_persistent_session_ds:share_topic_filter() }). @@ -35,10 +36,14 @@ %%-------------------------------------------------------------------- -spec lookup_leader( - emqx_ds_shared_sub_proto:agent(), emqx_persistent_session_ds:share_topic_filter() + emqx_ds_shared_sub_proto:agent(), + emqx_ds_shared_sub_proto:agent_metadata(), + emqx_persistent_session_ds:share_topic_filter() ) -> ok. -lookup_leader(Agent, TopicFilter) -> - gen_server:cast(?MODULE, #lookup_leader{agent = Agent, topic_filter = TopicFilter}). +lookup_leader(Agent, AgentMetadata, TopicFilter) -> + gen_server:cast(?MODULE, #lookup_leader{ + agent = Agent, agent_metadata = AgentMetadata, topic_filter = TopicFilter + }). %%-------------------------------------------------------------------- %% Internal API @@ -66,8 +71,10 @@ init([]) -> handle_call(_Request, _From, State) -> {reply, {error, unknown_request}, State}. -handle_cast(#lookup_leader{agent = Agent, topic_filter = TopicFilter}, State) -> - State1 = do_lookup_leader(Agent, TopicFilter, State), +handle_cast( + #lookup_leader{agent = Agent, agent_metadata = AgentMetadata, topic_filter = TopicFilter}, State +) -> + State1 = do_lookup_leader(Agent, AgentMetadata, TopicFilter, State), {noreply, State1}. handle_info(_Info, State) -> @@ -80,7 +87,7 @@ terminate(_Reason, _State) -> %% Internal functions %%-------------------------------------------------------------------- -do_lookup_leader(Agent, TopicFilter, State) -> +do_lookup_leader(Agent, AgentMetadata, TopicFilter, State) -> %% TODO https://emqx.atlassian.net/browse/EMQX-12309 %% Cluster-wide unique leader election should be implemented Id = emqx_ds_shared_sub_leader:id(TopicFilter), @@ -107,5 +114,7 @@ do_lookup_leader(Agent, TopicFilter, State) -> topic_filter => TopicFilter, leader => LeaderPid }), - ok = emqx_ds_shared_sub_proto:agent_connect_leader(LeaderPid, Agent, TopicFilter), + ok = emqx_ds_shared_sub_proto:agent_connect_leader( + LeaderPid, Agent, AgentMetadata, TopicFilter + ), State. diff --git a/apps/emqx_ds_shared_sub/src/proto/emqx_ds_shared_sub_proto_v1.erl b/apps/emqx_ds_shared_sub/src/proto/emqx_ds_shared_sub_proto_v1.erl index b0a132ea5..01c704cb0 100644 --- a/apps/emqx_ds_shared_sub/src/proto/emqx_ds_shared_sub_proto_v1.erl +++ b/apps/emqx_ds_shared_sub/src/proto/emqx_ds_shared_sub_proto_v1.erl @@ -11,7 +11,7 @@ -export([ introduced_in/0, - agent_connect_leader/4, + agent_connect_leader/5, agent_update_stream_states/5, agent_update_stream_states/6, @@ -29,11 +29,12 @@ introduced_in() -> node(), emqx_ds_shared_sub_proto:leader(), emqx_ds_shared_sub_proto:agent(), + emqx_ds_shared_sub_proto:agent_metadata(), emqx_ds_shared_sub_proto:topic_filter() ) -> ok. -agent_connect_leader(Node, ToLeader, FromAgent, TopicFilter) -> +agent_connect_leader(Node, ToLeader, FromAgent, AgentMetadata, TopicFilter) -> erpc:cast(Node, emqx_ds_shared_sub_proto, agent_connect_leader, [ - ToLeader, FromAgent, TopicFilter + ToLeader, FromAgent, AgentMetadata, TopicFilter ]). -spec agent_update_stream_states( From d32f282feb5c2822e3cbfb7673422ccec720484f Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Thu, 27 Jun 2024 21:17:53 +0300 Subject: [PATCH 087/131] feat(queue): add graceful disconnect --- apps/emqx/src/emqx_persistent_session_ds.erl | 7 +- ...emqx_persistent_session_ds_shared_subs.erl | 68 +++++++++++++------ ...ersistent_session_ds_shared_subs_agent.erl | 9 ++- ...tent_session_ds_shared_subs_null_agent.erl | 4 ++ .../src/emqx_ds_shared_sub_agent.erl | 14 ++++ .../src/emqx_ds_shared_sub_group_sm.erl | 42 ++++++++++-- .../src/emqx_ds_shared_sub_leader.erl | 36 ++++++++++ .../src/emqx_ds_shared_sub_proto.erl | 18 +++++ .../src/emqx_ds_shared_sub_proto.hrl | 15 ++++ .../src/proto/emqx_ds_shared_sub_proto_v1.erl | 12 ++++ .../test/emqx_ds_shared_sub_SUITE.erl | 32 +++++++++ 11 files changed, 229 insertions(+), 28 deletions(-) diff --git a/apps/emqx/src/emqx_persistent_session_ds.erl b/apps/emqx/src/emqx_persistent_session_ds.erl index 32c291eec..dc4a74b43 100644 --- a/apps/emqx/src/emqx_persistent_session_ds.erl +++ b/apps/emqx/src/emqx_persistent_session_ds.erl @@ -757,7 +757,7 @@ skip_batch(StreamKey, SRS0, Session = #{s := S0}, ClientInfo, Reason) -> %%-------------------------------------------------------------------- -spec disconnect(session(), emqx_types:conninfo()) -> {shutdown, session()}. -disconnect(Session = #{id := Id, s := S0}, ConnInfo) -> +disconnect(Session = #{id := Id, s := S0, shared_sub_s := SharedSubS0}, ConnInfo) -> S1 = maybe_set_offline_info(S0, Id), S2 = emqx_persistent_session_ds_state:set_last_alive_at(now_ms(), S1), S3 = @@ -767,8 +767,9 @@ disconnect(Session = #{id := Id, s := S0}, ConnInfo) -> _ -> S2 end, - S = emqx_persistent_session_ds_state:commit(S3), - {shutdown, Session#{s => S}}. + {S4, SharedSubS} = emqx_persistent_session_ds_shared_subs:on_disconnect(S3, SharedSubS0), + S = emqx_persistent_session_ds_state:commit(S4), + {shutdown, Session#{s => S, shared_sub_s => SharedSubS}}. -spec terminate(Reason :: term(), session()) -> ok. terminate(_Reason, Session = #{id := Id, s := S}) -> diff --git a/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl index f3aaa146e..0274b9b9e 100644 --- a/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl +++ b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl @@ -15,6 +15,7 @@ on_subscribe/3, on_unsubscribe/4, + on_disconnect/2, on_streams_replayed/2, on_info/3, @@ -118,29 +119,20 @@ renew_streams(S0, #{agent := Agent0} = SharedSubS0) -> t() ) -> {emqx_persistent_session_ds_state:t(), t()}. on_streams_replayed(S, #{agent := Agent0} = SharedSubS0) -> - %% TODO - %% Is it sufficient for a report? - Progress = fold_shared_stream_states( - fun(TopicFilter, Stream, SRS, Acc) -> - #srs{it_begin = BeginIt} = SRS, - - StreamProgress = #{ - topic_filter => TopicFilter, - stream => Stream, - iterator => BeginIt, - use_finished => is_use_finished(S, SRS) - }, - [StreamProgress | Acc] - end, - [], - S - ), + Progresses = stream_progresses(S), Agent1 = emqx_persistent_session_ds_shared_subs_agent:on_stream_progress( - Agent0, Progress + Agent0, Progresses ), SharedSubS1 = SharedSubS0#{agent => Agent1}, {S, SharedSubS1}. +on_disconnect(S0, #{agent := Agent0} = SharedSubS0) -> + S1 = revoke_all_streams(S0), + Progresses = stream_progresses(S1), + Agent1 = emqx_persistent_session_ds_shared_subs_agent:on_disconnect(Agent0, Progresses), + SharedSubS1 = SharedSubS0#{agent => Agent1}, + {S1, SharedSubS1}. + -spec on_info(emqx_persistent_session_ds_state:t(), t(), term()) -> {emqx_persistent_session_ds_state:t(), t()}. on_info(S, #{agent := Agent0} = SharedSubS0, Info) -> @@ -157,6 +149,30 @@ to_map(_S, _SharedSubS) -> %% Internal functions %%-------------------------------------------------------------------- +stream_progresses(S) -> + fold_shared_stream_states( + fun(TopicFilter, Stream, SRS, Acc) -> + #srs{it_begin = BeginIt} = SRS, + + case is_stream_fully_acked(S, SRS) of + true -> + %% TODO + %% Is it sufficient for a report? + StreamProgress = #{ + topic_filter => TopicFilter, + stream => Stream, + iterator => BeginIt, + use_finished => is_use_finished(S, SRS) + }, + [StreamProgress | Acc]; + false -> + Acc + end + end, + [], + S + ). + fold_shared_subs(Fun, Acc, S) -> emqx_persistent_session_ds_state:fold_subscriptions( fun @@ -322,6 +338,15 @@ revoke_stream( end end. +revoke_all_streams(S0) -> + fold_shared_stream_states( + fun(TopicFilter, Stream, _SRS, S) -> + revoke_stream(#{topic_filter => TopicFilter, stream => Stream}, S) + end, + S0, + S0 + ). + -spec to_agent_subscription( emqx_persistent_session_ds_state:t(), emqx_persistent_session_ds:subscription() ) -> @@ -339,5 +364,8 @@ agent_opts(#{session_id := SessionId}) -> now_ms() -> erlang:system_time(millisecond). -is_use_finished(S, #srs{unsubscribed = Unsubscribed} = SRS) -> - Unsubscribed andalso emqx_persistent_session_ds_stream_scheduler:is_fully_acked(SRS, S). +is_use_finished(S, #srs{unsubscribed = Unsubscribed}) -> + Unsubscribed. + +is_stream_fully_acked(S, SRS) -> + emqx_persistent_session_ds_stream_scheduler:is_fully_acked(SRS, S). diff --git a/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs_agent.erl b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs_agent.erl index 97b38d0f2..72b4fa22d 100644 --- a/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs_agent.erl +++ b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs_agent.erl @@ -44,7 +44,8 @@ -type stream_progress() :: #{ topic_filter := topic_filter(), stream := emqx_ds:stream(), - iterator := emqx_ds:iterator() + iterator := emqx_ds:iterator(), + use_finished := boolean() }. -export_type([ @@ -63,6 +64,7 @@ on_unsubscribe/2, on_stream_progress/2, on_info/2, + on_disconnect/2, renew_streams/1 ]). @@ -81,6 +83,7 @@ -callback on_subscribe(t(), topic_filter(), emqx_types:subopts()) -> {ok, t()} | {error, term()}. -callback on_unsubscribe(t(), topic_filter()) -> t(). +-callback on_disconnect(t(), [stream_progress()]) -> t(). -callback renew_streams(t()) -> {[stream_lease_event()], t()}. -callback on_stream_progress(t(), [stream_progress()]) -> t(). -callback on_info(t(), term()) -> t(). @@ -106,6 +109,10 @@ on_subscribe(Agent, TopicFilter, SubOpts) -> on_unsubscribe(Agent, TopicFilter) -> ?shared_subs_agent:on_unsubscribe(Agent, TopicFilter). +-spec on_disconnect(t(), [stream_progress()]) -> t(). +on_disconnect(Agent, StreamProgresses) -> + ?shared_subs_agent:on_disconnect(Agent, StreamProgresses). + -spec renew_streams(t()) -> {[stream_lease_event()], t()}. renew_streams(Agent) -> ?shared_subs_agent:renew_streams(Agent). diff --git a/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs_null_agent.erl b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs_null_agent.erl index e158c19e2..5bdae08da 100644 --- a/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs_null_agent.erl +++ b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs_null_agent.erl @@ -14,6 +14,7 @@ on_unsubscribe/2, on_stream_progress/2, on_info/2, + on_disconnect/1, renew_streams/1 ]). @@ -36,6 +37,9 @@ on_subscribe(_Agent, _TopicFilter, _SubOpts) -> on_unsubscribe(Agent, _TopicFilter) -> Agent. +on_disconnect(Agent) -> + Agent. + renew_streams(Agent) -> {[], Agent}. diff --git a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_agent.erl b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_agent.erl index 11cb011d3..0e8d17614 100644 --- a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_agent.erl +++ b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_agent.erl @@ -17,6 +17,7 @@ on_unsubscribe/2, on_stream_progress/2, on_info/2, + on_disconnect/2, renew_streams/1 ]). @@ -68,6 +69,19 @@ on_stream_progress(State, StreamProgresses) -> maps:to_list(ProgressesByGroup) ). +on_disconnect(#{groups := Groups0} = State, StreamProgresses) -> + ProgressesByGroup = stream_progresses_by_group(StreamProgresses), + Groups1 = maps:fold( + fun(Group, GroupSM0, GroupsAcc) -> + GroupProgresses = maps:get(Group, ProgressesByGroup, []), + GroupSM1 = emqx_ds_shared_sub_group_sm:handle_disconnect(GroupSM0, GroupProgresses), + GroupsAcc#{Group => GroupSM1} + end, + #{}, + Groups0 + ), + State#{groups => Groups1}. + on_info(State, ?leader_lease_streams_match(Group, Leader, StreamProgresses, Version)) -> ?SLOG(info, #{ msg => leader_lease_streams, diff --git a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_group_sm.erl b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_group_sm.erl index cd029d8df..3932aa6ce 100644 --- a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_group_sm.erl +++ b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_group_sm.erl @@ -27,7 +27,8 @@ %% API fetch_stream_events/1, - handle_stream_progress/2 + handle_stream_progress/2, + handle_disconnect/2 ]). -export_type([ @@ -72,8 +73,9 @@ -define(connecting, connecting). -define(replaying, replaying). -define(updating, updating). +-define(disconnected, disconnected). --type state() :: ?connecting | ?replaying | ?updating. +-type state() :: ?connecting | ?replaying | ?updating | ?disconnected. -type connecting_data() :: #{}. -type replaying_data() :: #{ @@ -169,6 +171,18 @@ fetch_stream_events( ), {GSM#{stream_lease_events => []}, Events1}. +-spec handle_disconnect(group_sm(), emqx_ds_shared_sub_proto:agent_stream_progress()) -> group_sm(). +handle_disconnect(#{state := ?connecting} = GSM, _StreamProgresses) -> + transition(GSM, ?disconnected, #{}); +handle_disconnect( + #{agent := Agent, state_data := #{leader := Leader, version := Version} = StateData} = GSM, + StreamProgresses +) -> + ok = emqx_ds_shared_sub_proto:agent_disconnect( + Leader, Agent, StreamProgresses, Version + ), + transition(GSM, ?disconnected, StateData). + %%----------------------------------------------------------------------- %% Event Handlers %%----------------------------------------------------------------------- @@ -229,6 +243,12 @@ handle_updating(GSM0) -> ), GSM2. +%%----------------------------------------------------------------------- +%% Disconnected state + +handle_disconnected(GSM) -> + GSM. + %%----------------------------------------------------------------------- %% Common handlers @@ -301,6 +321,10 @@ handle_leader_update_streams( _StreamProgresses ) -> ensure_state_timeout(GSM, renew_lease_timeout, ?RENEW_LEASE_TIMEOUT); +handle_leader_update_streams( + #{state := ?disconnected} = GSM, _VersionOld, _VersionNew, _StreamProgresses +) -> + GSM; handle_leader_update_streams(GSM, VersionOld, VersionNew, _StreamProgresses) -> ?tp(warning, shared_sub_group_sm_unexpected_leader_update_streams, #{ gsm => GSM, @@ -335,6 +359,10 @@ handle_leader_renew_stream_lease( VersionNew ) -> ensure_state_timeout(GSM, renew_lease_timeout, ?RENEW_LEASE_TIMEOUT); +handle_leader_renew_stream_lease( + #{state := ?disconnected} = GSM, _VersionOld, _VersionNew +) -> + GSM; handle_leader_renew_stream_lease(GSM, VersionOld, VersionNew) -> ?tp(warning, shared_sub_group_sm_unexpected_leader_renew_stream_lease, #{ gsm => GSM, @@ -344,6 +372,8 @@ handle_leader_renew_stream_lease(GSM, VersionOld, VersionNew) -> %% Unexpected versions or state transition(GSM, ?connecting, #{}). +-spec handle_stream_progress(group_sm(), emqx_ds_shared_sub_proto:agent_stream_progress()) -> + group_sm(). handle_stream_progress(#{state := ?connecting} = GSM, _StreamProgresses) -> GSM; handle_stream_progress( @@ -376,7 +406,9 @@ handle_stream_progress( ok = emqx_ds_shared_sub_proto:agent_update_stream_states( Leader, Agent, StreamProgresses, PrevVersion, Version ), - ensure_state_timeout(GSM, update_stream_state_timeout, ?MIN_UPDATE_STREAM_STATE_INTERVAL). + ensure_state_timeout(GSM, update_stream_state_timeout, ?MIN_UPDATE_STREAM_STATE_INTERVAL); +handle_stream_progress(#{state := ?disconnected} = GSM, _StreamProgresses) -> + GSM. handle_leader_invalidate(GSM) -> transition(GSM, ?connecting, #{}). @@ -485,7 +517,9 @@ run_enter_callback(#{state := ?connecting} = GSM) -> run_enter_callback(#{state := ?replaying} = GSM) -> handle_replaying(GSM); run_enter_callback(#{state := ?updating} = GSM) -> - handle_updating(GSM). + handle_updating(GSM); +run_enter_callback(#{state := ?disconnected} = GSM) -> + handle_disconnected(GSM). progresses_to_lease_events(StreamProgresses) -> lists:map( diff --git a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl index 379c0278f..2bbdc67d8 100644 --- a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl +++ b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl @@ -213,6 +213,19 @@ handle_event( update_agent_stream_states(Data0, Agent, StreamProgresses, VersionOld, VersionNew) end), {keep_state, Data1}; +handle_event( + info, + ?agent_disconnect_match(Agent, StreamProgresses, Version), + ?leader_active, + Data0 +) -> + % ?tp(warning, shared_sub_leader_disconnect, #{ + % agent => Agent, version => Version + % }), + Data1 = with_agent(Data0, Agent, fun() -> + disconnect_agent(Data0, Agent, StreamProgresses, Version) + end), + {keep_state, Data1}; %%-------------------------------------------------------------------- %% fallback handle_event(enter, _OldState, _State, _Data) -> @@ -399,6 +412,28 @@ assign_initial_streams_to_agent(Data, Agent, AgentMetadata, AssignCount) -> ), set_agent_state(Data1, Agent, AgentState). +%%-------------------------------------------------------------------- +%% Disconnect agent gracefully + +disconnect_agent(Data0, Agent, AgentStreamProgresses, Version) -> + case get_agent_state(Data0, Agent) of + #{version := Version} -> + ?tp(warning, shared_sub_leader_disconnect_agent, #{ + agent => Agent, + version => Version + }), + Data1 = update_agent_stream_states(Data0, Agent, AgentStreamProgresses, Version), + Data2 = drop_agent(Data1, Agent), + Data2; + _ -> + ?tp(warning, shared_sub_leader_unexpected_disconnect, #{ + agent => Agent, + version => Version + }), + Data1 = drop_agent(Data0, Agent), + Data1 + end. + %%-------------------------------------------------------------------- %% Drop agents that stopped reporting progress @@ -790,6 +825,7 @@ drop_agent(#{agents := Agents} = Data0, Agent) -> #{streams := Streams, revoked_streams := RevokedStreams} = AgentState, AllStreams = Streams ++ RevokedStreams, Data1 = unassign_streams(Data0, AllStreams), + ?tp(warning, shared_sub_leader_drop_agent, #{agent => Agent}), Data1#{agents => maps:remove(Agent, Agents)}. invalidate_agent(#{group := Group}, Agent) -> diff --git a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_proto.erl b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_proto.erl index ec0a25f14..53a6693b2 100644 --- a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_proto.erl +++ b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_proto.erl @@ -16,6 +16,7 @@ agent_connect_leader/4, agent_update_stream_states/4, agent_update_stream_states/5, + agent_disconnect/4, leader_lease_streams/5, leader_renew_stream_lease/3, @@ -124,6 +125,23 @@ agent_update_stream_states(ToLeader, FromAgent, StreamProgresses, VersionOld, Ve ?leader_node(ToLeader), ToLeader, FromAgent, StreamProgresses, VersionOld, VersionNew ). +agent_disconnect(ToLeader, FromAgent, StreamProgresses, Version) when + ?is_local_leader(ToLeader) +-> + ?tp(warning, shared_sub_proto_msg, #{ + type => agent_disconnect, + to_leader => ToLeader, + from_agent => FromAgent, + stream_progresses => format_streams(StreamProgresses), + version => Version + }), + _ = erlang:send(ToLeader, ?agent_disconnect(FromAgent, StreamProgresses, Version)), + ok; +agent_disconnect(ToLeader, FromAgent, StreamProgresses, Version) -> + emqx_ds_shared_sub_proto_v1:agent_disconnect( + ?leader_node(ToLeader), ToLeader, FromAgent, StreamProgresses, Version + ). + %% leader -> agent messages -spec leader_lease_streams(agent(), group(), leader(), list(stream_progress()), version()) -> ok. diff --git a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_proto.hrl b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_proto.hrl index a2cf284f3..f8158c918 100644 --- a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_proto.hrl +++ b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_proto.hrl @@ -16,6 +16,7 @@ -define(agent_update_stream_states_msg, agent_update_stream_states). -define(agent_connect_leader_timeout_msg, agent_connect_leader_timeout). -define(agent_renew_stream_lease_timeout_msg, agent_renew_stream_lease_timeout). +-define(agent_disconnect_msg, agent_disconnect). %% Agent messages sent to the leader. %% Leader talks to many agents, `agent` field is used to identify the sender. @@ -64,6 +65,20 @@ agent := Agent }). +-define(agent_disconnect(Agent, StreamStates, Version), #{ + type => ?agent_disconnect_msg, + stream_states => StreamStates, + version => Version, + agent => Agent +}). + +-define(agent_disconnect_match(Agent, StreamStates, Version), #{ + type := ?agent_disconnect_msg, + stream_states := StreamStates, + version := Version, + agent := Agent +}). + %% leader messages, sent from the leader to the agent %% Agent may have several shared subscriptions, so may talk to several leaders %% `group` field is used to identify the leader. diff --git a/apps/emqx_ds_shared_sub/src/proto/emqx_ds_shared_sub_proto_v1.erl b/apps/emqx_ds_shared_sub/src/proto/emqx_ds_shared_sub_proto_v1.erl index 01c704cb0..117b34e98 100644 --- a/apps/emqx_ds_shared_sub/src/proto/emqx_ds_shared_sub_proto_v1.erl +++ b/apps/emqx_ds_shared_sub/src/proto/emqx_ds_shared_sub_proto_v1.erl @@ -62,6 +62,18 @@ agent_update_stream_states(Node, ToLeader, FromAgent, StreamProgresses, VersionO ToLeader, FromAgent, StreamProgresses, VersionOld, VersionNew ]). +-spec agent_disconnect( + node(), + emqx_ds_shared_sub_proto:leader(), + emqx_ds_shared_sub_proto:agent(), + list(emqx_ds_shared_sub_proto:agent_stream_progress()), + emqx_ds_shared_sub_proto:version() +) -> ok. +agent_disconnect(Node, ToLeader, FromAgent, StreamProgresses, Version) -> + erpc:cast(Node, emqx_ds_shared_sub_proto, agent_disconnect, [ + ToLeader, FromAgent, StreamProgresses, Version + ]). + %% leader -> agent messages -spec leader_lease_streams( diff --git a/apps/emqx_ds_shared_sub/test/emqx_ds_shared_sub_SUITE.erl b/apps/emqx_ds_shared_sub/test/emqx_ds_shared_sub_SUITE.erl index 8bb460967..e9d83d4fb 100644 --- a/apps/emqx_ds_shared_sub/test/emqx_ds_shared_sub_SUITE.erl +++ b/apps/emqx_ds_shared_sub/test/emqx_ds_shared_sub_SUITE.erl @@ -152,6 +152,38 @@ t_stream_revoke(_Config) -> ok = emqtt:disconnect(ConnShared2), ok = emqtt:disconnect(ConnPub). +t_graceful_disconnect(_Config) -> + ConnShared1 = emqtt_connect_sub(<<"client_shared1">>), + {ok, _, _} = emqtt:subscribe(ConnShared1, <<"$share/gr4/topic7/#">>, 1), + + ConnShared2 = emqtt_connect_sub(<<"client_shared2">>), + {ok, _, _} = emqtt:subscribe(ConnShared2, <<"$share/gr4/topic7/#">>, 1), + + ConnPub = emqtt_connect_pub(<<"client_pub">>), + + {ok, _} = emqtt:publish(ConnPub, <<"topic7/1">>, <<"hello1">>, 1), + {ok, _} = emqtt:publish(ConnPub, <<"topic7/2">>, <<"hello2">>, 1), + + ?assertReceive({publish, #{payload := <<"hello1">>}}, 2_000), + ?assertReceive({publish, #{payload := <<"hello2">>}}, 2_000), + + ?assertWaitEvent( + ok = emqtt:disconnect(ConnShared1), + #{?snk_kind := shared_sub_leader_disconnect_agent}, + 1_000 + ), + + {ok, _} = emqtt:publish(ConnPub, <<"topic7/1">>, <<"hello3">>, 1), + {ok, _} = emqtt:publish(ConnPub, <<"topic7/2">>, <<"hello4">>, 1), + + %% Since the disconnect is graceful, the streams should rebalance quickly, + %% before the timeout. + ?assertReceive({publish, #{payload := <<"hello3">>}}, 2_000), + ?assertReceive({publish, #{payload := <<"hello4">>}}, 2_000), + + ok = emqtt:disconnect(ConnShared2), + ok = emqtt:disconnect(ConnPub). + t_lease_reconnect(_Config) -> ConnPub = emqtt_connect_pub(<<"client_pub">>), From a20d2623278755b61ce19cb9b3e391a45745a8d8 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Fri, 28 Jun 2024 13:41:10 +0300 Subject: [PATCH 088/131] feat(queue): send progress before fetching new messages --- apps/emqx/src/emqx_persistent_session_ds.erl | 9 ++++----- .../emqx_persistent_session_ds_shared_subs.erl | 6 +++--- .../emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl | 4 ++-- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/apps/emqx/src/emqx_persistent_session_ds.erl b/apps/emqx/src/emqx_persistent_session_ds.erl index dc4a74b43..62e6bdd26 100644 --- a/apps/emqx/src/emqx_persistent_session_ds.erl +++ b/apps/emqx/src/emqx_persistent_session_ds.erl @@ -987,14 +987,13 @@ do_ensure_all_iterators_closed(_DSSessionID) -> %% Normal replay: %%-------------------------------------------------------------------- -fetch_new_messages(Session0 = #{s := S0}, ClientInfo) -> +fetch_new_messages(Session0 = #{s := S0, shared_sub_s := SharedSubS0}, ClientInfo) -> + {S1, SharedSubS1} = emqx_persistent_session_ds_shared_subs:on_streams_replay(S0, SharedSubS0), LFS = maps:get(last_fetched_stream, Session0, beginning), - ItStream = emqx_persistent_session_ds_stream_scheduler:iter_next_streams(LFS, S0), + ItStream = emqx_persistent_session_ds_stream_scheduler:iter_next_streams(LFS, S1), BatchSize = get_config(ClientInfo, [batch_size]), Session1 = fetch_new_messages(ItStream, BatchSize, Session0, ClientInfo), - #{s := S1, shared_sub_s := SharedSubS0} = Session1, - {S2, SharedSubS1} = emqx_persistent_session_ds_shared_subs:on_streams_replayed(S1, SharedSubS0), - Session1#{s => S2, shared_sub_s => SharedSubS1}. + Session1#{shared_sub_s => SharedSubS1}. fetch_new_messages(ItStream0, BatchSize, Session0, ClientInfo) -> #{inflight := Inflight} = Session0, diff --git a/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl index 0274b9b9e..616414112 100644 --- a/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl +++ b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl @@ -17,7 +17,7 @@ on_unsubscribe/4, on_disconnect/2, - on_streams_replayed/2, + on_streams_replay/2, on_info/3, renew_streams/2, @@ -114,11 +114,11 @@ renew_streams(S0, #{agent := Agent0} = SharedSubS0) -> SharedSubS1 = SharedSubS0#{agent => Agent1}, {S1, SharedSubS1}. --spec on_streams_replayed( +-spec on_streams_replay( emqx_persistent_session_ds_state:t(), t() ) -> {emqx_persistent_session_ds_state:t(), t()}. -on_streams_replayed(S, #{agent := Agent0} = SharedSubS0) -> +on_streams_replay(S, #{agent := Agent0} = SharedSubS0) -> Progresses = stream_progresses(S), Agent1 = emqx_persistent_session_ds_shared_subs_agent:on_stream_progress( Agent0, Progresses diff --git a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl index 2bbdc67d8..d563c0115 100644 --- a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl +++ b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl @@ -336,8 +336,8 @@ select_streams_for_revoke( ) -> %% TODO %% Some intellectual logic should be used regarding: - %% * shard ids (better spread shards across different streams); - %% * stream stats (how much data was replayed from stream, + %% * shard ids (better do not mix shards in the same agent); + %% * stream stats (how much data was replayed from stream), %% heavy streams should be distributed across different agents); %% * data locality (agents better preserve streams with data available on the agent's node) lists:sublist(shuffle(Streams), RevokeCount). From 8dce530d1552c64fe12e39609bc6f3c7ad2ca5c2 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Fri, 28 Jun 2024 19:36:45 +0300 Subject: [PATCH 089/131] feat(queue): fix progress reporting and more tests We test reassignment during the intensive replay --- ...emqx_persistent_session_ds_shared_subs.erl | 4 +- .../src/emqx_ds_shared_sub_leader.erl | 18 ++-- .../test/emqx_ds_shared_sub_SUITE.erl | 101 +++++++++++++++++- 3 files changed, 110 insertions(+), 13 deletions(-) diff --git a/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl index 616414112..bf0798e1a 100644 --- a/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl +++ b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl @@ -152,7 +152,7 @@ to_map(_S, _SharedSubS) -> stream_progresses(S) -> fold_shared_stream_states( fun(TopicFilter, Stream, SRS, Acc) -> - #srs{it_begin = BeginIt} = SRS, + #srs{it_end = EndIt} = SRS, case is_stream_fully_acked(S, SRS) of true -> @@ -161,7 +161,7 @@ stream_progresses(S) -> StreamProgress = #{ topic_filter => TopicFilter, stream => Stream, - iterator => BeginIt, + iterator => EndIt, use_finished => is_use_finished(S, SRS) }, [StreamProgress | Acc]; diff --git a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl index d563c0115..ecd06846c 100644 --- a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl +++ b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl @@ -540,7 +540,7 @@ update_stream_progresses( }. clean_revoked_streams( - Data0, #{revoked_streams := RevokedStreams0} = AgentState0, ReceivedStreamProgresses + Data0, _Agent, #{revoked_streams := RevokedStreams0} = AgentState0, ReceivedStreamProgresses ) -> FinishedReportedStreams = maps:from_list( lists:filtermap( @@ -569,13 +569,7 @@ clean_revoked_streams( {AgentState1, Data1}. unassign_streams(#{stream_owners := StreamOwners0} = Data, Streams) -> - StreamOwners1 = lists:foldl( - fun(Stream, StreamOwnersAcc) -> - maps:remove(Stream, StreamOwnersAcc) - end, - StreamOwners0, - Streams - ), + StreamOwners1 = maps:without(Streams, StreamOwners0), Data#{ stream_owners => StreamOwners1 }. @@ -591,7 +585,9 @@ update_agent_stream_states(Data0, Agent, AgentStreamProgresses, VersionOld, Vers %% Client started updating Data1 = update_stream_progresses(Data0, Agent, AgentStreamProgresses), AgentState1 = update_agent_timeout(AgentState0), - {AgentState2, Data2} = clean_revoked_streams(Data1, AgentState1, AgentStreamProgresses), + {AgentState2, Data2} = clean_revoked_streams( + Data1, Agent, AgentState1, AgentStreamProgresses + ), AgentState3 = case AgentState2 of #{revoked_streams := []} -> @@ -603,7 +599,9 @@ update_agent_stream_states(Data0, Agent, AgentStreamProgresses, VersionOld, Vers {?updating, AgentPrevVersion, AgentVersion} -> Data1 = update_stream_progresses(Data0, Agent, AgentStreamProgresses), AgentState1 = update_agent_timeout(AgentState0), - {AgentState2, Data2} = clean_revoked_streams(Data1, AgentState1, AgentStreamProgresses), + {AgentState2, Data2} = clean_revoked_streams( + Data1, Agent, AgentState1, AgentStreamProgresses + ), AgentState3 = case AgentState2 of #{revoked_streams := []} -> diff --git a/apps/emqx_ds_shared_sub/test/emqx_ds_shared_sub_SUITE.erl b/apps/emqx_ds_shared_sub/test/emqx_ds_shared_sub_SUITE.erl index e9d83d4fb..3e80b44a9 100644 --- a/apps/emqx_ds_shared_sub/test/emqx_ds_shared_sub_SUITE.erl +++ b/apps/emqx_ds_shared_sub/test/emqx_ds_shared_sub_SUITE.erl @@ -10,7 +10,6 @@ -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). --include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("emqx/include/asserts.hrl"). all() -> @@ -184,6 +183,89 @@ t_graceful_disconnect(_Config) -> ok = emqtt:disconnect(ConnShared2), ok = emqtt:disconnect(ConnPub). +t_intensive_reassign(_Config) -> + ConnPub = emqtt_connect_pub(<<"client_pub">>), + + ConnShared1 = emqtt_connect_sub(<<"client_shared1">>), + {ok, _, _} = emqtt:subscribe(ConnShared1, <<"$share/gr8/topic8/#">>, 1), + + ct:sleep(1000), + + NPubs = 10_000, + + Topics = [<<"topic8/1">>, <<"topic8/2">>, <<"topic8/3">>], + ok = publish_n(ConnPub, Topics, 1, NPubs), + + Self = self(), + _ = spawn_link(fun() -> + ok = publish_n(ConnPub, Topics, NPubs + 1, 2 * NPubs), + Self ! publish_done + end), + + ConnShared2 = emqtt_connect_sub(<<"client_shared2">>), + ConnShared3 = emqtt_connect_sub(<<"client_shared3">>), + {ok, _, _} = emqtt:subscribe(ConnShared2, <<"$share/gr8/topic8/#">>, 1), + {ok, _, _} = emqtt:subscribe(ConnShared3, <<"$share/gr8/topic8/#">>, 1), + + receive + publish_done -> ok + end, + + Pubs = drain_publishes(), + + ClientByBid = fun(Pid) -> + case Pid of + ConnShared1 -> <<"client_shared1">>; + ConnShared2 -> <<"client_shared2">>; + ConnShared3 -> <<"client_shared3">> + end + end, + + Messages = lists:foldl( + fun(#{payload := Payload, client_pid := Pid}, Acc) -> + maps:update_with( + binary_to_integer(Payload), + fun(Clients) -> + [ClientByBid(Pid) | Clients] + end, + [ClientByBid(Pid)], + Acc + ) + end, + #{}, + Pubs + ), + + Missing = lists:filter( + fun(N) -> not maps:is_key(N, Messages) end, + lists:seq(1, 2 * NPubs) + ), + Duplicate = lists:filtermap( + fun(N) -> + case Messages of + #{N := [_]} -> false; + #{N := [_ | _] = Clients} -> {true, {N, Clients}}; + _ -> false + end + end, + lists:seq(1, 2 * NPubs) + ), + + ?assertEqual( + [], + Missing + ), + + ?assertEqual( + [], + Duplicate + ), + + ok = emqtt:disconnect(ConnShared1), + ok = emqtt:disconnect(ConnShared2), + ok = emqtt:disconnect(ConnShared3), + ok = emqtt:disconnect(ConnPub). + t_lease_reconnect(_Config) -> ConnPub = emqtt_connect_pub(<<"client_pub">>), @@ -265,3 +347,20 @@ terminate_leaders() -> ok = supervisor:terminate_child(emqx_ds_shared_sub_sup, emqx_ds_shared_sub_leader_sup), {ok, _} = supervisor:restart_child(emqx_ds_shared_sub_sup, emqx_ds_shared_sub_leader_sup), ok. + +publish_n(_Conn, _Topics, From, To) when From > To -> + ok; +publish_n(Conn, [Topic | RestTopics], From, To) -> + {ok, _} = emqtt:publish(Conn, Topic, integer_to_binary(From), 1), + publish_n(Conn, RestTopics ++ [Topic], From + 1, To). + +drain_publishes() -> + drain_publishes([]). + +drain_publishes(Acc) -> + receive + {publish, Msg} -> + drain_publishes([Msg | Acc]) + after 5_000 -> + lists:reverse(Acc) + end. From 7658e081c53699ca7080dfef68c89cc0e91ef92c Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Fri, 28 Jun 2024 20:16:00 +0300 Subject: [PATCH 090/131] feat(queue): move design docs to the EIP --- apps/emqx_ds_shared_sub/README.md | 13 ++++--------- .../images/groupsm_leader_communication.png | Bin 327420 -> 0 bytes 2 files changed, 4 insertions(+), 9 deletions(-) delete mode 100644 apps/emqx_ds_shared_sub/docs/images/groupsm_leader_communication.png diff --git a/apps/emqx_ds_shared_sub/README.md b/apps/emqx_ds_shared_sub/README.md index a5b08ee26..456d5fe52 100644 --- a/apps/emqx_ds_shared_sub/README.md +++ b/apps/emqx_ds_shared_sub/README.md @@ -4,20 +4,15 @@ This application makes durable session capable to cooperatively replay messages # General layout and interaction with session +The general idea is described in the [EIP 0028](https://github.com/emqx/eip/blob/main/active/0028-durable-shared-subscriptions.md). + +On the code level, the application is organized in the following way: + ![General layout](docs/images/ds_shared_subs.png) * The nesting reflects nesting/ownership of entity states. * The bold arrow represent the [most complex interaction](https://github.com/emqx/eip/blob/main/active/0028-durable-shared-subscriptions.md#shared-subscription-session-handler), between session-side group subscription state machine (**GroupSM**) and the shared subscription leader (**Leader**). -# GroupSM and Leader communication - -The target state of GroupSM and its representation in Leader is `replaying`. That is, when the GroupSM and the Leader agree on the leased streams, Leader sends lease confirmations to the GroupSM, the GroupSM sends iteration updates. - -Other states are used to gracefully reassign streams to the GroupSM. - -Below is the sequence diagram of the interaction. - -![GroupSM and Leader communication](docs/images/groupsm_leader_communication.png) # Contributing diff --git a/apps/emqx_ds_shared_sub/docs/images/groupsm_leader_communication.png b/apps/emqx_ds_shared_sub/docs/images/groupsm_leader_communication.png deleted file mode 100644 index 48040ccea1e609f4228f405a6f581d3afa2b4579..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 327420 zcmeEv2_RML`nN=hQ&EO?_OjNy-t|7y@A*B?yY`W7>PoX2IT>kaXl8Fw zme-=8p(E1JFby!wfFnJ%^+WK_G-oZPO*E;G7q-*TL@2r_Y(?ovkSMVJhzC5siUKy1;KQelc}AvpuMFF9D?85Iam;O5-csq=ZFZ2hzSTu2#84F zg+;kV<;0M`gv13Ughlnp=bP@bv_}V2bMPeC+M05UD2fXT!B9MErsf2D7YAo6ZqZHf zd5gV^r7ipmhv9Dx9Q?H%{x2+KEG%v;$qPTqIXc=}>ROs@CBSSH#YH6rMJ3=c&sJr; znku)*X875bU}FjYC|R1@I3Sm7wsLZ?gCmN(GM4%c1c`{2E(_Q4J za?;W?ceQYrbkW*@mr~p!rQ^9pi~KZhmQKzD2YdA4#RSC##mT>L@o=;xAKm5Pu*(*1 zhz#j$ZVEq=RueuVN6Z|YEG(VSLogK)ZcznpAvyRB@}KBtbiv_+y(vtabm=x&Ljhs( z^#p9;cWiQV+rHUW*ICKcMpKR8scY#tc{FTHyFE%yrjAx>4i=>4wb(-*R!UrqJf?*Q z`GB~P6#2j|C+s%pU6BJgf+sp3VbZ)sgm$?SEG(VL*P>M;^YCtq9aP8lSZ{y3SAWU&#EAb9a(k}oa;cydGnE~ZYqAlQ&>`HSa~#Qyb_|DxJBS=ySq5ZorZl%GcnkVu9+|DT%N zU(tfUB-1bH4oMF{k3Opb@&VJGAE*UlsJeVp4TMR->hIKp?-~2ErfqWoEc4kriD3Q+ zStD2<@yULNY!B#F-{*R;n~@w4#UXxunxX)TeyqDf_4JpD6`~ksL8T#ref~zY5SBzk z$j{>^B+171OD#V|`O)R4AhB;OKW0Ne&+4P0`fICiX>TEipj)teb6ZnqXM*{}&^!@H zVa^tFush%~N^Jgmpr%A^(lvsR;9!yavzML_tVvOQo23&0kQ6BiK}Pt2z8lMhz7@eq zD*Sb~g}FM&T`2fWh)P>9kWUTmuw&mH=)Y$QpC>ZO!HZ!I9djv@UBzE^@PB?w%pNc^ z!n_z}ALO9<3&s>Q@%-^-g_#oBL`1|93;9zRL$Z%45SU_Se_~ZX(11|i{!Q#d3c=r_ zy+{i3uK`g~MlsAqU^a`G8iKukc>}^c0hI;qB-mRR+gh3;eADF20@89wv8Xe4Q(M9= zgx4|$BS2!qCM0<$0NEmk{=$x6VS#V~UrhZc#x(Nn{@(0=GQP&VFI9|K`-Su$qXaO4 z{{-1Tn(+M3NKO=!3l_z&sP^lL4qOM~J*Y&d1eV~4Q-gy%atQMc6f|-MRZ{=!o1O$} zSE&5N(2rE&!_J^aXFs9kiB1upmgO#jGl^yV$4O0WlG$N0`t`(ymMy5PP8E%E61GTG zvmm%}i%9H3e&m4~0Ev}xb}@Ca1Y;zh0h9>UN&i3-6~V$G=3^z$+>A=vlqCEox6-M6 z6IxIr*zfuei_Xab)QDXF6M(3NCC`5^(!mTB6D3ynpb{l^%FiuH5v&sMXGNvI!f-<} zN6FaW=N$-rBKW;|9woIST?XCL10TRFf3QJ1SvnIuP0dJM zMF{+IM9Su*Rvht7+~V?Z1Zif0bC5m_q$mdd!NJCIvxBVz(l%%BV2^N+(6(qh=@6RB z|E(=Y!lEdIMeSG=3#U}#r`Y<>ZOEdiKc-@)CgPd@4=_%G)f6!P^_q)?sk0TStBlfV zM1k}Fg&rg*p_2HzZ&qM_a$r80V-?3px?dicV-li5hCe?r7stW`>Nmu&at)Q=z+4Hn z+nAhA{RHtJ1ukD({vXf)0d3XaRj3sbT)S2jITiC2l#KpQl6caIe`5;-mN$Oefsp*~ zU&!pKR0gYNQmYJh>_0;QOm2b@!xA^l&;0l%2+>J>7vk6xW4HWeIR}xctzGi43F04D zi4Yf`nAB2&6_sjZIGb9vF%O9e`oHfDu^Ih9Z-{m5!bTO*iJnbRSV9tWb^jOdM(Hra zMx-FKwIX6)ce8Nj-{>p9yVaK*^M7?;Ny!ya5I|{)#Ps>c`^w4vE|eM|cFSMZS7M16 zmF2;%0gO!g2RcnL%-3KL5{tCIp40pSgcRRoZ|P3zlOU0S2odnli+f^N5{5Z|FJu8& zLFFfpdsC4LR$AZbK&rQ!qkIh<-tCI)n1eoJQ7O{@s1K2=^b?4u2~04hTu&aBd^&ck z2_<~;nSbT(+~UA7A^S8DrTgj)|Hig~Uz--g=xj`}F<(XHo4!bkVFrPPqyIjl!!QX| zT$$pDCh}xILEUfi?v^hBYD^^5j^!uZ-6A%%O!PlhUzl9*6T=cADpSG^QHv52?@we& z{}vbk(ERCtlREQ}oj%}bgfTf{DDMjp;ftOWyY45F{}*81WK8$ddr}IyL}h;1Qd65B zZ0{t8iJxfwr}oWZEfgfY^*`D{A%S^MDv4r;s3nR?`6m)63cCM}ikdDj+LR3vYHnIBqIfuYnFMr6J2fm-Q zC;7C$Ktz2v8;?YcZ@(ktM{nsCo4T!aa!v+?@6B!^|H$T?$=Qn(x*Aj-LKyS!Ke>xQ z_fh^GE@EO)ADWQAt3XF_ER+JBpoHYwzd~b^KW|5BU82)+oj%hPhmJ9D| zv2@3Je@LBq|F-^6zins=GRK%YYA^9?010d_2$h#W%nFCL<)bZD6S4VU=?0>dvM_Vs z0a$?2=JOd=kZ=M8R{#h=U(l)4@1u9mf+7VEql!(6rC&qKUr_D;2r&L`rH7S(-2Uql|E!V}6%i5o$$>Gp091jJ z>{Fp%R4@*Ln&NRe!zaEi( zWb{F4@oz>SvA;b-{T?I#Z0g^LK9U#`#b}maHu{K9EnNIB(QFv;POLV4hk7CcR>pK-w722r6#w5!8Y2jZ&dxedtAl7w@IEHJ-^3c zOsa-_H!Ap^vH#o|A-_SoD>0?TI!iHa{AIhNum;Nsu!Y>(;OuB>PinNmUhIqYXCUt- z5g@r z`&93X}7+d-;>lV5K4 zm2x3DF8z>B83{B|`)1t%iu3=-0wA^kR8b!5qWz!Vmmo~(BcQT)4BJs#Jmv*{qNqJJ z`6XKn73=fYa;6C_a^J`APq1yuhtK)dgx;7RY1nAk`KDq2GczAyA&fl3^zv^=eK25) z0$t$mCT$YWgE$3qHQ&T3f4KvZN@p;SOszB6vHuQ33n+V% z&i;6uB8ojChEt|&X8d(=%9r+_exnZpeRBm;c6Qoj#skZbe62Y44=>mOd_>+eA+Xcb zj$jLmhMcNwY3pW*d|wjP@5wJQMSd$R;t1>~8nVQ?|BH;9{8+%Jjk3z7wCBW*52rVVpIYQ$rXbGw}whpG~;-h;vzR|Tq>4iW5`k%BxkPjmi zGF3MO)++=8HMg`NucCmdg|(~m66MQyFv9u2Pd;E2IF$hYw`*FWU#MdLBF+{%29TzK38I|3q5oGQdm|4`s1g-{B~xK`>LJ)%fKlhh9i zk|nW=bFMkGm3jd{{f#&~(Y3N^yKbyF-N0{=B2h=zYD|k`Y z`LLYrCrZzz@@E)Z|Ml*2vZ_t0eoPf2IgbC8H>OW`Hz1{$fRs>3ZwUNcIzaaT|6g9P z{sTJD(Mpv()vnRMKZC&DJ_N-Y(k=(eZcS>%!}_NF(~3uW*^-k3@*aK#ZXrZ7KL3Zk zw#muV(Mrw1!V-#+ROgU>Ye6vGSy^j-zimlrNdfg$AjY$ ziM2Sa%)5E%vb-bvACLDI7B**%kDR~ygwX9ga4of_e^0})eCL5W8^aoBLmQhL`3D~b z!vBs2(9kk)`Azyu_8`5)^6=91wP!ZdOrv9#o%oj{EgjBw1r7lM{e z9uUT7RxO)3c_d`uE6Zes!r0sd11HS^`6w%-uw>HBNgwY8(iGu!YIAwMFn5|FR-cLU zC6B4-VD;R=V*o}wn7 zK|`vld27tUzTv92mz=h}f!t!UmmZU^J#+cjJ(TU3+0L$tFx zML*V#uyBgE!5eWo6@_j(Sr(&IAEs-xBUK$vwHxkh&wd#qtz#rS^NG;z7Ww?P=%B7M z#=g8)-R$bKvsOiVms{7QE|Pw8&Y>`4xWDtRZXy%fI(i%5>M;nxd!%BiU>dlYI?1bkn82Na7^PJsA zFr(a&TnnMNW}o->S7i*YN-`K98*MzC7{#orJ;A?XEtOuw2IPtr;G}>u}C4dq>7g z7r|hWB%PFq?#v2xLcV6v5pnI_yG!=2c4lmOdt=-_%eu=i+bsBgts#crU*=#L4IX8r;v$X zqg%aQ4{?3(?fGt>dhed`+GphZ(Xz8VE=YRk`&aj?W0d9&w$d}P*GjdXe;GiOZ1%25 z&3(g}|MAVaMw@0(^wrE-+hWRgq}>?vw6%_^zFh8+n%}do-m^0$$Z)(Z=2-Tc8=uOD z2fA&1hWmK6CDr5%GRzJb?k-_-yYt7cYc*`Y7mNh>8#r9{+V=EvMy!tz;oY&aMhiwvP>)w=bVb^uc9qaku73KNRZ+@3SN`zbEfOLF{nwl8qaxcUCzg#uBN0 z=GNSe4Qjddd&fsp6oWY}hi@MX_k49ise$2&zC%G(){dCVEnBaLDOxXbt<=nRe6cAY zk#TTC)8R>Y$1EsgAst5?Cdi+C9M57VpqI%N{=W0WzUs*saT~9{G99kYLw&hF-afB= zWsW+|yR40P(q==+jZ_u|##nsIzQm!d_<)!D{I+y%koolPR@^)BMhEczZZ~T5aGMYQ zwtH`?s!*IliMUFVN*G^BYJ8dM%*A`6MlKLJtmOuytW+Wcgz0sQ^6*DIOS#V&v{b)g zd~7HZUT*Kh*dsPu(@ygoo^_x*{ni}&SK-y+<;ubVi7GNd*Z-Id4d2o&V^Jl{jy1ub zVqYSLx;(j%4nv(lz#ZO2MC@n-U zRxl?E$0^(w-^8OjDdC!3N9q+b7;sI44IS>%kQBq@^`jfp;1VgqGtVH^}A%1%6 z4HcG}q!ZbT@FK?+-<@nTezFz;yxkAu7rhq?RC;m$;aO2NfA#2Sp|~xlR`zTUOD_ry zv8x~kKZ@MgOuuRC>D5Pqw=<^3u*l2tNbGvNutHfdY@oaToNrHc$~l>luB# z3t~_5gqOda-%XS7l)u(rh~co9f=1`2L<0!N87!Jd7G|b@1i8F1D)-dpwA9fN>VI)) z$({9VZiK$=@#+`4UY{`zxe#FzIG6RyYZ=PvGd?<4pWAwIfz1tep0^FTR~Mwj(&z6T zZeMk7oof=oMVncr{W(0D+OdjtI(&K~q!7XD|F&T8qHEjJq6^Z#-Q0^VpU|ok-llMA z#~DFG68wYs-2iZ+Yx{)#<6eigejlVKI zF2dUVu~cjAjq%a9X6_QTxq=&xmS^V7*H~ZHZE*(tQ2%MMjx9lvm)ja2uU$_HGy(GMCJ= z>P*SW2I!Zksan()RoY}muV44bzP&uo{zUK2M-jQ>HJL?M0u`m(_s}b?%O+G1GZOu{ zo9ENv03q#uQ3Y3arQ|d@`9~0|cM>bAyJFe1*QO@bwWhN2&gfusvvp;H z<__AXW0N8-e8g|(Dlftx&~|zi%)KvhVIq6gWb^}?qlGXW^X~MW-*3VvlX{+QiaT`L z>}SF!*|WJ=k6HEgyGNI-Cj%0$ZG>qMi{=yG(j8jjd#!#n&7rwa!mX>e&+xG5@~LFY z4rZ?KQrdNA#8%){24nuxqe+Zw^jfs>HfFnSglPvjr^xI5LcV zKb92u4m{gcA0plJd*b508soh?274M)d`E{7)vLF?`e<3z2F`s`Zko$2D`dG7PA@QY zek3m8^h$ZbT1S&&={~^x^?-#Jcy&LPewzuQZ*5M`&Uslo05=zmkB#WWsjwk9$o%a> z6uwJ9Iw$?|sOZVH%MVR)#x%6|XYE;{DxjbJaG=3iqu%;*imK@&iF(^p+un6tPRnLl zu6Wz19#XxoeRDS`t4kjW@qO+;Gui!ZQ>vnZf8D5|dc|`eAy-Jx>MIj*c0=!8NVv2;-9M;{+xcX~WqQp!lI z{5_a1f21cr#rTG=TaBUnYs--Yu_>wSt3tn;qRQd}OeqRM>^*`h2D#-2G>(*hc(T!< zG0)X`4ZVNxSh6NwvcYk4LjETZk$zQ@_U1(k_C5tGvW1C5{+z_!Z&aZ1w#;jO*!zG) z5Z%29Nr%EMXJYwo7fn%VKiOH+()J}96?i^_<*>TD$o)K|BPkCbUpi2>#H&lV-tvrL z*Ml`TTrLGypD%A|jTqL^c^-R0C#S(FNP6(o#}9VkQFF{gr0%G19Ch)6;Lrt#-?dYk z9@i1TXvKJ;`g(+nS2oi_v6R%PsGA`h$M$qSySgjGvv9cMvfF*`GuL?CMRz@RshABo zZ8xrsSf1x(C%e|3iWU9Go<`EDr}T*|iyu2#XbS`)0Jh>r+M`=fxHxTP-BUs>C;m%I70r*)+c zBsz^-HDvlj7kPDM^gLnA?HiMSeL`vR?$+7W$5mWZKk4DDFB2st>>Fw~jP_N8y02HC zoGXBZ>{*~npu+)cU@-S-k4sT#;Yq_ym$V}{_%x((tTD1xixA6zRP7-U#%l(pgWWhc z`g$i`WWBLHFQmOJ=1^54?k@cn#+-n8OH>K0IM9&Bt(=D9J0JdTQ135(Eo% zD0hkLDB!7gnP2c#yln3z!nK~pY*R)uoYl^=R~^u0%wQe5t>Mb#n>J`86R($LU1EQf z*i0|?c-E@(&)IdG!8@3x9a|(_EtuD~Y|zpppaLcTlkC=e zTaK)1U@b;4b(PeHa7ge@6}&5EDyfS;VaWL+F1pN^lbfj3#EC1)C{oSe(%5`0n?zwhLu%J(C@M?8rd1gVB>RFKz zwR5-~S1h*Jx@f-}2BkEGbkQaw(K|v>7aWnh@{XXSjUZ6_V7@T$WsP-^b*&CB_r!JA z=d7t_KVZg~i5D|dS)PHHU{fur+4bt)2?@vNH}M0~j}J_7i45U6MWNN!d2b6xY$JUJ zGQjA$yBK?8g2fCqmg;QFl+9&l(=Dr0uZY(;RzSQkeDJqFT6_g<#HueHTHH|`$*8DW zk~Ufi*@xt&%NAvaSJ7MW6-8+FE+@|GUOfcFRvyh(?hPyIfg5@Wh`}bOS{ryIyS}!v z^@3mU1S<5`X9Gjfh-8@3ZBhG>sLXdDNp?&AJA(gvq1T{1Sr^=!o-Ok1$!)EJkZjyj z{DQM2woFw{s2@=N_I$VQVuR8}G27H?j?5+cj`r);n*|93UWgHar>xKl*REctcZvVO zluRF@5$!aGeY`1WV>Oj5gTu>v+#2)U-7O-uZJ81vhS=l1wbt%Uf8G9oq^d$Kt8N;M>iVq2s&pT{(h57awku&6MGv9I3F z%#>KFzunMAm0nI)$jrLCE-Sk`yFPC)()qEJ{cY|0S4?jmtj)YdR6D2lC_=n~QlkM`FX>j1Xg?l1)NPX;*H)2gOsF5Q`Y)wbpKk)G`- z+How~pU4cJU$px+OON}hRT|lgWjt9s3BA&^cixu9U-*=%9_sdbq&+?|uxiVyYrr!# zdTQ|1g=ypu#M#`&d$&oL7Sn>6zfoa4Z$ECu3s}l8t_eR z=5<{okCJt6UE5cBehS^#-RBM-BqAV{T|loE0Dsq}l?RmQ>FEa`PUUYL?5N^*D;zTE z0Iua|0ffbE{Hs?V?XHYZ?+nGM?+!Y#1Q-dMW|dD#Ifco$-;*hq-(?s zw^_a@K2Uw#<3p(1di8@-O$Nln#>SRgQkr$}WI#fp&&VCm&FN#Oh#v_#*N4jR-1$eV z&zA3IJlZ#|M^^v@gpQN2Gd+IYN}i%8y+xIGf)}JXw?3Be8nAT_e;l9Z4K$YEE3LAT z4If|4zifP?a3nXnsIoP3Z%_4OMwR=#!R`&a4;gy4$ED6e2u>5dCC7gI!rJj6>O0N) z)k#HOBzg?gnje#@7&?L1HFOPY-19Q{TxPY)Mzz`aGUXVsuE?GzVC`oKOZ#?R^jPT@ zl5JO?r>VVMDJ1i(=&m}H;LNDW8mmkE?sic)jJyauGK@7(;9(G$szs z+xVVsJ{;))hcg)GdRg$jz>&o9C+Ccg4BRORPt0}}4v8+4F^m38e) z05x&9^_Y$vTP#yYV%&VSz00!1T;Bm>rFuV0%XMCr2jEHfPLoGxMOSgh+||;S{K9IK z&F2bN`5-e2CfE%SIvt6e0%lE_go;|)HbS9TywcQpIO##I5$`ZJY+`*`i$8t$OT~O} zYO3SLI%S->KuR3z2HvM51KBpl?J;-cTf($&**7?>QB z83^~1(hpFPHo|%V{2zBzUA6;vu5)##8j`v5d&e^n`Bd*NU{SsdRd78IDX9r=;kZ7n z=|0}7;bR-e(mBPgugzHK@#=&GB)lREu0mY~$&yY+tQ|6ol|jlNx0Se8xA#T@BCiCH;Xlel%s#B*P?>nsp^TT=? z^XmiI_`4#g(E9oRf0iSv+VZz@IYR6{(g0h-nYm zl6fceaRpIrfj)q}vv02MdXhb7(Yjg%4nqs`yCDLu=ZQkrd@q41uxU70U-^tskM}gCr(B?lDTovd2b8S{0T=on&}*60}0FadyYV&W_j} zT(&}S;zX!0yjEP8XjOS~6L(nf0?CMgRl$3-KGxhAv*eVDHBpKQW*ti4>GA+fU@njc zsgfO(yn1#8AMQY5|3%LZUXD5Rmbju&MFaYpa52J31QleLuAFWr>L_H=^?&R-|H3yu@t~fVVO+8}5$;Rxz?e7Idu;FR z$BBvl4%d4Ro^!fp`|OHYO}UF)a9sz)fC9*3`qt4gC-{Lh=%4PH2z!pF(E>IMm$0%p zQGGfIGQTcIwHFmU@$yB-=adg?aL$U0c8$t(!+D3t39b752l7=ChiYEa} zZ^Q%KlVineu|VaD-g#_r+X#vHL|Zii3r(sS&dbBR4AIZ73abDr%ycfET) zexbiD_+U$1q|HQb>koE@{5zq=%hirl)M@b{jDM#j@3kbL5Zd%2p|DA8cZGZ|wGk?5 z)llRg>8sF~UpW5il!hDb6XZngk2o{p`lDHohppCIrNWeDvnV;rmM@$y&8UbWW5|2} z2w#qB`fjG##gHd#Qw7dzj{gJ{xMu~kMEJZTifX*CG|P&c^=PgqB-~5FC${*;6YQYC z?~&`tY1ltL);u26ovLd@Q<LW*qrHx&8iJq#+}ZIs*xKSL)O1nKIX}+rofd2&Ns3DmFb z2w0Mxb+a|N8IETzu7JXjwk^wts9r|LK41FHdXap-a((i+kC}Jy;2Rt2A21PFk0Q)g z&su>EMgS|r2O(Np*YUn3p0jtl5oA1)h$KwO<*9r55W39f?en62YA?gLutqC87+|Jyza;W!Z3eaLDsf(64@N_?S&U0p2_xbU0kjYx#S;KuQf$MOqHx2`F$0LI8|Z}q*DFn--Spd3sh zA~-^A(h928G*>$WUJAJ&ZgX1SK&CKZjvS5B_$C_vz**2z@i5qZBFymMJq~WV-3f{{ zyQMz}80J;7;uGooL&lP6l3Y8hiM$SwLj>nqJzCYB$q_2qwC5Hyv2cv+c#CUe3w_=S zmem`R*0-46fbq`GCl|YmSA`)EKXPB=c#D!pdp{Hzb^ym?)?JGi>POHw&Fb33ZOh{` z53&Jj=L1)(Ew!;Jc!Jiy3X}bmr#XG*oJxwFbKx6M~ zu^eo5ZQBcSEIxK!ebTV0?aBHNs$JX%lo|8zXQ=Xv9#?@>UtsR@6l%p4gm~&)(}0+1J%n7J&Dw>&buHL(HMK!G%@*7A!jqIkn*))}z*p zio19ZYwgKOyyDiahkKrhbUSP|hN7!oTET!*Q1mLk2=#=-gGg75bT%N!b9wR^HR7cm z#Y}5lZd63Kf?M3I^|sZ7j+sXQFkae3jK{2;?*}W>cPdbkhd_t-*1U#jeV%dHdD`%S z(gWB1s_&JXz14FxWJ<#^%SOEi!wQpp^&9y)O2EqCp4-a^q|A| z)|!I8^4c3N@zXaM(`3e+s%AbCxk2lMjT-PrI?3B#WJ|5d^|MoWe5Pa2s<+v_$>Z)o z^WMEVYu6xI+2NCZiaPB#mh(Z#ab-F_vs1hMYVOuEg7yOZK22#x1$DvJr3c;>xml{9?LQ>uHA~4U<^0~UkIL& z#sIaYX9BF3qxUv$fIb_w7qg1+oav9w3l!l!ZKe2W8D1Tf)zmx@3>i=6yFKao zhTBBDlieXN%xh7a1CPa>asY*eNwK|Epz)`JCvagx(KQc_tCn5*A$y(W1$pBQ)oI3WiMIQB>$JDn z=+fs)SuPLPdPmGs3owk9(01rv9k=&mn)c<>NPE+{b?XQ6nskA~seNLqq{^+o(NI=e z6Uai19;W9+w^GJNnmLqW5Fam?`4-ux5R#e8veIYL46^AOyvQD#L)5bhJTR8EJ?KUu zv>g%4@Y+fE48&CfZ>Z6^$BN$}B{#H3oY=Q!u*SeSSifFArie8*aSre|*ZaGkiN3JA zC~SVP(dmIek4&#Ze|PhZ8(Dy+Dn}%2YfTVtg2ULmtZC27*)^&5jd{5?38R*3ueTC? znb=pS1M_v4-X>4i`V8@Ig4G2sIhVdXxdau>@+`%b}6vh`_;fZ4r!j%l}YKOpU+jnS=z9S)IuzRFSNL z>3%@8NjtQO@DDzQ()x{BJ>G~p9#D|Uw|hFD?SN_0=KCd2PNg&_@3?A~KL;x8IpLB; z_z)Re{7f?9dIBo7{wG9;W_Q^Kd25$w-#9fMum>xlJ9y#Nn+9Vkr)6bU3yYV{ajGkM|ax(1yC-lUohtef)UvhxwAWVDJx&?y^%q$sPRLBP4s1m>lc+!wdhl*6a%Xv58|o^mY^w+>x!HZF@e zk1hCd#5z~&)!|82qezZsP*JIjcz2Q+9uES4m%B$dVLthY)H0ZqN|4tQ&$dhFej{cw z3?JYyY6}^GgxU@|_1085sl`ZxC$E^NO3=LyMBsty>JSeEyyPbGo1&_ArY{tpsuRpOjob7eV!Z!xQz%#ew$E9Lydv+nL`6dbQ-*7M zx#cPNp^fk=G?0BxGhk{J=;nN0E^Nlk zj>h+sKZf!j?=exakyzw(Z)xfri0>z%dpQ^H1I?#xZZlPtgyItS03DN(Ppr-0$>8}az}jb8}b+$GU0RNQ()JvK&oZBwG&;v)18L zk6CaFB<)>r47;9OT$Eo(tmIv-bvQP9oi_vrog2PBPGbZu$8si?LW@rYm zg`z{x6HmM1{j=NPrJyx3o_*;^FWdpCL|DBKJPXS!uY*^)E7%Xgx)!2^EmRfb z?2VcWeeCWXUuw^2yseTwI5`if8|VmJ*R=cYBBUiz^3fWG^H8vQUVTpMth9UTBIk#~ z8C-&E{fWP&8sBK@?pw9s4|?ur!v(-E>1^Bfyeai&8dROi+Cwwx$RUz$88FpsiS=8| z$+XWs<7vUamE*G!K#PPSQZ8u~t$G^jq-7h1o==y2pNNzRI*>9!X>x$uyvmkV#!!6o z$-5l}(DQxvT)8fAe`a#0jPx_DPAu3G_mr>9tc`okdD~GD!rd&4T=EZubW&SBco3~R z%U08^R6lQ3mm+xf3fLghZ*t0miK-FL0(xVTezO?uB&KoyKD?xPO#Vja(@UF$qvh`o zSu5w0#8bX1ibb(VPu(W>^125q2emB^=T=-uJU^MEuK`owUZzUVkt=wpCEt$q1T2&&|4ojB>hK87~hu3^@HcYVY4DP zo3noOXiKU8IaxOBq0u!EA&sIS6xMrJ(1*iM=g9c$jKNB$j_?Z$2e$_M`N8ESB}s^!weT>ETb z=tKD@L$-+wW7*I0s^uB839GI|W$wxe)7b{K4P=-=jTv-&OK|)Qv5?9ydzVI>w=~v2KlWp zPFY@L=8Vs}<8J_Jvkd-}maICpZYi_yG{xNtT?fk-Qq4wd1tKI^-snR|DuC@DrvJSp z%-1}XYH4IwqRT7t@%C;3*b4QBzfjCI{IipHxf$y9=gMhj7omd<_DijFFHooJ+{?ID5+PtPTjtQ!) zueDqgm}ot)OLdQ}4ufQghiU(;Yn5ZF2MtVV1*?u6=P#8xm5mT6FJOkn`n!Px`jkdi zDAvdspa8OT^u_xP&i-~ye&Nbj%bo|C?26sO!1X*#mRa?qNWi_E0ne(l1lv0#U1lJP z3|!m26xN1ooDQtRT6^W@RH12SI^^$#?Hvu~T<#}3#x%rUIaJ;B>FuRVE)H~6oOaT5|wrbKSY@kXuIz06!w~ct5<{?+(Ikeq6=8^10S!kw2Uc%({kOwJG&e zr$goM)96mle5I>V zm%W)2bth*)Cu-|Eb~*{jCK2&w-M_nkI@xybZS;qA(+j(QoL}{3+&UJg*T#Ee?l91i zZ*}Hki0nFa#mjpwcB_-MaoGy=nd(T-w13@FwfhUuwUbPHPdvHAZ;px z>!yg4+sIE;-p!(X2))g^i>n_WL0yB#QbbIfKa1&lo0*69{TH_m=>zhJ>*&(o_EO)A zze;M4D6Qq|HPFnb&=$)6u6sJHQh6)Z=je|I_01Le!!gl^bZ7kF{!cQI0lZ z*yOB6T?b1Dz4`9Nt<4&*D-MWRrl+SLXTgPOwLW4PLT&PO>s5`*nEJt!Df1jx&_RI~e#Qm|fxAr_jCNl|Z+274=oBLXL=K_8GFcfDg(-3g7mBP6%CzPlVReHU|k zpO(gXQa!JQO4Ri-Ep8ct)CXRxsn3Cn8}+E%7xtfm9qeqWbB6||fqS^s zPa8t9Lu51yIPZ1@A`jKmCuTj58=X3jOj#yXBa{E? zSr0GD)I%d$TdENr9~!4;F>r+rceekHbr}k>0^eL!yG*4C_YV8fCPIifH-_xc_~TOR zG6wlgsIARACPKYWlItq%5n71551Xc^9+YK8NzAOCU2hz?R1T>dnUuxSp8lV3mI zws#RacwXMuGw$vt%8P%oMzV)c8^jYjt&9pI5w!s_aO|TGQQ%5Ou}p3JftS=mnD3A8$vmTa z%7+vLMST3tSs2hT#x_8Bv6kxboea39W`HN5raZEGloVbrK3pdj({LRH(kVO+Uxx*V8#F!{9g)5KtYy)A<9}3Ni z%pa?pEQV?CE@Rd~?We1vcsG^0yz-MJy4P(b-Y3iRY~jrehW+R)8AmcPUrM@S2kX6R zb+7wkWK;P1nljPmW{2QC=*Yt|HAZgrL`C(wXrKE0&yMuH23I6)qHGNrkTm1ugFgUi zL<*}*F5){r_uoGcmfe6#{?T-m%{a`xy}uhcpIY+QWm6FeZS`It_e$Rjix6aVp02@k zm#S)fe5JoZ6KKgwa*+P&Ld9<9TH^Md3&lI{UMnd6a5K5w4S=#1<5 zQvZY*TGJxQQU9rVvE_Ocue|TN6rcN9PG>okVecT6MUtbUxITn!dC5jrK+gO*@dMk* z>s9)4E=>fvGKGNe;sbcSo)doeRb>4Yt>+UUcQD?b+&r`xO z3W$<@Lk|X?s;Q+ZNS0lx&zWpX4OApvPnPI@-X?oE%1=>=5Q^Uht?$cYqb2Otl%;$i z84-)O!JoR3_r_cuOy>^eZvGAz(n7}&JGS>XQXcqV-?1-cKL{SU$24>WELgIf^mJd< z%;)2D-gO}Jb3G)p{F;93S@ZlQ=zD_;ISo}ce!iWq81%J4DE9D?N+7=uw z<<>O*E1G~|OioW{j$uYIlsw0CX1bGT)~qs|@nZI8ZPG-XzLY=yy*p~k&CB=9 zq&>-)wG$%9*@{Nmi{wI-8TvTJD?36lX$|`AN__p2 zO{9qv3-3BA(mf={s-YF1G2uHEKUqGX9P;B4(^?&OMxqhTdyFos8=I`35-pup`;BL0 zT^W@*%s+&Nkk>CyX`G|n^cY<&VU07oT02u$?dFa|)!W;%g#hJivei95OC#NFIdG6H zr-uV8Q{M!kIm{Xj>23;$Uq8-@msz7i*Yrvv%%x241PyHAZ$xjaUl3d; z4_gaK&X7+-w41Ver0F$WM^{DCYf)CII~oeS_XZ|zq>P6=?(3e}gCwKOt^bg)V*|$K zeVDJ>IC1#~vF8|FtUUVzDGdfnZ%L#<$5D*v+q2^4I<1e+SIORItWVxLLIZt}kA1hb zbg}SvbNMvd#83I?1jjvBh(Df*O>mD^;&RFakp~`YLLc~s%&CahD`?~u+<2*SHm2ra z3>OC8kE09tMG}cLMZaCNU6~xg3&_SDQ?`|$kc#F}uj_DIe5eEh`l5&Gi{j;+@(2Tz z?0TiB$2U^?_A=6XK1ETdgs(#amv6Npx94NEg0SVsE;H>-~OF? z=XlYzRF$zaKr7>P%S)8&Vol6&X=p7On3c=KX~~q*Rt+VtMX08<@42#e6-5L5K}9_rPo1)jCf!`^ zinQZe+DoYMZj#|!xF1_QF6K2+;<2o%^p;Rhv0ua+*i80yHE)~LIx8^3lgyQ(*RZgH z9WyjlvP#@1yQB2Wj(l{KSj#2?lN zNfUmwO|nJgBC2E_15YaqFeBhP3d;REcL*;p83MNVl7{39Dyp15cRkXwzP)WkFGg{8 zJuqy^&_q#P;N@-uuQq8LwC|AJuuYQtb|5*y_u)N5lE{ZG?fKyD`rYO6|z&4s@A<_anRc?4Y-70P5=X z(8|{h-G3of>Ck^R0G)^Vb{_rXd{BVPg3UcTdGjr!3&K)YtfDw9X5}CxbpRBS0suy! z=4EUd8#xyb#+08z*qX_XF@kpi^GLLxqv9jd{MW}rS6(LsEn0XzlzI7q% zIue|u)j&kHuS>(SUD~xsC2{CI@k)k$syGSgAVwo_k*eh*m;a`^SFV>s z^mb7ao6^KHu%j;2hdgPk&_F&59 zoDMBcgnf_?Sn=?ZGRpxd&0ia|ZcMbxen!+PI5M|VrMGt{#o$^I6wqpagoBMt==Mj? zuef&7oPtz!guQ`A+jps`|M*at4;ux9Xg5Ukq0S(u&p zpJz5%lz9Hy>?WOJ(jwK=hp64E;DeVEyf#UK4rT$9R8382INh>+L2xK1yebX4ugVKj zp(SCfq0@bCGLH>J6q!S)keP@?;aexVzxR25@B4j!Joi73 z_H|w7u!dtD>sSNZ9Icm!xzz5YMm6Z-X*6m_IA0w`$W<~s^@A_)rC2Xm2j8nh8~%w-28O07Zr)yLJ} zGZzJtFDYAgq%?3xAXh%hT8jO=VZHxkZcrtopSq-Up=BKc=zofpjLiR@HiW< z5_R$xLwiQa38?Nce%DoWVhi6d_Nq`O-Y`{<7FuSifT0!O4N7Q11xqSqalM@RPD3oyt{$j;fiCGV@Pb??)dBWEgs}vIU$Pmub3#1 zWKNmeq>=XDVsw3~YR>gko;Y ze($^4)oZK0E5(N8El}E!dL*|{m2Ld^O*n&0jA-}r7!?wGLxe@dye#-LXfUeoVVwa_ z=k}7RbiA`}`JR}hB)mBf#C+s@K(XpB6v(W+l*m9uXSA}K%9V%NS;l82y?O1vu`_dl73z?~pFbpc~`a(oVPV4VyZv+h7F zU<@|oTky3FgDo;IHqI!wVKOeHc*P>38E~zffXVjWapvnb$;W@pQOUpFH;DKTfj%Df z`->l6`k+ZJpQ-9aGKR{bK9(lEH89lP=_%6^r}> zeNP%6!Oe_ZebQxI@J~#qi;*-)3OYmdQ`_m+I%<2@@D%i;V-2f+S>w(|lYh`a-JWS(* z76D@rI0SC48?E2WuH&Ed>4zZbC_?RYE35FfK0Fx`hU3s|=JwJMYVt%H?dsL#$G{oo z$z-;4s1#tin@q26h}nX|v}aBZZgap#E?vkCM%1t{1~R{1sxvTwLOSn~B@gU@sDSnu z>)Xi9!yH@25_h)*r3KM@;fQ?IJNX{LkXY2XumBgKhMeUCo$2lM0>&4hZP8UVhPY3l z%hX8e3>$N&ixw$j4Vto=FP5W@LlD*b*>*nsy?;WSRS7fLu$VichS|J1;2PL6ZM_UI;lDI zWkO?Rg?TCTvwuW#FsLnR68a*%>l~*rI(==-3b`W%Aiq#{-+)K`j-Zqc+;dCU!(JTe z_l-jvU^}PP?Pv%~SW|^yj2`DToP-SIMdKRG)M=YO=U~TGe0lylY&%T-OY!~m_4*EE zbj?{cK|#%^0)z@WB`1t1fCcVerYdoAUr2qVVD;$nBi!F=w-sZp9=W`C&u@k^Wn~;xA&_U;A|jX`H%OV@^56}zT%f!a>?g}7Ou+BMDdu)}NE}6p< zv)i~SL62&)^4`u$aax6J(GIlFtn&1rLD8oW4pS$n+nj$X)$!Jo)q5DP9(Ha$`84E1 z&A=LG?z&Gq%>VcqLgx$UZn!@3ylH*1n7l?ok^tUtOnRUT9!Eo9p>V!5%JwswZ&Kk} z?|{VeZ=_&CokEvcsfJNRC>U>A85*~3@2jPh#hrYXrKs~&Y z$j}}g<*p1SkIN8R58KIXofLvVaeDuzDgjw^5^H)4M;K?`hfy^&h?s|#=^F3vVGLpO z%ZG2Xe^n=pRi|Tty!k-byyU~mnI}5&M_*3eF>&uP0^P#CQ6ZfjJM+>Rk6f(*?zG0y zLQ46}z(nXiDHDaW4XJ?vLHpL8Y?u$x74{%cSmu~Vf~|lqfTIVc-EAq0jzixg4y$}(u6hUr#?t%7&_QqUY>>sV4`_#E z52r*c#uLJ)pT5wUAp9jAx?-*UTcTesOmvl}tor?&9D9}Xkk?FMCEbwm#3=MSw$s_Y zQfguTz-FlQJ$qmFlgDd@jNi|^*a(wHs&snWpd|`!x#OQ-v#z+{qNkm@D#I~ImQzRN z1M9ZVc-+9YmkH}{? zWl1qNgl*<8N$&wdTQbXa%!D>gg;~qhm%kYW0sqEcf7y;C7rQL4xJ(j3-NmPz#lbXq z)wnn(EpTBM`3SYlGO5h$M^K`pLDTI`{J~(4_1Pa;pR{`J({0{v?fJubUmk$x{pb2$ zohM6*CHayQT)y?dxetS^lj-=vU!%KNL{~bQy zZkejaMqG(n+jOs>-V{)mIQqDf$k3iM=i&DE-{=1i1fWrJaE^8;ZqqSdT~K#S65UA4 zJ;zY6d&vL05OXx~>>xFapt9h6^tP&j&3 z7G=Fi`2a+eGDw8k|5!e0EC>{(AHU@|T)Fv_ejk0dh^URXN6jWH%L^WoAK_x@@MMUT zd=5T+7pWB^Eh`sD=cawwq$QzgrofZ;-)JB9S&^#_1sq8&FG z+Ks~8<^OIX3;Mkn86`QOHf*FR6W^LP0&O!MCJ!JFT>0@}?a1Qv1Fz0A zNcycDR6Sf?T-%D70hoZC9KBw$b71{odULssi24jcmpt+3gj>)g-A)K;`jI+0DET6p z(eB5ZsTK)vu_tdC^l8#=WVGsOQLQ1#P%LXOJW6UT0v**AK0=@abUWH7q^PtlB}zDa z?M!UV3((;3GHQ@v;>^Mb4BvpGqS!-|2Qtv*-x8e=BOB>F6UjT{zYhDwdTP1(NcBq-B^nEmd#%q z3;-U64P&XE=oc>jWV7;n@t)4ij%>(>u?InrL=G~LX%iII1DwG(91!{Mc5&rWfu729 zxy4UyMLmu}ivza+8FjFk z!4EHiY?(bM=y^KnRDt%G$iwvLeOwV~CR_kEY~U}Gi$@(9)iwfUklqaJvl727bfe>iW;Q_1*YAn!}U1k8BRnZ|v5;BpJw;##(`R{tU_TPFpE93SCHc zy9F-suuCA_@HWY9dI#eWXlTg$aP7SP1(O)aK2705HOl?~ZMn~uy=QN89Ql?X&Tc|^ts zeUH)c#|xDW*t5-#PdOQS6swPf@l9j(bq(={z8K6$Vz&jM{7AZ6qL={G>g;?4kM3LOQtJs?Im z*?FqnNHzC9>CzWKFx|R9cqpt7a!CpfL-%t%H z=8tN6lID%%LJCzTj%VB7&4X_9y74FHKpAC-{3FHpTf+4Io5~Q^K6q^Q$NqfBnZ?0Y zyl5e$PSW#Bo}Z!faF0_pmAXYSM7V{e(AGiM%P*QM%s+?}*d{cBQhructw_k<-dtCY zk_m%CS#)`Vtg~%d3O<&uE|kEM9hfi$XyQ)d4usakFnv%sZ=Vq!92>YYVdV|Vr>rsq z0OAdHo}&+rS7n9VGkIaGE%PHDI)Z^+V;zTOLfmb-=(jU-qDQ)tN{-qhrKLYnXa z8C|4<2ZIruDcSXdM4`M(YdZ2H$sXqWPoKJfcf0_i?Td5%C+mnLxj+&kTe8XaV zhRT%M@=Hns1Q7>!yvBA=<|f|*uyEn{opC>b4dXmvahv=S@zkJ z>Y9P;a*6a*Poac$P|ErAGuOOjm&%1su`m!>yZmGS)9~oy#fi7Rm2SG~R3E+U2kGFd z?LNh82fVQNj3%G7cU1Imu~HQ+9b#H!wOXk?9oQ{5|3Rg#^R9SDe}Uw+y$BOar4=mE zzm$H=!j|qKy=x~ib!bpNe?6g33j^ra|c}y0NNpI@TI@ynLMnSkYLX{ht6dmnV1+|wl zOw`E-WXbUrrIwWsuYRId&fha@aOEthgAG#tRWHfY9Y}cLm&DE(X z6MFx45BLXhS5>mS_584Mps(l|Xjc6!nY*-S=YXEq+FPS51dEI7;q5Pe5Ju1V@D+eI zp^iTg6~CA0(@Q57!K?AF1V-t(vRd$7#i=8=VNO>f$2L-pH$W0(hvR2oh^ukJVYHe*s z1E+ca9h<&#iN$m62jaKIZHP&{m1s+)K8#0!0ky{ zn<=bX3z%L?Zd&qhdfMFcJ9)+Yp2ofA%;s*8T2H-_drY?ZN@bVe;$<-#oPh!A2bQt2 z8G~k29Xi3coOBo#Vb+~Ek^8*roL7H68pPy{*+FSV#9f>2ZJS+U_}f!fWwPuN_7N%I zY4Atm;iOIh@G?Eqz(dMv!mS)G|?a#b{;BiEGk>wRw_vT)t!dZ2|xM`Pg0&UeU29Nq*qZi|+ zNFua~Td1`79@cY&?h`JA=3>87JBe>PZu5$pG22f+4f_hCHO6Z!&$m<> zsK(%)W{9oqf7UWrv;0PpubIKS=kZXvM(<&33Wo*X*C_p8>@hdWwK*Ggwur@Ce|k?K zo~t{%D)dy-PP$BJo-4l?x=oDqZ7j*O@_v_i5ULJ9Wb@=h-OYQexBBg|H?~l7MJQ7B zJevyIe&~|HNnBCNnF3kemXcDY$nAC8ZE(RnAANOOdFunBEuTb^Ch(Dp+FJfB`AT&t zGxu&G(ABiV$dGf8di66+G2pc+Hfo*@(&vhX#+LInN>f^WD&O;wp=F!3IP8JD7DgOv z0$(|{Ltu(N{^I=`lU+}&QeI!TpLl0}cB%8g$RHFRntW~-DRC)KY5g|8U$FY^IQ_mF?*`l~It}R<>d1Dm6`$Y)XwF4o)Py7-~Xr#43>9v?4B5xRui`=Rf z>YcO0`wxCZsFaTqRTZsgqy9?6K&D4WyTu3wF+MjkX%@T8 zWDh|NUK0PdfrWzP!^*22O05~iVs*3ug*h5CX-)+ax^6}7hIrC=)GxznjGE*K4>{UX zzJ4VpRYNExDeu6MwolgAcNvyG3qBY8$Fd{h98uO#D}X;ek+O=YZp(@_FHt zR3Q}FTBGl8=B(D1<9fNx_>$sV4>|{weAs?%{`iNpyAP)7zs}IIV7#`4xldp&?>TYtz-SVbD0khl;&TMw(!9lPcOPAeM_cfOHx&FVjr@hLXHXf?cfJTFk?f z0IPqR>8^>M^Pt&-H*LuDj~G*vSw(JgT5l-?!@=q=MhX$0)vLbk_XpfjL6n|+cL&8{ zsnutyZGc&g^y!oayW5ShA1-5 zpAK-Qshjz3Z{#c}P;6usuZ^w>>iF5ObSQjYxcD`@k^J((^H(T4&^b4x%MhS4arY1& zp~v~v7?tf@2BoV{6FOXxp8m`EmGc}nF+OAQ=fd6apLrFcw~N(3jus)e_CES_yWw!* zD|(?4@~GJGt5&Djp|Cs_Rbdt$GV}e8JPR3uzwEb{dzPlNTR1*w zKoyBtnQZD;n)n3}wUJ*e_Iyj?yWGQf5rSqfe@7>$zG+{-+*3=CT7Pjjpy4wM@Y5E@ zd|c7Tv#|{C(^M@iM0(@8p{l}*dZI){GY}Z;RS>FytK@dKGWtGx~e>LOr^3W zNMRVcRFPM_59fOdtzQP8l5_TDqbpV|=ZM@0P(UM>C{7L+6ttcwYKAt#fQ-IDCgX-) zW2=Bj)5x=BMZ{l@4nvSmCF4w=NC{my{q(xkCAU62$KyFj1drSg#OAd@jQ`c|f;}rW zy5@>pesERT7hl9>C4jp4Z+%98{d!5>Xbl@c2)*WPZ%F(5FC2d-ML>TZl)*$BX(d#; z^zqrb3F{9|H&t<5I>FmxUYO(S7WnGw@56+hLfgrVC|&pJH7I?OQ|6f8?@bnZ#wV6; zF%~_j|5Xd$)f&c4VlIsGl}nMasi8%xc@ylSTghD&^L-cE81%(2y*;v^WKg5jB3=eF zO~|VcQXhzQozR~*`PN$g`LtTFg;$5M?i`JpbWwDK?8jQs)88^fjumQ@{~@u4%Fh}y zN2TRZePKB=Qg8fV`~$0j%|jg>@vzI|mbM~a+T4^13YnfbC7k7pb`E_HE&pRU`;#un zVWFjXu?;!>dU>_GbJqj~LaLTJk3UsvDJJ=~T>*uO3~z00SN!+x0r!O9ZE@>v!$P+}P*2otS(Khp` zHJ^$u#^d)u(lZ|J0)?*er-=IjsvL4=;0!JLb3wg;bSNs>&{CMsS8qV=N!7CF92DU% zm_}j_kUjXOI{}rR3AGh!nRXg4 zOumKL#6dx(BnW8UC_0;vnb$n9eWVFLPPgSCF^86ww{LttBbw9@jxRvSp1!} zM$GBE#aL*(^5|eg9$uy*>bL4%j$|fYqmK+@C8VdB^6oW?i>=3e)l$R%Le!mhcO^Ay z%+?cpY*a4;*p8jQ!vV@0EfQ&|kGV5!eDL;+8+-eZo0f~ry^mn(`sfccXslP5otg!8 z#}1iGz*ib7dcM1yi~Nco7qIRS^gcsYP=s^?T<$#jWKbJBulglhr8I*X6O+28FSD#= zXpnaJ;UXRUK&9C~nL37ZXWooHCUr9<3EgJK|Mc(Mnzs#Je+w03mh-wJik`zwLYF_K zFLLex&g0uVEtiCp=Gmn~*P*t@T&Dzo7{ai1*qdJd&jo%y(SjI2Ri2t<$CVz?h?sb) z_dx#UskgI-nIbN?y9N@-+LY%%Y(nA$v$MH~LMNn641cF#M|jsChpd&-7BQAb@yjC$ zYYrynZa1BRm`Y&^T9NCLYQR9SqXr4k=Ueh8_7EQNkd)CsV$o1sB7?9#p}72$AzpYm zmOeIL_376wA3j&m0U4=3>jdNGsR)7&WHL81!J^Fb83dO|gv_<4U#8*sS_mdpX#EPo zm)M6uSzA&yqz>H?v<8CFTWqk(VhAivf^@jo&&=$S#u-L}LF-7*#&HX}yqD{lZ^aBl z?%%^84u|0e1jLQsAt)*{Cm8IjF^ZlqmD;pfsnm>yvlXHR^c!!cuLGxb50QezwooF4 zR3Bh2mHNp%M#BBCBVRY4a(YjFQQM)=_vgIA?00S+3m8p;K1pHqS33T({JA$f+mb4e zHJx}Jd**zC=0B`9SSA=Cvhfr~8LEWP_j?SVm8(h6Za&y}vTiWfF*oI@Q9c z7{`>8UirqS&eTrbiq19n>GLXT#R=V%r~QAI9h|KidY^u%0Oi1N7M2bMwjCP_c1_W4 zBQ^y@(*-H0+Sh=guS3%vKS(dJ--&Oy%~Xe07PdS4-WK1C?3vV7o{a;b$hYxN^D5?W zSAa@E&nAilEdl7ve&{hLyd01^XqNe13cH3ZeeXbX6 zKBGZn%VA^rqV}DCU+)WibqEGI)n7wjcK-WdAa&x(%nDdG)+T}-erPuu#}Tx($*j6ImO@+&hPFsSmNoL|A%MUaQqG0VB?50RlSH&yeD})PTPSB5vZcAF~VSK~WEihsM7r ztkxVW{T8rwe4x{_zHHavOxlQ%j;jl#UQPoY6@@2XROfzF9Ch2Y7h9;-M7a}Feva?H zj>bm71EZGHy;WDi8tPElfy|p%;2JOb54mIYC#_hm>a9&*^zbm%(ESG8 zQMhA8D!Zj??h?SfSHxIpn7CTyRuyA}4InZh(i0@K9?gT#x| zIf!LfsnHSnhUL`h!w+1`&UyBOVTw5^KM+Du?#@?%qOP2xqRwa@u7==vVx)Z6`pO3J?$tje7)k=-XPNa*!1*}E`98}diVHwj zx+Phf)Y#@WAjUX}R(ub}BxXZJOA3^zP*gbuQI#iMEuSK)g2mPRPISt1lTI;09h8RR z$)MxnWvl%mP3+v8b><`t7cd|9L#UdhLu<{jIo2l}Lkc{B*cu-VCE`^Rmn(pzU~Z)+&VS zph~m~I(+mW;SS@AZRm4CLshdFr3MXggUeL_iF?N;gzmGxZ?rpsvLqr2-*4_y>0lxZ=fn}_Z`8q8G=+>TF%@-E8h^;uBl z5^$RP;VDMy;f-7$EeaEiCZVJy9`JJi5NR3aVCDiRcRFn>wB0L3D6ixdGv69eSwF%7 zT{NmWRa(;y(6e2GR(3w<0hLJsH}2a{*q{>!ypY;w9n281K@5p%CrnDyebpXtvy)Kp z@ZNLQ2F=*)6{}+ht{v!3_MKB&0)ITVoXM?3%E|^0+yeoiEeP>Bj+c0f8EgM;tu?wN z+P-f=&&w%+dm{?N#=&)~tNd};ah_l6qR0aLNB19wL(T^;Ry@lsXvLjTdDUT5sa1bm zLGkeGR*k?N4h(hcw*Z_%iXz$CQB(WAM)v%kJwx$5=aXC}fq$TF9PeUM1?wTY2iUx? zPvYMc3|)ki>glCfBbbd_)Ze3q2VGcTWq!5l? zIvrw{DXZf%e&%WhRt<64X(_u4J;g37P^uk_uMMT(%Hpx)0LZD$3zJ1lyfJ@Hu&!I4 zuMpG349fpvl&rk6SOn9B_DL`m9I}#GylLS-e|%Dkz`5zoBk3{%LrfIhLoqjuF8h?8 z79+_Tnx@8C0&6wQr~q!`!1Pd<>W5ngT}O8#fw#u6=rc&0n6=O-JOp$1@bFv11Aj3? z^FSCCQUu1oq*`b#h8u!E61yo|l%%Aj7j+v%kdt$U7S(-)FkTQ@^egJzs zF0F;xcwY#`rZ-N6jD)@rG4=I9=633J?|FZ(qV{kJjlAkE8|>Vy-nLC!e4nxYPHY1| znbGjeK%z};JJc7>`3=Ncw(^C=ZpWPRRQTSN<^!oYG+5JTBu;CP>iRT7w~XHz z(`M;KhoN3l_-@9!t5;lLT*t%;T&;RP?K7PJ=QhdzCgb4gO=nd>Ya3oe643P&+>OOYi2peWQ0)^PmWbmc+HE__$o`D82#k}-0FFIG$bBdgpPuA_USiqi^AU& z(Mzyw1h>RC+Sxsa_hqiO?%RAdBWm%u?=({pqY#74^~c!MM#nyz0;T?4M2v6?0aWm+ zt~vLWEiiG=$G(W+vBAojM9;}1eaQu?DX6}oDU1covmrd;m+G=DOv^I)t~QJmw5T5} zt85vofmm3?5XUS}NcpD{$q%Bx(?*+VV=lD8?7+`# ze?U)E zg%+R@%fRtA9+41UY;%7zAF@6N6pf_zNlwd&1ywk{JW}757+QHoLTwR)7)w9#H)N z=JsW%9<3T}0zk*LE9lJislV2Hk2y(T_ck)V@*xAt38txG6u;%YPF$vFk?Xcf2m} z6x<)(1Vt;o#2XfN6O(rchm&MR z3_<%Tv*I_Mpz>ug-vQNg^idLR{u8BB|*uUd4qScF*?Hh%=NC zys2{74u(<{fm<(+Ae`8@?t-}(%^+0?sfiYOd@;AcGH~+aGjAC7*eRnKtUm=h>Sc`I zpYE{1I;XX7U*B@>hwKujw|JgK4yE<|92Y7*^aK(=4^x!|-UK(&Pm4Y3N+Oy5zavsh zWCuJ>?B0BY$woE=X4wgm-W$apvQu-M^sMhmoWBu$(zVy6cvpbzVY0FY?j}lmm~<4C zag85?so!bceDq0fGp9l~Kj_m&BA_R~R5R6@CQHB~{@ad5kXtTvpbSiQKL>!-^%&of zQA=?K5Kk_rNUogSNt<+59Q?|NJRPw;$4(_H$`TLf_A#z*I^88#bpB7vDyU%iAnmU$rx!XP%@T z+RG7>o4N55kI#9w?btX8Td32xzz+n)`+=>PfckUR`ixX zDcZ~b`#)uyxuK%kUO|bI^kQ}t(U{0c!sE2fIrIL$yj4_=BXhDoe3gO1YR8z z>Z_xpgH7;##DVXb(1yC$=;hCs|9(Fg<3<1k?)h6$^C~iqkl1CqRLs7?7o3Osva3lt zn4Z^iKT%I4jAByZd|45+n*+mwEJN0QDnH@u++43GTFb1_?@|_UWBRR@7oa9(_~suF zrQA5>5gU-85oEs3tACbIdf5vFUH9-B0HIz>_1N#PHmcZFQ|&I8_dVKBoZbrw`yYX*@W};q$2_HQ?tN7)1LJA@q+t zaE3P#&q1rm6PkBWLsLc`YeyGi2)X_U4ihP>Z-US!Z9L?)*4m#Dg(<|?Y1Z;{C91Ui zNR1VLaOoADcb`|6a7zX-kUxNbS3sH66ga$am8f%lw~jI;u(rK)^i6qNJOf4hD)YZq z1a1l!d5(RXe82wq>s5dOCL!Ff_Lh5&*$^^yz4mNHDeK&%+Uo74(Tl6Om{rXh zquELEW^&6M|95o|{GY$MP{L1a$YYU0Z z$o}uMxAGxb7x8IF&>@1mTmZ2>V|Ltc-d<)rEB({CyrWZ;hQBE>VzI zeGeO94eb%yg^GBC)#a}0GTu?*mJi3*pmv(oC``(S-|U6sVFROC=mU&kWk>He8J?}O z8ApA(uqxkHz8G^xvd9&Da&H{w*p;q%4@wjbp>=k#XZUBP*9%5}&J7Pu&)% zyXJXer>Ri8naR7$^ZTBghay$FE1v+JsTGP9xQG{Ec;9ywWg5H~3B zrMFJx>1mH$MYZ?$1T8d$hov@vs;Fz{^M=4>aB!{JS>S6!`&Z2^ae^ zv5UmLb^dV9BLo!mz>r^Op!+$Fen>mC*fEgCX7WO7A(SHR@vVeiarp&*&_g?u9yi<{ zm>`j~fJIn88jI-VO=%6n(dQ-~H*8T!UVGCu%PIkQS;*Mi8R*lZsQ`}3xg z6D7m`F6tUEYV;|`zO1xFehA};I9Tzw&Vx0egYFVQ#+VK)W(%0Q*n-gZllsf`BUeYO5aj6kYo{=FD!fr8g&OL18t2^ zXPF`??Sd^6B;Wh@$`9yY1(KpGU4*>#`F)|=w89|?5Obo| zJgw4mA@H@MN0A>O2z30z-{@~>7co>s3M(o5$*rPEl+s34_rHIHQLKiy?H$Om_TR8$ zbKEERH4{@N*h6{wyhKQU0Qo8EIAumko5}AWF%$w`@DmPmAJChZ`F7tw0WGF&&M@R! zcW(4`_2qx4ge>j)-^^d+X%mhtZ&F znH{ID>@k^bYxaC&ljocLzzu_Jm}qmx!f`7R_Ki3X88UHHRKF7{hzxFyy!Mz%PMY9+ zkO4kdT;W6Px&0a~1Mc%Yz`tlES-(w;cQkndeEv^vv3DXtiR^X>$eouB6n?^wXr9n~ zp}89~t`(XKNt7Gw+lNXoL+pI2Xvt|w@VXG zwYu}Fm1t&Z`cv$0S3oXWYHwRsl@ZwhBkev(Li^~q8zE`6y!ycwAA{LdV6+%%qFVLl zjcYa6d9}HegWqT#vG0y{f1p*d^fh-PQ=>C0peNKIcm%S8W5WSBQQ|%bGBPm+rKhR! zwvt`3rjD56KTkgV&iuWXmIN91vBC_!E)Fr98;E?HM@RH#Zk|Wd+NIAj+(??dF*a{h zR2I(xNG27kJX;FjSc<0c1HMFN1F}2xq>&wX<61vo(Wd zL3@BQlJl-!;O<4Wr&`h2V4)CeCws#7_jGE~QoQ8$DlSL7K_GD`yED>>k~$qk?U8)L z=n_LVvQ%O*|GTBSUvNCSqjkX928Yjprr@#IT!-#1K(Z6cCK^h4f*vtUE`QD&l^lNW z3-6}=iZ9M6;$zEg$AA-Yp8CFkpQU^wI+o%|AT;-?LFKez^{}K zipM4Pd+8z{qPgvP@&R03?oR>6cs_dJsiJO1@$oF}g+oAyo~~}t$;jx!0+>w-W7PU5 zG*5AKRMx7(E7x=Ok%bTl3rsMla~&>S-+9EYVpFPZIEQes|-P>xLI#_d+H1${Nk&`gnDCgL%8pR`MI0m8M zJ`UE~F4-}&0hOA13?wQwe`2!>O}#tDc$9I_s~9Q;45jMf#EGr}wHnF&PY9g(&8f3)cBVxcHm zW-fz`RC&o%h*-^PR|S{3Xl4UY z|AMvg9l>9tit`SH+yg55OP75SmIbm>A=iF`dq}ywlQ@LC^n`c+;>U9g0_OCkf!RX! z4jE-$ci*K69;ZuqH|%fvGqb`Fi{s84t$;->i*Kml%vLD5=Qs!m<=`8SqXN9ICGzIi zxX$=XiLrz{G7mNiV2FK>xfmDAnK{7u{x%d^{OIHxocP5aTWDy&lpqgHR&FwtLylbu zRI)L)i8Vhzp5PZ3{pVC+Mbl94;{Rb=GAbYC#IBwvVgg2iI@dN=_HNjyY{fJ@BLo}s zj!(_hJwua8xSAk-QFvQa2)(;8uy0ZZfEGs>($#04&0wf+xRTC0I3`o;AZ_&YlQy2} z`IIY}WW!{QCB6WfzKcP#O~O(>vmF0{bj<@RJGrm43SHFHwu=(0^4$(BJ}tWs+vzH~ zaJW}TDv@launs}%-gnH<>ewf)uY$*BuUryVi=t^9j^3%D8OwQxz-J$~-x%X9OKpRV zW8X1!<^w-R=7gg*E_t0R^3j%v34LsB@)m3nUks~|2iHG?YArquFl*!!r&^0$quIVA z7nsuliI5xlD59!O5piqw(jz7+?nyQpYcc4KFm9aCu6ljgVdZ_?j&=3`$~3nGX+x&P zevaGWqTpjvb%M<}NL2VJ3l5~&b5EsarZKddu0{);Een6kOfXH&;P@KM@GYTB4hCA8 zyF01rIGl+|E1469242?`+8qDp9pMBKF0Jjt0`HED-;O=!pmkQCjfpT;dOz61|Htl{ z+uWYalqgFoF+RFQ@L3(cN8;kKL#q^JwI+uPI&N1W%8L7(nEFFgrVT^H8|VaCw~JhD zXe1dOyk^hA2FJ0Bc~;A|1o zBP;0O)U}~8hRQ*oJH5?`vv^|oNpp9H&J*MaX zfN&Si!v0g}EDZAe|C|LJe)g2Po0?CY$MCZ=(6P2x9NR!g8X8cNBbI$|b%JO$`ucy{21+|@k+^EsImgMN7E`LMHLLO=A@Yy{CoBtS;8sk1>^C*qM7A1b1X;O z;$xpt7_bdvQB}$MaEBb2C-=wqhr^2K^^fbt+hl{QoY>wXT^idmwx13soJ24%bQP9q zXL=8mM(G#@?RzXThsO63h!KW0QSO=;%w<1=K*?{s*+#tYd7FW(XnQ^)+NyjUGcNi( z%Svg5spzHa4S)nfkEv}xr5sLi3=ePkDQhhJ|qjv>?Gcr_DRH z%dT+#cyFG+(p5G(o+;0`svjw?YY@+l@dgpShuiW3f}#zgBV)}-kgF`!3SVixz3M5( z_cFbnLyIU{{YN~;`s4`{&hbK-Bx2-4{0SQ0@PcGV0KxEg-lWQ1DGPaQ-dHf_iv#AC zqIw_bu;|{&h?{3uW-=m<8e-J7$JV{u(ORQNi+cGQ(s5L&M79QYGspDRJo^(`T5406 zS2#Yok?7S2RTWPoX7wM$1CAiccgUnAJenzLY#6mf-oJ+Ju`rbWn9rv$0ENtC?Y7It z`E&2CVl|THG%2zGVN!;l8-!4}^y0Y>u}HHNN8;7TKNtDQ8eth*#vtfa#W%;Md@Yf{^50ndC=vs#F*sOKTKn-}moNojIJq)CQgg$|Oa-{8jLE{x%j5hJGOPrbXGHFH()!YRN}`zoWd?kmzWB{irzq%)m7BR0(eSiU}c z@I5YiYAr@MkwZhixHus{bIuYL9Twi*ai>4bkv?x=>{HQOPJWEhtFvd7vJhB#o#oxV zBi0*w+)YH-u$nmG7Ct(1${O?8z3oqiPvPP%owcm^$yMXmztaoleC}Zxs*XI`wsw6& zknT^GLAqhwPFph-n+DbyAU)j1l6f5=9W&)G1!?#4E5;9KTTVn}&=KdfQZ9cD#eZsA z4@OlC{D_@|mezL2nL3&IGR0W#l+UwU02z51GsZT=+qU2Em5;E+u5UrBWV^628LQ=e z5X7{7I<*ACD6yvv%_jd~epX}%cQ}O*z-vpiH-Db}|Mc756P@qh#=QF@x6CCEnojN8 z=jVI#vgoJAV-KliFnlH2)_)q)F~rNnJB)}4|7gqSV+rLIo@f)X&Yl7Kv89bI%?OY# zUV0shx)VX&AgjSS&$kmsF5fum`-m98)|Pjr+csoe2C-kNdCvsTLBm@q=|a$*9f0!V z`^WLWroxKn*tG79_K#mB#aV}*FM_JV4n8JhB!FX-VJ5aH8sgqi_(#M(>V{UtC39ts z!NX=4Pgx011vK;LjRE6<7+)PVk1;F8tzP4l%(8RG_Wl-PO&T}&y7rEn?h3@ZqKAXW z%N4R!lJ;nNW0Wd}T=SSzXE5{z!4f70nt0U)iao~T$+Zx39 zBY)l!$E+acgEl^o*9y6(oaclF zGKq+5F{!o?K%=D5SNJWH*1oBBGW#i_CSB~=n+xrvp&=$+)5xrPezgmL<@w=<4$`w39-L#1fND?P84QDzz)?Zps~!h-G2P@{>F- z$q(HJgD1tsAKVIrl;9^6Doa9m)vPJIS9YeV%u&+W>+EHnjyUx@y!{FdE+*lPLpf2S zS-0E-AT!~D{tGEtsuNTjU?_lfwgT7Um9?d*DuTk-2sgYx2=OO8j~&$_8UQaJuU05A z4dqC;Ju=K|)X|(e_wNu@qx^TcJH$NaQ`KNmKko447{uRa(}U92SuF@-eX#}rAQ03j z^uFr)HF${bcDTAwLfdv!SySohQ@8LZIe*Ov@$TB;t)$IGtS@6fGRU#X8?|%U-eOSu zMJF)}aQd)=EXkYoD~PY&A5F7jy2+S&;)IDSn3wmHsnYMT3$2j{ge`48LcC?QY&WFV9Pb6BiLnn}ekPwvkZB zkcuKVm@du&nEBn_y>j${$ApZ!csB5vT(Tmno|C7KiDzFb)7mQu-~NRs_}IUI)QdHs z{~d;}mpzD(a!sBHp_4e}Y%-ajk1OcZ{-BJ|#EgaFr!6m9{zV`^2ZBXg|1?@Ce0-&g zsIT~=!sU?0@311gM*Efi?;jcoZH3pR7ce0o|NbU0uGfEa*Rce7cwzX>li5nD zDYw2g{^bJL$!)&N#OFuufBkxP;ud(5oAKG>7~K&rK-5(=YDZQDblQ1yt7i{56i{!T zt{~jzi(p+kTP>mXp#*;^8B+gy>-yR~S=?Gn^Y`3ju~DH_u*r5LAL9s8%tsX ziEM3pI8W-0EgV(fXYwycuN8m~bkJ}QY7022==iw~7@L_nEmH!O?%!Y&x1|-ws-NRo zU&*^I`stW|zk2`XSD%fF{RU-$Z@lfG6C7c{BC&ttA@poSiA@&emZ12}9TiK8mVnw+ zz3w&DZ!vP>8-LkwD5c=g-;WjCd>z3HT!`()#;fRQbUg(7_1ea+bN_Sr?q8}qSM0{7 z8y(5yzZ(o6%gA9Iu&A*9uDMlv@b~_nsPAf9V4an-~Er>uYKVjR8ni& zHf_-oh>$RdDp(a?CwZV7bxrZv=E^;LejAvSL434~9JLSY)izG4-OxHkZ9N^c*<22r z3k)OhJ{UN9V*A;+H-1sdbItef!fmA;`a|V03*$atKIaXBR9D+H@W2mv)EIS0I5y>M zq@(6~sVXVsnYPv6<4PO1We?0P=ZDA^F4eR|niMT>7;MhlvVpS;QX6LR^xzG@zZXPb8iN7P{blgb!+rTMma-H1HzyFcaCug{uM^^#ie{HlMa~tMtX;CZH>-VH zsIi;-o_!+$ckra$@`iP(wUcVyO?yLCQ5;tuHI}(b zLjp#8eOAPNg3cFe|Byt$zy^ft5F6aA!NjT?(KsP@dxkpc0Z!cRV-G#@&h1$S-fcR^ zO9${6SSsA_P4XbR05w2bLAy_Ebk5*l6uhsPH!4U+S-0%;)M?zGRobcd5FmlNhlD^V zDO-;H&bcw-lvhgr{Ucf0BM00-5h&^P0`5Nvui{gh3FIhCCMr^1%d7&{rdEG2Sr8oorEtuG9)9WnVeHG}sa(7FBNa;KG|7;08!O2WsSFu2D`S*G zD08L^aY99=sEEwlSTYMINl^-sGM3C!Wz76r_p|q=_x+yt_r4#09VdG~&wa0Z-D_Rz zTGv7}R3iwgMx5P%Q!cz)rAfPfKZO2fUl0Q%#@I0!$xTdr2IfMFM)S1AH7KJU81LOn zy7}1Yju5UXoc*(iSvhuEI13*(GTf{Qfs^$Nct0H=HGpPs;wx2Uk!<3FBUkL%yr;t( zVh`+}i48?qHloaHsNwlWMxc2}=ze_I%Dv80I1EF=j@$uY0ifI4LX_T`oMMs#t`XTN z37%XEuiyu0OzL0TS#6-TL_vcoLKo)NhsQ&$b|O|z4JsQN_EhJfu&n~^Fjd?~qNqvU z$5*a~=n!fm@4h_=>~^?^p+`i(&-aSf6a@M0P^-3Gmd>y)=}#cuVfWJ`7h_s-cRF?{9PO#S+FHcxI%M~4F(QFwgWnnA zP5Sgfgcwg91y`~znmqI3$3m6s$31Xemyd9}U)4R{lT3FQMm38qXXOb3hrsgj9tEGb z&GaNr860>^9>N0EAFa#SM@~ckrcn1Z3bHnHw9?7#%cCT(ly3k{ElV`kSN(w)zlgBm zKwFVr$MaC3K6bNZp%nGKf1H6+#53SKE()0Tt2(2J7bI?CDlw37m^k+RXj(F!te9&yycY!<1)TKCTd%yDEH#3IY@)Dpj zTVH6ES23H9z~E1ynROXX&3k?5UFj|w%YVCGV15}+@|<$)mBg*{8srtqVgQ57i`b)l zL_{*a( zo!w>Zb5l2fHu-&_^(^1$|oHKqB+ zzV2U~i2h|DGHO6U30zpy)4}>yC9oiOpgD}eITfVrrdQf)jxNlhIIr!qO+8-iFAl8R z%LH-9Ixq2$PoZ$NJKX673~R$;E=JLSVNTLQSaK2%l2~a*Q9eXcdS~ZuebV1Y%u^;6 zxr9i;Haj9uqKLxy;{NfV0El$L&kZI~anTiJ+?R66L`^24DdPJ+BT5ERkg}|g@*hH* zS_DKD7H19FEjPP*ymIT-pEHqxcvw{8N()N4z#~Nmghg^_GWk7GXn>BK9MG@K6$(2# z8RYV#yK73$pG#9s8Uj*uBe4`g}jnaaSH|R7TER_)X>^n@H*FhUY*gTX@!0v1}u=5^aaWiUIc5+758n|BqNbuU&l$7bU1| zR&%Q*fCTeP;KT!~Nv%$~IeF+x+>c{+q*#9gIgCU?c`*MYHcnm=;PBT_#+x5>kOvN< zumQ%och_YANeyvipUA%~{&I1TbZ^l9Y&c^@^|-8(*U@6j03M0Xi;QY95w zkgn5BO47SveT~juT{fQU2hQk|JL2tcWU^0X(TIOeOfdr6;y!#ga{byzKH$46M;{p6aJL8tME>f**iIzeR(>!Nwn=_RhxxEKAoq7mTiMksw=pyK6)TdsMwB z#Td3ErPF$W3K*<4>MeVrPudlFNJ>+EQ2cHlU-x~eWada1j}zsZ;eekd#LX8*^x71- zlXau6N&GfhZOipI`PVjcgNl)|uIbI>3iLm@ZGc5R!jM??vwEv=xcf=V$-cTGXkHU@ zKJy}MapcMin(q{AE{*OG@rt<^nIG{W637byy6WmQCo&`KdF%q^?xXYq4CeZA<+su< zBNt$%d=W^G@=bRyB1%~r*0qdz1UfRJ^L@q^3mrmq>gmHE@- zETE!OI{F!6OnVg3-vO2SWLauHo(X14|h-PmTaf-yG9452bvGIs9Q=^q@sljuM6W7EDXDm{$IyEBm3bM zG%Nv$8jb-gsCbWk$zwrZ$g~7$(zVxC1BM@AYdm>$7&q()!BtARf)NMZ6?lAg2&Iip z@gUmP1j*x`Ef+wl(G^mgzmOPiaHQtGmU_vI<6MdU`vKXa2)hVVcwZ2)b}3*bnvhjaqn4r{!XnS8S!CZNnU z21BIqpH~0YckbZOknbjs)#)H0hILomV|U@u#dlBOp*j#>e+$%=C!WE4l}4fO9~~ zers$og*r-@mTS`kI2WA2nJMKw^z|GhxHF%sd{!G@{C_Ow)qUqrqE`E4qr{bI2sf9~cypx|CDpemUJt0vJ;`v=YAhO$+l<7YSMAR~; zYGH`NTpwZ@K?nnYC=5E3P_(IVV2ZOO?Oy^r<`*>IBIr)-Br{7IfK1$#UK6^=yj0Yz z(<^N!hatx=1|TD%15=wYkhI^STEPKt+@!^z4Wu%8B?Ska5*`YM zQ-2p{%mahbt|>_Y#qd(?u-3VE_DVJ{8$(ga4)rRlF4sfmRBXJQ2p}jCLC~}(oWjh` zXM+9<$`|&LISYg6LW?R9$@;x^P5P}PS*N#s0Q=u|g<9Po`1kWtH)1WmuK&9wY}105 z{=VBYopBqbFt-=w1vu_HlbhdqK*h*Y(tj=W9_rMD`2!|4hgIX)Tw&YU|Je3e5{7dB zcA)r;p^Ov34o4x#;{GEf<94$KxxG(-Tt$r5WwVcg=9Cf+_Y;is~JQL!6s6l2dnZ%b9_?qDs!i;3?)Vx}dl)(CfC-`hp6geZqya?F@+*mHqH`Eh+4ux_A}3GZHV5S1L3>G1=nK zrT1b&t%!n?1AG9aBV`0vo-Iu15H3ANqxZGEil*@9ntp<>98&QC4}{;@pi##sq~3Ho zDcQpLycIRH7aRm@y7;X+5FsC5fX`_;HTdB}`vkIC;qtMAMh>r-RVNH`sl8^3ZCMj7 z)*kk2FwyIXeHkg=7X0q~RYogv_&j}ddI5BDy?|o?3RBh0#?@MFrk4B z|0%&z5EB3^qQ52#xvl&O6l8+@=Qt7zS~da&gmLK0$wwIx&|d9c_;0d5+cVUZJP!0N zb=E^5yZ}w>M9q5K?HPQ~)qIp-Q2kfV7u}BF>2vo&jGGUZIWjAp0#BFU zHv&qgcF=w9syxG9mL7 z?xExlJAJ|!#8*yjhJ_peRQVYqW-F*}N_83igh1~5H7yW&^+r4p@K;fg{tewB4wj{4 z5PRSIO9wUM5TN~^*w(7tj-|d3C8g#ZHF#;aFwly*9wBo_oT3*Z;aGQaazj=&jt{$F4s|HG#)Hg8$NH3TDoUC8B!S zv7Ayk|4Z9YWFVJ#3SAChBx7j)gb-5Ke0$ghI^a?q)#4p$cOgFZ7bHG{8bYKepyk9( zdgk9+^w%O%JuToy-qz+V=+K7g2+6TZ*tKHZ~g!X1t^;=21tc8qZ3CJDcKi}f7%V=*m{D_fxm=4wte%#as!3w+P zQ51MVGT>16E)U{-k3C3#4AJogtqgw2yuF^5xAIq){aOiY19>HO?LKAJnGtH$Ne&L3 zTy}mjgKU@SG{ju)K;)C?o8@Ck1^M_!eImu6u2m69mhY1P3Vr(F7IWo_|($UBCq&H$!GuQUgQ;tXP*PBEK&obR6COnJwVX2dzt9}VVTLr z?&t2}0M1G;jz#U;Btqf>JY(e$8vdxb3-r|ADXVGe1GY{TpYhKZ^W+Yl``RkJxGLS; zu3r$yIh+10U&;GkxD;kG!x{U_m;pVkJUVIJ6`ma+Xt#Cb8?n~2DL(SU3)zK-r==y( zD-oW#CLSHH=$P1go>a$RJO;xRWRr6+&!ul_N4*g2UT``<)nRE-@eb*a6p#hy{G&77ntc>{SJ7u_YTDUcmUx^i~r0}n%- zd?F>uhYyqX4au{dQ0oT@ab?V-f}?Hhxgum9##>~^(GlA~Rz=Y(XhWxS{WZD4Iy^Lt ztwFX0&Wt}b62>=;xPD0{-kgq5k~tv0_HG*}^06AtJvyCI*RNSus5#QF`D1ETehZLW z>||S8*bTsSCa;x?>ycw=yz`uvW$9g|(zaKRUYvZ6JM;s<5Bq?`Y zeMy7nXp#vOf{Fq5 zY4V@l;b{hF(&r+FeFTd~JH$v_`Se}NSP!QzKMK*%3;=}hAo&dh6+1=5$$FQezGjUR z%W&L*da>WAz$zR8VB#>&X%T*x1LOR-yQuzCRQw7na>I_ds-dI@P)MVDM_{JNDCkI3 zNOTlfpHJ|fz3&Y)Hk9DHjX@H5nUP2B3Ce^($4s+%{AW{7#O^n1p}HJ94{{WcP^j_j zN1-;mW&^T^_wmy3X&;RXqLR`^1YSI?S1kk?)Y6lXM_xv<;6|G>{%gq}Eq5$}bfJ|J z3IAYZ3?N;UrMiEq2kh=bhwZaFoR6Ye_!50M1#t52d=}H+WfBbKB0@7#ie_R0F zM-m|r`4s`uSS#Ln^(4ycvy~%xW+BXKhg!=eG+5&;H@D;r^l?6rZ$7b4jMKw7F9psR`hv7$f!^RXw%HR%$cQtuUG zTrTEywi4=rM})uMb|twOO?RTP^fwq8A__)gFXeA0RJbnyPw@ebG+d~t)Yj2C4>37S zeEJ7S->?0F6YV2-QT{N-P*wCLr*J4dgocb_qzE%!7IPYwBNPqVOE|ezs@A|~s}0kZ zb>Fp<$c#D=7b`}EoNytEduqejt{|O3N>+6!u2bS!0AvjKy@l%g2z4&SM@K+J*7ts@n(Gwoj9W_{xXhU0A?wxB+xNpeWG* z<*v)nP*zB@QtE z(?S0%RvAbB3q{SJAi8~X75FU|;=fmuHeZfio`}HJP$}qPzrBn61t0=F1}rl>+FVpOuzffT3(zuEB`iJf34Sd>H=ph~Q*N~dSgV86|WD<`d&Le_fu z4x?u~sUV06fg(;>^x;d^Lg?^^a1r=}`hAHhYaHGh62|{ZH-#R3N9IGh&q)`F;yPQ# zp{`eq5_AozVXkuM4A@GJ@Ap9iKT-L9dwJEoapWKIB{$Kuqa~cI525d*-Va6gU63mx z1VZZ?8UW=vHvs^<4;o{4{KDtxy#H$K)jM=ho(_EGlDA_(V)#^KXOxwbQwUTy7K&96 z-g*L-$oyZ+^r&tp7f4T_5(wBzG#is_)q-erudGfqV@fcsDkKOJQN=(?x*duVx+3+c zL-gYNIBOv12ur4Ax->S4kUj%PT*pkH^zKLdtt$>&nMtsdGY^lt=MyBuK+y{!d_sBW zZ*lxcu6k8)%V{lymFpx{E=n3y+7SySk(w&%WD6RFDDL(9CsPiZmms@1cBliWhUkiw zv8B;{w64=z_qaEy{C1Z-U<$P5TA$(;g79w93jUGSo zUciG+{(%SmbGSrZcRp64gA95NqkZM+ffT{SB1A<2Ud>nl)ZE(RKk zf=VZ3hvV!l>%3DAB2$Y)ohOqJ{&x}xU1qhiYoE<`j{xoATxYKm0-!#1c3Z7OFo+GY z+{&*%dDgAW8-D@(qD!}3vma2`@G#h>4C&xXjI9hfsfVry-_b-CmR|3>)CP2faJSJB z0%t#Rn^sBj~O z3KY;`4(Y3JpP>YfumOGGiNml$5z3}NnKb{)Q6wJ!&sS4QN#3M}h;fEA1*Gs}$3I%G zlK|A*zw`(*eGmX(k(!0_?(}x;b#VP3?**L}kd|vimF15gZ2KWN0al-M!wL+J`wS(! zF&Mx?v;1j(3~GVt>vfZ_;e@@Il=xP{gkC#^0pP~midn-wqUYqgW6eCr4?~F6FexkB zv(bCzNYCsaN;5&kY=yb)i*h+SNV@dko)zgeVGh#Uhwb3Ku7HN|$?n%Ij)`F_JDT@} zt|xSad=y-#l(aP_QIErDY0BMbd_9W9%wPV6_@V=@V><3YCLD*MJB1*;S&zg!(M^Pc zg-!(t_JECQcI0YW%ETDsSK+JW0D^&a$tYTPnZtTlP zRKO3=rBhFRu8jnKkr)L+!=n3LVJys9;vfOcNBn(=a9#AfH#3||M6yZLc< zoAWD5od(-M_|P0i21cPl<1)DiK^6ywAzT@9d->1EEC^2V?88vmyJ85kgi3}L$TlVO zzmBSh@G~xGx2)H-_Zb-|@)neyf)J#+mTsA3REbi=1bZZ|V;U)p%VkskW;hJu$dRh9 zxm~j>D|1yHs5}lqY^ng=cX&nI17?CG5{0` zf6xWymo2E><%27X9(R@R>dAMx9kOPHa&L%1-;>KKV3pcV;3tZ$fjP(M zLl-28&a-=psmCz);s>9V^S}o$f)v`43ir^i=>7~GAT*;7@vE~TJ+$N){Lc)G*Zt@# zxWeU_k_MnYSwT2{;&UD%bXkA86pO^gQ6-b4SKzNfNd2h5y!4<16Qn zcE_;jh%P(_Nehta3NoIpMFJh_i}#_0!S7GTXSKeK7Fq? z8h_n!=QO9y4I1$$iLCvVwEx?ceBUAyDEC10aN|ua*kcB))#qKCz$iZL5+(@OqHnJ! zUXsxc{#pb?aET3e_9BpdTmcP#a2-;0H>zov-xXt2TMQ|M;o$rD9JHh5GVmBhwK@kd zQ$oxB6Z@>mS-%(GO9r&DcLQD!6v&Y*N4?|>@F9?xRDM7P^}Tt-TShEJDWM*a!`w)c z!-PjVO1y6$@$88~xB1-*|6w|$M;%x61twGM1M z&g4YCKO`}klXJ7^tq>}**P@$6eXF(U5BTwo_y$oaYQ6}fr#L`%h(|4m>{USQcZtpK z$Tt4WD_+U-i0^Rj@mz4OT*Mci>FhAkOP_Ue)f#+0)*gKwXZH0u9I1ziL2Twet8!X0QSRi5x$YRjO!$+nv-~fXc&5LE?n4&zH{UdVI@YN{jk_Wdj413 zL<{JYRQ!QG+244t-)m0<_pJD`WgU!)Pkbguc6s%qb%UuFp|?>*_3w5mSpH@Y#+^+t}WgAju84>T$pd63oKpqpu=tApbkLIIMBEEQ~lgvBBXQA z0iggH8swio?YcaxpKfr_X3fF-5f_I0oqR301G&UUL=$;bE;2wm`I_>?g;9t*_iAhb zt{XQ*GpwqCvfG3{2q0RRhE{TDUZKRK&OkVF0+CxU=M?HIk2N12e0TplLQslfltOgh zgJ@TJ%u)p!dUmdLwzQrk+KtD+jY4Eq~ixD zK<8S2ObcGbS-u4NYYmz^mJ7{Y_JEKG_ot4|6jG1V22O~u{RZlhGXN5fh$Hh51~SN+ zv|6FoId{Jj$OtE3T3Va-ayP&;6L(K;caZU%bq0p9z-N~%{C?~iv+7uO36OR)T7meC z`2!cmnwYd>AE5azK=W8GOaSz}7??CQ`;;UFa)E9>ffU7{DdH-4#fy?RwexaKtbF*K zeU!~tA;6!w;pk1sV3CpFxgU!LZmIG8edN{ zo0CFl0)^Aydlz6?K1D&q58;QRsXtIak5oYSFSVdayl>qo^;)|zACT#&xS7OOWQx2Covn;P#w6{1c>#^GH6mOVjK5zzCH5; z&Cm?iYN9Tr(h12ngnVXsCzmz^vWN2y=o+0TY;yw}cHY7Mk@q9|otyI+EFS%7TBV>x zX$;fsZ3~QZB$8UIWPQLoI#F*0sGknEl)UN2s99T;6%H?;NVbc zxbnMzvifzu9_St1VG0A-qLBv1pm!kk*M-act9N7f!d#bKGT(usGpRh-%46tk(!aCW zHOFcg21TReyDUZ1$=Y@q6eN1M5GeV3%6OV zKFQ}%t8OsIZ9RRNHtN%uy40N{EtS>z4?dkx0Xv(V1&yPBZv?;#P$W(t z7?SR<0J5QrwwbcL1k5)$wA~=-1YLYmb(F^FXyfCPwJ`&YT5}U!T_77g?$f%}J?#LJ zr6>fzxvl9|foaAwW-4Wsk+{=MCEU|>_iPm`G$AWlb``XHt0Kbie)G1SU(qyYk;$Qc zGjdJ3C-A`(X}}0Sd+H~N3$XlypG>eqglZc@@ouQnEsQ|T!A&Lhowy>E8UQTn-mDb2 zbBfw=heM}};7RYXEFWYJHbHHRvQ`x$6yahzN|O~pb?$~LMYe`d2E}Uqt4A3h1hiEw z666~+qq4o;#U5B`15fCN=UP}0+$+dVmIr8eWo{BHvmZ+cg>u8|vH*V8gLErhckh~H z>X1!2CXyd6DtZsmgq;0}BiIcG7syU7U(x9PG$YT%f{Rz0OCRNlGUp0tDKsepF*m*( zm=;}-k}erEF_y1@cySqOK=#Syqm7>~gaBH~TlVT#IT5cw2IjQt;d2WbO_Cwa+BFS8 zeD2+U@kC0Pyjc7Q??a^R>-IO%V>p{AWrq_v!XWq zom}1*surCfI5{9Y-^VB-g>w{K-ZwI5b>U8J`p^-ot4QWQQf{YE$PQ`fyhSR;gJ&ti zH98incf9BFb-BV_GrRx@O&-hpmUPZyrl+Y%e6n9OV~*+i`}A9c*?!znM#qu5C}4gl z>QaLDr=HW%AVuzB7P4c$JZVA~Ad_dis~rj+4Fn`jj4-729pjKvcTzubKia}B9s(dk2}RG#vA6@Oz4z+jbdN!kz?jebdpf7w?-rP7E(>_5 z7=2Zt5B)24m<$W$KS}XheLkywXy!7FVqnbh>zg=Wx zBXf~(W_;1XP*&+rTd%iYWVxfQ^PZDRk>M&=->KW05lCkD%awlZrr%=kt-|l}(2IC4 zX_@1eYfhRdX&uEiS?yiU9S>rV)b&5jWhuZR?;qqflCZjT>5jcKjmc$@{B{c&z&Ef> zn{fX&$eoq{(FLl}-}h6WY+Cy075Vdi-CXtZTd{;B_V3g>HqSs7Y*A8(L(@O;zc1z_ za~f9wp9)Uq(%qy>)w3cBPx*eFzBaEfe{1r4O3d9uN(oAV{1rcK9)>##^N;WwCN-UA z$eGSJc}Pd^{W#&k`M*BAjx;SOd2R=@uj9Auj0{8)I1;YONybH~$#9G76rOKY zkIx~1UqKADdf~+xILTK*HM`Bkv5s4n236VT=HwJCj6X9#1HXt=1G63PHYY`wC}Y#o z`%IGDzf|T9MX}pnkfNFZy+}2M3;Z9n#OSX(Z@oWZ^v2}g){o8_BdgH6J^7t>8<*@z z9zQXn%{aWcl1j?>*sYQc>Xi00B~h)B*L5 zUnlz1oI4pPncN>oJX$hNP~M<(@KXPGEwP}Yn}Nov3$mf2saB7kbg6Z0b;O{u+^Wpi2H76}@bGlEvmAy&3)xxc&Typm-qfnDjMEir-ee)@0?K~MlIA(4jRITu|f zjhmA${HXsG$-k?dg00`5n(B+o9q`}%z7cO2gxc#mIw>a3xGkqCH$GkRqBIbf`SUcxL?5#O52m4Ynv}- zI|c^6dBS%!{%gj|ElIr&5<9;Fft&KNp%#1+D)$B`|uhDTVFRQy-s8qzgw={qbfng>u0HYH@owc z%0Lc702;Bkz}5Y2=OWPS=Z|W)Yw(0ChI6WH&t*`G)z+T9zPvEMz#x9&;H>RZ)S07Y zt-@57p2k9T8x5sfVzcAera&HAWS~r;sm!USP|S-H(Kj*bi;7g03`#QMWp+rhGhsLo zb=1U*dm%_m+}sAJ!Wu?%^&Dz}m+Oxe96oTBc~Dbq;)&jYhoXAcE6i<=tSWQW=yfa- z1LlCE8~6HgoOLFPV+cQ|@KRhuI_$nPtBDBL)aiq#GoRkUGPTyB$lIirJimHC3ZnaE z&k!!mVHW1Jq`H?UeL|$PoC8&O%yAvJti+@gtNk?Www69f;(e*`nw9y5zi4Vvy=6)g zWtix_Mh*XHd8TNIYF#nj=z#lI{f(UWw$JhiGkiJ`WsvJtct<^bb7q0_RCg&dxpPtnaI+N4LFh%wS;$Hh8m_#!AI9) z{Z}%@%vI>M>kU%(YMYN<+F4R`c;t~vhIUhtJeA#-06fHMT(fGRtgXb6d8#owEHx&> zjPIaExEjOm@Ak!S6=Jn1Qj#{-9f%&SVrtb(a49UCU`bv>puLbccv3toXWh`9qN71o zf^O)CVbHpUM27LX5O$wVr{uO{)~VxC8+Hn&;7Cxn+AHJt195IqmO0WVtT4#upM3nr z=EHxnGVqPQWLHy2_5hkdd6XuM?RwJ^`<+rp>xG)gERerHXb&MR_&MnL_#KozZRg}5 zxe6I1{{b6-L^X`|B`egGp1jURrMq$Swx4Fa^l2@^L6jkXH`n<$MQ)y1)-y2a*8l>F z4^4c1y3S&CA)LfclcuC`^Y+cWO)1R*c%V{vRO!}n)gzTc*-2IIIPqBZ*i4AI+8#~o z%qPtq=)2B98=UjNZ8+WP;?%G)IP!19JvYyXO4cfAoaM*Tbi-fg$4Fhpi9gW_i56lj zpyUSHP0@Nj<^Nfw=qf(CA_C(Pob)G-o_fe*0`J=Zk)qG6;{7Ja189bT7d7m0| zOf@e{k7Nx~{ho<;EVWcwNSOHU5Xc|zzjNRz=QEfT5r3gf*K*J0(c6bTvzqyLiILKH z=Hn3iytR8mSj>wFlN6xs!_^wd`Cz&_i}R_9)(#w4bfacca8mY(q$@MYLHF1>hb#?^ zysLxVLMFob(t-p`_Iu>8tMMmgK~Qvfp{0nM+hzOrkN3EPh07jM@KN#LzP&O$Hfk86 z_635$rCX)7yCW{DUbBV)p2vcPKj)0!Xy0B0V*$7o_jJMVY=QbqPzrKoI7zOfkUunc zm6e-2%>LD$Llvi+k;FE-L5*F7h51 ziu1_$!Nr9;nA=$$RqOFya=Dvp1+vsN+UDD8Z za7Y{#;*Lpbgo`{zO2zyS1cOeRKTJq!6U%L%+!&lgQOo{WUL=!mM|={?o%RULj5nFn zkuzL2*2W0750l%1~a4a}iX{Y4qBHEid?kg_6Q zeCy&A%1F6p1MGqC&oIvx?M|e>d%rQ+UkR&Li_d%lUT5 zyC(eu@h1>-ge{nH;w+C*)N`ZoT_L&cK<=#g%FG-pc2Qv?K4D0lTR~^o(8E1uDEc~0 zFgL?l0Ne!X3dMcqggHA`Pyc%Ktm|T%)x||>|`boF;1$1 zr_6AT394*PxLkL{!6UaVt9GG@YrgchS-5LRezHV*2ERf#|E2q!)T%;%jhrLzKm1VC z++P}cf*ZORFr=tKQf%{D6e~(ZMK?Xv_#&OpsTRF~JH{8(zpCpzo0e?cU3NZus`0ZE z2vJ z2G?t*$`E^@+6H&LOaaDR_xnt!G29ua)Fs_I8_2DRm4V-v2tD>88ic|?k~AF)R!~5I zaW@08Fi`w)&r1ZQ>ux8&;R3He?xZ4l-Bv{+NZEc-=sUXkH3JFXm~$cq@@L|wL8*#R z{Gh&ndmF}}@g`#a-m@mQB#wPz3Gul?q+buQV#TRACk*uO?IV4X96LYpqf(yTbj5y4 z_s|zx`lqaZA?yF_4#7B>Rs{)RX%n$t;k>1Ht;E}6L|oJvSC8{M;g$UAIS7S;{%gDU zlV~-G?WDp$N4Dj2^bMJJ3E!wU6tQhB;G-sOHb-Q*Ir`3w55$%DPW z+*v`exb(?y8%h5g{_e|k47ch%LBIFS6xqwC(+_+8a>p#5PlycjKdPhZ#sszPE5iN`d(L_8ZDq~4AGvw%r=Em&c-$bmW`Vgu#y@P9S zMBK1Qk!{Mn+lWQ2T?9*tbl;RiD$@A+I#KFaGNmDeWe$4pUd_IQDMc>;*LsI~EA639 z5Y{EEp|CIUyJpgM%>1~x&mjcligr~Zp^${$TP_xtEb4*=bOGy}cd{Ds%~2aUs#gZL zryuDhD`Vd|OrXuzm*na*cztSBDcf@!jwEKRekwe^qI$JkhJb!t4%!<$vtjx*C^Uo+I*0L|Fi6Yd}GTnZIx8L-Rp2>O9MvLYl&$ z;I)tdqmY%_0I^Gv{R&1C&my}Uvqd>p!s}R)V{eD5T7D^TDC_yu_(!ZkD10LAp}iw0 zV%UE2{0*g$63{GJ0DkC-XIqLg$b+b_)(fYjLo-7^4J3lE(%!=Pqko*@=lE6A)aoPd=?1E0zPGBUBAVr>;eJcQ@*n zT|dM07aFG}V$6R~1xc%11#VRw0?|x>61{W@rbK1JG*NX^)At*lO@CVvXw@!s*rvu% zp&}i!C?IHo;{_W|rR3ulZF7}nRye_bB`V^vKzfwMKshW?7a(NARct^bMq|nlHwpDp zuc~z{6jX@(j~3IMI`|>s2k8GzBtNnd0EXHWr{0atG@WRclH%>1-;j9Z9Ma~2{ypA` zRXh|X2k+e7jvdy-9R$zPyYw$-9Di`DPT3L&h;;&ZKOC9(;o*@yMT%&Klmw86l1JD-ECH{H##2|Z6;hNx__4$QXTPgDP))+KTq1n#NFV1^}cQ_ z2*`v9TNm+OIdx?r_1r zk==ms{sSYlE$c@S3a+}l0`UmFL_%!C&XpzofSNi3J1@*g##Lrt;`!txLV25uo#3~~ zV-nZyQ96_g!ZAb84YKmEbHFQgLp+$vO-X?tM51%}zRjx$nAJwdTN*;nZ`+U zKVSZj`*H5=hPu-D!tD4cDqN<_Jz*ce<||C8C`BM29L&v%rhtq+Vq)|8wVN&i9hqNb zWzg$!eNPV%S9+nu6_&rCe9d?JjUmvby7Mb#bzfzRe$8}5j{RAhzFy1(KyLxNrth_u zmaOsh_0%_*XWp)E0RiYSB&&nfB3;J~^_5gu(x_!()hzUkpKg7e)#!F(1#LmpBf_Pc zI>(nwVsOcM!=Q;!5ht2Q;B^Ok1&IQ1Mdk$}L zOzsjxR&R(&KSP-_W6%h$RKW2#Dy1{<%}VK5e}+rbdTH0c)Vk3NOAf|;2bF_;bKqs# zb}B&S`I$!}2r2dhlPBKa^UPscImTAKt49(~qUTZAuX$%wO;$Af%R^d$c2Z(-?jhj` zVP@Gib6m_M=>C)eBW$plOW$N);-M?YxiSUrEox#t8}S4i$oWW?vF=3jheaTWI7=R% zN)^@XHwk5>Vox}LPSrk8b3<->poK@HU;Fc(OM7Q>wriQpjE7plo7{~ekPeiW{RzPp z`ow<^F<4i3-p$IAnqW+Zkg|S?O*XnF{vQm9L(QH1sGPy1n9#&$SUh1fLU#4!{F}^F1g+yfYg|RYlawv<^V(UyPZyfhYEz}Cz|SsWKGdf0Q;_6W4``V>07mwc_HW&*Y53E<4u4Q~$x zjW*ODlrHXoo7%v7lfh+VD_K(6$lLguP;L6pK%g~F*NFG3xzdc)YW`285-zGYoLjrm zIkktQ)&O*pM>40SYkp61u&>^efb!Gbn%*cZy_)~cVn;gRPl)FVZf|4oZk_0Sp~=4a z9-Op|majg%;&{RY&u}$FWk0()?zbg_Up+=UQJFHRCFx0`5$WiTX$3Zy8??Uvrgw%W zY*wh)*aHL370nr}lL^`~6tREMpN>xwK29a{xv7b$yt=}jdWFP%4EZ;el&&RSS?rp` zpeIyj_KX|cdo4SUgWXrixu8)s+gYVbz;~x{ZGW*5224BAQ?>|@%e%B@b?kZ@`$Rv* zAauXxSrmirdFz+4&eJKC;$OK`6f{g9& zHactlY6g}OUE~HWlLN=2@$EqD9hy?J#C@BxrE1PW=)6#*s^eLb6Br=_bp9Z2nEgZRp0cQw`QdftU2J8pbZDhL22XGnWC?#wuwVVqRTxSO;oPmV40r@Aww1g6@K@COwZYE1{XDQw?$VNt zYMC@~4HY`5v1DLjWWyzmCX&!ASqIz*IL+ARyVoIDI|AA=346VJYl3$v;f1b41vtTNHw8)byj^vdG(zV_ZZAPEk{bH#75TEY5tHGL>`4RsB`l8uEe^p{H+ z-wU11Pm#TU1LG+XVuG*e`Vv$_YGkIj%Rp(T3vP>;-RjWZ*mdCb-tS2J4;nmNRx2XX zfMYkl;=~-R3;GS_rtkeAL?pzY8Eq6vNf8W*gSdmh&$~SL_a7s0SO~lIMdJvFhulFj zc!Q|Vv>2xNUZ{8h4IjUZR-hqf7>!$9+WSq!=hygrLf3QAW|WP~Rq)cS?B5^ym?6bk zC1Is#mW}<>*q!gREu@9^SS?z5HM%#mY&fm&$rifPPMBS&t8KVV%D5Bx)-QKNYMRJM z4s}G_i zb~8ez;aKX9s6RiRPlfvMTLHKxtbV?=pG;~84-PTCL{-_13Up2Ufhbq^p+RH!wza=U z^(h>ETIkWqQ*Kb>_?_bek>D*CCntP>X!KU05p)|pQDBecL#6P9V>o-POO_LUj zRMFO_FoGv}COU&os7&WEgp7x~n^=Mjl05)^Ia!A9d!{zTA3C@joe5|G#v?wDR$rM!it8^ty zMDXfZ*ugWiwVgz1+dwdr<;6KSbVk@-0Fi$jE?(9nUcaxdE&?xMuySSLGP3`<7M;`c zI*YO^zt6WpXKvCpA=*ZT7b@xKOEMq%%JVCz^cxY5W!o{UwW-Y}ghdA-smpVq7F-5t zP^WT=_7||o_@X%(FcW(?1ak>9AdiR`zwt2D#2#1}3cvIrZ1s=Uz#KV9Z(#|`t>pE@ zf|zNcNyUTF51`owNy@V+gVK5zXiRk>Fdr$nt$s=HSs4a;o}QT-C{V2oo?zLRxK%w0 zi=yP%RS3cD^M4C&70q!+3n~G_wNJhyKDl3Ox&`EpJE6g0RJgP+qGf2i%(Uha=zy4m z`d~RQpm{5L&Z4OKYKk)<8lxvh{bOi-utSeBd*nf(ik*tF-caC=4&|{32p~(LNon5B ztNlGoh@&P9f_{O-ZSBFxm46(m)_qaeMCjnDAtEk&*%Ap^J+hNG@&sFe#-Cv{;F0m6 z8BAl=(9(jKMj03!MoUaug>dqJhnoW4i|+JIa~EL|LZh}5Li7`aMTq-9i!gG7R`x1E z74Q4&FgG+%U3xervJ(W+e;m8qni(CINe!UA=`TBK69|{Zv=#_B8XqxiaV426gi2V5 z#&5C!>hx0}g9oyrDzVMV-uG+LVM^^=8aCOZ!xg!Otm{^VW<3z#f+o^!PJ|0~q*eF( z0ANrI0`0}1K0hCZqCF%|yJ`xRFzk{qUSQ6G%SbSPnn2v+#Igtx({(`;lsRV7nd63O z1a+lhITRwU6E=#G`1BtC^7KR`sF`108vA|FNo>`+C1mAT*Z<+KW1=xbJ0yt5`*PU{Fs&39u(bdhA2&ue{iD3OkzYnynPn6}kFS+klR`uzb!7GZkQ5-v(6x zkn^!rxNQXLZDxernaiWQ zm%v*3fe5*yc((oWrR)XR^m3#zhdM4k<=U5g+uW9AkXI)BdX5U+2B%`RQ$D`Ce~7U& z>4AFGhUGk;<;&g6>}t$=Iaskk9icxMgPVl@r1H>xY6tDC+*QQ+gr<(iwO6rJ8fuay z&>16iSgqnbkh3h2hmekkjxBYBb(JD*%;K&65aqi;Re{B%&mPE6Ep8#9HgF+~ z%pv@094L+}bI?OthIBp=QSa~qnk`1aC-&GCgadPFE|cCsA#9g2Mz~9vMjt%~VF_KD>GgR_=olF+1-z_v=bdUA zqhkAU6i=oHYrX>p%X=ZbQ_PlkyZ1G(LON3@9O3(c_>2=q9dc?)v`T>8w(aL%@6 zP7^o{egtc}CPnaii7gbp)H=J{fFRff?c2+A;EmlAR8{)nGC*9=2dndN7rKD7tOKkA zUm^cM9uRjG%~k3eK?gQ!_wfW-?gq10axZns8c$suigqrYa_<*tKuT;k2{Ght>fmwaYfS^Ffo13YFflxYg?jh6FZ{NHfi# zZWAobnOO!C1{~CxB6YSj6kUWKBlxr4>b+6AoiK3t+50kO)_YrqQu?e2c(McGLfrFu zO9nAb;p2iYIX2?*(6<{aLUwq#obp}LR$(8%ls$_XUT!+(dF<|eVl3) zz^mdvu1p@aedk;K%H>(3bjmEm$@^g&Lai5)_yFi8NCi^W&c32@9|C;%`=fS4K^Ljx z8pB0>*+tB9*&vs{XtH&5KF9%W+z8{U2- zZnov!-g72xAnkF)u3fB0aZBvXkDg<$p14oMgl~HIqX~2(d57Y`WP0oX3fZM}==#z% z@L3}1^0w^@#l0Z{pS&+QWkZ@3`gaCl+aw97@e$rjiNtP>#0?Zc;eoCm@32aX|0th_WVi%0&-+?W^N%%L_9cSDnLGrCr6lB%Iq zi-xNuUMIk~#n-R#>l0k!VS38T<1j*DLb&9se@i0^JQUZd3OJPZ05ah}Fz-w<)@ees zv&&%e=0X^k3ZyV~)=Jud!Y*BcOsxELd2+mlN>=hgfFD3xuH5E?-mmitC~Kbq%&G+X z!1=H6o`vv7r&_!D{WV_=l~7_m>?XlsIe7VESp<0O6AYA>?c!v)RfZd%LJDJ#9|2kP zGd7pIp*GOJL)dtTmi^W}M0H0)P#lDn<6+tb%7*H{$ud{qAb?SNhxHZ35*9Mgda(8y z79sF$CF{Sb{V2oM5+QTI&A;)XzY~D}>=`C=ByCs%2k{UybF}GMt+Plp2F-POYjN4T zimzIC6Q#~A77dl*WP?XigavH>F1tvaAE z5Ix$0k7P+nvV3!wBNxO7?V)XChK&|(ESh~sCC(3-bL_pnW$1)v3N7}6ZxjiU(vsUgMxKZk!zX;BhFlAP$YWO2RT|dtW!p;>4rYX*X!gfp8yOKd%xYj-$It zz+%@;2#lPakr3{Kom8?s+~+xVpco#?k~wWUFye6!uoosk8+UR(?SBlM@sQ7azS7_c zgllhd=V+kXEj$MueR~iq0DwdJ%C#aWIGG?Ao{O2oo4Mdn%NDU{5Sc@8@zvm!$Xzw2XZKhJ*J@3W8J zalF6d_`UD({daGD)>`-HzOVbb&g(qS3w)mjKszdWAK~0i#UN50b%C@aC#k1Fl40!u zEyyaQg?}5dgc8ln@AQX_LXs-Fw^0I!vE4bxU)A-W4XEza5UAQbKo+#6b`jZ|#oh9V zj%Y=67yQ)uq|>Q2hP!p1ASq5pH}l>NfxBO8%^-Umo5JtaTE{d*cYhYCSo%Qf)PMUF zs8JRXl|kJDhwd}abD`4kXOh3W_iqTXy-}{Qzg7oOsgrT;Law;stpSo67sppi521XU z`~X2=#zg_qFx$U;b42c|Qx2?tQ=KH^-j9=&#a0G>4&oQ~{sZ#0{@80+w1ex+Htbc{ zNc}E|wn=yN+0p63iKz+Xd&~-BZ`rx(LuPc?UxGm1d!Rnlqt8Lzs;s}-stctS^PRjC zNGRRGp7F9BFK+T*aJcIE{h}K#t@MmcPQ%J_grRv`|X*+YBpfrSaXzA)c*1zA1hV2Eua9c*cnSvaWf zp!e5MrY(JQxi{eNj@1EM9d)`zO>*p$d=Uyv?FjAK%* z*j*Yk0wRT!m`BFn3eN5ESjfEf_F;k)EIv;WvVXi#aP}ko^tNpuVD-4WU--z8K#_47 z+ltbTd&`3z2b4hohe^S9tYsl)&L z>Hn<2|A{M*DlvNpI>(D}zpp0?fMjzN%%%0{&zVsYX4v6CDN~BruT((@8wKMst*|$< zF)&BV3Cgxdh)1QobGv+Xz}~6Q&Kl?8+>0pi53wg`Rpi%_ARn_Y=JmOVj5a}BmPkV8 zq5bWI_$E8nfz4e)B9imYs!lZ#~zZ@NvH!#3fEJ|8g>-|SZoInpU z<8QA63rva)SFPj)gpGZ8Lgdb7`Q7a~V}(^OfD9_Go)oA@t|*{@wn?b}+9QeeDD+1Vy?)$TE32)5a{R|qK#Sx`c5Wx&?X8bE1p8X_du04eq`Yn=rsj^7Vop)ipjX+g+M zWiau`eEut_58Z)sDq&Y3wl2{HLfkpy(N^~SxbVP2g}c4{)jz=vUfHarq@(gV^n zJL{r%7wq);U6vj>Z|P8b)gyBs5EyUgInsuQiGC(d&n$90re8cttIx3V9|rcjc5dy? zrq6NWOehLs1}L*i3a~pg+nCn@dku2IH3%Y`Pq!(cGy^|?Cqqg9q4C}PAB`_)d?)_5 zX?%HJ6|J%To5mOYzhC3qV7egQ7E?`rhbk6}>I1oO?q{iq zwP>S`AS55hrtZ?56^4&?0CMgXKMWy8JB$YoA7p1!l)JtOLfGfTue-OxUeh^c1EnN0 ze+R_v>3%JCX6VOXJy~TLYQS=Lh{;5`=OW+)(3Q z;H7N{ik;aNgs5&XAGrC7Ar-YI*Lz&5rM2nF=}&hNFf~d(DZiuc4G!@)WW?Vb+prpu z=L^};UhfH-Y0h8xCo13yA8+7eiA_Kj&6!T}qsIY=Tsc%4`R+2FSX7Bv4<+mH`YzIkz442%M z@bctoPN`R5)o^LA=T_H!`bONx!v_Q1hO;O@>rwH4fy%c8tMdKR7=q>auTY0yr%A5f zPYo6!PRz0yI(R0taGf~;!BolcHW-i?UBqt*FR`!+SK?n9*)6VCks=aBVB!++Ban=t zGm!Mp1cFz$!jk`34ZLn# z!U^bGM&Ek)tE|v5&g633VfW6_%b5CwX~JOIcRuVRjBVK0MjenITbz^ac=Pb~RaWB2EjsI6EL%>tNJl0TLSSu*UZ##?8eFIdV5HtYO4U=LJ5HxeN59-wP&?ncMPgJ#)|K@0@(knWwyr5P6M3#z!EJp;Zv zTkBmid1{a^DACRx>-oh60MiAdtYE)@*SknB0p_N(sK0#$^o(s`)n}9xrMNYKQ!Bej zj^MvFP@`dt(s5vtQkHwwy$y=lGttWTs{k0+AkuAhAyx7_J1UX3MeZCmS$8o$e6tox z^q)Tt9SRJoblen1ndNhA+ye1c`B$ffK!-PNsO6TU?_r)-h{CNVG|Es)&6hqT)=z%) zlN}?430nXrCVT-8%v5V_j%Oy-8b3>}FRs6rkXk!yL{W>%J zGsL-?sAX#4A7_&E6w`1?zL$jY$zsna*k)QfR-z42p7g+N8#}d3R9jrBow(Ju3ZRkroX0<`Tu}+O<>q%F3ph`dE9jqg!=PsAsl$ z!O7=b_J{r}XU%Iafs)9(Kj!VVqDupJvq0#NbbyWdwRjmtJ5b7gJh;!KxJHM4(m0&i zh9{y}58l40qOOSvOMD?YEPgF;ID=xIF0mfep9>0VQcjQO2U=X@9*0%4H>sqV&l7v2 zkFlZ52l(Kz=tJ@h3dL6JQOX;S^+~H?;4qi5Va1zb8cy!EUKxJai2{NqTzViy-C72q zx%Wq2XdRm2y(?D<-^HH6NypYZWXleHOp01yUSJclEbqTi)JjIq@HXkS+x_H|X9I$o zK3>0vm~AEy=<2sfVY;tfWRhBDP3q9Yaz0{2%@7YJuqeBWQxd3A!L9D2GkGibtN+x4 zHV;Y@w#QTYE|j)wUp7w}{iYqx&;1AOu-A6J-?FT=P_!hKR1k|mIdZb(hPGDcrsfm| z@mdCpoHR_ArnRdKHnM82L2I{Z&%-68l<*Iizcb9SoQp!K)tRcQr&l1bc=lSh=?R6Q2Oux*VN*j+#nU3U5PzN>Al(Jcpq_MTv zP7IT@4@9t<${$C6+bRF_^!sJ>1*KI9Wge+||Ae9qSLov9CC~Yr@5p_T<15=+m$H~U z;RV<##s`;xUYUp|Vq7?5tsi#9Vq(crc~}uar(_TV-}0Q6QDY9uQw7nig%?qD6^Gf7Oh_A98~HfxG4oB@zkIH8kX4my1` zT#U1!e*l$Kg;fq%p=5U{k8GoBGrJ*2mt{4MaF+GYO)&XmxIe>-?S;J52r*-^1+2%L z+OR>y&-Sc;27|bcNTgAc&x_{EQoM3jPpU>ap`IWL4LXTx(rjPIuGpzk<5<>uwOc+rb*oJUT`j_x8y!I z+4fliDFwLz^^_-m@yOg~uuy02lb#5YB#(ds^0FIowDWkJb1Q4+ZIh8_BAwQ5@-ewc z0Xx16nc?==;YjlOIFtbyW18gbM6xVLp!f@x4(-cC3&5pa!mrZ0PuW>wtHW&c!Y`$7 zsJ?Kzl*7J2LsRc^s`9`hu<|IP&9m*a@u5v1aN9XH>51#^MfZpmyx5y;#ucT@ta7&o zYgp+eE1K>%r+Z~=rEg^rFLPP!i#w+e%9!StPNi;n^y>7Ge47)Rbbg0k5gly;7ZAb2 zJZ)xDmaB`SLM^VDv;0^ywGtQJ{SFt4Z}~*;rx4NSL4^_+wrP%x%J+~ZeBi~JT<~i2 ztShr8$#K16BhDtZcoC+AAs9egC9!0OPU~;8BryiyZOZ8Wf1xoJpB<4bJ2AT%w5)q; zGGcqT7&k4*yzA>nL3w%F(V$C~ZIm9s(y2n+Aqv9E$#y7Y&evg_j^MI;tR4>@>m$b$<$dz(&1P-| z7jG-*52A3T9@ig|5=l*XnQqHcd38c_^VlMQON-}x?!PA96apH@Pd?IvDI$!r(?tm8 zuM@~5IPeuJDEat%%uezQq04&kgfsY5LG`V!-MXU}ZTmYjy}(V`rqOb=%440miFKI~ zjhJsrg&%fevHPyE64R)q??YG6CqaoDJ1s@lkfgDaq^3_yK&BhvQ}|qxBN*mFtSu8+ ziQ^u}FNzqDyRVf-B3&2xMlpT4FQVf-MJe(5hI5kAsbzg~p_?S5oH6ZZ_o5<6DSc4M za*_p!TJnfZGSPjvrIU4NKQ;W6A^Ab&Yy+d9(xPj7gt3z8oC4C5W{`I56gW1k2>U)nE%irYo!&i-YR3bKx$Y5wrIECu(MHH zw*?(h7wxi_4@&DUSy|@nV&A4BRzMYcg9n^Qx;oUx8bn4LFQ1h5QKBx-q#80LCKb6# zn56mj*`%{jOiJpR_>*~$t{51&1=|8oQF)^rzF7z=j!pxn<>AB|E5w2UX1 z)Xpd16;?LJfA$JCTsK@-GqVHHi!G2*PZV%%8<{UZl3#+#x63;sgH@bA#+U=YDaHPC z$FjvnO#K(6(Q2)uHTr&!<(}oPd<09?6d-CQr(qPA-+hr?TU1??vYVb;vI?FgPBh2B z$g-8>`u+NN-ikg2iZo|iL*Wv)`$l2r?}wA3XgRcnwWBOkh{x+n-y7s*$X<|gQ;S2! zl6a#GfPWFM+;`qh&F7+2|BR0;_3?@tdEuwd#vOZ~?z}~TbujjwAc0ROtIwFmDiu-S?BEI9#}#9hsS-1fQ}?+*ion!p~HY9qe>5T=$6i=RQo8fcmZ0VyTcWo3bP zq-xh}crKz2#Q#z(j}hCRGVPz|n;WD=dV65(7)qH+UA!q&Da=Jh6M-*Y$(_0>@BZ8< z2Ybqd|AhbcJ`%sP6Bmb(kx+T;@fRKTh6Aqmr@+*4{_1dqYfWixc`BufK)%Ty?lQT; zk!M(qTgQJZKC}vIkEjo!&c?N$kGP2AeHNpOn$B2pNM;tCtFlW%*A$f+=@nhddxwaR z=@lE|Iq+u}pgwbeNk2PQo-{-%K(ua^5vzzVj!-dG3&X2Lo%=dcmM>~`9RS~*6zO*j z!w)cgUL0HerOeDXf3YG?wmNBgSL7$eQG-X*6`Tz`bra2*I!*iUBg41BQ>6TP&|dS0 z%%gOlkW5VSa*fWieXRbe_|+cVR*vHlTbpT)B8@tvzILW)hIp6|Fn#X1Gb@yBBkMM{ zj#xalR(#I^74X;$EFcnyf6a@q$kNfYDbb{Xq_ZVydCH)SVUAS4q-&;PNS?=^J|4AI zySv?S_R6z`&On2_{GNMml?BSn?ez>#t7oWPO><6$>cUV{7m+TV^wR)=w4Uy=iObF@ zl#j1T^gJC9|Sa@7K~yCW3;x~fBDt9)mB z%j=-zB+?)R_nb!*QV0mXJA|-7q;dez72#q6rQ&zyg4vgkRryEkE+8Bu(Ib;ZS#z6+ z*_jZ~e5$Xl%olZ30A^dfKwJ&3)&bR;BX z!3Tcz{5T7Z-?l+zh9Kv~T*Lzu-h4j70=v-vj(MOLL_S}3mO}2DH+X{D_cFeNo1w9O z2;u}j9^#h-6F*ZnkJo8(u03oFat?cq4t_TK3uw%oL)eH$2hcZBZ{+W5-WC8S^*cL~ zqfsk`AoHw(8;OmqBF%On^1JG~&ct=N!PpvTPRsb>k+KW2jepd|f_9?6ZqW~{SB3Y` za9J$-c?=~Y9@~mav`Pm$_tu*QAVDfWy37(Iqb3QD&gxma5Rt*1%3(tM+{p9e;a$*8 z7*3tM{Ko;~b+AUiiJ0Uhu-~3Vm{D0=CNemR0U4eEOMHOG%<&JIhf7|EO%S4)(uI&} z&)r46Mqs#|DY{}gjP+2VMeYC~IaE(1=^3q~tj-Z-z&;TeI&)k-r5B#~+|(6>K|k>F zkO^cm+;W#yHvqLY029-n9pV^*JAijEA}U}9xHo*xtMSb4&0pSJbqV?$)=tkbim;OQCT$C_ z*TD86ppUWIo&-nUauPpmM_82-5N7bwC2v%iPW^z3qJ+V4jWR1az$Wq$GIBkG@fio< z(KdH-U9X|X=p^Eyzt!QkX-!yqUpQGNWDO@OXmbqw-+y*geD=(G+zx5Mm$E~aR1fBd zp+8m~aL!~3@SNP$O$b1y=+f`3-R$U;HCmVxx(E>JH~o1SIr)Z)X|G8I>?s2qu)dQw z4KF?SFiAEKnqfbTd$!^(IXbARF9VPzMKTuL8yzy~t+S(@sS*Faj~cE| zOM@+1iBjW~<-g@R!@#FXP{lF3oseS4{9NR(vH*QLLoSg_7Anc4c7TkZQ|%%IErgTz zUE1G#ddRN;Oa`HkRtea`t0Rw6*5n5@QhFYqalixx5YVt+>W9|S)ZGXx#usvoY>KSJ zHZa^*K6ny<;O`Zcn#EFT`eeycrZLM(fo0{{ejTJdjJ`_Dd8{@HQffC~9zg7_@aIG9 zWh&0wAMiB8>B!FS-%LmUAqb#j?>Q4K0OQApa@aXN*JH*4=&eT60R$RQ=@f)WtGlXI z29OLNHIY;k7@kd6Zw_ExO%FIHMeCo0$Hr(h4ve8u2o|r(#U$2$5Et`TTmCzs7rBYmsHLvfKL6j8yux|4&2ka+~}H0u}9GOEr^O;2#^a zW!_?fJC&-$-w~lV&PfR^05=(#(sW^Lt9%`EJC$BFxJxa%8Jh=r@FPAYw{R3g%QYVE zlQI1Em8ahK`6Fb+@L#@lf9WdD^n^cdkOHjZ-W=0^L;AuMdbIsEn51A=UPB(hUQ8IT z=sZQ?E6Uf<3b!Z+4qjzgxlE)nLmf6;1xoycdspI`(^<$TC)Dq|7igxaH%~>#VF&_0 zCEpCk(|A$cn4EFh{%9OnGsa%Hp`0}bk-T5#y8qoKwhoPjr?RHK01(9|y+}$83yYq* z?Sajw=1yT9+MRA3Tg7r{cndt7jsx}5DIHJ%L`y_&iC>cOw*N&Dws;UvM1nD z;GY?GSOcci4bqFGUCGqaK@+;raVOwZWUZTf%vEzU=!(4PkRvG25e??rMr5&@(y9$f zqs?QCdKGF-9=DtI5gWwIRN$%h*nIH9*Cx)?3}RGdWl+#3VMIMdt(g!`q-$)rI%*YH ztLk51jSx?)k(9AWvc}d3r~Urd${f*8Zy+OhkDQ{N_fH4i;}x<7vPL3jvl$OY@u{xi@6A*@mukyfXKEo{0;DKZx5;JkIHTKJNmZN-YOM%?5|@Eo@gf^|yc8DSHy zxK^EjT<_L5WTnUCQ5t#KHzi(x32jerxh4$By`lW&UBkg+*Dl3Ul9kSpeN!jvioo7Z z@yGumN)zL;h=!NmlKRdo)sz~53e=hJR(~Z4o2x=>$7Sx3sbjj`T$4;Mzfc*_tlDc@R{zd)~U^k;VmFVZb>f5OVmIO54P>O!BkPp-Uu&$W~GjAz~BN!kxLdk zk;;IYWb|S%B|>aRymbUtuTGld=14=Vu-HFo_y23GNVH;u{egupEyP@R{mL!yGJ#eO=o))&pEFQ-=pR0Y6my_U?eG!3 z&x`xNMC7l(gpx7w5erRJuHQ?!$5`&{zV@00*H_ULy~!ATi+-BC?9@IR>(pOAP8#39 z^P|5JMqQ3gRTn{6^1Cv9hhRCQ@nO?_Iiq^-oj!J(O8xbHaVyeYkv)uCL7<7{)Op%m z2Q{AR5uD9aB!TLiBm`Y*7Ue{T%Tut6V~G3L#d(XHq^6;E-03hI7CO7a#|^?@7NUdj z&TuR1qYAKZT?l#WeS|QDPkH4ole2xe?VatfGOY5jnI8K@s}HHWUPVUFTn`hw-K)uo z6S21dF>!>Hz`REL!TXWpj#^;AhJUy75dJe?`DK6=WMgX=*a_^C7!9 z-AP|NHH{bc%~vB>eVgulY15weB|6%l)Vo2BWmSa9_@9C`}#QW-g11Y+zCidX%niaH!zr z;B$4F?E!K=tynRwprZxIeTitf3dHkX))D0C7FjyLW#A0L1n+rwC$`ACvCE(KB-y?; z3}UXDe51K_Px`Qt#|XI|BBHp-2D&nI1aVRcU(Y_ic8$%c#ML8#eNrNk)YOPIPmSOW zw~dUUnnsrwVQ?Z_!-GMjMtukaG#;NxnME|K4v+#O14UGNMXJ?U=Q*0i_I%l3H%t#{ zbwrx9I)A7reWAKVzjn?ac3Q*h+7rY>DhyM=vau7@7|dqB<5 z!42ZL7r|DjdMS0V^z%sV&VbO0nxe@p7~Z=)qIM35g;|HlG{_HtM$8YFIeztLvi}Zx ztbq?OLSzE5Ji4<1o@377CF=&0t&8VItIBqf8F|aKQ~I9Y+M?(d=)u8H{(yK^&DfB+ z`#zq-{@1O4@7_BLc4OBw(yX-gknBt#I(;sAM5mYsu%E(c0hkyGG2V!>0(Tt!3K%oE zDTp9+wj-svyBnbTbwxY~<{YBx#$FwfHACkH{Eg)xa@G!1v~rnXWBIz|Uqb)u%f6LKsWu9l`mi%2IrJ-$ z0YQi&4hFiE^jGykrMwQ7_T{ikHx_)@U^=xkBHu>@Q!w49?JvJiiXl|)Nq4HOr4i~6 zt+9R3f0X1uLxK%S>clHM^n|z_Ko)L@3KK!aV63`kf7L1^QdNj&s^gFsfYEVb&$Ip3 zVBFtYMbdQ+BOM5mgVlR-mWtGIgxsNhXU}Xz{fA8HJ*D+$?^!=2MEHq+5T=7;i zk~VS)z}&}ybd`kEQd&+JQF=@#=!yCU^cO+v#xVu>k2o@5Wp@Z662sjeJ}}DZUw!LD z=$_38ftZ*n`s7yUc-I47XWg>|+{Y_3)Nz)h!MmnCqz3aSS(BLd`6=G<6IhX~ZmnN;)XR@T(;4Tsuocf?s|^Xnt9hC&oi zze+mMo>CF}ukXsxbz|O5vR=prl7n9zfu>$4v|p9P6fT^>Zk*EH6R-FD1>A`7pnMQ# zb2VanVF|?RJkVl`%x0E$!+bR^bv~=k*C3M9b*K*@a)2{=6QPSEt)6(r*(_ByuQ{rt zdtQ4isDrqf65YiEI%&Jp1kc>@G}eC7+wkmNDCmW|hh8Geez9EK6~fF%(lk>XCT!NM zsj5ZLZuS6lX%jUK@}_Je@l@Oy1byycCToSp;uQJ$YkwdZXBp^qnfMB%tJH_}Ll5>K zHf+A^Z{_lv{bsL@ItoQhY0+omQu)HM| zoYZSK5M_Qskwp6U29C2nKVI05y}I>bwqP!zZfx+u3&he6mX0vxX0SK@JY?n~6shuI z&4SnrV3UGKLswQ8A@nrH)9oiyb%ZPpf<4etxCH8D(LW%Fj3eMQG8s&yRQ>=oe`Cwi8RrN5^{paJth5$cP+(03V{pO7v9@;d% zE6@v3CBS^ud_P!wUYERm9ytC|8_it=j?4kKLd5{>y zzL<;F6J<-n;1%o;BD^MhrC;LWdH-sEO6EI=EGky4)cgxb{g?C;N>+n#d=+tdHILvcP?!)KXD}GNd7MX< z-Hs4%BzMI_R@%@`?adKgFYO~HS(y8pi8g!T_`}fE`hSs0|5csK7rZ}j^lFgC=f_8+ zN#l^P+cb7d?445FwwH(?Wl!u0+@3dSxORBkKCX&K5ApYr_Ryb@m7pt7A|@uL!~dp6 zefw*p=2ey^_`eU73TCR;Pwsw?{dwbp$5$Jj>^>XD_6t)t?ZzXdzl&U`tlIkaK3nM$ zA2MuV2I{D2=#BK?Cj}Z629yPUyvYLjo+*yGOoXTNAv})a^>>{U!Bu`l8Ud^d_s*ae zIo|Gx?dd}O#K^y1fa!5570_;Khbph~L;?82JiGcF+_+|7-fmdv?DKG`SDB|`z$?aA zf=Eo3%%;+r?^>Dr#SCP;ZnD|2g>Bwq&rh73^`xc*JY?$*AF)1EvWj$%Pnm-8dWmUP z^cpy=#=rofoM;VGnpmk<1xM<{As}VqNw%<*O=fAnAy&-7P3k{s%BIzJLg>?XaeK)C zR(N36$NW<#5&7f+Uu}HzCGmM6gz^At=B;s+M#Q^(2KaayQ}qIuAA5d!@&M-9`#KZS z31v}#A8cXyuoc!ADz%VNUhr_u=d|9df`Nk#9p|2m?*0t9^MWR%{)v!)E^X6^Fh=`4 zM3hc<1?NR|hab74Cwoi)eWwpjiMi%jio_P{P5)xNjKqrJ=4a<}F|7*wKR53JZyJNF zf9Z9J$CnmQ)a@_lN#D+~8}MJ^wIsOUmD!=jtBI)m8V60yfxkmeQ&$a z4qc;xUIC|*%0gq82R>6m1v-ce2%dc^I{a7=lZt*`kW$%)&py z5MF(~=7%c({g31EEfx(hgi78Ll-=%E03}I24Zf%XFp()Y&L&Om=d06t)Zmzi1rPuA)2mS4d1DD8cq%gy1X#tNn;2nr4K_d?m;8%*9xS*%WLj&{SY(TA<0 z@NfymhHCxGC-?L~#3XnN53X4)F&Ro?;)HFtB6U!pkI8;-VC)-PVLVeIt%HIWRkCQX=(|5x(H$} zT?pU**(LYjN5f?EF%Ifq&2`xIc3?Vq5$iC=AFshSJ_D!T;%!BaPS>WIxXXY9Vf6l^y~uuB(dLN>nfZA z;m5y!_igccNDlL5v%p(9@llE%B>e5rvbs?^?&GB&#kLUX_R2aWVFwOwBW|c#@M$MW zdmKHX)`v-_Nrty9td<%LKmGnDnMDKS)Dij96PSt>2ktb&s6H;ArDHCC_#RD*k?L#l zBvBz|2C{kq9$kv5$wVR*?#~hzh?I2c$4!IIOp0QsYm?TV*gJZO3%G(DHb@uG1$we0 z6Vu4RxOUHyhOhM`Di`64Uw?N+kjC-d!OnxYbe+Ls9H&zcNn{+XSGNil>m61z@wOj0 zR_(tRW1I#q{dOuk15cTry|eNRPVuXX*{p1v)(qhFv>pc!1#-m%sEUG>^nWhGyot!i zDIN@C_2F*=Au^(1D_*0{CH{6S<{YnA_woIGE&BUvXrHYY`SHgg_GHegdEN_P{Ym@D z&@7m*?c2w{dL~y#5Bo$PHO{PhVd{e>K&Ct6sIoImXJ@kN@_|LK}|q-1#=pA23^5Ppp7Jrp_G) zN7C~pz3R_+z&Mp!T7PH7-xZ|p3e@bTS&Xi6$9Z8!Q`7-%z|VrmX_oU%RP!gAgxT4E-eR+Vyu@&Ftrl=KD@p z>Rrl-_nglPA;7+(Jc8fvLC3T8w0D0xeTQy#H1z+*+o1Is_$f$`$8jl*mX1auF%h@< zDcf}Yq}^a%qJUeIUIJ$SOc99x_BEH+#O+{QB_A@obaWG$xXtwR7*B+tprD`FEUrxp ziy>}isNih3TSiH;n|gMiSFS7FFs164$k}2Z%^HsZU*woNJt*EPJGG5vP+O{GB8(;b zgDb!b7$E0$>_yt;MKHC6PcUa@;ObW5=tVl^dtyi6Teco>mgD-EPt*4#C`^O#glNx8 z3GDuOjQ_Xc1sqL!G`p68LKDlJ;N7hVmKuA%{IRFT*^BVYQMj7BGN$6Y71HjSa!h6I zCcREz&94iDC|y(t4Jg(Jszh_uc`{6lFW6&`J$c9CHboo=9ZUu_G4v)-|wLe z!Z0L?{D>QZ0B|{iQ4zzSAhIs5c7$Bd67Yl~(XAPkyEe!r;{pvA46yn+oD>5xOB0hd z@GeRnpzkB)Jn0}NcB)?0om79lP@hfLTcnvYrcsB0W2{kf#|8IqH^0Ca%-jBmO6ZNZ z!JJu>rrV(Pj6r%z$e3YEU{V%}Zb4hR{mSPAUaNbmHKZX!?c}$y z;CR<|XC+-!s}a}(BeCG`5QA`=kAoSElYw`qG7Sxl1YOvHA+_^QQ*g5eP9AO#4^!NI zbzbCbis-hMrAks@-`6-$2qKk|uwQJo39<6VnL=`G+1k zsM8r|`o+4_PUS4TK&y`C@Kq|kS5B-*Jozn0@cQUa@q{2T+Z{a&e%2OiV_uF$aMogh z=`@%CmQ~}HS`~fEYpD*ne9mKA;=h3#u^)yH?bbO85e@!~HzrUuwqXo#hvJ@Ky zP6ZiHD)VYzm`d$tUw#`nY(`y9yUlD#=BR%}C7rMt1Ch&Fm-FoPg+r*jNSL zXG^{AqYFnT`AgC5*BdcNsrlD>pXM*{^}c!7J#V#jR6?AmOk%=WWiAz0wL|E zZLr60|LU_Y*?dv*Wx$7NO(^16lb>Z>32U$8G=G zV;q0`L5C>}I>`k^VY*s7{Ud5(0qvGdG981iE8v51B1DPmCixN*&6z3jL@+)kox{N) zoW}hXEn<)&<+tqVOMNfYRA&8Pl~HMi5G30nm9o2#4vxvq^-fap7%YzP;GC+0G(L`g z-hF)nrK9utG`SQ|0tJVX?1=t)zQo@~?X(zm#4%C;9BA7R=7L8K*k7?1R{|uLS3crH{yP6RJ5nS786E+b*I+-U5(L%lY}q;L zF0YW$(WiM#{bryAF}iu_-T5Nazn1x$4GuBEZLxbhy?67T(P+(Ds|a7NO%$+Yc6%l6 z{wrSf{(cnUjsxl^p9`Q_vCxVba-zNY`e11!`g!Z69>ojMVAVRZz=W7E-`NKGCXb<^ zp|GhQ@B?G&YXmrpe}muJcFBh&s`iF@_|XJNycBNzv6E{5>aEhz?d9n z+GL?6uz2U*-dwfKOn?Vk--|Q24V7HCiqi;=q)5pL4w?bFUk_XP+lLWq1g1YCQIMdM zmu9V!8uB%-K$|Cx5K6BOzev1whOWm#2?xn&NHzg%dPnFO7t*zzZMF9!}m zHPHqhhc<|fNi;-r3|JVVo-)1<)NMR-;|jF~ErO`s;0%O$NZb*}Cjs82$D{TZ$q;VG zu2+DuFYexys3P3LO}aH;I1bvg5Ucw@n*_aq0unE zY|jy1Pr-IZ->zhtGjgmt7`+Fmw~SF55mn+9CCR=sEh!iAJ_m~Z!Xo}H&qx%*h3G)$ z0>si`qO;)EWeL6r9G%nQCsyGa;zq!?0wb_3X%VzuKHm}ovh~6Fmv2w5spwtwJueb) z2Rzy=!Qi@ZNIj3Eu$kctT=#s$8VFGhu)qBFMD-q$pKH*xDpk}cUn;fu!!?E-mSECG zkbn=9QSt!8)_d^?&*1)2Gf4j#6y6;^h#v)mkJ^xY-|{g>X>$W+`7+*}Zyz8;&UcQ1 zePPib_m6#X!el<33Ay_-x&8jeJL4TwAFxA7?_USUHG6mug6x~9nxAPLW*`L!x-}8V z+=gb=Xanp!cG&G*zZ#q<5KL$gDcyK}D{Tl`ZEdi@t&j|hJq_{zYmuMduEq+uy*W&Q zIZS@-wPGxPJ4{7lcl!Xz@n+%-_a(SUE24uWPD;+nKyW@+lo#4V*_OCp?;bn-=LY7a zzWWd|{SYp}qtkU|*$M`wL6IQF-zD{xrAkWG6O7ihrVqXuusAw#30VfgM)f-wxR zvINj((WwpYO*6znlpV(ris8tFFOjXUT?Xag2WVcE|LzKmvRKidA5TTr!7!8_CB>N= z$lVeV(3&C${^;#si}_rz02|~7)o|1r+8DK}Dl6wpg{g2S5YKzQeq{qw>H$^&FcEz5 zgByaJVL_y*V>khso;`#y7*Gntw?TL}SrIpId!T2Jm^%y@sQ33miRAeF z&YG^iofl-?cB{*&S40881=t`XLJ1m-UX1#Qkr?9ZWLI|wYIiK23VflAwT^Os$1qmh zq58u@*kxaW z;C7?z{i<)Du=wmajI-JJp0I0|2eW|m-$?_HR=X(wE9^+Vqp7w(SM%@k0SZ{gvBEx7 zG?Nw^1DV)B4%jqBxM&t{fJ@8;@Klc!03L1SYO0gD`7z+v08n zFsNGn0@~C}S8&RHM|sCdvoUB^t{NnTmf$K{JTVKvVrdiN=ICxm!mNA|yhmS-=AE-n zF9<54J(j-)XMX5MAL@977P$2OAb1D#>yg5)5*suNg$2??5+rEL6$WvRu&8#5fdvw;PyaHxV*w>n7R_a1MXY(#B z>lr4fO2j~wSW<5)NGe)9FK`a7Ws6U`RxkW3ISzR|UcV_x=l$vS~9cngR6>7yVXL< zXkoym1)d(`E&NcLKA%R5n)|cTOPYR>zoNvy9O>h4fH3lJ!OW z5sK@)+)SZy>BG$jxhkKYAJG~u!+rF*|H*~-R>oA99O8z+M`2-{k=@nj1J}DeJlEbI zTBL)0)`B1cWs{hNGnLuQY*EkB)65r`0OaP9Wyh8Xq{K&fuNjTX-DzCUHhE9?#o2kn z(L8s-1a-li=-it>6!@9bP=C}HZ}KcJ4u?qxHE@w~ex!Rt>h|<~A3yId>&@DqlU(hv z3q9E1qH+o%I$VVh-VD(21Z%mrpz_CMMrGnNx=NyuP&<_SE5Xc&Cm zM4RMd@Np)?<*De7Cwr_x`1wu&J)M%6`uLO%@DMk;wE@bxB4U;G2*1e1a~^m4;)sof zBNm`Lzxb>n#S>N1i*$G8YxQjNkH@fwcil;1=M9v+| z*-~35tKH5e8)MV^8kC?ws*qy4AcvO?z@{yKIM+g?6oA#yJHs*n<%pEhug?4Ahw4&v z=R}G2?_Yrg(uz`{1om&Bjr0N9Y);CG6`vSyK0JQfbT{NxeNdq{==aQwjZ6a>+jh6d zcZU`vHVe<6o~-$%;80QQW2`uHrG>y^!78>J+9bNq1ZGuqmGW4(tN@lS+tuU`c&?gO z$=><_s(T*`g!`}+^KP~rzFe+>dXFtwAs61&X^eVzxvSP~Az^#|#u}dmMpjJr$TaJ< z);jraRGmTEv&Jm0#~@bXZZ+>3k9d~K#F`?}K&(J4Hj$E+o@lL)!S&-ZU5QBv{T%y4 z7Dw)f+?9$Sj6C2htZ?RmsqjR=IlF`~LZ2g;iJb;69~0vqfz}$eBT>(fd-kfyXzg@N zLdaBQ4O^&3n=Pk{ZnxweZ4n7>h*Ep|=u{cQ)o6nt-S2s8puHR73oDj_7e4SPkBfDt z-=Fs}dLd;a0W^B;NrF(1EO0ZPHawCb8RD58xBxEIc88yt4hVK%?u7w_!SZPlg(pgk z`sxk3HT4fOzCw}5_4aMv9HHe)P}a0o{tzj$>I4&SqQ`%Jxl}4@(Kkc8Ji-)fXNnGH z$y@`eT(2YtPYE!BMh2(?SuJMdHnqmX<)&WRO~=ZQ&VP3aUr{nA9$~ybTvgHW{KPM#6Iq8TCa(l*-5SlO^a&r3EHX0 ztYjkn{HVLOQM{g-52TI5P#QmXh4SlKHbs9Tg1-pX&D!fWp zHt4tCQHz?^1vT)zHoM$GeDd1oA*D~Pw_t?8YEYVzt zRvN1N*Rn3pBTjlkv?YiHXQZFgN%!_lYHoArpxW94Jyjn2 zwwcEGbyrUb_ZnSgP*hiI7oJqmwFR4lJ7q7CHe?YH0oEvMxze(V?xR~6KPG99rb~tt z&i58?XWaI>L7ldU(KBUh*;Ku|xtynn${mMk1nK%rQH62E&sY78ZPoRa`-N@^Ley5~^BcUq3714asZ4FC^oFkp0uaV-_RV4mpE>UO z#TzAh7ff5DDsn$iw70oDd)hvGN80TN3O}OcTK>#>dP?RQo#FFEL6;VsdW`1~cqQH> zL%6&odH3L>djxJ|eiXl&cT(s6ty4?9ei@FL+G^`@CnfzDv`I2k1w)Ja=+7L+7L>)p z4^#QoG|k#p$DK$r3q`rcAI4*hg=ix6QPeTiiC)cmJ)Cxnr$3hEqM3)oG8D;L==jzf zj0>fVr?YIW8YU;7BJN|k-s(L^x#eP^9g)}F4k*1zv6h0wd$Ow`=h9$!4DkiK!Iuti zyU0a&wECYE-pGnI)|bV(%5YWbIC>@}2gx1XgxWhq!ivMJ)W0{Nd0+g*`FL9olP#*5 zT^ej2Qz(=rC0}G`8!5TlU>T7ciRwkGC2`%z zdXzlT*J`7t9l>j0kV|J#D0@4zV!;AobF}|qqxC#P#e#a~Sha0yqB{L@F#m9K|9mE= zG_8!HTvGY`nSp#Sy(rVnNbN<+D7EByy1-+#GuHORa;<5^BrRe(3W`mA#2R7rVS+y2 zi`s4M2V6p`@+0_2J%ne=Vav~K-W-U)XqHAdava(0NDwksv>sL9-PBJ<%~ncG(40^KQ&+a25i;N98Rb(O zC?V~Sx|(X0w0h_Bd7q(;#Rt-00q$dM6o%fC(35E zNL7+7VN7Jwcy=EI>#>Yh|Gb?^8Q1V<*hUBC{qmTxOUGL8^7kpV=60sQh?*Ci!lh zUkI-SdPp@}t~0|SWTcT&27JpRxh6G9-jPLwT#znnS8HNBCtH#icX_2Fs@Y{CH4>xF z^lbLrMmvQD*kcx+ruMN-$H-TTphr_GbhN{FU8`*(;Y-=JstbVic%`s-bNso$?0L2_ z!!C=3hUl|SQCGBFq<+{I^v}hdjk>c6CV^x9P2HRBL%hMyoBN+&mKlqU9odx?>8ZmT zww_c;uj}bnYS^LSiT)?wGAYUm8$sd3A3Kw*C%h9rUu@q=CZWOr4?(v! z3&Lc@x%a+>*~KU3fIM7$xOtRhgS4#h4k{c)4_fjad7?!<)2zzSO3q<9o(R`FveH?3 z;V~21^d3%XYtg(BC}uJp^l?@9=r7v4)?3L5QmGY@xNiC!Zox2vqIU7;v{5Hzufd6z zw@2g|YY~klO#f_pq1~P{d+gG=G}81%Lp}Z-gFu76m-8 z{TT(lo1BL!xGZ|l(bQA^V!zhv;lH>;8e8SJAx=7L#>yzI#vpz-wDI1M5b{?^DxFr$ zstT7>+~5mYnH;k>S>|Fnh+0ai6v_KmZlu#6R`W8OunO+2ck$U8scji~d7LA`yoJS+ zN=-BPw{kC9U@|yEaecQWzHVxKi7d22zxd?%Opzh2H+`#4&(OB{Y%MRQ_=Id2qIoK6 z_+%)J;eA*<+T}>B&G{)454nvxhvOh;v>zmn9zHou(W~?9cCKR!6C6?fSh(=GZ@aAx zgOmBf=F(UET1;(rw03oAu|tWEoA7$d6|x+igDV+S8nag=6!ow66>qDL7%+CPd=}Ml zrm40(k4fuRIW{XKz0 z#zLEC9Z4^$JYug#KmG;Zlra3|HbR#2PPrXQ=WWZbY5u_K#VdE@w0kr*RxCsUG2NvX zbWX{u9DM~O6EDA)K%*J4dpc|XsHXTPa|J_hEA{d*&$-o$rEJ?DPu=AnHe$^D2Eu>M zp!%x0|A(;m3~MqAyGE6P5l}~pprBYF^d?A)TLk~?UfrO^? zj-wO_(g{eDk=_}p(!c!#nCpAbIq!A;LY}twz1zCiTK7V$Qz2zl*t@*f{`x}Ms-LB~ zxgXpg&{261Oduld-%3mX)+yxl$eufYM@iVUT%iOWpWQ+ju}TaRKo$ zpn_vV*$q^KjcVjcN;I6#33|&n&c9&Nde=-(SYtu`GyUJuuZ#&Zm*)#u7(V9!D$Xd? zF~mI}{Xb7cjxg*VXRLbT?DNnk`spva0L9w5IkaIP23qwoD3vFexq!pvmh2#7zQ0w| zYv&L{h23Vt207Ne8}82uG8$NPr|4vf@zW6wZ*vBgUOtW~n6$4mX=rRv-peikKe!+> z+s5c}IG5E0av@hV6&(jep26bq^;edOCOVxpvl-#v03>98q1zE}kado$^)4~D$497v z3s;=osI!L6VbXYg)vHS(PV;wJ*a%{KwLDG`gl`P(sJK4yDuOWymMxr(v{t65AML** zq;H#q5nd=gk6Z#~1GgadLrC1?g`20ObhbW&(qN0_>ZtT8t6j7K1c=M1K0_*Zq}s)u z^SnU&?@U?z9j{B?IxC)s@iu+GBEhC=-Zf}9InGm$th&35aNq|RfIH?UnqLR3^CT^t zJ~rvMq#ptGODh4yj7*=4;WxQYAC~-4mWkG%tc|8TX>sBISBtN+Tj;QPUiXT-nRTdH zRG?lM__qEVp0X-I?7iI< zMk#(?6IYcn>%OhT!fA~C{O@wB3!1{u0aN+gl6w83LIp!x=Z;>3%UL5fp=|5XJ98`J z{8jIlY!70lMmCjMG6!Of*^S2w*^M)Ds&MwE@3KPgIbQWEPN|$ShO--AFI0(1R+TZ= zG5tBh-0RmHbCU?em!{%(aQ;%N6fJStlWDfOl8J!(*iM676Ny@PZPim3Iy3>buuAo? zsgC&T4o-xHl(L%&pg9@!}v(yyQMM( z)%L^adbb+ESwHjPrP3vBt!G(&ys0)})Y*9ku2<0hUHxUCFzEZ?Y#13#J9Z9X4mUqD zA6_>ZhJgj!U>pPYuTot@z02xee<9jrDPy?7Ke*NzbkGZkI{CxdBQf7cF0gaP)=iWf zs_Ez6@-4u^xy?HTKNU=fKH~XZs5lac808Ka^d{@B<2daJ;On7$Q4}>!juQneQU?vn zgt2@4&*wb>j=bU{Q?Y~)OE7EJqwCmsELtPGKB?5o-KyW94%7IQE07TOw&vLe=7LzH zLRprBOskH^8o(uTozlJb!`=vwr_6fD4VdWC`lGfDTn9XjpXm)7nO`sLNIWPQvYju{ zdjcN8J19!%&$9?$_D99z;^X`O2HZ4FuIah=jj~zWGK*wX^M8LSXFw^hX`!Ts{wIevv10vG*@_8 z%*ME(FQflOm$Z`xb zT;Ya8i=vQ9x_@lpy7KBZf;R{0(L4h(TB01{>9Si{|SZF3%4{)-YIF4wbVF4FJNFtGt8#I_)aB$M*SEQnlIgJdee%8l)*0fMc zjIF7v&{R5bi7V3L@S3EDjXT+cEQpn$A%BXLzli%sg@YWDq4;th-6DVB`kIH)k+?gR1xF5l+x_jSgC+$p=#??-M{j65a5rV5H40QKw|1b-wUE9j`Y;uI z0L@nk=y`aQkio+92>>5Quf%oE9+9UJWbZ>UARFi?MBIVDug9H^W&G)}B?4HCrnMuk zI}%CWQ)!Pf0MRc@2Wl1#K5T~U9W=kep5-|uAU=`5;e2!hXYV?Ix$-Do!=kl+g8|($ zd51HH%eKP=eANw5jR#~0kxoKHH8mVtadz0o*{9v4_6s`-f@EV=8&$Ep;no3UE=#iy z@7TO5I}K@z?~t15r4>(9w8!Pvc%DwdGpxWE>_#-lQ4=>C^HSn&$KJ*=2Aep(dN!TU z0QK|KO(=Gh!*I4QVx?}H;_ozEehu=KPvwWm73syma#LoJVG~|; zzvF}8ACDT55Bs&Hi!^+Y(sS`OEAwUagq2qwojlzTLEA13>M#Q%_*_%&;U6YvSwgsy z8>l1!$NQ-~CAnhT4eFZ~FOZavT%K6<9scgy`GtD&`tW&)jMp3k^pjttv2eaA_HNb% zS_dNcj;x|A^5ROuDYlnT`%?tdC`ERD%>UKTYuh27&$pcAVfKYV^kMQt$>|s_s1Zsv zpfC&L$Bt9;6k|>5_$8r&X!6^Xu?Brg?a0A#c4@r-mJKJ{YI^ZNtGk;s)N{uJ3U3c0 zMfa^(+?(WNJ<^})gNs8kDo$-Qu6+o#b_q(7;xxGn_OhGRp=Kqw>`TlhL1sk>;@jVFJicI|CTal+B+6*{G$k#@-_k6u{O{jYa;v2_Ehfw` z2P$~>Yx_>esX=cz840~zNFrg_y!A2khuCQO9A&M3HE9O${l=YMRYekDCMx%he-ai7 zi5cVtj^|plk+5@sSMyZ9ipoo)&nwKK|Ch!MHH|sBfv;|JeT$c7s!ap|FSx;%h?;ynzzzB8moUy?2Mjphv43|!bxC;HB>G)B?Bdv|# z)#Pu?kQdX4`6_mGJf_w6#mtRIqQKFev!w8GZ}V%CbRKi`H9Q;q+HO!suPVGt&EU9B z>vLUs=$8tE$aOJ?i^CsQD^Gdc;Q&k{l)AUK_n@3TDJD(@YahR0YV99Y;?Nt*Q)cly ztVZz{fEFk0!bN0B-3RV#!hW>Knbd{fJ}MW;OsA&VCt9nk{g)kZ6pzMOO-m+yBRl`) zQWD_xo*C4oz(~KOhQ7h(xR)l!PBlWVjyKrI3SMI(UM9|?(2bftNIZfD`4Dis%6+4* zN(yuRMSfHQx(Rl?top79J3y}y!WymAcai54{%>)5K}Gymkdd!gyD(Y%_ z()cycFO;Mxym1Yh7JeJa0HTDdqGJm@L^;3kJy9Eh*A zG$KtmNUA_UpCVn^wDRZCb<)8q8Q7&izWyfD1DzK4ODT5N!$jJ_MlsG#pNax>+9~pb z2{s@idv*POcg(5l?~ECa8+V=Z#a_LMri$d>pW^e@`|mM5!COe9^?4k$NgQ?%d7-DQ zc&xYCC_OL@lI$x!M5xj;Uul;j)0-Da~x4MyIc zVfZh^Hj(_*NMBPY3S6&6?teOU4?K^8w2a-vQhSsL!=ZLl=AbD~PN6PH zi={_Ef&hpS7SaHcV95}2?!xNfHU)?p!@z7&)~9-6)Z~N<07E{Qle`Hk{=QBEX{K#p zcLR~$tS-pIr|$n0CZ`ltINC6YxqpGwb6~8%*q$b8G)I;YG(R~Hu*EDlcrG^f(P{EM z0Ah8MAFZy!!i~r~_W>`vFbPzQq(%I4bH7OLp?q|ksrh)DTTuHG)K3W+bojF4rz!jJ zEMh>o$|k8S!V#Ld13dgl5J5#%lf7l&Ht-e{0&IvD$=anB2u_;@K8m7e`93^N0;T92 zEQ*+fHb4rUSt!Z1ew!y-YEL_+@eov$O$1{3K8jK!l`c9+O(6s+9sNh~jJE%6X zgQPX?Wi)jrc--_fXfBC{gN>qg@KR2|0LCD*AjmQQ<4Y2)00^MJPk2C_6($+KQb^|l zk#!aqhFB{Boq(Ch2{6?spv)HU1>WC)1!XcQn==5D5g&28`(y+d1U>|e`AL_7?J2<^ zdk8ljp0PaK8F+bf4442;15K{@(97u|KmoA^wRRu0z`Cakxw+RskPd$_G_hyrC30w| z*g;h|N`Pscsj(Mez+*@ZD+uL~Sq6nHii~5*)z;Cj&Y5acSUFBDEr&@4E&gKiMlZQUnMWiziM|PR2vy(c$;0)M2r^VI_NneBJ>1ow*wa*%g_-d#;`ZTZ@!bv{LD=5cQfl3)XNlw*1 zb>mY*%iNL@E46Ee0?!jQ_Kg^UBtQhvkSP27pLRH%_qa)#1}UGOTFE;N7rtpic9W4` z$8jDAy&mzaM?qOzTfkV90F0_A$>qw4MNPuAaldG;kRBz-Wm6%*z&1; z5J)RNt$@ilO3l^Es?Jshm9Jj}`m)WHW0j{{g{QXU)J0DqBpFJVeD4K_Ma)!eVwSQ6 zz~X*mPALQVs<@A6wQ?7;s>B*T&$zlfncSODs|nNCcy^TVW-pTVo&0(fuQ z=E@M3Gni{Tmmd=t*5TSkBRG!cQISPm^~TM=Vcl8ht~gsGdKqO=6L!go|<2bU`VQ8MK zI}4=wQHt##R}%Me1PCDGcFcw{atp!&W8jE7?q36Ua;wtqW!WzSULOy|a-26Lyci6s z*|{%uMksFY#{`_x3+&5ytp5CcYIH8Ub>nDf0f(lNUMIS6wtp^(s;ZUM3yi)t)l?;a2 zlk{HX?<3ey*Q&7Rco?+C>@p)E-AAuh`E25CD~->ZzjpZT_X^)YWuyD6^~IBlD;q}{ z-V6wpH6jc0g;2%ZnOf~*g)@aE0R0}qAbBu_zDUH)Ehr-z=O$W&4z|i-UH4PG?|m08 zX6ASGRPYGcEh@v$Uw`LWW6N|@?)_pkm=`EgL|)-~mJ$53u($38|`_eTS3XG(zF4ij{+mX6g)myNs5h8-{TiFz)70Cw4|(=RS` zZJXvt994V`g9hq&hxshqqu~u?=c|4TGWW(<%iNLcT4jknh=o$3rWTVrkA+1Mca54( z1|lX=FFp<<{Gv2KhoZwg6%UdNF*Kn??SMuD2Qw(h=DWQ1gs9xw<;rsVsw1}wp&6%IqhyHpDsat*B zw3wXG&zJWpgcu%8xnCt;RfRSz#pH^+pQ9U0-_nxy0}@9%_6n!G3UK%^&YD0N;h*mKJE)~^V9|+7SXZhA1vC3?1z3Pt(dG=n3j?ffmpV&{ zaA6IPIj~ZE98?Bc;Gnx|z~Z*LAUOLQn~ZU%E2xE03H*_@oa)$vG_B}_;zR@0ZzP=N z_=T-D4vhJL&T_~}L@)OCD}^dEq;4hup2PXm88{4C%)(f~BD;FL$db-qOG~Y!H8%Fu z5wOx$q957aUkfxO28tOzNS~7A^8c8bPNxX|ChkoPH58dS%jba>_h2!Mx9}FI$$odr zTjsp>@K>|7id=Oi9AoSl^0|7`=PP9pMOPez*c_ec5|-fT1I=B{LJk?{oGrvowlukxhM8LMA|W09b52VW=1Fc!mLK3R$vLkZ$kVh&$OG<>LwGJ4qu3FSKX!QP-9P@c?AXYsqt3u<2hP6FAF ztJA(hj*v+0U0_MQM85*7_y0SSa~bz%{1Dik8@96QGmiav#p!;Kg}F7MX*TWmC5b>e z7nG*Dl6uq}_hUMnGb~=ZsM-ViO^~?ip+OMfp2HOEt+A2Q)ZaAwaepQ>XHO4o0P((Y z@YO$p{YfW6zSFYuGi!gMkfc4aB1kJIb&d0nOA#-*QOD0XPWLb(!nB1U97$Z4iVHR1 z512HgX<<1-%0r4X5}}tl@2;-}9qx{8hPoBbaR>OBq+R1zJaB3+w-RpzhwFTAD2@m{OQ}Pnw_B%fGlt5O%HsT8+NXVJKd{T zjvW>AS^Q4uEcMyvcg1UFf1WHeDfyMR+z+yvycAvL>7?B!x?5}Zp1wHNYW#^b3fNQ+}5#Bf;X85Up*7N2LK`E zZ|X-u7*xT^1_5`vF2_Om!^pM3YTAj|HkMDMM-)UEX59U(Y-Ix!2W1o`29|`;yn@a2 z$vtRaY|DYWRa1U>b77#P)y3AZ45p&#Za*s3PN_U_H7jn7FDxO&Y(yQz8e?=za%y}vfYhPApmb%Sye4S^IW zDzG0&*y4OndQIk&*Ss_jqTX_lBP5!z;#7u!Pf0ka#DhaP(gZelhHQ}j2tgSfdqmn& z$-5olqRCqnOTt*B&=w z^7Ql{PXXwC@x?J<^0l3C3tE>iCY^x$;_T^2&lZO2jxULq?|PD_B(8RnM66)jI%KAL zmE~gam%pUHr92)F9ko({QG4MNqa$Q`X>up8@On09+*<}> zE0cQQ5@2?eyt?1j`vVZC>n2k@Z0D#zzFE?Cq|$wK{0=CIyGTzCKpAlbr^wDzlF|Sb^b$|$5tKOX zt{KdKXv9&*V11d>kA(1^{$b>6**+`K#V9LQzJwxAX(Qo%!2d$&lu*+dJ6ZXPx#Qk0it;grA^$GrxD6y%oHL*sNAsS*`yZ<~7v^*lXs006 zp1mISI_X{z3U>4={-ZGwe?lM0eVn`~P+qtsn&X2sL4vM7fjL$$%dZVDyTnqDbK zVKBjWE8%>$T$$FJXVdPv?&nC@g8aL^! zqduVddj9yto;>_EnDjKzd{CODnHDI71y}a9Y>qp{@VF%mmv3|&)wsHM6yK(>Wd^Sr;TeI|FoC}8Z*ym5n#t0 z0-f}nYJpdOIpn@$ylg#gM;{D8f@VmwNIJOzUm)w`!kz<{^e4-bYeSAjNx;i+6c}FQ z?q7ocqy|=W65x=Y2K)t_7c1?XIBC$i-A*AiMV8-23c4WAtl!~rN2nWU*RQeA@M4Ha z^aBV&J8+@4p8(L3b355tzUbe!yU+p$i7z`XK@o)Z{E`k#Wf7l_Kgd)^*t990<+p*P zB7B18Umc;VZ0GxMwg6B0s1wcOPy=j#^Aj<^IH?^-0OG=L!9qE8(wqjy!H62Qh6BgP ztSW%+EC%3r;|%@DZoXy?iYHtxdJ#*FslkrD*(u7)%j;_34G&771;!vyPrXY#?$l>b zwg7S<4xQCyJb@9Q7$h5bW>G9R1r}pb$ORB8CcSit66mmi^kTpa81aZNJOD<(yb=ES z40=Br`+(xc5i&~URlj8BrjX2FFdFOa4vtDg)(~*Q`_#UPL)TUh;b=>^Rz!~gVXLIV zu-k95e%xhI#{-$l1$b4F*-=PO0ez1I)VRhF`&{LUGx9_CqwC9%a7OlU&ofy_4AlC5 z#K)+vhjM}HLreqhO!35PI)vX6aAB9YaOdBrEL)JL2tZ{9K7RE4*-sT~-iM^;xm!z! z;PEK8TgZziCCi}j{Q3;)G=s}R*@9(YflC5P%14V&?#@&Smbh1}Lb5Xz(+%VR$9%Y3 z3E)3<0CeYLOX{S2>zO8LApcQH`vGr`z06gDDj&4X8%(!I2SPckHnK=9>sT8*pM?p~ zW)IiL;3DmP)_n6@fDaQ!%oztqRGq~UhpT!^KsGA@n9OwnzF?gd1~pTR(A=SSI9vT* zV=kZ=1NMHk#A}t9<(=W8&G9p7AF(dH5L%R5mSGLf^u=Kjt3{?daQpgLv1~~T$VdfL zX^ow~(`8&vPPpNKxoJ50cz;Nb4slV`)plAEN|lm^lPzgDIWr)3KIe~jJ6v)4pzwy@ z^Fy5agSt0mm`A+;ka7mK zIB1hUPE`2Lqz>#&a>@yX^A)RAq+5cGwc&mHdeLj`b%lAqVWaVufxaaGNJs`}mBA5! zactg`xG?VsitXYqviPv@ilct0EmvZH$DPmSk6b>R^+e0`cj>`&il?@S*R1ZYb>suE ztZl9UP2=NsKCokDw-u%!NsAXl!t_P`I) zrx0J!u3il6@(bd;@%4Rg9OUfJ8;n`I1Om*`PV^O?A$zx_z+Jr-Qu{ov`~>uO2;-oG z7X$PNDH286(P#pgGsrj;3I*OfN2bgurcQZD=!Z$uH(G?cZMr9E)Ps1pdkFzGhO~0J z;v&C@E#PbT?~|lT+v$EX4_>Lf$CY-U1j5*7OHpiLJMV(#+&fR0EZUFrhWHTeCjG2y^WW?2lQO+zz{*dWDF=NzXK{5W8Ay>ln>;dt~obh+V|`D#HO@ zMK+%OoN78vDXMi!h#@%t^>wb~w1AZt zK25{;M31{>Ksu4%hmO!2WkONnwsjWuU&fZ7#c`)W2LxL??o1^`<>4GNK&^iwm1q(3 zy31F}vdqmZZFiNQN*tW6zJFs1J&jt(dCk#+DRUQu(c0g_kOB`Vh zeCLEpE$96q+K2?_A_yAm<-gNl(GHgBhZJEp&$6Cjgddd#bc#`(4XE-S|9lAnmGg+x z8k)}HyU<{N_H8*Gt^ml#e6A!x@|IAV*30Iq!YY=t`%(sUtdKiW93`DG+FU(3hMu6I z_rR`!`>tA&q64rXGs;fWhZHd{?0EY@62Nk>I#(7}3KiAhT||1ecIwA`*WC;+-{Ll; zV%q0^>jg}0hlKjjx5saU>d6FJ`Xl9jbuqw<(&_Zkfra+F!}L0L9vsf~zp&&bq|32n zx`QQe(9nWA5B^M5YxQ5?YFr%j=vkZj7t2m`8gsg1eXXapX7x4Z_7;#*5tI%Lg?-#e zPGiJv%$?0PvC{C&l+^F_cRc%U%>S?IL$B-j?KQR}cF9>x>F#}K6E*A~sRmG7OM&~Dv_T!W|$7-4f!9gKA7^#*Q7xT5YzTxuCWSq&|G#_xvIf^~e$WoG-c3SGBg&pu$QCjzHIPrh{|yJ|7y@3`J|>EC!^YeFkJ4l z7P>10xPHuDH{4NRdcGt3JK_*PO}0$myiLkw2Eod#fl*S0u7u`wk-OV&Wj>!58q~gSL&=Y{138M;@5`{+-< zOv@6{x6$(Yc}6Sw@wFbHHgg$8adp*Fgx<67EMpOg9>$S+p6JPgpDBw}r3(QZyK3X+ zLh7|c8adbqtf&d#$_Zn8-f3Px+8qHONst$1P!MiCLn!6Qrr$5+{h?w-5F9tl`|Vp- zAiW%3Kpn+v4ISJoX4lDOh{|}E14CZ0dfDY##M1K~_>Pa0Yg`q|&KXnx-01A5h)MsO ze(TPo6ydn_(kc^*adxxypeAz#pVVG8_;%Ub5i#UN^9+~^Vly5L8LZ;^5aT6-TjniF zq1=(C6jS@tjj_`x5JO9wK&*}C(~rkk6cI~Vv|R)+LHJ>? zX^NNG=f?}Rq89j|rOZnlKbt3Ur6>#7KxQrm*@KtSo_Lq&YhgrnGGOID1zK#s{~^)* zlu2wixENIEU~mOQXV$jD*N^#yvKOxs8*nZrC_0q@YYks#pu`ELDHDw5B}W-j z=WE0-8hWK708zASme&+JAB?j|Fp2%jaJ}G?m?e!ggt~hH{6Ip!-O z+o2zJ)i9w(Nm}?pEXN$*+gL-ynEhr#f934)i5{H1ZiDpV>$0wIX9&W4IX+Db;kfzn z-tlQU+M@WxupQRSqZsJ-l)V? zM}6FMOnI4s1Qi|XyITKmilJn4)k3$jtl$O~t>LS{BV?06KeOi!ko7=?h1_a|A!ZIk z2g0e|+NuKQFvP~YSnml5)aB5-(jXmo(_#1BWP0op*@PgwK69(K)x4{%M9< zt48n}-A#gto_P3}*=f}Nw@_}myqq2NK&g%*`-`gZ?a%OG9I~>yx(ZO8hD*ouIZ*^I zi!u|jXM4n|3mzW!>Dnc0<{q_I)~Bpdu$R;txJ z@FFM~9u>8Stf_uo-$2wh{bOUNF_hlfnX7C*XLZ}rNDMR5%fNo0xQ9XPJnGrd)q%?Z zZw$jbI}6_*?gFp-dwSUEw3qktepmh0#Lmff@9WFt1PVPe>aqy%QG{3Vhv+fIx-Tn#^{mgqg>h(c8|vSVWdGMDUJOh>?J*#VMdKfzeu-v$TfijDZbQgirft23^r zEOu%fLytH}@J9)94=eM*d|+J2FRfxhZ84SzyH8I~+k)hM;nf*A{!SSoP_DpA&8x~i z*RCe*T0*Jh#~ZTfM|ARqxYsp}&oIRgU^#s$eE4|V$nq$trXM$(R>tQ7CKQEz8wM6* z`Fj`zpS#+g&f8q*bUl<#owjT~ztV3!a}X?h!H=Dj`>s}V+{74at{8{?Lh;;s>D|x<6Dxux8=)E@r#(IIkBYI^ z&P$IiGx4a-v1@{Lb!;Wwp$nxi%Hy4S)1VhZ5j32>%c3K+cBk-EGdT%M z1i+jbLN%2I2fOKcn}BVA*Pve8j@(2*lt` ztmQ7@iOu}WXw5=dQ20CNoQ5B6TUhhZ@Z0#(z3^~pqoUAsB~6SKUW!*|AodgDssLl8 zR+2wq*0a6cS%!Apb*;~r_^f9mjv0Y zlOzV<$bJ=~__0ViEi~L2$f_1TvX>-1d1)=J`7U)uBsDUBlX#`cR8s?&qhh7eGHbT~ z17xwTJxmii8P8pOAV$eQs(=F!Me5eXFjJAN%|iIcbQbn&J9K=lBu>a3m*KYu%j7Xi zA@z@W!<$?QWZhMC3V{f0OrgR43mGb^0Gu~P`o3T#4|9Rw&T?Djg3h2EAyLnn2W4&W zkbFC&z&k2!ahm$Mri$HFI*LI0haiYOTZ+yCZQ?BMQJ`VM*c;(q7{f@+ASja^sp)rH za+R*v z#I&J3L#AVX@TLs-k}dhC&-r`FsOk#-;Ns4Ap;E=^oV86ybfYa3-Y$vjQy~Aof8}CJa@06Wi5Nryt(E8 z(S2vU5F<|~CDn3CmAS+w0Z7rVGI2I`I6vgFxAjCS6C}s{u?*4@ZVf z@}3}u`N{(OR0}q)<`jv(>AvJeTsgi<=BZ8lrrYo~!+qCsbvJ{D;bOyQS_U}Zecq!B zBP#FyjAuE$wSJeX?dO+QMh^BU39`ai4v;a3WYAuwN{IZfp$q7iwCp4bG6_pPN5WDy zKOKI)W&7=NGDR<_0zybP7TnYYX>_GJrse`hZP#g&_t)~yn?fUXaAJ-2e+aECkb7|9!#xs-#WCe5<4)wi3c ziQ-VL^u@2GaS@<_s62?BL!ctA(waf&NzDf_Q~zTw4t0mo*Rq* z*hg<*{1CygkG0-|K1vZrgHL1zc4OT^>Q^YbV$c(J{<}yKx+sht>km)u!155vO!F0$=T>VuKV8Lk!4PMqc6x+&@+yBOZ=o~2cYzJWx zFSO)^HHN#^9~R_ub?d3T$wC$|l2kSyR{qL>G9sifWPtKNNLwpBv)CZAV5B+);u62x zC!c2!cx0@{tQ;G$s(P7p*=CVSA#@3tGoH#Dli*+$dW7tb-yl+|1Eu#?i}=G*8A5&` z47bMI225I_kh|v-!K;~%BA37pLKvKHKR~t})+yanZ;D1O|gQsw4?fn~xyPg_Fr@ki_WsxQWH*1Uumu1F3!+RRbT7Z@|HGbZH!r z8=fZtvoy@LE(m#o>?la)k3;06^)Gz4!0>Q+Q~fJ@a>eCSTHs6Dz0q-|?b zYk+hZ1Jy*-3rZmm7N&WPs5EnLZVgj}?}OAh{@kN8LZt5otrkrIg+5P$6(OCm7+3t; z;NNt|+d^W%4T~8C5StI5sco@Iwhn$YkP8$j5|StbZK)l1_IzIBAZ8Kd)8?}e_*sdN z@$3z(NgA^h*X3YXNLw$Z5J_!gpHI6(8o*E}!b$zJ+=Se?NNmG;DEIAK%|G}t)SBUA zyQYE6w9Qn7EV8uJ4mV(>u``@A|tc`Oni*rUL^hp(NESw(&Om|h5dae0P~}#Q0`;9 z`%NHHl0yTo?gm*DPXkk~+`;s6cKp2^;Jh}r*QAS2Vr-1l*%3kRj zX+(+kQwE%b6?Aq1Gis?7-p>KN#h#{s2Aq3$4rKg_S%9ryTRzb1*#cYQ;z4bYF-d@G z(k?oJ)OgujN@Y=AX8;*&18nD$Arj>^)?cLnXCH?1ZfpSZHU`Sv)t7^`O)>y)FAEVLQ@JU1 zP)*A4;;+?{va{2B{SO|WOlFgb$BE3<9E=lw=&YOuybctyo`x_FP6)`f1!=J&uN!}R z0gH*@f+w2%WwC>hU819&7#5;jdqOazHNGTcy#ypezPp@(tPX0N_w(4X&5>6glHkia zh^m{J&p#k4;LP2(DeKHO0&;mZ#8g&mo|YB(xq>UftSIH1*aJd-=NGaAlr$EKyIeM2DW6?gI`DttlYahU_rALDZ&qDsmFW+*t26$ZMVvmL6duh zz@xBE-3_@(K95qi7afYv;rH|Xo;GkUmDGCTC5*RE(rc$Fe}nA+?ZVB{3$gE;hGz%l zW%R_rHu6dJZi$|yvR|D=Bo1WK#?*MKI8J9T`44e${$zVQS)rmcwF_pn#eNxeL7AUA z#`-%+c??7dBg!wWa|6uwgyZi~^oDX@s;Adg zj@28Lb=&Ja{2ru2e9k-UwN`IZ&cuH0WRe&ntvJQxzfZ9Qgongx834aGX8W^%?^0(( zA5R_UTHMMJx8$+W7+ai}<~NV4w$!~6inH&DI+gxdeiYv=5b&3gNtuiM*iPO57 zuH%(s1!jsfM#hFkU%+e5lZZwl=5VufCz}_2QkT&nw0`uUI{Slu5(KsvxQaOa z2;6n93F$0TzGU79+1nn7Xu!91wtD;J7?h0uUWIJcZ12z62Y}jl`d`4&uz)9ZxX7}h zS_HQgSj}jKZ>8w`iKFH6__->2<46S>DW>=zmItBAki}$~+lsXMR2Xj8d;=saA%%3M zoM|Q8Hb(_eCX;)8q#0maQ&E3{Aybr;>{5->w{`$!%)Fao^BCBA`tbk~#E=&@0>?kbEGHZ@eR%;f_UXksHNKCmBEX8+sD z-8yD7TwK}Dc!&VrV!)le9iA#hbm<)iv_u|LIhJa`OZZ}|hg$-Mahn^t5jdzefd-$9 zg6nM8`_b7TNOJ+E&Jxo-a1>0P-Rap76^QI~g=P~-GP3`+a7wQ1f>7M}=bSw1;9^Dr z9qX{WkfbuXnfd*NlUVqCEHWDMTB+f~gSEffD=jQnH5A8tO)6Tn1o_10m$(80VtoE>HNe2*oTK01L@4tn0a&TK|NpzedqUgfTSi>fV2>i z_ALjyhy}KEc^;`II#W-8r*Jz2K)lhT^^x)I3}A|lt_H@71(O(1L;Tx^2Vbh&MXm5! zXXQS*TmgG83#5^e(a*GugiYj2lEimuLow)NntE{;N*N5i4|oTverS)3D}Tj?(IE(K z5OIlDXDUWif3q9pm|YG%wJhrs0Be3%he5pouI!@Sn{ufrRkfc-HOQn(&rv-D>LlLJROK_7%}!d zv~CPr&4NX*A^@M}ph$F(($-mp>RKiNh?Mu0nitNkS~%P500iCel2;v8&*17TY1RSqT>DRbQ=hc&8Eh7h_<@t1i3nAD|MIGx-o+)e(NAK8`dxrFybVo(dj1 z=Ad*0N_z<>^F$>QmX0N$C5|0BrV!FdtUkXJ7&s<;$B@DKgt`!3`2<3F* zbhD)(uZE1hX( zx`sz|FFfqgqnZ+&#<3AL7e%&IC`r%&`&8JQxlM5g*kUY@f z`*6ju^y9r$2L1jA*f`VDcF+=Ko+%l-Tlw^`ch$xi~bq`mm3c#)s~#R%*1h2!Uy zjppcXB=eZ3ZTMxn`TFGFt9N#NMo5xsIn3gZ`T2N;=xF2=O?Zz86&V5eHsY1d1v0 zi^4qJqMrI^j;XwVXohK?bfqrc$%H2%@*qp+odxYieZVe8Hel!S)l9v1%Jw^1K|+r#DUq z@6pSad;jW^Jo&WjV=ESofN*-Vc=8PD1Xtjmjm^tzTVEcnX>pr(-fr_)G&@VS#BtJ1G~qCiUO%`;LE)b9H~FnJhmP z(zZbSzeXjNn9YgMYMbAE8}|n&k8yAv&SW0Mh#B9_qpssHV@x@j3{R0CRjeDoq+m~FV_bPhhX$aKy-v3nRgXqq z{(tkqjc5|d7#PF&ZMj(pP)%mL`KJ8?pjBw;yz3n_Z9fu`ZFW4;3yB({CzFEB@wcIT zTigr!s4l*D^PN|y=(;(G_sJ|+7$vJ*#AO4P zG6I|-DCKCKJ)mX>ZaGKGvea?^de1LURBO?4IsRP5-YDJv$4|L$cXP=R+OCbHJ71$7 zO^7~XooiOwP*I&sD-p+}@CX-if!?dZx@Qw4RB?5xF1KEV>?Wly;HBjE~B z&+>D}wWSA1w~Zj&2Q-kakvw6FU-;3O0r)VFs24%0VPc`m>R9N1;+>Yf8oW&BLLb^gyaxJ+K-WNFJQDX&0?@x&t(Q` zM=iTOwn2D5CL8kKCe6j=zS~_gCleM#uY;;TX!t*`U}gpr4*S<_b*gBsxLC8m&ZAW} z(C8XTUPhV;R*swH$4XB`%K#Uz#zqDx7WX*JqIr#OXX-1K=YaZX{`h6YMSC2LH@diw zXI8``%c#ODIc`-?FcwiS+jYs<<;4wzIAuv+-DDOQ!FaWl4Hk>}B6e zn6VTpOO{Zw7L`3)D!ZbDWbJ$2W9a#PzQ6DD`PXaAyqEhv_c_XS7n(P-9z6OZ#cI;Ted2A(yL?cKHKS6>Sz4a~^Yjig&&*aZs|8 zoW%N~#{n$t#(R&1=jqXC-~~r3$$IS>&XtX)9YZ(P3=zB}khhXEncXaDnU+Qzvy+H< z%EX(PbNH0HXQGTzRPn+sQ`-IOeezATZq(;xyxe*Bh%mMdXL=7etM%^Tm!zimZ!~$W znCAeFZ+rBlj;n6PtoV3F)?i^!Vqvo^MkgVAeIGMXGhg;;b>pciR61EV!Fm?d0#k|l zs(k6M{q~GianLjA8GkHayMowDbMF<`7g7;5Niu%PI)$5kiiy58m6q1mWqg$T7F|FM zYr`Hr(o4KX#SgQ1%GGmJ^2@EGF`Ma@so4_tQ~BnbZ&)wm_u_JpUs{|rNXpXg_0_TB zqw0&GS7J)dT5l+=u!$Kogq_*TriKt|;ID3El(VOU zq_fAXC1m>E1vH(8_BDr`Gl`DSiAr*fe#h*$UUl6SqptF%cfRq}!!EKF61#6s!5Hj# z`k@fAn-Re=h>WG&&bh0FC5ZJAWs6IL+6gImG0Up>5+&|me0P62Q8#x(=IyMkyQy*j9ttI>ifZCXlqD*7#9M)qcp@o55%se# zxEl7blkm~8nQY3fGq|PI#u`4R9h>2Ghsd8Dwz>>bmd68S{)2m?>oSV#0G_Ed@6iBO z?mln63%u*kfT@9hZNWd6=H-+I*=xFgO1#?o8kH#Bliia?vze~OfcLF~@FB9)o1+oq*;tY<4k0(&krAJE zLEhB^`SS$J1xhDsDiGPF91@rQ+&UX4O7_rKk&jv1?58F7Z=s$Du-Pd`3K0g>u(6Z8 zjV#JM&hAnExC40@mlc;|p0?d*NVR{)Oa0%UhU+VIQ-C#P?-+}ctJ%aZm5;M8-Mw%C zUP6HBp}R*CvF^Kt)26x-esZFJ=A5{5!1FQnF|CFkJ+}UsUaxhyPU|8n=$-rJ9{WCQ z|C&%e{N&6CKWxs|^fz+#*iqygN6e@X8SQTqud@E41u2UW3x4UiICHn4)6Xwoq{`*t*elDUW>QhV9&sJw^Nn~`(bjK*Cf%LfPZ2wY(l;=AQ$2Y3BL`wMs&*&*BTAwi~ zmXEUHMT&f%3~NNMVl+F3mUcz~Bbykg`ULly?>(cxOI#IKHO^HyFx#}Uya_x>DZ4O0 zi1IR_3n+%}xEt?=_U`6?-6+HmcVa=VZ4dHRf3gT^VoQ5)0FOvP*R*)$$gW-DF}KQ_ z_OI`oK#KCJBQ>6=C8l>zNnSmode}%QmeS1Qy+T{4c%P^-?4++&8YnbbH)=}Rxk=C2 z-vq8F+*uryjb|w?*pi`@ol34Ah1Do7d@C&ps!50T7W*ics*MY_&mQN%9h4>Y%?ExVP@AaWqks*bWP9lmH8 zOxFjEG&|@FjKHXBS;SsiuW1<|W*KM;CpmULI{kK`MBWOz;&xyal@%l}8T5meQ23mQ z{${+yHgWc=eiu>et7x2({0y_tt-Y0&8 zuTT@{#)z?}tmzB7EPZ0yP0SDJXn`a}@|PP6BcSxw=5cvoD0=_cK%sMQXZXtConta0 zj~=U`W-bGW+zcux+fFP0l@G)ppd#N6UG_Y2#H{BEbnzyYX7@YnN8}3QpwPB+xqo#r z+p+s8iTl3I6wC+XZhStDQl_Omv4Pn~Hg<@q86`)h)PLE)0d>5BZQ0T5@kpm@hmU?_ zcnD4xkuQ#b7OALhN4eQq(&;RwX+)!coA3COW|>}sw^mT))vBv2l(F9+_jUp8;Pr;O ziACp*oS2ADhcEd+6I6HfZAvppusAPgBVIEw2*B3z7+C8x4Xpl>MGSrt5CTGG`PJyojRaK=xG}-q}c3?C10ltE~bq7bg{9#6MN|zWJ8fs}$^c3b7{Z=?v0@!pj zIJ>#{%bfpkG+hhb<^AiMV_i766UCATI3dwB*P_}myp#~B*G04|UJDMJAy1(^e)Tgz z46u70js9hyBN zneE*EV2zR#&aK!}iIM@;>K>5*b7DJKxDiK(>+%N)@3gZck)hbRZRwra_(KJ7gGta? zlQjG#RinJ*`2?5K0fV?Dk2qdMVEL3g4?{eV>DZ?W>=F=K!hx; z!dif=pc#t#WNvEy>*dYhZ0o%IQNlUg5>P-Ua?3&C+pn^RX!;fT6}$Mv^r>7^g3-B~ zd9LC4`ZQ~ahi4vEa|0n|fjp3AO08!#wMo`0=$e|1@m~4iwCL+9dzYB^<(-V8jYWm` z0B_|Ux-;2t$8AP#A~JWO>5k_rw0La13E~bzDTW>Al~&IT4*tak(n$} z-FnQ|#afY(mIb)xA|C(c774?s{n33H$9Z>dzM6mQLCxR35w9O4cbn&vkEQOry7wo& z2*z@__dz~EoYXQ222>|ivoAc$h*A0A6elIE&3lN?H~R^~QgGNyp8$cpHh9~f&qE<- zOoY}!t^HcJwDYuLvxMeg0H3WuKs(BOHgV{mJhb*FhL=^aju3K9;o$qPEXpjWiNU`n zTGDS%TJaA9U8wk2SCP4f$*W7e^UM<_5tfN}Y65B!_|Kp!%)L7SOPkz}Os$-ONY(o! zEBiwce*i&ItHly}RwC#xIZj=9#j0f^=?%qzJ@&>yXWGQQ!`LJN@vYN*mI6m=!h#A+ zSCvWAokZk_=WvQ(tmu3rincvj-u*mVj7~%pU7Je~aaFl}ezA0_BF&p{t?$mxsMJWo9L6IBTjV6iNzg1?>UHn3FIx|Ds=O#*+-?$Dy~bF2{VC&&ZA_Zp^&vvGHP%S5 z!2VqxURK*k)#ZeI<4uIsra7v0y(Ibw>79H~>I0pF+H%3E-_7KituMYufsmMYJ8FQ1fFbNUtskBc!+jJqc)HyC~34P+|&0>j-K@88n&Zf63 zHS&<2DI@tnk*_q_S3JhTNlQ8*dMYzoUR#)6I!LaIu%zR5;_3T4Wi4~kjF^63p!IpS z%CUzbqq92q!RV}~!*@CZ@&>0K)U`c2ExA;2Be^DIXb^@ZCIq#*MWUBh$M^d?V;4)x zOs15Nr+o!2M((S*8?5?ABmIcoaYw*jN{xn`2dV zb#7a>&~A*6kObC9G1$~C@+DX0ZE=RliNQiRJZ;L7 zX*GUyNpCzn@-i{_1Q}(5qKP`{HToOC#RKEj${>$Wt>S|GlM(tKK1Cz0;;ifFIGBw>J+}4C;O?=sPW)IDmK}eU81HYye%*`bk@2Au z)itC8tpJroYT==*xM2FAn8)MwDl_K@`T&|xgr-Fn#}ESHJc zedeH6mAtJLiT2i|G0b*fe8>K@>-cd{0~9&;7QZF+%}iu|E!~p*WBR+X*UC|hBtn4( zsn#Fim=;&A@}=%qyN@MjUm6aNXd?EP6@@Y4%hFp9c0{-4sB}oZ6D|J0|DzZl4z{#1 zjUncG&X?({rVsBK5`xTJ?F{mL5l@4gjqKAI+~1{S zf)Ze(3|`9i$hkT6eQ$h7a@xaUAemM_uuGYS72H#1fc1;IY?uPHK9{(hQ;*^!Pene) zv4)sq3Rn2D9ZFh1FMDfYnF-<-Ff2j2v~L*|znuvoZx2ETiN=+UE*{a;9XHojy2vb0 z-#CXBPZv_`Fo`6%w|JF;JS)7Fwad&^*{95_3qKUiAB5VT52l*?BBJwSgvksQm96GQ zk`JQdf^*#WZILScy*w&VPCltV`qSGRwv9B{xFkS%zqTp@5a;*J9>2GgJ>MOE1WByR zb)OI~8Jk3D4~(+l5$dy^!5f#UR~5ZRde)k0>Y_p`2F(2Gw{~U=kj-0MgC3b!`E^YF z7?EX{Fj*nPGa1!>u5~Tz7<<$&r<0{uixltKBx-0q$u^4I$->5R7U-p?Q^b+<)XPML zey=o15W5XRsJZpIjiuQg8b?(zpKXn$8Eg8gUHdRcx=6_HD9 zQh@F9aeIA!MS7W$JRF%1T7B^D_V7ysJ$wzlKhytffiT&wTgNDb&kO|)}aERXmw zL!iz*rP=rDo@aiKLRisP$P$>``Qd^3Z3Vv^ml&_zgs4YG#IxegP!S}gFtIZWOn-UC zDQWki;fVCJ-ZM|Eu{U1}7Pq6U*#x8e&BeVjn&S$QXF)KLr>x6tva&Rv?;c!LU|kBL zio+HyZaH^`W!W7sDUNCzrogHF%l6qpx5?L|Z|9wkI8F+cz0Q}9`F>B6k6dkOB~3EV zEif=A(B@~75uK5U3TKU6W%OCi z^EJx&iN2{fZ)jv5yzJIu$o0@Ba_V*>k8Nh)QPM49O4vSGu~E_Kt|CR%oA0IbZ<+2D zm%jL=M%Us9y~Lh~;YCItzjx+0~Ynt;IpvtQw~L>m#JJ`L1Em%cS%T6b&{(X>Z|td zi@F0kx?0z>?tBZ(M$&hD5|$gy{R76&G-jx1I|%)3tRU~6&)GG6f_Yrp%4?xE+p`&p z^ViEzmw9@n;YG6$(HyD-@R`bpLPPvSDs4*Iwdm^63r2JATgZiHk4`qNFP55VCDir4Pxuc^N%Ye^5 zx46sQuR}YEZdcpWOE%FS9=e57a|(bW&_R^iDHx8716YIS$DtL&P%;WTarO3bo~f9r zp|s>a_84;J{c?x9=GGIy+AWd}OxCzCY6%nEHCt=_E_fX?@%{vE;kZLAI{D}n?b?!Z zuSwGr{Nss8gLoW!HOc|!mxAcO!!Lrv)Gynhy_ike=pIcY*}T%}uRa*8V`lNU{+}oh zf?);Vu!8@@ePh|Ohcl-C!QSyIguHF}b4SA0+miNg3l&>~LZD7{qGW`-*8C8o%(4TZ_~WIFf)1Cw9vt5bbk;# zyMjqgFxVa4SbS^DRKZrpRqhpqpT0QPPuAPFPo9Ht!))slcJ_l0GPEA5)u(^e`XHpH z=UI*_Ha9(+cV+C}L&jLI2|O{y>Wrlu_Nf8plZ9Rm?i6jxLwNv%W=_&N`U&;uFkkM@ zcAD}HQm2=zS~Zrhi^(^L!x*^XvP7+W+)k0 z-kkD$J6~Kp_M=cbXT;DeWJdc1`^c>eL*_5J-PE{gg74zJ&t(_Az`mSdxFe!0sT+I- z-St|fEB~w@`k|047i~4mr+MNj-2OF>edK6urSi*jK5tP^tZ4`Tq&P{vhYy||>L9w6 zFG+hG&fjxF)iWhp^Ag}kd)b-q6l5D#4GnVICo{+AoPIlP4zP{PF2Z;uHYwwHDick| z42vsrWA&6TlwAay<@LNX<0eAGGM2U$ z7Tqm4gJkT6Ayg#Q)V!c+gYs~h+E}}MPa0|bsPcADKSem=a&Y%L-l|Q@bi=W!f1L|J zabDEE!*qy}(x~pGA~o^`fQ=>f#?)f3W#p@YirHc4G{e_wR334<9ZMw zr0~j9e}3qZlqPaL6u7sD$|UIc)h=O){LNTv4fo5f7Hqt`o;zqkrioBsV(#9KIE(ya z7P$OSnw6sY79;C)7u74&F6vK}bF*tFZ(Tg)`Nd)W9>l@ap&P+|-g3JsX`33W7Nv!$ zeuS3sZvk|gx)5Ah-@~3T;{IR6iEsaavD6k?(1!I&As;vbe_tt5)xyaot|u(=(J?U^)^|xN;9#&x00v z<{KRsQ-0MN!rdoJReuWdGirOMw|j^xcQzr7C20F(OwQ{9==Pkw{(p2wq6J^@yzltg zAVHCdb!hWCSJ}793{3o+?|`CqX|*r2a3U)ufa?$yQvdza)_Z+9M@LXug@ao(yC{GB zacu2TANh*RZ&`afY!G9CoHV@smNn1bx0H|*@P%#-9U8xfC z-%?D0>g|Y1&>Y4fAdZe4+58W@0BEH+%z@`&y>Q&Mz1@7F8EPz~A_?_n~zk2Qd*(LK>2zm`(?% zBp7G@l+m;iO`6o!uz zIDG01GV&17B}aQ4`V#M|y=BN|{pIoAWm*eB!hE|0ULmW<@D1%3CTWXuuOi1QNFmk@ z+N7B;=1#dqP4ztf=RaN<Qp~*AvO6An7g-s-%=NxHT)b;R5m}G{ zH>hiy{>olXgajI3dJtY{_xXb6CIENr(n)O6G5MwEjN@_uLH_j~86^{XK)Hv!dEzV1 zbpMQ~tuIAIyY`ABDY|4eUBO{ToBhb5A;zojPb0eyRN8@0jvbhm%YrLgs9y7%j(h&A zR|*gXUmOS$MB1KTai0OT(+BJE;I!6K;Y3Sp$hk^V=pFa<$azf_M}5{AN@1Z0fkJf` z@=~>gJim4}SwE?JlJm8*+Rk_e`Wcx{^EBJq%vNwFA%Y?hp$ z;#a0@;+dH!?R2uWKM)qsuDelxe@l(}TQ`(KDH3y4QPx16c8AQ0fGyi^smc5NI&Vh1 z0v^5M>bDt-Mjt6l;n)rH1IT4_(bWhNIgVJK-w`h& zOq}W~m2VNoO+Z6HhFj6AFkeSEySN>E#)ybx=wO}cYrh~TcoGr+dJ;&70Un!z{yTx;XOgZMz)PR%mNV0B&nSW}L<6TD(Z>4eHSOYcSw zsr$$LD^HhPIBR(lJ1A20{z~4C8tH)?GrFC7Hv*sy^&6gxQsRPYH+>Bs5pdky?$z{% z;-RRxdGFoxD>J%ZJ9E`}cz(T*8`0cz#6k_xXV{ORyxpA z>oEQJp3nK43BMkX(NP~_dCb5pNjIID; zHJRrl>}?suQSU1vek6~cddP30O`|gM(yr!=t)00O%&-U_Axh|%*SFd=#fy_o@-5jb zc3f6$l+U+10%0uTx_J$p(#wGJBekCHZv~I`D6k%^g2mqKv4Yhw) zqXzXn)!5D2zxL2>I>hYaH#3!#U|M%IXT4odF{kRrL0{Hb`k&jH*$sT zn*!uC^ad*Zc5vX`N1DVT0}V~Suzx1Z_r1qDGjkKIFm-CC@u!!Ak;r$oYEm1qz20_} z>tC)xVax!*YhI~k%_roCPt3qWrLF59P)!AJ6uo+FWcL`&fxV1tKFn}ni(VIUo)L91 zzQD2Z)Q)bWaK)ze*Pf#7A*Q0NgrjG#gJZJax03tb6X>>FGgT z+BPB2i$(WEqtLDvdbDg`Rt@2L=DY7583X(4){7t2G|G~Msw(S5C%@JT}E1=v9w8&P+M?N?yNM+lJA*N;dd zmGYg_ay!Wp;TH(W2Hdh~2V!X$D;^S^d&)kEU`Cjz9d$4P27C)#SVTnkz~}g90mR|W zUpS~;~3)uu@2X-*o>7BWzXJ$J>gK0~{gs9d`$#Ea`oFKH7 zx{o%vo{)$5!~3uJgQCVC?2CUIn2^PcN>EkdBxMOn@8M(zATux#%#QmtRoHt?i-@)( zO$p}`Wt>gV?+F-@P8#at#lfEaBVT_?Ub2gaH705N5(pd1N13f;XNk- ztL`jst7aHi0!-5CIS-s1JMe}O*|F%>^z85!(!T)l^~0cgH(wkQQSP$Q7`@4I3&}q3 za@I+=_T@ft_U*QL8>fOOHHj$tfs)*gycGte#AH*PD-h1w^)%PCNFvwlnG7iDo(;@{7TXta@ev3kMFTom7%J9$ zzHFR))0)xKcmK>TX(qj-BGh9FIIUC{4T8tBv1h zBta9JT1-y}LEbXL_|cRm-}C*cT>wQz_nUdZ;X}EkUju=q4l#6C{f0n&qNat9tpJ>P zX9x78FyEIjXXgZVDO5$wb4<>ks;f_F$$|#24_z@MEJAx z9bAp$IdMQZaV|^8tq|gbCvL?DCkm~s(~ArrQjeTmmHsB+{$+X7bcm&Y;lP7=Nw{8b zlF}77a%^Y+r2%KgW6Ku`z{fNZl4m8!3`80W=UWn3a@QfQ3=#S4HH0CpjXfN|MXg(j zv!CruA)oVPxu7`iIIAsk#*(2$P9uyqTB9BIp379lyrmg_0@{d@2t?1lh)AuYr{+Nl zoD?f+)>Bvoz`q>>IZmSLoBLqAYxE%L$a+NMHC$)q;|zGZkVEIkANuy3(4VdsAZMrx zb@L^Ec1?Ai zsm!2snW9%y<+364S!27(;X0duv2pfDZrUh#w}&RkT}&GXfVTqk491ZsK{?YS7sNK4 zqoIq@j;dVG768KlelHk$#NNMvU>K-i)w`i^%B6gh+s?Q@K07QrE8xV@csWe-^{qRa z9-S^YUuM}`>^wOp_=vE{4-#NIAU+f+&RSeu7)H2c%uRV$_gsAY>@aKU-;l(_y0$#n z+g{b#!glmv(JBc2`Ta6ZBWywkxLjG^Ld4NCqDva3Ygq$wya=sV(&&jLcU=dA%b|YJsNhSuX8xYF3*od>02c z%wxqlOn=%v*Y?5m4e~7#Bq`597;PXh2*+Vg_$CSAWzi7R_P&et+{pMt#$(s635+%; z*_%Fdv2Z)q&8?vfiIUdF9{9hY6T#BJeS*&A~Xp?#HK>0j#GA9U_pE zI6?7J9>TZOs}JW+PkAKH$ya=N`b)zSdE=DlHPoSQX;I}C1lJJ?+9Z*K^1WgZuCD70 z^A<*c`y*+fJQqFQnQQiOdKMwX0CQpB<$>{=OA zr(B0aA^W&wmQDm~$B1uJaeCt63w7?JB@7!`pU@F%-1=xA3L^XfS{C>ke>{AR7()&e z=_9p!tfWM83vq9Q-o}f+@|b9Eto#L*NFPk`M$h1y=HkcKDhXse5h#>jAZFG?)x#W@ zfDiOisB|Gq%a3L(x3j3b@peiBaEe@^|BXY^_jtq$eHpr8$dm_(b9JHO2wAnCI-Ng_t(o}P@d>i;H zD2^Zlh7be=EcHFl2M-|h-M-6d$>{?oeKVq6N@MGiG#H|rMBUi)c+Ec$tB2qVBLqZ! zZm)r^{{ev#Ltt^0eYa0e8-cJVff!D-O!**0ZBg$^@aIWDOx_C-ZX+b)2&yC`@?LJ# ztTX7Z-gLu?!Fc3$=oGxoFZ8_W!S->5l7lSL2o53-Y_}%JO%nFJ1k=Ma2*siR9OPoR z8k^6EzWFo`;tFaFv_{v3FwrTu`q20(8;Op4$uALoyZ0Ts$lHavw+^bdGYTbp;p$l%kMAB!;h%I zww=rmUaPg5!RPpJ(|T%f;1p?&8>~J?U}|k)esh9k=3z%aD0zRG?2=qhp?G&uw=35@ zj}bb`z7YIDj=rdMF_5d{P9p60u#$a8&!~Pn5BlHjX71SWtB{I3l>hbNoWT7n|6R^$ zi3IR+7PaXso<>32PHO6Fp+ru{m3ltn$x~;yCe_J81%r(D8^LrJ*c(W~z^!e9C!fp< zgfDO?%B+E|ltNdVs%(jz)sWzzYv+X<-u1DapUieU>*CXK=w(O%*St~j_1hH1BoNal zA)MQ95NR<@LVwUur~W+NwpLeBikVH+KoGF~j*ePtC%qhJ%)`s2BlMqUPAe^++fa^PxE=0j`E9yjNW-v*ouu9xclGH3K}JMjfyUtq$lAaA81L# zjNDace4_Zhh}iLZC)o0yFKp5Dx>O16$lgUfqzcl9?)yGX`2hAYO#m&Kx<-a^Nef^3 z0&i^Scq?_{4IG$S)LV$TwmpJR6^b3+^qjf3wa2F9G@5}8>jF=tEb`DiaGn$4s>?I3 z{x?>sTEx_FD4kI9TqjP9?Pz{%66c`YAd@sqz>tJNszj~O^@_Ck>jI8CGO_y zYi@Lq1gR-MRwclE9}T_0wNUJbUD*#luYvQ@g@lBSs4q?hvc->im^0F7BOmSI&TgKD zjrdk>aA&!I6(11f9qq!TJB5a#4eJ3%H{9}>cO$*Kdx4rNeBKSCs8r?QvR)sLQJF!& zq}gxXZ2-#`r|uI6+I<`3aM$0`y92(lOOi4CZ|_^1n(9ZcMoXyBX=tSKabl)_2CBB{ z=>xdWh0#HF<;OXMR!u4KAtg=RV|kfxrp)Vy;+H^yrl)Vj_AOWd%=%x+;YFsqfw+BO zzj@KB{3}+?QC8W8u>aC^r$AMlMl%FPBBT5elce9GM*(o_lK`B^IxH&$dDxqwvA@wv zgCaHnkma{(3p=f_-X>1tam5#N|Ined+m~S+H5mxxURecp%@SZOgh0k%1MR_+-KZRA0s%*V^;%-o5fi5@8#3=i3_-3k>u?MAi_RQ+`c z;OxwbG~2W|l@qeH*p7cbvi}$-9%gp(QGC-x~*CP&4)ya@3%PRm`$s z3pZ^VYE49E_B@$p#3+)KUxMJD`f{IXB>_ErjyHRJ=J?1Qs6IaWQVYw7L4c?K>m4o7 zX{kCMQkJ%1b0VFjgKwkC5cV-7WV8bm`WpA!d<%TXgM+XC7RtPzsnNNb2X`xuk)|Qb zqiC;{J!vCTWq13Ml-Mv5V z`Za@1oVDIstN|!EuV>JbA_VRnjk8J~P2W?|k zidRqLy}av9Q(up+z;Lyko2rqRX0Q&|yK_Lfi3$-498Vlt(3Ci?@OLLLQxRomWnGUg z0lYK~LY>=aBQ4zN#cLTTcpW9y$53Uv!t=`)hq(@ZvLWPs8BUamFp~D2Nic=i8P2Pw z`^4ezoefg~)Yr=X=B0J8Ly6yRNu+2&kl?S?wqIXc&^o1lF7Zlof|ekBPWX6=(@6oK z6iR=T-q6y+^mvMeD)>75ksoA`*5th(iVMESwYACV8(A_X622CzskQy@*B+!sqeS1r z1VTH6dm~m-RUzCeX);{WY9o>`g|Dq`hWSSq2n-OgahKZ?+`8eKUf+uS*LyOiMl9nM zI?3Nz@VT^uSbwJ9;Ec{Dg7grm#n4(yTzHyi?`01@7#G=)<;F_(hi^NzW#coOXiC?v z4gHZDr@e_Y=5_De3=u65^Dn}#MmLLnI5E{PmF*JG?YkY5t2UFcr@eev`Z$seE{KaWa;I>3`p z`>b_@DKJqrvbO!hnCzQ%E&x=;6idZ|S`M>#I1^7u%%R@NM$)%#G`5)nsUaeG>}*=n zYi3nV<8Wn@TdJp!J}LCr8hnBBeD=0YtfjXjq8f>ZW@rE5qITSacRSqA{8wnMOomr? zcqbiU4}jLqh^_}{(GzJe_CwNR2M0146T*Q>x$lRAgvTcJU;9INU9Jc|5*lDMUS+&vv#`vK(cfx zQSQ?w5fad`Fa=o`wxr8fBWT|EVAE*Ed=ODioKwe}) z_IG{7wEQPB_!!}X+Cf9)$T$+${$Ke=ofM;hmgXcd**I>earJdM4!ug|gIW|SJZO{4 zBH-#^U6|zd1DDTO`XVl+SD?ymYD$oFF1v3ET?!;~ZUVrS7!KHT4XvE?G9zTH5_gN9Y`+ascns!!y|n-+o#{nVgHRC= zJDa75?F~whjN3uEUwjP6>s`7aR#JO=c>e5vyp1QFw?0E?O_`N4X`LlPsN$g;qP!N` zXoh%^6a&2o!<>C2>#-a(DP;lrvIl11tn2B@FPjiX+O&x)&9nHsKeTzJ5y>gwf%Z)Ylg{IQDu@0@qc`UHbCu{h0B*1DMJ!8$xv+roR}X{w zR{Q*B_sJ&G?gzv*VXD#2R=3tKwIO=XT>6_}_~pLFJlpWz>YF?CfQ4|r;y#Sj{!nhQG=#*R3W3- zYes7!(Qzjg7cdKu?#Iq3w@K*fh|i3^qwqWs*I?A|!DLs|7BcHb&@kkq0(xq@tysj& z9>p_68oB60n9+X}`mAo`?+Wo|+* zCdyNA)a>n*7TJwGADFoI>Vp@vbJA8BnDGnzP!kYAd5Uy@y?zBh%D#0gp20s3aKt?j zHujk%olwgJq19&tM!BD$Fp@^bl|l{YF%l?O(|&wAw+&^?2NIY1GzV24}{o-jVyJjj&nr8X;@hab3z$$O*p@v#w;$0gaTwI%Z zk5qzMY;aV)nAGa!i3S^rw0qSo+XPOhN@YDpJ0NX`u)P=N%YjnMiGU*U;3eA5Ny5#d z(%WZ?df-RxAWocG@&pw^xwYe8jcY-l+ncRF(FEOW#ul88sO?fy2U0f|R5OU&s7x=r zgyjLTZVV->UsU4$4I>>-U+vL5T<&Nwc7Qgl?U-eYI*SG}XYw>mSKU&70pNiTJvgkr z4*-4GV5Jss+%8^|YduvCrQp->rcGKfL-Wo7X8}j^nXVdy;C+eco{IpB-#$DUp|H4N zbm{T=kbm^-nx$q}4TT><65pM&0KFNeoQH0?@*vr(e)l)ugvJaF$8iuv0*y;#;hV_Q zF~px$?JQu~b3>s%7OkHVMW*KLH;@{5nYmB$&w+-mXAup~;)VIooVwm7!x21^@4w=d zvb)>m9vtb<5Igf`$Zx;hK}8YNPhkAn#lLjL6NU5)m{DBnfFn9vA^KiGLMCrcb-UQ@ECv(YRm6O_ zvV#E}yQ;FYzaiPVGUoD!K52{S$V!uT9WO%2&~=-L#E`@9tHg5Dd++Pq9U?-t{_G0y zt|TLlD11)~5t9J5F6Qmv+)Eg9$@5qi?5TB`$Bz11UjDqbT(q1gAky#NzCRFB&S5m( zmlP?Nqtvr8w>zqWO}0zZD>XhvkWVlTSKves^W-~!k-i6zR10Dh{4oUTK@s=BBV>ZB zRs`qqNsy~HYebYTpJ1v zt3K`cz5N$!m`_@7h2Nr=?;`v#`>L*It?Oi{mqyGj>U?U&o0cBgW}{XrDP%0p?nBXwdPhcu^xmSkvdD?3hsmzx5Tq=9ZgJoGb<@gqQSbfRJ#@pssnQX z%V|Tr*m~jRPKM(b7m*=_ZIK#A!I6nP3H%AET0Hce81}JOrMQx(Nh%GXo6?b~)@Ig= z6p0k%@tcjPxWx~;SGCzh+z9I46tFsUXG3un(HzAHiL6V|8}+nZGj#7+fs<@4sk{H= zjHi>GN(1w(zu#FjW=pt{L{Ytlp(>KU;oHMUcsKt9blN`6o-KeZbkr}NqTKIFm&K(bBb!Jm7TKPT=H_`omx`HfiC=QWv(c}$TeMjW%Wem! z;lQC-dA2s%GAbG85t&CS8sd+gO&k%G7Ysse9wo|ZpHcypc(E%cz;T&qov)?Xp^qE- z0G%hhQ4*Z|_ig!~fIu1lP z_Mj`yp|wxyHi&-nKjPXHF4cCi%m0F7t4rPvGU5Ve$1;DcYsSq(O!eP6F|xue;`HOO z00Jucg|9=Nz`um%AVZ0)S^?s zq9%zqr#RDQC1DO6Sgu5WIpip~@ZezSgITdE?K&;XvC*qJmWE+y|9WY^hIjg`<>Ta} zZ@0CiF-p!QpKoCX4s=|K{DI3E0>|;wnLI&qjXeG*N-6qQLFNpLZgV`A)Veae`BdZy zwdVYHLj!k^3H1ypX$z~t2EmJLVd)A6-GlSB(Qn(7e|hmdCdc7WNp%dzltYjbpB&=55*y>6d-r?UCHQ6tE$f@_D)8?bL(Ml+u#B- zW$lC`bJKj`%JGEag_9!}mj;YsVdtPT+B3hAwK(G#Ne%4uyAEAp!|>7%%4!P*2?uSn zGS>=4CM_2mJ|PyB6g_YkG(e%*VD zF|~hk_FyAvTI37X=mJrNN4pl%)b`K7^RP$ouO!P%2hl3D?R$)#JpJ&WZ|!)4qPdcq zH`1gyEK#_qvAG*V&kbt*xmKo?Ss6?knjc14A6Dx6uzYb%(weq7YcA~F)Nzm@IFL)h zBdg*VF}UuH$ZX0_Dh8n8_7>?kunTemGfm;R*k3j{0;e!+N?7yK3cPne ztqct!VNV^!VO!a%qcccJ1cyj6T4E~PvwXZXW!589CDeY;mCkA5vK5$QehQtgl?Vsw zyI`UDm&%Xg-HMXgieM;wesYufuEYOd%_~VthJE;?LdwIAAg>?>hBW+Jo*NPC z*krs<$4FMJLPAj)Mq#y_W4VTX(B_|argU*9=qVG)N&o0r&9=?;IaU5JVvD;GPrUB@ z_~kawV0DpTU0=55UvX8-q>te~*nNbI*qY2J=T&7lPd_$4lEpS-g4Nd=j2HSQp{Vq_ zDH>*t$GZ9TAEH;dUp107y@=}!vzQCT~dVkmo>X^|C&8zxF%X7VZwJoG%mjN zYw)Aix8B|86mLBVgJsl^x%>CsEdD)>N6NE#ItFl2p;2_RcsapW`)ADGgQ&UI{0|U^ zidyW0ues?t`bOd+vyL~%MY05o&56%iYfT4ST)x-hbB(rSDhg*%6wCg^q+!XB@JuLt zK~wtV;$KK529Q>U)cb482&>|;D+o=`7grYpV}wT~wYMB%ck2kn1-M`^rB_k81lB|4 z*_TUO^Or$@_WgOAknJ072GUga(NdL1PW6hQPi58AT_~r|*%mu7GvpMUhdVxxOUn3X zLPZ(Q!(y-4s;HE@NgR6!JDx>{S@V3X*&tuxXI4%Cv(D*1@Fe)}q$eukehFgdgKl1b3~B zbm88N21F--1)6?9a)Vj2DF6%o?X89eYw5k<0hvpBR=SS20l!Y{;0#|<&F2|m z!3T#L73z5vhD>OK%-I<_G+y)4L5xMIYA+yZ%j1-=(c%^#4!){mgJ!pjF+Q0sS*gBT zxn1+s4Q9N8(HD%h*2>C!)t}~s1El`x=#S*+C)W6Mlrf=7TRNjqBxQi@NxH=XDyJP~ zgZd|!9lsjx%fjwbf9y|UBnS30eiy&Cd_U{tugrr+q(H_hl?Z<2XkzA-XuPiD+a6H^ zo|+Xl4a~Fb7o|EzN+a>-LR&lA`}FM-Ge?phqALiI1RB-(mvgx)MdXo^g=T-fuyk8@Qmzc1!+PbA1hjrVeS58wI!+Faxq^KtBi%1F(5p_21;5 z0yb}!-AG!*4jJ4#f9V$f%}0!A`RbjTXj(#EmQc#~u244`trnih-H|5xj6>K6e0yN) zLzLQbAlgLL{O8bv+Xr&~d422IQq@ zMKa@Y8FxDg_f`BtTh$OM)1+P7G{F_?`M5orKj)k~MLm0(Sm7x9!n}OoV`WyW50~K5;*m;I`7pu@aZw90XY`^|LO*1U)vYG zzU$(I)heJ6?dAIhUAB{T4%?-IcKE5bGCJjH7Y^lYX$N4#=N^dgGD6Z+%#DbcQlTEq zBvn}(wbB#ESGD{zq#`ziYq~Z%B)(PtDk`8Z(*Yf~8eP%W6EA))^u~=u{#^6+$Rj(Y ztT3UaLgE!cWpp(7JaalyRCEb`sL^LO7sII~ z6X-rl*YlZ#p}EuhrwU)CBwD@F>1y?a2;9Gi^dhLVcb6f~Ap2I?_u%MA0{(T94hgRj zPBhA1DF2Cxc%9jE3VV!zC-g&Vo{~^7b=%9JU2d6wDeu#>sfbS3_xc_#^D13u4JWLA zRqPTVAJk8d&;AM{;7m?mjP0*Bw<<5vKA404+1wvx{#Ea5{b>5R*>B^|dHGI{mFcxj zJ1!}M3n$r8RMKP})ks4!vrp6m(t#(v+UF~XRk=3{ibv<~^}3rH`ROv)YDuW6Z+%-2 zjdJa{w!E`R=bvhm!RTf{M@(BqOUmYiu6D6jSqa%g6_ZesD2mm=4@D(f;+sV}Zv9LT zoJ~AZPB?0ormoi&y=H{WUVV4e%nG)q4l zOx~A-GJ;A~52}3@y%xhfs~`n!G0e46AGAuD7Z|X78z$^K4;XBMlZe~f{v9iZE6pj2 zU6oTrNsK7@Y5Gs8Td+iW$!29sIX**O{nNwNd0Dcud>xKBZqcsx%u_P$)q&_gd;R*$ zFMap!zg1dk)9u-1(e?iBx3Ash##n_ipK)Tv&56K&4pSvnr$;sb{M1v3&fRS-mlgdL z5Hz8LnHMq^Gm(n+-$h!{#x7uLibCRpwU;7{6;61%v&|t4ScO;7f$yEV>zzi0wP{IS z3p_78MYT~Rkp;c_tjf30K_=F#06%u=sYWPmLSou|0=_}oL_lXi1c&V{vHu=>=t27R ziM;n-=up9O)ID<>f|pf~vHqV0MURD>O`gkut2QIfN2Oz{Lm`V1$+(=fv@6R8pQK;R zVk+T@NM90z1aM5YQHB0^M>p=c2RpL~>#>d;H_u8!Mjt7lY{ugCnLH1@s(alac61Gi zf2dtqYHHgZ>T%>m=-vqJe2>Aq&s$3h&!tbD%aie6UDgdd+Kd?2J#fk7ao$778n_?Q zO3Pjmhpa(nL$Qhc2QO_y(8u~E9Flj50a z5lPcN*P_g^um2oIZW!Bv$9@4t^6C6BzwAMwA>Tcv!^9ixZZ2%~*akF)fcp-}$DOn{ zggNH9@0=Ykq8g0x%2K3DWj+MDQ^EK5sD+&S+lHH^bdhv;CwA{_InpsPC7yMyl-01* zCi>W*xphui5bYhypT~=$8Jr`xMxHJF$z^Ny5&08^_-7!AAI_vZtcNg0ipgLGo9~qqJp^Z(Di&l zb7CmlaJY8E*Lg*;Lr-)h^_k7ppY3CBLm7x7c^A-9-sSw_Jfm+@ww^ev?F*uZ?zsx( zy0=GPl1Kq6Mp5kA#y=1-=mW)*Fzm&E?DQ2r{LS<7Pt63dpXW9)KB4Th0IEqhyL;&% zKWDWi>C3Ax2E`h&X z>^Zf*lFg-1L0EN~%@-+{;<}IbRd6jzuKl3H@}NTL&}b#4`z)B;li4RF+o^Jox?w@*B?N z?2q$0cv8=ayRwIrGxvDS6&mW$S8DJ8Pm!Ma96>r3FnoIeMo&6i&u z`MfRKxC*ovDBtG-sAdvSe5HrxTRNKgMx4WAr2ehh?7iM?fe;`SmIo>NpgU$8EmJAq z29>RH_=>AQzfNBG(OEADI68OFVKxr4aN(6=SPnuE~F5AHo5Zi1!%4>RdXk ze=+P4-gf!YuU)gzxnK6~1TTX4h6j?R+L`aTVo% zmVyUbD_F#tvlS}c`sW@Ny=F%uRD_7?J@H9;_bWnGO~|JE^A3V24}@D@Jl?C;R5X!Ev_j&6Sx%#s-xEDb03Kd&#nhm zvHzE!5GRteRoK8iT{18g7LKj+NJXf0WpjO`Zls)xw+#fM9aADM3M zZm*RyE<(9Zc)_>Q)H1w!#CBIwm%X^Otz1j$ECEZk@8DSk4A)GN*C?G9t=+S&2#pz! z-A>jV1TTc}QNI}9J@6m0BZoOYX_KDfDP5d`8ib(A(=b)F?d_lb<(Q|o6k4HM16DEW zI8y6ezj~_FP@OyYjU_;faIlz4s0_5;5`krdqxJwKwBq|Z8|Gowc1nVfT^uypJ=B05 z(8ifxKylo#nUjD85W{A*i7XO#{=@CzTmSm{7ApLZN1xw5XP@sq^WR#$y2aBtxTu&D2KcnRh3j} z(bT6@#}Blx++*WyA_+a4%j(K&nsyez*lF>u>HLCIX$?Q9!7-x`ZMEcnaAs_UCa3w& zL??MQ4l2zDHVBy;WNb~7h0<5Bp0jR3zn!@B>6P732%ZPgCG6Ad*6Wm4|2bT1zie8` zd>x=qvo|LGq13eYBYi-w!Af22z*Yx(YQ2u|_!Y%9;;IBbKdgLbocP zN(g&u84j%hzxO(BFCD-^55=N?*@E-O#@!ecL#KG3Y0~KqyD=}I0uHVh05rrc?!Lqn zC<_<9Q@1l*lTpZh`AnxDslP8kXH3mCN_1{`Zs$9vaN!)%HWE%&4=n6KKgb=>vIR-y ztL=%pBR)$JNRVIntwSJxGtZ56CO)5vF$X$6vsY$34M{_{bV5fEAj*>t->Z(bD?j>B zDV$M7Ij&N_v9Z)o^P-Q`AU4*UkMrkZpQI0$0|em7N4^h2z*NxqFQ&pxBsutoBnLK2 z5kO&5m7%YRy~*?@{kWh8-MU3H)z_mNubJMQubc=y^Mqb{*ulDp)oApU$fs*wl~P>Q zjGtwHl5#H2%?c*Fb}G}HUxEIt_4gP?3rd^si4C6st(dZY%~>JvA*=&)lkuD{?$9wv zV2TuF{N>6&Ph&^nV) zPntx+u{-bR|FTo@t%@9HHR3m9Q1d;sHM$}}+|5=t+O&Ur>ZER9!SR&~btpfSw5&R# z6HxDHUSznqnzO_|7y#jSOY0cBP;94|v0-ImLvmINlD2xT!#;jaq4_EHQCtd>$jzQ4 ziV=fC;{>E@w~`SoNUzLY2JjZCuHxQAJ5-ZbQ4bsniT>$*eOt>Z>8vk?GOs-GI$c^+ zTGy@T;|)#qk7to`ay7{6!L_$xLEqGneCor`1h{FdkF^9bOiQ(;NDitj{a2zEE4a{9r^^l$AhNfO>BG$$LhAa;G$rN*Hxn7vDErUTTm(HM+CY7^ zm?JLHif8iW-$V3xhq5W7m?}D(l_QIiEd(IA<&mYN2|5YP`+QDhx-bVak2GANPi31Q?}S#U4AWe4%{Os2Ol)(rO#dNdy00QlZ#gHN^#p| zQKVFGLkxOvsPI=EkqV*vZE$iBRJHm^7~9O|l*YB7c=?Gx+8|W8+e`nXi?VJuiON0Y za4}5&%4jE@n{;wsr=$~y@9euLJUV*GDd+2fNa7%TMq5YSg3&N09%-idV#xXQTUS<0 zD{mP^-{_wy@G?Vn7v3;up7RQ&K)bg3X|lT?WtpgXUVlr4ruZO+@m$YxIOjw+Ra&1* zv68rgk;-Yvh-wEF@?xEbOs4=|ocBTPukWW!8U@lM?Cx4UY#qNod%3_WOp5&qOE8Kd zbBpC$7zY)d0XjH!Ao~biPe8}vGfkAf(@yGvVN^~_pc`PE0EmawFQB@nW>e0c?L9M9 z#F6~L_pUR2_lm53AS!l@iG$(ISl^QNY<%Yk^2s`b2(NH{+e%%!= zF>80dmb$o&ScXdzXx$$|%x{^gdQ2s06bd1^v2@GH?oNe~EE@jf%NKi&Docrn_*B^s zDHbo&hE@nhukqFjkm-7;@xZ`tX!)ZwL*+X+wxQ}jEAI>~#W|x+B|9CCT-%MPpBd?; zjjly@bw8XQ=&gBzR`QF-?Asm0oF47Kl|hAkS+dUaA1P6YsLxEv>Z39=ji!0X8}|4p zZCr?4^G(yx_|UNphfCNwmP!%ZZuU*4LcWMW$2v8hiApZ3h5D?ciw?93-yJn$R6TLNzwHUx=f`Hmf~UmB zTG-ZP)uTC4vbGBk>!oTqNLZ_b4ALD{An10;QH;#WmYQD2C?nNEZEW15q%b;SrW^od zSn3sja8wyRsjJX7{qUyyk{zk?btS93nRi&mWJ)1G);tr>UGKPpGN4emt&7)TMw>Hi zK^D-~y{j$BtkAu#&ab|bQ=*`fe#a#QW6}4xw{#(vQ67-QS8uV0QpOzYiDwZ==343c zzI@|ahjmf=Q*o^m{*ts7OrF@|o8lVkGclE+E7d}n=_=4YPhGuwiY2Qqwj^Ol&rZ4Z zW!F08A*RZ-oNmce9r3R@A8rYMrVW4OrNwceMCUJ zReMg|S--=Rr&BY-cue=F7|%r46Y8GL2l>R*?@yc3LpQSdcPC=YfP)H?=OF-|ed|cB z?^sudrr_~C!d)tcqjPM8G65qnJ~_U5W%NsNMAq@{{u%jM^Ann(c{(mLa&_^|WnJ-7 z1yutgMMp|_^U;lySI8|ui}Jozc?U(L6tpnpo`4Apx1F@||8or5N7|;#5p#wk)sUb_F{(-l9~IcM8?!ktasiYYs& zn3bt&^SZf&sQNrC)H4j7Zvie)nBtvmp&p-p^9q$D=9U@m?bW&K zV|CT2s--B0vb>A)Sp5y(-ifP=@3h0(0l#Y3tJ-2#NZC}uuyTmqtovI0Ces2c*H(k# zyG3E2o4@IDPr&{IcYL`D9f^&~OttNHP9qJxYv$Vi#*d4XYdA74N;r4Y=kvbV_AQ_7~hIU=eb!iJ$;Beha(1PzIJoxz3I!$;9eFsi1-TXO9xSs(YwwsY zhHyyxs?ZD44qP*(PbP3);H#$^EjV>N46K0Rnmo3ss>P`wblbw<+5 zHHW=oJSzXD!D=7?24;&t&*h8OWMp&;D)#twIRyGeRGJ1f82vnFD_gyxKXyDwrTLNmdKK$!zy7~Q^X4a!r)UaJD(O$d(Y&eQ&L0%@xoW>2 zSC)d4Sj=IKHG8yT?EOwWHSldRRlCtUSydA6gE{Iaqoux9Q?l+FQ^-(gV&}$5p3J!! zb{o3oqe*g0*$i`4q7jRr7enlfsvM4>vHm@_lX5h@N@nBYw3Jr$4rOIF`x%AlKO87{ z*a}Bzn$*yusaU4_Z+E2Z`Lh~9Y68dPbPB)bB28`5)+l; zGO|KBu62*%buRay{V;-tA%gQ;Q|=a|#vh>HB?s;Z1`lO`YDbq?XEfrZB z3HLj-u~na>Hox2s{ZV?^+_)mf^n|U#vGqeWD_)csxrc|33#$~P^!_06n!?!7N}0g3 z3vxw_)0~Xv1@SswSuqb#C!rXa_fC-`3>UU0d~ZQup-di0%u(!13MrAgj{?K$u>V0~ zieF6flr~x2m+CQ5TabKP`hO3@6q!{IP}YIN>UXs+SBy7`7uz%52f0s3hO2=>iL#z8 z*_p7!dOCDT!uFk+)7z#U55*0&!?5EylYL1f8-Ztla{hgzh&nNA3v?N@Zyjz3@|&@H ze7Wa^t~V4gW}AkC+BIkAoVKs^@T7+Z=xmgo)b%Qkdfx0H+i_p}B0!L}!(;#J@I)AF zMANzR$5p_2f^GX$_wxA*hWnRn!@@}t%`aOtkEE+HDlWN8tSBE@0hZZLs#iU^3m@oJ zcP(gFr6h)m=W7auoC~4sxuc_-6&!P8Z}>>9&!ZFwH-B`+pu2du@xL>`o3=4=}%yKnfEZ1<19g7>*nzeJCz}A!AIZI zZS^?J)a(o%D5!VnjOA|T43V+O1NuCxm1WCV!NIUDD|?aZrBtavhANYO8=pe7vJR(? zO0@#^^{X)c6HE$m7>*JRn>1UfPu29p>JunvFmdyCO-VUa>!IYwxzS3M_q4!vq^?%- z$^$)x_T<~KWG>H+{8L~Sv@z7DvGzK%v`l=9p0ImZ!4+f zA3Y~tdhDg?IGWtGU{v zzHHLDCRrtrysJ(gVkRxMORV_As9SxcN{>r=_R4k@WfA;Xq}!UZd06@Ppw70frpQ?!VkrTkg?!W(Fk#76V^H9XlsosFejDrT9 z%w48SbA9raw6X3=U0W|dVCPg;P5rl55b-Xz$x(L;7in~o)^cRNGOKd>+wYZ$jM=V| z_4~>ryJ~gcc+4wKi}WPx$UieTQ7l${owxHOqmz;&YbOcq6o__FP+RACIO_uPtpzTU zPggOCsyt(P@j@w2u0WniKLC9*)os>j_Wc=ErksikB)!$=M1}s-!`M<5llpx!_yiOG zOiYpaZ|$lf#P{;`L|(JB&$Jww(qUPOP`v|>BGI&=-C75k)AlRaGNc^|?UZ9@)F)HX z7C&;DJp0kLaAuu|Wtjl-(13t|M`vO;{rw*A8nJnM{7g&rKo7A1zIO5j0?ARl$o9bJJ2)l)ul}*zQW^5{F z9Xl)fc~>dHEHqzxZtl;Ob8hJ35$bXUJ135p>SZ;8#}ST#^sW5P#fsw4r`OoeSF*_# zm(%yw6ASUCcs4vU1SPk>R1U;(x93q&_TlDXTNR;aKFziqs8mhtQz)Ye+ ztT_dVSgx;{?+pX@W&kG7P@DTWrQYWU6`5S$c)bnX-yVF?M?Ob>NBjJjaK zQTbU-PKY*MKJ`&dL)4?xS!Yb0ozC*6Opw_$^b|a~G}h+4wM_62+IMgf`fvO^IVdGe0GU{9pLODhlim?C z)d6E03eA;%<-#A1H%zFuG)kCN6O2)};u1?XOE2k{CCJHcTp9$Dr?k(`J^NmEEjZ1kVD5tI@26L4Y?5jJ3kH@9 zEd_qAoPlVaj~I<>ulmnP)F>CkGRc$HmY1Yv+n2R!yCoRguLRDU4AqAZUO1q!@*2&T zm8cmJhrLPlH147;agJ>qX6I^LXNEQzlEj%A8a^%N&gFUuB_n}+Es0LY;S5ETS<8mr zlm_qZBSIY+Z|r7f9sQP`sqt$qMV2$M#XkJkkE4X%6j>OC+-o_)+PE&mzpvi2xV%zv zQjf%t(F|HzLPY6`T>xzP`sOfw5-m*#S#7K~jSzM`&SMbD;z`%Y)52f;qHe1B5H;~z zoZGhalHQb|q`oZF=TPSVLFGB^>Uj4GJa-b8FRFZ>^=;g9g5gI^Q39h889dLzltInX#;>Q~M*W z&L$x*jscVbCU**bJIeENnmC@^>pIKpOHex6cV7cXfTak-Xs8=UlHyX>38tvWc{a-i zQrs=w&(-b9`s+bI6-?O5p?Rb8EzzA-#RGHVvFdqdO**sI^-l7=j4r9F+LJP|J-A3N^!TiEumqGIX>~@f zKRjCVXu!FJd&oAwaPlTpu+%bAgN7>HyW;H^$wg+3Zx{vE}q}=EWJ1{*$495s6)cX$d@4X0rRmC(l?%3YB!f;&M zrI{nQ0&FS%6yza}sBI7opOZM`*)aIxo9I6cNu@pUC6(U~_B14KtG!OldqK^+gvg-c z`Nqt9T3V=Ul0WdIk{85RPYU&pMM$3b@{dB_y|+%bjPXc;R{TF|^C5EDN0qX>$zCV9mz58ldxu`%yGy!CwCA`OMyJN^bj&Q@Z z;}w%Q^YN1*kJ4wwKdRZ~DQ>+ZkNdc%8P&XcCes7=5_j8;kXN@x{hNcVy;aqj|9Bg> zdl-cG+hsHw*SugOqWyQiiR0fY)EqC-9VLeUM+uq?$Mz~x0W#LO{&_H_`P%S7WQx*6 z4;F-+vHMMZE9almbMu}+ckDdYx&|3!6(!u5I}+#q)8|fO*n2Bko;`i;X8fZJJQpbb zLh}3f#NPU^5j4naVxNc+UI&Kmb8Ee;hlj`IT(^1xhEO?vIN%z~V`qWDjzq%T0qH-`0O;auiOe&w7#cawlof-c;@VtEQWWya~_Rw|@$RcY#n{k5}bpaKSGh zfmE5Ze{n3$dm|THTSE*k%>gn_M{sMJyIswa`R^z0X>xN8>f@!<(NO_H+LqkI6dKAw zWcV^A$$E zFEN$m@)ZK>ffUP2GP(o)-w0TM@`xzBP;=TT{35ho{GVI*9HlOz(`4fGo@)CDqB~d) zy5JT4aCYaMP}Z>NPm%8iaiyP;Vw0yuEE~zn0Yp)a$eXSF92o(p%U$Qu{i46+Ki-(J z=zQ@44^Bd-;-`7>lNKZHFhk}?7d9mP7Q)rr96jzojetNrfNCt9d;n1;-fT9k8)NAP zsM#Ex^#%x8s*2ZVl+*$E>%f^#aX}jbFLct zdP6{@7=<1@PrYEVwm0xizCr}`VLGmCa^s+(i<>T=o+-SBvXF*34R5p z@#1H9er{_?jIhiB15_q@c=28uT5G^(scrjXoT;<22Mj=UtbuV8H<08UU|R1&e>|=l zI{ZT*i#TehtgHcwk-i!kvtU!{0@T#yP3%H9{CS_nBLT;hD>e2up1 zeyyhB)WpH=W(<0tAB{&B-EUGJ|2+TM5-%V!9fm)uH^q1Nwl6ZdiE!J(Lj&J|=I*Y0 zb8w=#8`c{%4YqWoxMyfVu@un*dIoSNJEm=D7=8t#<(SYYlH|pGZM^vP@DYW7?#m>J zzj=*J8SG*b#vF0UwtE2!%QWBqAisrG$A48%O07JBRVot)Gk=I&7wGGEHKn&py+NWO zd1u&UZg$ogl=Tf~T^<~s?vCDr@W>yzTK(y2*wdzeTqqPA!0-V8D6@6|VV|8sZzm9A z>iP<0_pv(zox-p-9)(5V=Scy-Hte0mpZ{AMdB_N8seucSn*+pd4mRS9ImU0_H{zDQ z7CHXRC*^NDjVR{hPAL{hm}b-B2LN|51VPdGmAfF>S`9mB50^1bmv9rVX#*c~siQfZK4G&k_U;Bo%x6UE} zKbZ+)+*g+`o{Zl|_{)#s_#Fs@XE%T+x&hnW14JU6ZnBAKe7Gd@A-zEu_W@C8{OrWE zldk^rya~n}3&kJ+Iz)zZ0a-vyis94Kci?c({39j`A9}y}B>rrlV7s`V5WGau;9nOy zunUN2M<3|RiFhtrl{ZkRTDfOz&c+{}t~>G@KenZp@v}YOdUTbb%=20p{{VozO#|b6 zUPE$4ulyrpSOv&3_xes;Fn1#5dL97&aQ$Z@?*LEN0}5l_voGWRAC`cY8lp8A0a{ww zE+T*{+MTQ`r9J-H=43Wrv@@6j_dUEX@wX;2mH2?vljxERTkWGScwxq|y(=wA>@|J> zktt6*fq!klEB*k-h2*%Klc0{?s#6yi{DKd+Dk$&cPkP2yzD7dWBqN4gW$Zbjg4Xu} zcsl#9bKy3@mAy^y?1lV6!lic&7o=$h9G5yNE%uUYZe^*~aOJfT`3a|ZsISO^B@ zxx*fk|DD@Vb9+1pN7B$*-I^pUp7^--%(z|d%I_%d`IEOS>C(ovENF43wvr0|XO*W8 zZT<^z8uT_jXYj)39eRGA>D=LK`siZ6K8MiM!Ez$KeQ zgY%}q4b3C`JAsyP6;up+=Y{#*!@LdYq)^Uu+MxjxZuTjPsAF2?WEC~k;FtS6a&4!O4#!+-5ajZc{U&rOKv~|H~`Ah-s{*@yydZS z09jTcw0eZoG*h|Y20?(p8o-h3U(g< z*Q`m@KmO&qALDI>4=;M9`M_U&1%U4lR`ta1U>q0|^lp8*OS3jghZf2Rn~frM(Hq@u z7lykDU^p!j{OEGOXSRwWtXN9aJ{nzh4OP11=-sU;=^?}V4gLBgUJb-wzRP4C^pj5A zT8ews#55fDpN73_g#-Vb!fq>;(}V@>452+-E8?J*@)gjg3s;^4G;c#(7YRLf>7;xr zyaA;<1To3{E;ok|#K+|TK)uN?doQ5<;Duz|8U1<}e`@od{GX>BJk}U0-b3IN{}VxK zBKJ%#i#H%(N}eGXKHOhOfd5Tz*#{0^CkSneC7|qz2n;`6HeIWtKmIx4`krRIj2hqJ zy^9bT0WZj(k;j9`Z?df#m<-xGgmJ(6#RM1;->)FtJ&k~;%K@Xjo(km$69bCg-KnItU9LzP-Nh$IO;;FGB*LoLl}+ zK=Wx}TDafp*-NzO2Fv;tYxC`K#%gMw5 z!a`Mov=3fhg-C_b7Q`X%B3ix094kP(LtM1mpUgtY;9mPxz(BMLR z$^h5h_Q1o_8&WK-4+Jk8WGg4(>TzEfaixjA-aD+ugdyt02O}FT<~w7;U#>qKIKqsn zTPbv0x0ph7Y)x(^SbqiCqi;p^J6@?q+3xW274UhN+QhzXVn4-qmdI|~bU!GN)64cd zgA+1@;sY*&E&S2If~&0{v&;_ylrUAWBhTn?<(4eh+?b&vW@GZBFHinvaHrehj( z?NT2~_89ILti~kwcmCLK>}(5aVs&%RTZ?x@t@O(-r{H4rFAVsvXyjs5A$&y%=!nfe zkMA7K)l6NW^Ig`iH+7uJe!9v8e!2UXK{lsCf!e5{KK*A_jD|Ce*VN1S!$SQ%aBSb& z1z_dsU_%qO6XDR!zSIcRP-=)pA7agF&*G7oB2?(c$*x&Y?dWAj7O z^t1hpam84f8%HQdrDE8~?0!!+#aR+U3N3Ej=jvs?L#%Hdpji2v-VEY+B(g4|I)8ky z9vDMNoW1}7lq*tqMiMRoj-YGZ5ElgYuq-WKDTJN1x@K#Jl}9R>X7_GMWsZcL=k;djpL!DgRSWukY+HOi-7{GB{}+q3VA1`fDhFw(c+1C*g36Yjp02|9>%Pn{u+ExWfl_GXIkG>(JGSz2X(>o>dn-D5QCE=3_aO-e@FwXBF1iUp8fPWu?D##?Y>0Mb|IRap| z{boSVT>z;2s^biD&jGC@5vk6gQhy(Vf_>WHkEjBy@ct6S>+e1_KI*c+!TFqnu%gN1 zL*TdGWlX|`UVQ&)y!7|@hiV`eh_>f&JH9(m>e5lbxozfuMyLd9oO104F4SD@?O#}M zI4M|BoG99MkYLm#LjpVU9JQ`^O-Tl`#By(~qI%08M3}c63J5uOn`u2D3$x^rHCSQA z29eQ>EnkT+)=yB^7KbgZqJfW5SW;8S2(okck3mLBC8K6@#JCFb|FufE$a-s$+XwJY zZcrX6Fdr+2MccQd>0;mlr}RCmCT!@3kV0e&ckkW50{^q@%^4i8;ziUo@Z|9Hcjj?Q zgC{rjwXaHw*{y<@%>Nqg!8;(><^{MX-Mlu31wLIG4@*H{d+zxKC7~wJCXwq!!}C*+^M_wvs_0#PRjsi2k~J_Z9}!H%O~iH3xe zPb?CwQ;n55i1#sIEJ$nUF5#Vr{1nPsmxX6h2 zPf2qZm6mP9k_{>H;yfuDodd*q@%Gx8dtY@LxZ; zMs}U_?<;gQ90E;_k(Y5>6|v zBlsnVXlcMjV`Q)S2w9EXpww!=^6r4E$sd(UmkWVCC-k!~y@xVd;9liXPF?svga(`p z2C_BC(SdU_3Y)3s6(1l18TBKy6%%HYXI z_}w0sxssdL0e`kp7J?)~o#6r?i$kbdiReROfGFp@$5F`~Rgm5?)8kO%)= znN$Z@pUI@j?NzPPF)HkY8|XWz)gs?~=gUPAklT;bSl=tS84s|vH{y#=%ESkadYJn7QukFolxf^MgnloldlOy| z5$Dyv%a=V0W&qv@scVVJ7x*tj;KY}>frcFoxpzF#eFKtwCalgrjZ@?wqwq5&kRXJ} zUu05J!F|OF97>{8lE=++epdgK0!r!}E~^>GeZ`maL{24G!BTKjXnzvL5^zpj`)u!7 z{G25|`+V?ktI8I+aw&D;gs-oLB-BeX0qm=aeTShQxCT_$O7K<67Tzy+<&sP;BzL6p z0R>p1WU?TOIQ+o0Y1|Z6O+gXTYNb;X9 zT@skRvp2KPj4l@we&On#f-;xsOhY^keiiXc9CbA%+G|!{JVe7tn8ibT-9Hj9C;kFw z3Wj`a(o}@@Elqq~ZdD){A2}i!jQ(Ep8*3=yL=_|hvE>=Tz#dYU+0#m)_zrN6s6Ghn z@4bg5%qa9a(J#VM?n-DfAa|%I#Ek22O7OR6>E3T_3Ou4CEUD+Pq!7zNquPy-YORAG zv7FsA*DunL(u?*2C+Lu(h+Bj^e4oN`H+pV7hFRN7;Zb^ z)88F3Ryo2E@IO%+ht7Xix#1?SFRNjE=Z?P!x{DISiKx zx`SzH_^Hy9<<$r}bYBBm1u@I{o_Fw1PAZc}tn`pRk__G6zoauf^8s-Jg8{i8^W_aW zZsgqW+B)4Pe4xsndt~euKm6lLI``kAmE=3Z)9?l_)JC3+%I{h+So3K~46!&&WxFL^akj#5xC?QY~!A`x! zi0|NkQnSh;>{3cX2Y*1n=^mssLCATlbC}YAw^ttj?gjmgxBtF7&c6(+V=)Ukqz8cS zpW|>*5MY?**hRmPOS3%4s+sZw3AxJwQBy&`^Xu_B+AH~o9oX_`fyL$yMAAB3-la%u z-#we&FSQ>JVj#{rSWS5UefK~4$O9%rgv*QK`KpXj>^;N+dhOZBR^vRRl?$LKU}tBG zaA94Ley<4>=!O8{)n#DrIS+d1ywyk*xDEb~17MWe=AN@g#;p1%Ro7>+$(9EoXsjWy z{lZ4xnlRbapJ8zVRU8EQs9_vO2U6e7o{$g<#Max zB@w}>dF0B~*;=ib&ItPoq2v6AP#29gbhadY=K`-yY4h0uc+a7nXAZM)w-;p%Eq~!% z&Ogo^3aUDo?hS|T4kQn8|KdLFg(hGDZXD+Qk|@;lqJDS#=TYe#1O1%b4<6TIH`Ppa z0FBfVI8T^JF!1D@Jd1y=6R<_&pWbVEmd%aoTqvFhNOD0$Es!3mH`I}9KKFxu0m@aN zO9~9qBfC%owW|a9t4o&}xdX2|K31)BpxX(A1E5Ng-X#`%o4W9(h>Vn?vjs; z0QT54eaB{cr7QQO$}h$*{toT7Tk z<*5t#Iec@c&9}x*J!`M6SUMlJz5*zT%{ws_Ep1~BOmT<+UNLv}#2skvx^ic-5Z?+( zL6QU2qK&w+crU1E+o>L6$wCay>XU>@0rS^b2nrxOsm5bopyAvjMT<|L3J?J|U-zt8 zgBi_CMFV|Q!tZ+DQ0G(|-q1hZCSf8{G_ME;l{lSD5+MTYox|H5f*}i(-wjjKNl$^?>%8L0V>R>La z!GfZ7cMGRaLdMla2{$oV)V1=0C{>N&G|iG>QwcQzxT@hZ+fZM)FY*#LOb)&O81 zw@f>Qip6LX{hkD420%f;?3oGv-d^53Z*?`>ZQigj)qO$0+Uf@PFnAxX1(v3Lj__lyzLyK0y__-0L9x4;x)$<5QXKv7-I4=Oam zt@5@LorVz?tY-!7C{!p^JHn$k?5XNY?#ac)%185C%aK~B7zvB_A}4>&9hUiwz;Ce= z%x{)KNml4w=GC1AkNSMy9VAl=Qz9lJ&VSy(IdH!{wc#pxY{w_&NU$3n{?dOr;y+vWYt^=%j zHPU!~VpM-cuB)c+Q+Fw)mPM&i47=VOcRuGwPI-OJP$-%wrl5)ZuX7I*rIG{@I&Ly= zAo)p5V!+M7U{QQ0$LCX5n~wO(qFqV)X_(basez39v;ATfvhSP_QfA;j&{=ZHS1b;obf~*ASG&HHcB>jGgQiEt z$d&QUo?^d=kzXsoi}R$RIv79O`6ydj+3z@e3rE9tS+Q@MGP@odx#wIySq}Q4XSTt-)dO!DpgFM z^0!tww4SDxk09Bl9;!P2=Mt2oZB;W6l?3w^C2p@{D?brc(s>{=`gvxzo9HR}ZVryP zBj8@$)hG)CgUA7-dYiyzfBe#~fKTl@y4&dwkBNUI)j7B6J+hgi81Nb359h(vN7x(hTl)GcHHa9V62F}CwDWAzfbT3)d=9mB)U4wqs+2C4Z}H} zz?{~uQ17?Bb^4gppKl2zM>3}_mnco?7ExKGGqU-`BbLjXJbkR>2B`p%S8S6?Oj*c4zm$HcCy5iTb5L*)MUELoN5 z*Ql8f#YC-~l%z6V%$+c`ZEn?a4~sJiBs)P&?)&@bc&~fMkBq!{luzerTHhvvSC6b> z>XzX|kah>#I%oUSZMG!J(@B6nsFQA@pQlgEdhjTLpBH^1OC+Go*p6cv(SGuF%E~Dx z3d*+LD38p9laYf$0)m;Wks_lAj29nEKV?Jss;)Frs{MJnW$r;5iDgG;y-@Gfce}c( z<*RVT?yfod&H=R1%6Ev)H-~p)$PF_5Tk@Bc$`egKu*+{Ou#m8A~YE6k2mWSvL zCuSkJ#_*j&mPGKYDWym8HI>i(Ihv;*; z)_06~PTL!rxy3#Ww?(;9D5P`fUf|m9=GV4cj*jO03UHk2tynjJzyIea^OARhR7s_xezk$bgkd=-x*U5Z&f$X?qx@7 z@euEi!6XtYw-GDQoz&*rF}HQXLOHBQ9)u+1W7c z`WaopYaT4HzX`iRpAFZC)74Y=$zyz2>{xva?VZdVaWE)PmD0P2ocp?@vz7BI=Uj*$)M3 zbbNMb{fzAE6w1xtI)tG+x25)eoHhibxIOLj_>4hB+#F^#y1_EDpvd6#+$EJo1-nP~ zU>bfcltaKC0<53EgR1I=o1Zpr>5w&>p1jO`60tiu6GWAjR3yp0qDrB8ou?X1 z$=gKyq`b_>v&5a4a6cHn=db1pr?w2k!L`bUA{O~h(fnn zuM73|c33C|fCF#)xTffg^1mu%UQzu z5K9P~>5|vWtYNjQBMsIu>cYRJd%s4JMeqoq$w^2l5{pI-5cxijh+A`ToE_5E)Pj(4iF6z`7qOh{hn!s61A&LAnD47VT>3a zi_eWGk+tOsDcreTr|>eb!8q_kPhtp0aLUjXlUFjHacG-1nbO*DrZBjJT7=G8ilg@? zjjk~(b|N9sj`LA5XfX(d(Q)Dk;(FuEHp0UL9!Y%^-d-dYY7z}*gM_f{-N;x2O6)$MR>gU2(JMh zYB_h*x`TQTQ{SFTCTmjq#qO+=U{UTJ85*CMG>V}Tjmbh-3}9f&S&aua>QtD1`XKx znt60QFfLN_ zz>Q?SwajQ(NFCRe5xHE(er!lqubP?=_MM=^*9zktm8dfv8DkwH?*euIfW%+-u;8f% zG_z`yxN&oCwn<0MSSK_Yi*~hP2ipf30+A`quQKC5{Kz7)o0oscH~%eH@8P0IeGa-_{{_*bfCzQ zpzY6uL&O#abrHXK8Cs%Pe|;eam7zi!mbp zn#zF|Oi^^4Af3{Bdx7$+VNhC8kD_Dw9l_!*a$Tn<#dB(mn%v4ae9UfDfmY+UI}-zK zc7uRQi0z;(NkmKbss!o26rHukSY16U*3<6G%9~?D_rbWvrX-V>P)NH2RWTv45nSpi z!Wq3BCvH8$uB)PIyL?ERE5epF$CV_bIQh7vU(CUdr`Ky&?jZeyel9P8Bz~u57w&2j z#~NMXCz07?Zp)uWRe-x$vob@T zBxpRQJRN|zOQ(g-M$b~EWf6$4p$_>VrD2blgrx8Sr{yy1<({a{ zt>c!1fNiqvdksP%3scspdvb4n-h3dGRJ}p8)fTtY^Jz)cLZx0p?uBni@5PZ!-}hoW zkYCQ#{f=<_GvsN6*kMp{&m->wRd%qvQ$M28( zasOK{uh(^*=W!nAah}I>2;65qYz!!rrQ=PvdX*%}jhYM{=+`E2aVMTeaN={8G+X5> zYi8Do)J}}W`~M7QG|}7)AWu53XRWpzH5u2A*a|b|RcOSF*SnEBO6*jbv3%MR)!3}m zCiZaeHk)xc7GxPd=a1ZB{Eln?$q3JsA_gUx}) zQ};=*T3p6PLpLMoP|XW>n7ph3O*hApo>JW$dbZ7_6Uq^%D0&ycP|-BE*U@ERY(LJr zNK=i;CPnm_CA)#I*9`9rV^jQRW=;`R?yri9+MO3ooXns5xxsbWg4@~32yy?iYF5Ab zB*Sau$E8ERNi3-3hs~>8k#;eyk^*k!T(vfKFdiPW^Gth|*-oeIZ>Eztnul%S?PjrYAlUujFJ}0@W^j(nevpk17d)K{Oq|99j7%>y;BN?X@ zlPI&x-m|Jmt;=Dg6#J=qY9u90`OyK+mKNAeQc|A$gd65BfJ*I{mU*zZd5-}l_4&!e zULKN_jq&@XqEp?>leu!>R`JrLehsViE|=BWTzpw^?`UGmE_56`qZ`4$-Ef<+F6X1V z0kt3ADsL3I%{V_bcQ*&+HTygWaWFQUC(kMv%xk7d&oQo$)ut%u`%UFt-SLZYJduOO zO8GO*d0W=tiEGB^#)Krc_!jUaW#n(uKc5j80jSXnoOe=g-ak;@iH4F@R_41x;5k`E zMr$?QG^NwFzE9g68ianxaowRUEu>=~nOVH^HWLQvxnJg>)Di#>9WN@Qi<7z^jh=$) zff;O^qn|SQ167QBuWnwO2?o*rUS25!(y$l+f6y~*;$B#-lU|Dm7TgK4bY4Nt%EJ7m zo!Qy9#>f5gkC)OXdD|S>yW;6v2`E@GXh(TFn)9%swC_nD8=-7fp{y!y{Ox1^d`VuS zIn<1A4GjOKwTs$xY`unc7-V-ymQE@}@@%lPvZQM1y|n&zto2sL8yLTWZe%1{Ml$F5 zu%53}xacBzS>d70-2C^?zECa;VrJ-F`LH1IVe3%-)#x9k?%qrY3Yn^`xi|6YUOUe{ zKxcN$jsBsfHUgywJo}$D&3px};*|<}ROTA?A1N zEyaL0H*Fd)V)sK5Gl9gTp-=jMg4q2b)A=pYKgD2OSC$e>N=j%?oZ56@LF5rIz8kP% z5G^B+B45o00KbJkx_YCk7cc?peR?^b%_?L#oCHvgt;X&xXaWWZu)x%neTPZn`>`C@ z*do?Xj&_dEZq%_ABydhq&jOWJa86`IAZ;2)70N!VC6oS$3a!rQEfK;As z7Q=A_ltX$3d(l^=;ol9p2dCy~G=`6U6A2m}5?YcA|>qw@OSvZx(ui`t=X zCo(JRqA-k!yE8tf0{xv8O{B(N3OQnDQkQ64nTokEWiKRpemsYX3O!II4xQ;6s6Q^b z!q3vcy4KOFb~G9m8BjB$8MP+3OM+J!<4L%J*p0;N?#dF!Xkf7e1litsp8Nn7BJc~; z1W1AU;w1te6tfFko^?VbP|T_|(%5t|l^FAN@g>+hYys~wh`t)Jh=4Qe7f8mTo z%!L9)-C}H-;nL)?t>*nuL0n6Iju_LjXI_8jTmw0NUi)Wn2SlC@Ogh}mYBrR|m~`F* zlddw-%4Z`Hy6!nEfVevA8lH1#Mu*{75aZaoK}K=~j727LF||smvtaLrvae~W9Z=C#E<=gvm};|xPJWmJ{A7^mC3dGTO=$% z!1OIBh|nub;eTP7qw(@0>3c^Ve4n@FPlaMgK$>dp+juUb?_y-YLaD$*C6Jz!p@D9U1Sj;;==L1OD3*$b_*bP1 zUH4ki*Dxbey;(N56P_{pYTxW5W(>WF(THv_Y+8?700X|lo{fqvah$ zAkFIp*{Sidhi8x=U~S&l7VpQ^7z>=rizHc6SkA7c9D)xc%W)V}0bFd}prFo2ouT1hP zcOfGA2xam4A>L@L0a;ueYs6w}1#NOi-(zDy`ul3OL7Wd3zuB5%MPR59h*TG zSj}A7mGM*M;yz3aA+keg;5g}vRs9h~vLQ8ZgN*5v`d4Kph^JT+vcQcg=jxLC8r(4r znRNTbT`Q!SIjn?5z~&z%W5>zSx)+VzKbIgZ=}V(_n>s;NXmJq&OVnRBt=`S!83t!;P{g^uuyeFTPl3;9iMI7@moTOtN0KCG*L44^SUR143z z9?3*k`717>@3~(67BKazK^Dt9rZm@|@TxN+-55j5p2*4~t}Cqc|cGe|!a)s)i+V`lK{# z{@okM*u>~K=R9t*wR_;lj{n5Qfu2|rdk7%m4w2n*Rz8ln;vf5)Vgxn^cPR`%T7^AQ z-AP{Rl{D~(c%&09l-AB0Rl`IzI~6!e*_RofV#vWIY8ialLu}f5<5^{*we5`+z2dGl zzC#bi4lkr0Vm_XD6npd}KKU^cj&b#EV$ff1i3y+&@K^;ApYl`z=O~7T|NB0_Q$;%a zz|Y&F>o&pMyb_&E-hA#(&>0K@82R1y@yeliD{%z7bHqW+Q*b8mV@Z)%` zw)x_F4}<%nC)dclN=;9%TYE;hGeW=-$I}XYovWB4CC8NQL<#ulQ@9l1+2>S0J~U;( zzgaSg`!b2u6R?^nvR`^igzl6QqpN0~Jn1w&rnv{^W+JppLE7ki|#X zZKV0PIkj6_dSk$%YT)k5qbiJeHQH7qc9pc=!U&x)Ii2rhk$c=3zDw&LUSQ4Ut+4uz z3vd1JAN+(Uo!3nof?8KysQah{N*COIH2i7DRQsGm zJ2VE5OW<``fzc1&DYXBfEmQFTku!uk*bmo-tqi9B&bgo|ueb$e;^V| zaNd)BaC%4r3-;PiK2}j_`21=GJId@Mz3>n)M9|hS?AB{l;j@WsF5$HgSfqHmoptLo z+Suw^J6q9MYK(95@(LEWNN0dppi*MBK&I@yM<=D}GCwni^x1vqur_1ICqOg&O~ncc zZXSE5yO0F+TIz6ByJ4Pq6=7yAMFL@=(n#36-W?K ztG2ke6J$N}V^3B#^PRH?@fWVN^yooJiDqA3jP*&0eH+(=6Ni${ipTn-h-M?+SZ#^t zoyZG@HJj&xmK}lPRs9ezo`gop->e*hHHin$%Np;AGd?S98~`ldG~eKX!J=-^kR4z;0!lZ}tCM4r1 zf6c!&H>ghTQM94k7={OE$L9*BYorf$5t*i|%%`^LbbtcQBpCjf-u8nW&(P}1_GZp7 zXeY}Fd^{y$+T2zi((Ne%?YMifC&H#x0#4P4&z4tf=&HfLPBR zJ`|%#_73bpMd|T);H+TK-$@GTAWSjSGfZZxSzcC{K2}+X)fPbK6u0nK6_evxzw-F3 zM3j+*T4=8E{+vwqtrN-?7DoI{|3l!2KB3W z&nG?m-Fbmg`+(!nE&*s>7DKlp-+A>A-vAO1KtPKnBXIC3MgVJ@6V{{KZNN>cSl>^u zYVq?`5u3C7ti(;$c7l>ZFVI`ZBw{MgADFzg<;XRp2~q+VyS>xnEsOMk6H-Ge2}J-1 z>p{rSrK>Tn^Qp8#4o^Up;7t!$-%J8T&GYTus|d;d9|wHZhbpF%?mW`Z!N%5W&vBqE zYk#by=u`+^+)s7#2`IT$qYj|!0-U?wFKBc(LJN9nZWI#$14F*A!@~d;3$0~!1GmWD z7w&6abdgv=Hz2E3#tSrF!KL?+o}ugy!?8EDy*DuBaxpVTWnQHJL)9vfo)sSgNDroNE*g#Za8=RA~D> ztbUMEkFC!!pDD?{*Y5oFdj5{teq$gbnh8mjn$3Vqjl-M`bd3Z#L+ijcZsL6TcUi7B zhS;FFnc!^aN@StDUfizJ&?0Xz)<|Hehq+?oMTCGS%>*79>|RJ!}sHW_kRi$n1x;0stuH z&!BDT{>n0$tjXIYV1WRmJ%ID0z5c_`9O=WAe>Ow3C}lLLVd z6Mz37)nU!bLYtC?Lv{Ol-If=5>o?V(uS|W{R>Ay42q0TXGp`3Y`JX{x#!K#55>_WM zl{vg$DNcW&M4fs;FHKmM&6K;%&$;=19DlaAcGw~?iG3G_x!lH&^*QL5Zymk4cKbIl ziesFDS<9$z0Nt56n&Sg;T#cIc355W=V*d5$tMk<)4I!|^yk1Rzw{RWRg%?PL4iU=j zJ1*YH%w~*FGTZDT>^%GJ&NJON$7fnqdObGncJ`&IiRLWmX`Td_qeZ;$v9!tfxEk;T z$XI}Ym;fP|p741v^c9yG7~uk@wA`VQg7U@8ue#i0dna>|qR-WvatK!KTNd^eIK{&H z5;F;^tDFFp_G4fImgl$b)Dgd`pCEKr3>f$9FD3npSqI~**cuRtwf_qsm+$)P%D+<7 z`w5vQ^Jx!gvkKAfPk%txv~=~6UC2X1!3#9pu^=Ds5vp^nOE%^x(vT~J=4$`x1I#h2{NnUYJ zcy|YI3F_WFd#`uDo^RVLVGtGiE!V?$-EKRmHGMkJA{icu7$YSE-s~owrw1}@vVZVi zdU5l>!q{x6wzZdT11zbIxp#SF3VDxP%jW7Wk7WqXfWKZzw#%gH#cyK~ZH}!%4{gm_ zHFv2<`(@8775BA87`hsJ^~3tsFZXSD4G2-gSod2%!~Hs;}RhXrHeF8dn~2-Hf@)VrSp%|GXb4}GGTcZ z1MU78vb#g-EGEu?JS{;AM3zh%j~54BHMUE{M*3=a2A_G9w04Kk6Q+bEFpTUNddQ;} zWK03pai9~Zv`{cq3pnZx1+TZCoTU+v?uHou@)KC|NhKRPzd^I}SiMKnZ(r}do9?6- z0PcFtIl0%(8S}SgXjnsx9$$0$7<$Ph-+#W5CudCSgX_232Mb}Tj$Z7Nx*#ppRsh0J zukVLSh6|Dl0R7WaHe7pV@fTB)X86I0^UjQm3~mwpUKNnWvzfs7q#>P5<(WOG>D=g~ zh{#FH^&e4-XR=}*&uKXEsa1ICtK0G?#^@~eO`JMsFTYI_WGs_CQj~B5gif3ccCj7+ z2RUSK+MYe4X%tvfoYe-?;a%P~$S%Ctp!o1TxE31i*cx#@Lf7OWvw}TnPwJURA~xwe z)=i%f14EjtM)=oYSr_vmTml8VWMP8+HzLaZK&;5x-J7EXUQt}9j#(%l#qJdp1E%@QVJBh-jA;3~!HZGenqj=aF`VB}Mw2bMM5|E8+1z3sAn7%9Y<;qQKe%cCAWzH0E5-zlmi^+3!?wuaRbcG&FkKAf}2uSgC*MXF;iyB2%bd zyW{yAvmfs+?vw5UT$}pnT|X{X>Jd+#`wT^PON^315_V8nEPz$qovnmb90u1uRGHO@km*!^Bl`zA6SmtVx~a6x zK0e#U*aikW;dfFPlR_>!882(F>#@~L+39$iYJ^{&sneJ#VlWPSt@I#ckX2T(`RUSi zv1Y+mGm$UpE7Eq@F=iWrxQsT;FR=U<&>xl>dQ|YzBun_q}i~$ zuZOZM<|KH&)a|K@wcakOH!~}UI%q@VwWJ6F*A_sH*Rq4a^(wu^7E!I^5 z+Y$wnT+ZCDji!^bi-rPGqueV*Wc_K%pK~Brr(XB|er@BJWBMNcCV>fQCxlAQ^rdIJ z|L*vpk0zD!sX_Y+Rhl1HOjMpR^EL_I#d8)0KAc%IFh!2ChA!`0 zrEZRd#?cx6TusJV$boGpLq-Q=y2L}75NG|IY#zgq*iWf_+#ug_Ua;+x*D+48elx6R zxsg;RL0G?ex$`^<;^)2g;YGOp{VNBhHKqAdAs%=h$E|w>iW^@v7Z0|bQI^!nl}<|S zQ!SM;4mLl`9c=4rbncUO>Yb(!FK=I^@#d*yIL&{h<}ICUeBDscuB6X7ilegoQ|^^# zZF~AaNUc3MU!~=Hjb!Eg>nMQGAXZhuHVv@}E!ktAB275G92c#H4a->J|Bh zgtiSvE;V0e`S>G$w?B&s{H?rEFRdwb(8!#@<#Cqoxr9O1L1}6=lgmYls&f2JEn#ID zce(@~Z?slbH>TF&(`TtjG-Kw}RaTLFqb$og{HZC$AumTWNG)+|sG^5PI7i?`lgo~s z>RRT*yoFuC%o8fqXXnNCNARSm-h+@AxSkfSvOIj!FDy`eQ>VaV-Nlm?%3PVRj?u?U zcAh5R)09+@C(R(eJ>_7{3q@JEZh+Ya(g&VsqN+b;nyal)cf_uY)^J-Mj{u;(6Mh`t zcV17q^9oJD-Z6#g9+;AmQU?9pqzIm61`|+j1E_aQ>d3)F+r)y2Uz9J>-DP8o6v z^~q#tnA29M#%uD|Y6o2M>YirQvWy3`KWAv#@}Vv%Ib&+IXIwMBeh0mo&xQt^njee5 zn3S><$iu6u%Tj%q@*-5ji>5hJd0BCmv-fHBZBK4GXjm&=?tE4xG*+oM>WL4xUsy9w zW=fEXd9RorF5ikE9xv5{tX*+>6JE^G-}v+5IAu|Y zvXn@VUeu6pDj;uv<^yzyq~a@w&-$6M)G%5F>rnPdQs?{}h&pH$Dx|hWVwU!4`{)S~ z`h^J)hMAEK;xA`3XrnRAlz7va1!_LdhKt4r?l4Uy`MjEcTQhbco?J-tEBp2q_57z! zx)O4EJhJa`8GX2&x{rloaUy6j%_KzV?5oWZm1A6PYl;l>J`&TsN8XTjrk%lE_3cTv zB!=}9ymx850Q|)wy^&JpYo;-ECshBA<1J<_BsT2EQCn$yZu zVs`o%m7y<`csgbGicE&Z`wfx}Z$IAYL=M=sbE!TVaHM>G{zz?BGfxUwbH3k0w?j6C zq9DhG;UNS49v~}e?)oCBmsS=~yuC{rjgpjPTGqpe0nDb23A1S%NZX^bIwz;j)?(&t z-NASb)sK@lms1me7J-zd(lOk}qW zf=^ox;nUW=Ii7!nW;OZM#V?AcKh?{J%7>Q2@%N!NK#kih5AbFCY( zTc^@u9UV%vqGZK%3mM6bYBo+YC8=_q&2wzc-O5bC<#)gsB#eYh#N}nJRIf980q$L> zIdtKP2dfjuCOR+Xwi!Ta%@pN?Ond6srRP$dtBV$U#AYaoZaOT29wahacAc}VHce)I zea6mxdibRC^U^YE2gR)=SIgr`TW46PBwjABYZd-zx{tNQUv; z$?f6pV#pAyQlZ9nogzUv9L_yqr!sb>zO_)IELP?;H5LjC9j~sqrK=M@vty^nu#o z5-B1|>C=wJkM&Mg3~JMp&T1v1Iey6k+==>k%@Pz}v(8sujA1`4rW@F%dR|g@zrQtJTI;NRZ*sf_h7b9H;p>ssohFNNj zS58WGi@g)OW_20c?JyZ0kvNUxOk!B z1}47*J!#z>-Tf}GLTPIm^i*5epQkE_8NGjUS$L2^^PZqrSo0W2pl~G*rlzJmn^%4K z{(+QIJrlp7HyL%3n`zOd3+>5n9UY%Ht$VljabKSSRcKt9gRERis_Qui*@&e~8Zpq| z@>TK80+_bZFH=_; zSEnD$$>@5*(kb$>M>PneUN|Y9tFGk0uGI06E0NJMnox>vQtxgRSnZ$(|FN&@#HD8oY?uGTG@p)Pm$sjb>{LUqejxJZy)O|0@pc#nVNV%+bH!pauOCG zwP53�yOs2WKckuFWte<(eN?m@jdVwJ<8un69_JJQtVWbhQ?&)i%8WYqjp4)?5z7 zX7LRE?WG#$db~N30^j;89%hXaEY(qv{rGBMi*rjPeH%mCv}ECY!FS_>l>5$yC_6+x zZg>f|c{&fdn|uz=FbcAv&c^o`9yrVK=9h&- z0jm^8&RqC%H)~(N$zTm}_ee$8_HPGKWM5#KVPWDqHg&m%W$GN|7d)O7f)2)IIz%ero_2cDQcEsueW{vfrOKWJ)nkAW;J$Mj~k7ce}Jye;MiEod!eQy?eK0lGZ7*(HUrCaaKu}Es4 z0V)hB4+y$}y410!w-cH#YU-Vljk(!pkjyAv`%1~(oKs?pXH<)DMA=dHlfxjgVT%qe z??l7hmgZW7<=7TeXG_AQ$c(%mm5s6ANYgEM-L?>_y-T*n^9p%d=L?2;hA4(R48K)F zWoHs!eZ4=wsJcp9@0`R(yDYw}l=sL+(m6kJVy(m$m*h1}ug_^dItUwn;k#?ifeM`U zTGw8#GmU>36{y-A2+U%!5tQt4!E|=4t-Y}R#ze|ISGNdZ3+8I8YX`USdT#bJdzQK)IM$pL4vK z)n~vQH?T{Q-j>tYG*Mcp?J9F#PPwPK+P9N6fgFY=($N|wwO?AktF~R)yl(u5&oW#` zT}QT@*fC!SHdU!4Z|~A zS?-{_j4bG!c7&*ebOMGuTrGv^D{s$v-bd$bNu6I&Ppkn0P(0N!8YT(=F3A7oK-P1` za_ai|{Mf$QLf{+E#?3nxaOS^%HNI|j?cQz>1mO{Rjyb*jPk^C;or;rB6jv>SM2F8a z+3^CUUfvp(Ql)1aO*6T-b5X-M@~87-9Hazi{m%p-%0BD*AtZg%$c;CGP9A%De8rL} z5pLrUAX>(adN+Iq5U2!d>Nj=uM$)O2g^lQ=wWdGBKoluEi3U_en#T$~? zKk6iQ_;^y3Q(=|v|J$}gyV?8vHjJ_9v{zv8BL_lG@HzFHd!#y_WebWzB$`&t5y~-$ z21@Z9p~W_uVbDrC(o?wOF~;^PxOy-8V^so-=dGaoid3%AoxxkHY_GTYWMOgsfft{I z@IMN=I2odLXwRL5JzI&X;=w=luzvGTQg3?Ing-Mvy~pah8$%0-j^KUeE5h!_(VD{U zSAz*jHntm%yZ2%Xjlcjzg;z*s_|2^TeE*I{wc-pf=+A=aE3Q>n;K1v* zi&Ud2QlY^OVg_A4jUyUW2z(G5Ma*b}Mor2}URguDt_S-AZZvc$$BDcWQ|7U9Ci!nJ zd|2Cw=g+knlKO{Vtl`=!&9xUdFmX4)XMrh_@-s>4Y!zA58LW%_n+aWpu5$nXY8toX zU;682XxxeAW&hHh6&K@nQQp}0)Ai-WCvkD>x$Two%#`%kay*T#N>P&ge4*+6jq;aS z+&^!5@AX|*a-peuc}B3mzjL{#wEMcf{gvonj-?Ln*RM82jh-6)+~=|ob(JoFp9oJ= zA`q(S@R6bZI}n11Kw#A%oWP=SaB}^Rzns7{i9pD+UkL=&e+>k|WgfeU*3`yvP&xh^ zyc-DMT^k6#ng*t<6&PqVz(Ca+-plj)x>W)UbkCR&CvMsnAiSns7~G8_%l=xk)tiYm zyJswe1CIo{NU{u<253!3{>5DgSWmp~HX;(L3A;PNzawD_ff1lOLgFqg|0m9x23u%= z-fbMa^$~|?8xc6rsw?ZRl45klwe2eo2y3*)7q*tLd*R9|DIylFbqV3x=0va{-u>m; zWNXP#>PXvoQ}LtwE;bBEKycbf&L<7 zW}^-643({i!I#)ic-{z^IUG#mc??DpYXyMRSiHRStNaV_i|s)|P-gXOFcj@ckba11 z&M&KD79rpbNGlm^@O;~jz}=+$e^Ycb0a~Kq5Wjo)uPU=MObgWhXKj<~MtoM8b-3f|Z>1AWdyAfvZ0f?%@+ zH1w{cb}E?eCVc`G+cVc0Y8+^3=$J{XZI%0fNyiMVCI~iDFx(3_Nfzmu`DyoN!1pB> zgMold)<@)B?e-bo@RLQ+Fcp$OFWGPZ{_*4PKp3iR0S`%)!A(&aVyJ>RaDz^|cW9^( zZUyukKYdUnMmIr;hOmH<0*NtJWA3K+NVG z8fql!;J$CSlyd85!2Wzeb`Hq#1^MayZhx*k3GfVCgb;BXAM_Wo2jQ~<=wp9%erju| zG_Fl4Z<)YIZFzWb5VN|?-oUsYOaz9-No5XQ;}s%p)!z@O zLu`vDqIAYeymk|+x-YaflmLIs<#i!k8KEdXbr)p2>FCMiC2l`4Mt2>7iH?p)eo<(K zk(T^>qzQ&Y^Am!X61CeXG-VE8J&$Af0NTpDJXk?%o=U@DSHY{z>?r8+MVG+Z z+zO`BR7H%p}j`s&^o;`av(l{wxZ?N0XL7!G=|DS*+BT_2ao_(RH*u4}i z+8j569B{zyc1g!N$Y zd`eOPnuf(Y4<${!s(eYxfTDDS4=-E(u=rW*UlsjC1NBUh`x7WomL9A{GVpLK%o(#y zZJhN6Sip&LNW44qTe_+)gH+eATBAYWGIO5aqQ-0`e0D-)zeO1nOg=CZyYhwJuU%p+ zH4fe6bbCmuu&dT^6FHBYlc^Y6Dem9V5_6fKwgRQ;YmXj1vViqzS!XWI)0aj+SM}c|fUzKg}I9c2+D>Mw)AMga*<`?{i9}Udj!|*hx z<)|aIJytj=Nklx!aa~@Ikfn6AFup3#E6!SgLHe}j8Qk^f1ZN-7+ecCmUqv=RB&q=o z*=hW$ffAl5srlMrsjMhX*6!t0ItRs(}4q##QmF#j@bfd)s> zKYo{vRf`*e{A+_WDG>$zSCQ|ruIfeD(5}PIc20*bZ_yIEIBB1vWHuDA^}Sy&g$F0L zUHzA;E|Z)ed}ia3-s@S+zj(|SMSzJHmG@bHK9|_0*{?CPGLlJ{SJu>!Wf7Cmkf^eY(cfg<-QN^V*ilKk%(xN2TV6xbB_KxE zjeJk_O4yBk1@c#olUT8zqoO^AOWjYk>||?D`slTjJ3(UE5j3Tlqd$j#SL1cB)~xqUz?HF zuK5K}0N0RP0kVgV%Kyw+@;S!>L|~DItqs)KK`Ko@(6ue{CDd{F=xG7z!gnU0;UE0~ z+JesUC&8p0D5znIbm)OhUr++Z8kjk(eV^mis}=wzo=Q+x- zZ;71m3ZWRR;=eNgrx=_hN|dgr-wT=04x>-(dsE#%!CHeJEICCK=8+Q@gia)8Bnl{Q zjaU)eF+~BVX`|~^)K?1uRmqW$inFKNBK-a=M_OVz9^()?f-Zty(4bO~EJ(Z+p@_4} z=1p`}CNc~9tT9;D5~v8DF#ojxyH^i@-^U6TeSw&xc=VLtjfV!$t$adPy8bssONei_ z&lJu=Tg;1{@Xht$5IX{D%#YUSNT>Dx9uX*eKLUly&NCz7ZJ1S>=VV7U8hNW24Tfdc0CwR-J(+cV`a*V7h4 z(DZw9S6NvG`U&`Px2))-{kM$~6=C3pi+o@fCY_LbI`fgsiEG9XyA5?v+{F5!_mucrailov40h{!w<&k?^gLi@DNj#h<}W=Z$I`AvBPAVj3%lACQvQQ`!B-=`9opi zetI5TUt!hRzvN;#iF~E0es~alS%Cf;*Z=`07v2ps5y2#2H6Y$ma{X)4!lwPX|7`aJ zt>L&55QQs#j7hu0QDnfa3{dOHEoAP{7~8!i096Za%4|ep;7I-oZ0Vm>*l*FKv@~0r z@Mj(a$cZ~8MnbrxnimBK#*ol;vGf0dMEBQj*lzoA-hB!x`Y(63(T<>i#Nq^Oljz&^ z#-ytvM;*_s+^P_E{xK&Wb(b z$t_wD*XPJXU8ldyMSn2pA1#l1P1DR_sB2chSOc{> z-(R)*f8C8E)YLU9uZkiuYBlhZds;~aUUG2qI9S)R_{v3|=c1{Nmjh|@soq1R#cY0g zK9p9JrD5;cofC`J&abz;?+W2P;~5@e&QIu!nFlA7&W?zGw*TO}SRt$!hgS&AeDb5m z?w&ie7Gq!maOBsIcOoV5vK`0+@Jo=&xhaM5Po2+_Bad6fMFKuvzl8mtBtgyADCh4r zWwVZ)na`fJWR@?@=k3${!7cqp>&J`ZJ7EjLqdkET(2?nkGr*|F-=$*r$4P1^fw}cJ zgT;yS!b$h|AxgsXC-xl-I;mOa2h9Fty|3p z$Ijh+J5`u0cu$kcuAOV~Nuqo$7Ka;vrO3g<19Y{_3X>&k`E}84(N1^T*^4pc8fe5v zX_=%Z_5f9<7cNtJ1GPH*9JI(c;x@2oINYg-U*^1MV2l6qktv^40GPkP9FgG6L^%}$ zbMOU~4Pq#-k{MkHS5#RjkzF!^s<|k4RktPy*~1K?S(dT_qL+^8O;0*te>W`OQGrh)Z&vY&&B&cyjEI|)jNprj+v2Tz`|P= zzASzo9qO+Gy3ZC_Fl#XBx&To~Fmcj#kB;qwZs0KK4Tk=+MGK4H{17zPS;EnT8YLIJeZ!0AR{D_O*zKM9RMT%DZR3ws{8__jUA9c z(z`*|UN0W1#ecB5)*FWEjDXba4&>TY1OoWIthdut(Iruj414B~?)>Lx?|SyxyxrA} z4zlmp%l|pg#BL)7oR;c9C~B}>@`Y0M3NpxqQ}}m;X{Lr?1qH*Pj38>)cHxkRz*tAgAGCj z(G6_vBk40~$|LRN&QyAYY9V`e@+2MIOC;3v(VvkRt~jg>SUr$cERSoc|>CMy5C} zFaGL1cXDzF-PVEMV0RJRZ*sSzf<%9t-(QQ`hZqH)E#6u9DDb)Sc}HZI&DX*OU)_#b z0OVXptQE5YF5jsts(pX$a(jtlX73&f9V~f@+E)Zg79;duMk=>{RpHCq8{XJ2>{NOB z_)jHZO_&x9J3rWsnCR|&JEhW7G~mM3xjwaganjNv#^tBiHZ{#bUry1#M(w}d2Q@R9 zAQNzaQG-E?Ya@cqXu+PPU*c(eUm*X**k4B}u!BQOc$LKH3hY=-&GF4&iHXpuacS4oA8FV=*eN#EYsPu+)p#DJz52S)Wn~cX&SZDM%6( z{PO?_;E9cD6`An?gA3^Qzl%f6+rUA!2>kBy4wV&+Mt9Flc6VT>oTTt8-uvw!aD*LB z{J|h>A}&=n9XWmkHBn!%;cZCuQGP2@iP3pGUy*VNepixtgzhre{H{!3cG#mS=oQWR^{Bu>+LB%P`kf*83rRGNhS0Y@?7 zs7j-ADhlsh^Ak<^AL1WGu~9+KuSlf*seC-AaT01b!@KWoK`9P>CV-@iRY2%)B@jAX z(|_x5A+#wf>oYQO#Wc80#PZFRzZKJz2~Ceg18yFS%j`&KC%m%>eNRfOMuZ$fTi1$W zn##W_rVanD1EQ>{GEumx8~ZGWkW4cJVKzY!;xhYNi0i*x>tOFY^0#Cf zCn3b8Sx6G%TJp!?MSlqe8DZ+w8b z7R?Ym`D=3=x*Texz=K&{e)~C+Ufc^lw8GNvhee#QOI!Ym+gu{j#U79JaGQrv|DuCtdrIy=Xw3zo!b1HB6XqXr?Df&;MnG)Gpw zA%);7cV)E3Qq%O(_CCl-m62wfP@ZDK%}|GlHtq;%{jYMz+qic3Q(1WOfAbelV=Me7 zC;s;4^;hCZ*L`&7u(wyy+bm5v3flDO0pXne>M=#8FjGIzIBlQTS){k29N^V`mR8Oe%5rgtXVyYDn) zRY@ObIE@msjJTWx_xXpaC+s1SOTTr?!j%X%5jqu(d;>hBKN!NKe}W6cybopP4qH~- z+qI;~ptil=aU&EMN6%szBfDsm{0q__R7j#M)2Q$-fO3&F>?J za1C6qOyIsvAot40u?tDntP0g>d-bmG0ub?TEP-tMEyaDnm$O;~5Ba&6m*N}oQtzf{ zI+ovxC{-#8)H_uJ)#i7bw6>M!lG0?ad z|8n2@%?*DFA`Gy_R7nA=0K-ex5OMNvEqRy|!;5kf$)btHsxU1rS_hxPv`FKbJ#yN4 zz=K{5^pc@1JSY%6q1lV-Dft+Hg|QF9q? zNCojFQ!j6{hP3ucU9mLxVO8V0e&oXCFO|0bh4p4^r@*fEZBANk+Dhak_YmSd$IZcO zs3HwEXt_$EZ0iecA-;zQ;SFs$9hkLN>3PiE7Wv%VA;xkPjn6tn$p# z{;b@94fsg>IC|C#uLNK*|0Eg?lNvX4vNYwD$1g>H54}ZO30L**cZlTdpm%LWY-)rk z7K2K+d2qVH;ePB`&Np5LwOR|B*YH4T_?&Cmt+|GB1s-@Tqq<<}c7pOmF92#hn>C+n zMEs~t3e0GZF^WH;7V_ngKIAeCzZ6RS_wN61dmgQhSWWbska7b@8-5s10vbiz|8Jh< z^hpSv(a_g?R@v}tG`HTy3rslVDrEH7Q&MC%?j!1}9vYd-&c`SnDqEf03lpP=&*w#^ zQ!CGpZ*Gu$Y-bW?pA##YKMj+f4LpR)ETuYD*pc41no#TsF1PDtj57(8+3+K%JW65B ztJ{Fhz*Fr@3jK|wFgGA-%G$93bA$Jnv3M+gbinmaf{6#mD6EY({Ess2mmT-7?2RgcfmoyMnlca)4a$zIkyim2KIwOW1eybdXstg%8GYG5LcP>^|V zSNr|BwNAt`UuXl0wy4;{O{2+ZQ*wcC%sFo%1xFFV|3&qSWx{chN8ejFN}00n_^xy0Zu zlAS0)_YTM$5H2E19gfIwQ4-l`o-}8qY*a+9f;j&R3;oMqfaxG`9sKwBijVwuM1YxI zXaJ1D&5dvYTv!OSomXZqhYv%4Aljzo+{w&t>pD&?>rQ+QvjF4IS!93mr0)&kieBA# zGlL=fFps)+-!y}zQN7R$^RyW z0UyZ*HheAeM%J(Lv4RVzq7t~hrqwmv6zvGuBn+9OtMuw|TV_N;^URO$LFI*`XL=U- zow9=ka{&Zjxb4{c7pK0xSnF%wGWonQ`=SHz5_(>3;o7IwpLk*(ZfARsb(;;Y(<(X! z4&v8gT&OMzgQ;0Byuj7#&C<+8AfA(yI2dn3oV4Hk$4MKCrlw#1M@BvI<=?J5s}aJe z)^mvU%MwkI7c_4kX&F3_Kv1RUjpP00Zijnq1xMj7@_LtY30!khxJG+~bpZmUM`USg zhnu{i+;*^u2E0S}Rbi7GaXI;|q-M^n+cFU1?Y3~{1}uZUYWbu_qb6P$Sbn;){S+xI z0bj1ra4WOtA}ya$0H&=2=7^&WpkE>>HX22PFPls&nrjFWjc4F7 zD4x`=C`0zm33tfbHiB;H!H6Ucb;?S1otmoH-HX&B3@db(zVBIfVi7`06ooKB{;nr| zb2tD|qd5EB0dCw&V9`F;0z*!acS=JoWEc5;UNq(f3t)Bo3GVqP0{N3{{Y(_b?~j6D z=N6Xdu?ovbJ;ZdP$Tm;&3zBG6=M$}Ad* zIF~aH9k6K)hOTyo2e#l0OJ(xaXdz~^*r&Zu)wJw84PE++LwFRuyJ2w;*=93T#wm-9 zN>gkHB*7!=TGpjpp1oZiU$)i+0z@B;y-sH`p)K^i8 zQ=i5P&uf|UCv$p7sN^-=Re#IXCfj`c3tXPQ($)q|njU0~`&HuU(G33RYsgz83m11B&oH-1? z+|JWwiS!aMU>%qJfv{k{DP-OrYPsql@8J;?k>BiLeyoyN1oD8+W@FilEbM1=kIM4b zY^ez`nzC_OjeI$K;R00MzD;^sXlZuP0)lNiP`-o3qG_rhw1diy_-Co5dDVcQ%6XpM zsnLnqH0NtuSvyqUYUr^>a4S1!44kJ@sLlywQBIOGIxWXqC)ok3G_p1)IQKr)t@9xd zy$#DjMau9EOGt;#b)AoP`?=bKBfj9CoBE7?Rlc<6ZGI}Ie!AzS7krbwqv81AR<4_A z^)Tky01SC88*Q!4_>JKjSSMsgJBU3e&qf%&Wd*O(UgG?jmAar=wbW0Zf#}f!;`2p0 zfw_GQjzNwr+)SrING0Ib^E7Ord(UPH=2=@CgAuPiB=5+B&-zGyo0N)rLG3ha>YW>M zuZU+(wrM`kOq9m8&|!@Yg@Xs<+uppNTMT)pjOj%1^$F&V z^VS@!^q{25^fc+Iae8P@Ak~FStNUZBesz>DwPk{n<`-a6`G+cAV$O6fkv>HZF8r5w7=lZlkUVqPXPuT2ZHeP6XFvwwH#-Qtj( zr>zA%CEnd=`+s+_^NhEKhM>tip&ve1`;2Ztc66~1iI~{2-{FJ%#Q+%(3;u;BAzJmkNx+2TJpB?N*4a7HeS+(oQ4|`Gip66t{^~sCbV0|>)9=rOnm!bA$T|08t z>=1_dJ3_AbEAFiS0oSd;`Q_47#{1LS4ky_Q`@t_v|RDnZp&xj1-K|~>6!oA z_+)4V4{uO!+@w1F8SypZy3Vf)?X>wySU@l!hObIlHz$~rzB7Sq@$7y()ys9^^X%tU zq{Obm?KyJMd_;25%r%s%45rOIIRns1C-BfrUUMhYaNbhqtnLd&W!1WIWIpsJIGtOK zBZ(pMR#=ncL^1$#%KTxu=Fp{xX?FpQ6?}x3TiY{Tes;cDUO>uU{lXgB1!uFz@0XQ* z+v6|7mL#UGTz$&q1GN)ku(m3-Ottnp>IiKdnoAGBbU-1ZyZla5l0*kSc9 zd#vTN_t%mt`Z-qm1SZzh$tS)Q@o(u2-x$+I=5LYgh7EO)5g6}}#FdO{Tbm=)9?Ge% z*Gg^o3G*JDzbr?t?xd-^(@sdr}tnZ1M_KT%|)TI6vUhB(aCm zpk8uHmH9YmZCMsWy6c78)q*;~+Mx%r52jBH+&kO@tw42$(x9;BkW@ug4IU}wltP3I zUb_DzBL5iMv&08U+;>0kXvdZXC@X(g0aT0lS)R2oLN#0DFs$VdS%1*D~sGEiDj zN~OQYIX3R+e(vXae!qYBI(NpApX2j+$NEhNzmH&b9dvFP`Gxhveit5?)awf@jCobX zD&M22XbmiKe!hd{YAqwdjE(na6jmtRP`E~o7a-8gVxfwrilY>wJHCyj$MCW(8V^fTCl;J`zQz%}l5h=sl$ANp0+@r> zx`GQsj+W0PuTGk)Gz}>JGFVDZXgz-mTZSxW)+7?*|H-umm9t`zrmq^Ic>V2}0AQZk zv*0_zP>^4rfI)w`y`~NyS3}g>MLvf@*meu%iRNr6|bo%JwZTDe5a%n{>PUSfuWlYFgbx+x<|vbfgmY~livF$FKPHW6Jk@(hE)sP6ro(9~^ilX%O=8Kzr{P{F zX?x~xV}XjwKv?)QwWfXm*Z=0|pkf*Wt2rDd)&HFKoxn@Yz*TCMZF4+qRE^jXw%IM! zmWiQ`bYT_Z$4#VLwhhJ}AQ0_qGs+(J&qeYV`N$gU>+lJ=O!PJc2JTvS9&7EMkyL7Z zuIpIbe=m(06=}8$pf=v1R{IL|H`=bvB0e57BRG_EXp#5OTf;7sAAh|YP<;l%i;os0Th0QF*=ii0Umygjf{e5Dg163PN4n6vSlKLkCC8E2h9b)7~^WSY3GSlu-0O765MIh|uRW z@yqYO7r&G@@wHRZ_GV~MrhWm{ho1aXgi~Gu!5_gUHu~;T7btM$Ox#Tv=X;DfFmWoA zou8g(T_A<0Ys3ZjX4L@;8a@BYARL$D`5fJ0QQYGM@8o?kaIUk`jwRuG;bMY`lo<2$ zR+tO3ftOkNvp{@W&Ud)%!*~k9Aa{z zQ>1MdSlez*9J4|;y;ut_vkMRC5lH~jAVe*7omN3P9i1Lp9VJw(le4A1FsFon_D+*% zK+nP=Xu`&Lj+^p*nito(!&vv1OOZ#m!H)MnOC~zzm^-p%S>(LWw9hH!ixj@dCjflw{3j= zaU2@XvM9-visSG{nxkE(L)9ckfQO=`+o`2Ya7t?M14x=buA~}>${er2R0JY9F0){!d~Luogp;G%XS@{J=Q`>2i_#jg62bW7YxQ{%A9hBs}_G}54|A9)NYp;s%(7k;f*UgI#P6O z@(7?@lB{G7-!^|`uH&XP%5(SxhnZWN&4-aw885-`6r3_DD9*c%l!d%Afk@a|HnM#2 z`P<`iD0MMO1|59<@}VdOi|o(!;?zV#4cYOs{#Q{-nghQ6#TP?mB#J|I9~BO+;XH9` zhOkF32kWnt$Uq~ntOZ1?#;?>O_Q|duAnBJ-ePVCU7AW_k=HmTxHDS*99@eer5`^^R zUg4*5UUD+)S`5mT57+u%TGaWjJF&>JX{TJ-sID*%;r|FrQybF%`>DxEl&_n)Kv7AA zi0!(-BSeZm99In4c)}x!IScOoXyeU3;w>V^JHPd9gqmsO+tz* zdvq#H@1P^?nHF#vtx(8y_LdF z&>U)Lrx|=rtU3@_T^!i-I~&Zf-%+|wmN^m~zwIoEX1f4(bys?^Dpy|1@p=DSCdI6_ zoUrAd_irn1uF+5V6t9GRb>Q#rnATh{LcEU!#g-F3|MLHV>W*z=Vx-8klp0fpdHA`9 z)tqQo%(3ka6mf?VmY&sI&b}2Jz!E0%WRxj^Ya?Z@#m=t>LTqzofA%VGj;FmokMPLG z-%7|cK9#j`L{ZR0nt!9oWcsla6GB+FPT1y)x|#iE!nfy&{KxIL<~}|Nm;@Gd}>(X(nrg z*?%shUU#0Zq6`_&oc;ZT^Q=DIXg81Eap~_DvTT`|jW?DHYD7Uo$h}fJPjs3=y-DXo zA1lgsr$Dro%D)xemngjb8&br@iiUF7%G+;dhxt5tIMMb?zt8%s=Qas5?Ts!%*a^g` zx-wmMeiZlG-hN$+=cYPv3Ga9n<&mJ=t$YAkkm*Xb#!768=yw|7(V^t^;zttUvTA)|h93 zH6}oX7=5wbv&Iw^p;%+GT9U0XXU&{59Gh~dnDa}rtv-wlV)L;uM=bk!=cV2ctGs49 z?+Wu_hPHiAld1$k6Vnr7>Dq0+nO=oW-^#JXUMKP;6rpTsw;W;z3pCMB@9)V zSZp^}X<`mP%SgLBBpr2$UDW zp~bw^==&C?+pFH?C+kF5Dzb0Q!;@xAqvq8dePj{kdwJ|wxV&A~Rkn)`KPgW0(D zLwpbkoAL29?!XgDm~UEo7zHE( z%a}Y1Gz?tYKY4+P-bQ=79oP7pRB%8xBegHu<0BkF9>^v~T49JNjOPDi# zm$J`Ldip2{on^&{0!hG35EE?Q4+QA03=)pfM1KK@gqp|kX}JNT3^$M!zNch8o%f5e zB`vvTj!-xT^jF7$5dHk|E1wU5+Qs&J$R~DO0hIrKy_Z$h-J`r)O#MriXRox#oR&NS zlsCu0p4>fSCY1gp+2M5d?D1tG9Hi#kFO9YSoX1-H*Fw>2vlYF0fxf`&UU z?iDK+QjN?YgG3SzKaIs@3m>uD`0wo3M?)#L%H7B23*zOGoS{x`Ww-qs0rbWrT=_> zR2%U^zqucZjWGR^klYH70mz_jOlB_=D-dAeAV{jK4O;j){|5D)cyqp{khHHSQ8kMo zl3j$3mA`5*hugSg$F6`AZS7h97eM86>mLmo1P2Wh;xXrQAOTut0|+#*BhM$g8r9u2 zFkIyjC%8QfsrAqvoV0}5%Rt%IClP|k-ehl@KIhs