diff --git a/.ci/docker-compose-file/openldap/README.md b/.ci/docker-compose-file/openldap/README.md new file mode 100644 index 000000000..c91b5c1dc --- /dev/null +++ b/.ci/docker-compose-file/openldap/README.md @@ -0,0 +1,61 @@ +# LDAP authentication + +To run manual tests with the default docker-compose files. + +Expose openldap container port by uncommenting the `ports` config in `docker-compose-ldap.yaml ` + +To start openldap: + +``` +docker-compose -f ./.ci/docker-compose-file/docker-compose.yaml -f ./.ci/docker-compose-file/docker-compose-ldap.yaml up -docker +``` + +## LDAP database + +LDAP database is populated from below files: +``` +apps/emqx_ldap/test/data/emqx.io.ldif /usr/local/etc/openldap/schema/emqx.io.ldif +apps/emqx_ldap/test/data/emqx.schema /usr/local/etc/openldap/schema/emqx.schema +``` + +## Minimal EMQX config + +``` +authentication = [ + { + backend = ldap + base_dn = "uid=${username},ou=testdevice,dc=emqx,dc=io" + filter = "(& (objectClass=mqttUser) (uid=${username}))" + mechanism = password_based + method { + is_superuser_attribute = isSuperuser + password_attribute = userPassword + type = hash + } + password = public + pool_size = 8 + query_timeout = "5s" + request_timeout = "10s" + server = "localhost:1389" + username = "cn=root,dc=emqx,dc=io" + } +] +``` + +## Example ldapsearch command + +``` +ldapsearch -x -H ldap://localhost:389 -D "cn=root,dc=emqx,dc=io" -W -b "uid=mqttuser0007,ou=testdevice,dc=emqx,dc=io" "(&(objectClass=mqttUser)(uid=mqttuser0007))" +``` + +## Example mqttx command + +The client password hashes are generated from their username. + +``` +# disabled user +mqttx pub -t 't/1' -h localhost -p 1883 -m x -u mqttuser0006 -P mqttuser0006 + +# enabled super-user +mqttx pub -t 't/1' -h localhost -p 1883 -m x -u mqttuser0007 -P mqttuser0007 +``` diff --git a/.github/workflows/run_docker_tests.yaml b/.github/workflows/run_docker_tests.yaml index 9c695c4a6..17d5395b2 100644 --- a/.github/workflows/run_docker_tests.yaml +++ b/.github/workflows/run_docker_tests.yaml @@ -69,7 +69,6 @@ jobs: shell: bash env: EMQX_NAME: ${{ matrix.profile }} - _EMQX_TEST_DB_BACKEND: ${{ matrix.cluster_db_backend }} strategy: fail-fast: false @@ -78,15 +77,17 @@ jobs: - emqx - emqx-enterprise - emqx-elixir - cluster_db_backend: - - mnesia - - rlog steps: - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - name: Set up environment id: env run: | source env.sh + if [ "$EMQX_NAME" = "emqx-enterprise" ]; then + _EMQX_TEST_DB_BACKEND='rlog' + else + _EMQX_TEST_DB_BACKEND='mnesia' + fi PKG_VSN=$(docker run --rm -v $(pwd):$(pwd) -w $(pwd) -u $(id -u) "$EMQX_BUILDER" ./pkg-vsn.sh "$EMQX_NAME") echo "PKG_VSN=$PKG_VSN" >> "$GITHUB_ENV" - uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7 diff --git a/apps/emqx/include/emqx.hrl b/apps/emqx/include/emqx.hrl index 0e17f71f2..78cf3825e 100644 --- a/apps/emqx/include/emqx.hrl +++ b/apps/emqx/include/emqx.hrl @@ -65,9 +65,20 @@ %% Route %%-------------------------------------------------------------------- +-record(share_dest, { + session_id :: emqx_session:session_id(), + group :: emqx_types:group() +}). + -record(route, { topic :: binary(), - dest :: node() | {binary(), node()} | emqx_session:session_id() | emqx_external_broker:dest() + dest :: + node() + | {binary(), node()} + | emqx_session:session_id() + %% One session can also have multiple subscriptions to the same topic through different groups + | #share_dest{} + | emqx_external_broker:dest() }). %%-------------------------------------------------------------------- diff --git a/apps/emqx/include/logger.hrl b/apps/emqx/include/logger.hrl index 0083a9660..a65b05089 100644 --- a/apps/emqx/include/logger.hrl +++ b/apps/emqx/include/logger.hrl @@ -41,16 +41,20 @@ ). %% NOTE: do not forget to use atom for msg and add every used msg to -%% the default value of `log.thorttling.msgs` list. +%% the default value of `log.throttling.msgs` list. -define(SLOG_THROTTLE(Level, Data), ?SLOG_THROTTLE(Level, Data, #{}) ). -define(SLOG_THROTTLE(Level, Data, Meta), + ?SLOG_THROTTLE(Level, undefined, Data, Meta) +). + +-define(SLOG_THROTTLE(Level, UniqueKey, Data, Meta), case logger:allow(Level, ?MODULE) of true -> (fun(#{msg := __Msg} = __Data) -> - case emqx_log_throttler:allow(__Msg) of + case emqx_log_throttler:allow(__Msg, UniqueKey) of true -> logger:log(Level, __Data, Meta); false -> diff --git a/apps/emqx/priv/bpapi.versions b/apps/emqx/priv/bpapi.versions index f0a2adca7..02dd84f03 100644 --- a/apps/emqx/priv/bpapi.versions +++ b/apps/emqx/priv/bpapi.versions @@ -10,6 +10,7 @@ {emqx_bridge,5}. {emqx_bridge,6}. {emqx_broker,1}. +{emqx_cluster_link,1}. {emqx_cm,1}. {emqx_cm,2}. {emqx_cm,3}. @@ -26,6 +27,7 @@ {emqx_ds,2}. {emqx_ds,3}. {emqx_ds,4}. +{emqx_ds_shared_sub,1}. {emqx_eviction_agent,1}. {emqx_eviction_agent,2}. {emqx_eviction_agent,3}. diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index 7a6ec9810..b98728ed1 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -31,12 +31,11 @@ {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.11.3"}}}, {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.19.5"}}}, {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "3.3.1"}}}, - {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.43.1"}}}, + {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.43.2"}}}, {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.3"}}}, {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}, {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}}, - {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.10"}}}, - {ra, "2.7.3"} + {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.10"}}} ]}. {plugins, [{rebar3_proper, "0.12.1"}, rebar3_path_deps]}. diff --git a/apps/emqx/src/emqx_exclusive_subscription.erl b/apps/emqx/src/emqx_exclusive_subscription.erl index 3a9dc8014..095233fbb 100644 --- a/apps/emqx/src/emqx_exclusive_subscription.erl +++ b/apps/emqx/src/emqx_exclusive_subscription.erl @@ -117,6 +117,13 @@ try_subscribe(ClientId, Topic) -> write ), allow; + [#exclusive_subscription{clientid = ClientId, topic = Topic}] -> + %% Fixed the issue-13476 + %% In this feature, the user must manually call `unsubscribe` to release the lock, + %% but sometimes the node may go down for some reason, + %% then the client will reconnect to this node and resubscribe. + %% We need to allow resubscription, otherwise the lock will never be released. + allow; [_] -> deny end. diff --git a/apps/emqx/src/emqx_external_broker.erl b/apps/emqx/src/emqx_external_broker.erl index fe360a5b8..5fcee71f0 100644 --- a/apps/emqx/src/emqx_external_broker.erl +++ b/apps/emqx/src/emqx_external_broker.erl @@ -43,7 +43,9 @@ add_shared_route/2, delete_shared_route/2, add_persistent_route/2, - delete_persistent_route/2 + delete_persistent_route/2, + add_persistent_shared_route/3, + delete_persistent_shared_route/3 ]). -export_type([dest/0]). @@ -129,6 +131,12 @@ add_persistent_route(Topic, ID) -> delete_persistent_route(Topic, ID) -> ?safe_with_provider(?FUNCTION_NAME(Topic, ID), ok). +add_persistent_shared_route(Topic, Group, ID) -> + ?safe_with_provider(?FUNCTION_NAME(Topic, Group, ID), ok). + +delete_persistent_shared_route(Topic, Group, ID) -> + ?safe_with_provider(?FUNCTION_NAME(Topic, Group, ID), ok). + %%-------------------------------------------------------------------- %% Internal functions %%-------------------------------------------------------------------- diff --git a/apps/emqx/src/emqx_log_throttler.erl b/apps/emqx/src/emqx_log_throttler.erl index 3ebc268fa..928580e2b 100644 --- a/apps/emqx/src/emqx_log_throttler.erl +++ b/apps/emqx/src/emqx_log_throttler.erl @@ -25,7 +25,7 @@ -export([start_link/0]). %% throttler API --export([allow/1]). +-export([allow/2]). %% gen_server callbacks -export([ @@ -40,23 +40,29 @@ -define(SEQ_ID(Msg), {?MODULE, Msg}). -define(NEW_SEQ, atomics:new(1, [{signed, false}])). -define(GET_SEQ(Msg), persistent_term:get(?SEQ_ID(Msg), undefined)). +-define(ERASE_SEQ(Msg), persistent_term:erase(?SEQ_ID(Msg))). -define(RESET_SEQ(SeqRef), atomics:put(SeqRef, 1, 0)). -define(INC_SEQ(SeqRef), atomics:add(SeqRef, 1, 1)). -define(GET_DROPPED(SeqRef), atomics:get(SeqRef, 1) - 1). -define(IS_ALLOWED(SeqRef), atomics:add_get(SeqRef, 1, 1) =:= 1). --define(NEW_THROTTLE(Msg, SeqRef), persistent_term:put(?SEQ_ID(Msg), SeqRef)). - -define(MSGS_LIST, emqx:get_config([log, throttling, msgs], [])). -define(TIME_WINDOW_MS, timer:seconds(emqx:get_config([log, throttling, time_window], 60))). --spec allow(atom()) -> boolean(). -allow(Msg) when is_atom(Msg) -> +%% @doc Check if a throttled log message is allowed to pass down to the logger this time. +%% The Msg has to be an atom, and the second argument `UniqueKey' should be `undefined' +%% for predefined message IDs. +%% For relatively static resources created from configurations such as data integration +%% resource IDs `UniqueKey' should be of `binary()' type. +-spec allow(atom(), undefined | binary()) -> boolean(). +allow(Msg, UniqueKey) when + is_atom(Msg) andalso (is_binary(UniqueKey) orelse UniqueKey =:= undefined) +-> case emqx_logger:get_primary_log_level() of debug -> true; _ -> - do_allow(Msg) + do_allow(Msg, UniqueKey) end. -spec start_link() -> startlink_ret(). @@ -68,7 +74,8 @@ start_link() -> %%-------------------------------------------------------------------- init([]) -> - ok = lists:foreach(fun(Msg) -> ?NEW_THROTTLE(Msg, ?NEW_SEQ) end, ?MSGS_LIST), + process_flag(trap_exit, true), + ok = lists:foreach(fun new_throttler/1, ?MSGS_LIST), CurrentPeriodMs = ?TIME_WINDOW_MS, TimerRef = schedule_refresh(CurrentPeriodMs), {ok, #{timer_ref => TimerRef, current_period_ms => CurrentPeriodMs}}. @@ -86,16 +93,22 @@ handle_info(refresh, #{current_period_ms := PeriodMs} = State) -> DroppedStats = lists:foldl( fun(Msg, Acc) -> case ?GET_SEQ(Msg) of - %% Should not happen, unless the static ids list is updated at run-time. undefined -> - ?NEW_THROTTLE(Msg, ?NEW_SEQ), + %% Should not happen, unless the static ids list is updated at run-time. + new_throttler(Msg), ?tp(log_throttler_new_msg, #{throttled_msg => Msg}), Acc; + SeqMap when is_map(SeqMap) -> + maps:fold( + fun(Key, Ref, Acc0) -> + ID = iolist_to_binary([atom_to_binary(Msg), $:, Key]), + drop_stats(Ref, ID, Acc0) + end, + Acc, + SeqMap + ); SeqRef -> - Dropped = ?GET_DROPPED(SeqRef), - ok = ?RESET_SEQ(SeqRef), - ?tp(log_throttler_dropped, #{dropped_count => Dropped, throttled_msg => Msg}), - maybe_add_dropped(Msg, Dropped, Acc) + drop_stats(SeqRef, Msg, Acc) end end, #{}, @@ -112,7 +125,16 @@ handle_info(Info, State) -> ?SLOG(error, #{msg => "unxpected_info", info => Info}), {noreply, State}. +drop_stats(SeqRef, Msg, Acc) -> + Dropped = ?GET_DROPPED(SeqRef), + ok = ?RESET_SEQ(SeqRef), + ?tp(log_throttler_dropped, #{dropped_count => Dropped, throttled_msg => Msg}), + maybe_add_dropped(Msg, Dropped, Acc). + terminate(_Reason, _State) -> + %% atomics do not have delete/remove/release/deallocate API + %% after the reference is garbage-collected the resource is released + lists:foreach(fun(Msg) -> ?ERASE_SEQ(Msg) end, ?MSGS_LIST), ok. code_change(_OldVsn, State, _Extra) -> @@ -122,17 +144,27 @@ code_change(_OldVsn, State, _Extra) -> %% internal functions %%-------------------------------------------------------------------- -do_allow(Msg) -> +do_allow(Msg, UniqueKey) -> case persistent_term:get(?SEQ_ID(Msg), undefined) of undefined -> %% This is either a race condition (emqx_log_throttler is not started yet) %% or a developer mistake (msg used in ?SLOG_THROTTLE/2,3 macro is %% not added to the default value of `log.throttling.msgs`. - ?SLOG(info, #{ - msg => "missing_log_throttle_sequence", + ?SLOG(debug, #{ + msg => "log_throttle_disabled", throttled_msg => Msg }), true; + %% e.g: unrecoverable msg throttle according resource_id + SeqMap when is_map(SeqMap) -> + case maps:find(UniqueKey, SeqMap) of + {ok, SeqRef} -> + ?IS_ALLOWED(SeqRef); + error -> + SeqRef = ?NEW_SEQ, + new_throttler(Msg, SeqMap#{UniqueKey => SeqRef}), + true + end; SeqRef -> ?IS_ALLOWED(SeqRef) end. @@ -154,3 +186,11 @@ maybe_log_dropped(_DroppedStats, _PeriodMs) -> schedule_refresh(PeriodMs) -> ?tp(log_throttler_sched_refresh, #{new_period_ms => PeriodMs}), erlang:send_after(PeriodMs, ?MODULE, refresh). + +new_throttler(unrecoverable_resource_error = Msg) -> + new_throttler(Msg, #{}); +new_throttler(Msg) -> + new_throttler(Msg, ?NEW_SEQ). + +new_throttler(Msg, AtomicOrEmptyMap) -> + persistent_term:put(?SEQ_ID(Msg), AtomicOrEmptyMap). diff --git a/apps/emqx/src/emqx_persistent_session_ds.erl b/apps/emqx/src/emqx_persistent_session_ds.erl index 32c291eec..b86b44611 100644 --- a/apps/emqx/src/emqx_persistent_session_ds.erl +++ b/apps/emqx/src/emqx_persistent_session_ds.erl @@ -621,9 +621,13 @@ handle_timeout(ClientInfo, ?TIMER_RETRY_REPLAY, Session0) -> Session = replay_streams(Session0, ClientInfo), {ok, [], Session}; handle_timeout(ClientInfo, ?TIMER_GET_STREAMS, Session0 = #{s := S0, shared_sub_s := SharedSubS0}) -> - S1 = emqx_persistent_session_ds_subs:gc(S0), - S2 = emqx_persistent_session_ds_stream_scheduler:renew_streams(S1), - {S, SharedSubS} = emqx_persistent_session_ds_shared_subs:renew_streams(S2, SharedSubS0), + %% `gc` and `renew_streams` methods may drop unsubscribed streams. + %% Shared subscription handler must have a chance to see unsubscribed streams + %% in the fully replayed state. + {S1, SharedSubS1} = emqx_persistent_session_ds_shared_subs:pre_renew_streams(S0, SharedSubS0), + S2 = emqx_persistent_session_ds_subs:gc(S1), + S3 = emqx_persistent_session_ds_stream_scheduler:renew_streams(S2), + {S, SharedSubS} = emqx_persistent_session_ds_shared_subs:renew_streams(S3, SharedSubS1), Interval = get_config(ClientInfo, [renew_streams_interval]), Session = emqx_session:ensure_timer( ?TIMER_GET_STREAMS, @@ -757,7 +761,7 @@ skip_batch(StreamKey, SRS0, Session = #{s := S0}, ClientInfo, Reason) -> %%-------------------------------------------------------------------- -spec disconnect(session(), emqx_types:conninfo()) -> {shutdown, session()}. -disconnect(Session = #{id := Id, s := S0}, ConnInfo) -> +disconnect(Session = #{id := Id, s := S0, shared_sub_s := SharedSubS0}, ConnInfo) -> S1 = maybe_set_offline_info(S0, Id), S2 = emqx_persistent_session_ds_state:set_last_alive_at(now_ms(), S1), S3 = @@ -767,8 +771,9 @@ disconnect(Session = #{id := Id, s := S0}, ConnInfo) -> _ -> S2 end, - S = emqx_persistent_session_ds_state:commit(S3), - {shutdown, Session#{s => S}}. + {S4, SharedSubS} = emqx_persistent_session_ds_shared_subs:on_disconnect(S3, SharedSubS0), + S = emqx_persistent_session_ds_state:commit(S4), + {shutdown, Session#{s => S, shared_sub_s => SharedSubS}}. -spec terminate(Reason :: term(), session()) -> ok. terminate(_Reason, Session = #{id := Id, s := S}) -> @@ -816,10 +821,12 @@ list_client_subscriptions(ClientId) -> {error, not_found} end. --spec get_client_subscription(emqx_types:clientid(), emqx_types:topic()) -> +-spec get_client_subscription(emqx_types:clientid(), topic_filter() | share_topic_filter()) -> subscription() | undefined. -get_client_subscription(ClientId, Topic) -> - emqx_persistent_session_ds_subs:cold_get_subscription(ClientId, Topic). +get_client_subscription(ClientId, #share{} = ShareTopicFilter) -> + emqx_persistent_session_ds_shared_subs:cold_get_subscription(ClientId, ShareTopicFilter); +get_client_subscription(ClientId, TopicFilter) -> + emqx_persistent_session_ds_subs:cold_get_subscription(ClientId, TopicFilter). %%-------------------------------------------------------------------- %% Session tables operations @@ -986,14 +993,14 @@ do_ensure_all_iterators_closed(_DSSessionID) -> %% Normal replay: %%-------------------------------------------------------------------- -fetch_new_messages(Session0 = #{s := S0}, ClientInfo) -> - LFS = maps:get(last_fetched_stream, Session0, beginning), - ItStream = emqx_persistent_session_ds_stream_scheduler:iter_next_streams(LFS, S0), +fetch_new_messages(Session0 = #{s := S0, shared_sub_s := SharedSubS0}, ClientInfo) -> + {S1, SharedSubS1} = emqx_persistent_session_ds_shared_subs:on_streams_replay(S0, SharedSubS0), + Session1 = Session0#{s => S1, shared_sub_s => SharedSubS1}, + LFS = maps:get(last_fetched_stream, Session1, beginning), + ItStream = emqx_persistent_session_ds_stream_scheduler:iter_next_streams(LFS, S1), BatchSize = get_config(ClientInfo, [batch_size]), - Session1 = fetch_new_messages(ItStream, BatchSize, Session0, ClientInfo), - #{s := S1, shared_sub_s := SharedSubS0} = Session1, - {S2, SharedSubS1} = emqx_persistent_session_ds_shared_subs:on_streams_replayed(S1, SharedSubS0), - Session1#{s => S2, shared_sub_s => SharedSubS1}. + Session2 = fetch_new_messages(ItStream, BatchSize, Session1, ClientInfo), + Session2#{shared_sub_s => SharedSubS1}. fetch_new_messages(ItStream0, BatchSize, Session0, ClientInfo) -> #{inflight := Inflight} = Session0, diff --git a/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_router.erl b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_router.erl index b0ee14963..1b80a28d2 100644 --- a/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_router.erl +++ b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_router.erl @@ -17,7 +17,7 @@ -module(emqx_persistent_session_ds_router). -include("emqx.hrl"). --include("emqx_persistent_session_ds/emqx_ps_ds_int.hrl"). +-include("emqx_ps_ds_int.hrl"). -export([init_tables/0]). @@ -47,7 +47,7 @@ -endif. -type route() :: #ps_route{}. --type dest() :: emqx_persistent_session_ds:id(). +-type dest() :: emqx_persistent_session_ds:id() | #share_dest{}. -export_type([dest/0, route/0]). @@ -161,7 +161,7 @@ topics() -> print_routes(Topic) -> lists:foreach( fun(#ps_route{topic = To, dest = Dest}) -> - io:format("~ts -> ~ts~n", [To, Dest]) + io:format("~ts -> ~tp~n", [To, Dest]) end, match_routes(Topic) ). @@ -247,6 +247,8 @@ mk_filtertab_fold_fun(FoldFun) -> match_filters(Topic) -> emqx_topic_index:matches(Topic, ?PS_FILTERS_TAB, []). +get_dest_session_id(#share_dest{session_id = DSSessionId}) -> + DSSessionId; get_dest_session_id({_, DSSessionId}) -> DSSessionId; get_dest_session_id(DSSessionId) -> diff --git a/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl index c4e929640..5b54c6f73 100644 --- a/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl +++ b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs.erl @@ -2,11 +2,37 @@ %% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- +%% @doc This module +%% * handles creation and management of _shared_ subscriptions for the session; +%% * provides streams to the session; +%% * handles progress of stream replay. +%% +%% The logic is quite straightforward; most of the parts resemble the logic of the +%% `emqx_persistent_session_ds_subs` (subscribe/unsubscribe) and +%% `emqx_persistent_session_ds_scheduler` (providing new streams), +%% but some data is sent or received from the `emqx_persistent_session_ds_shared_subs_agent` +%% which communicates with remote shared subscription leaders. +%% +%% A tricky part is the concept of "scheduled actions". When we unsubscribe from a topic +%% we may have some streams that have unacked messages. So we do not have a reliable +%% progress for them. Sending the current progress to the leader and disconnecting +%% will lead to the duplication of messages. So after unsubscription, we need to wait +%% some time until all streams are acked, and only then we disconnect from the leader. +%% +%% For this purpose we have the `scheduled_actions` map in the state of the module. +%% We preserve there the streams that we need to wait for and collect their progress. +%% We also use `scheduled_actions` for resubscriptions. If a client quickly resubscribes +%% after unsubscription, we may still have the mentioned streams unacked. If we abandon +%% them, just connect to the leader, then it may lease us the same streams again, but with +%% the previous progress. So messages may duplicate. + -module(emqx_persistent_session_ds_shared_subs). -include("emqx_mqtt.hrl"). +-include("emqx.hrl"). -include("logger.hrl"). -include("session_internals.hrl"). + -include_lib("snabbkaffe/include/trace.hrl"). -export([ @@ -15,16 +41,51 @@ on_subscribe/3, on_unsubscribe/4, + on_disconnect/2, - on_streams_replayed/2, + on_streams_replay/2, on_info/3, + pre_renew_streams/2, renew_streams/2, to_map/2 ]). +%% Management API: +-export([ + cold_get_subscription/2 +]). + +-define(schedule_subscribe, schedule_subscribe). +-define(schedule_unsubscribe, schedule_unsubscribe). + +-type stream_key() :: {emqx_persistent_session_ds:id(), emqx_ds:stream()}. + +-type scheduled_action_type() :: + {?schedule_subscribe, emqx_types:subopts()} | ?schedule_unsubscribe. + +-type agent_stream_progress() :: #{ + stream := emqx_ds:stream(), + progress := progress(), + use_finished := boolean() +}. + +-type progress() :: + #{ + iterator := emqx_ds:iterator() + }. + +-type scheduled_action() :: #{ + type := scheduled_action_type(), + stream_keys_to_wait := [stream_key()], + progresses := [agent_stream_progress()] +}. + -type t() :: #{ - agent := emqx_persistent_session_ds_shared_subs_agent:t() + agent := emqx_persistent_session_ds_shared_subs_agent:t(), + scheduled_actions := #{ + share_topic_filter() => scheduled_action() + } }. -type share_topic_filter() :: emqx_persistent_session_ds:share_topic_filter(). -type opts() :: #{ @@ -34,184 +95,90 @@ -define(rank_x, rank_shared). -define(rank_y, 0). +-export_type([ + progress/0, + agent_stream_progress/0 +]). + %%-------------------------------------------------------------------- %% API %%-------------------------------------------------------------------- +%%-------------------------------------------------------------------- +%% new + -spec new(opts()) -> t(). new(Opts) -> #{ agent => emqx_persistent_session_ds_shared_subs_agent:new( agent_opts(Opts) - ) + ), + scheduled_actions => #{} }. +%%-------------------------------------------------------------------- +%% open + -spec open(emqx_persistent_session_ds_state:t(), opts()) -> {ok, emqx_persistent_session_ds_state:t(), t()}. -open(S, Opts) -> +open(S0, Opts) -> SharedSubscriptions = fold_shared_subs( - fun(#share{} = TopicFilter, Sub, Acc) -> - [{TopicFilter, to_agent_subscription(S, Sub)} | Acc] + fun(#share{} = ShareTopicFilter, Sub, Acc) -> + [{ShareTopicFilter, to_agent_subscription(S0, Sub)} | Acc] end, [], - S + S0 ), Agent = emqx_persistent_session_ds_shared_subs_agent:open( SharedSubscriptions, agent_opts(Opts) ), - SharedSubS = #{agent => Agent}, - {ok, S, SharedSubS}. + SharedSubS = #{agent => Agent, scheduled_actions => #{}}, + S1 = revoke_all_streams(S0), + {ok, S1, SharedSubS}. + +%%-------------------------------------------------------------------- +%% on_subscribe -spec on_subscribe( share_topic_filter(), emqx_types:subopts(), emqx_persistent_session_ds:session() ) -> {ok, emqx_persistent_session_ds_state:t(), t()} | {error, emqx_types:reason_code()}. -on_subscribe(TopicFilter, SubOpts, #{s := S} = Session) -> - Subscription = emqx_persistent_session_ds_state:get_subscription(TopicFilter, S), - on_subscribe(Subscription, TopicFilter, SubOpts, Session). - --spec on_unsubscribe( - emqx_persistent_session_ds:id(), - emqx_persistent_session_ds:topic_filter(), - emqx_persistent_session_ds_state:t(), - t() -) -> - {ok, emqx_persistent_session_ds_state:t(), t(), emqx_persistent_session_ds:subscription()} - | {error, emqx_types:reason_code()}. -on_unsubscribe(SessionId, TopicFilter, S0, #{agent := Agent0} = SharedSubS0) -> - case lookup(TopicFilter, S0) of - undefined -> - {error, ?RC_NO_SUBSCRIPTION_EXISTED}; - Subscription -> - ?tp(persistent_session_ds_subscription_delete, #{ - session_id => SessionId, topic_filter => TopicFilter - }), - Agent1 = emqx_persistent_session_ds_shared_subs_agent:on_unsubscribe( - Agent0, TopicFilter - ), - SharedSubS = SharedSubS0#{agent => Agent1}, - S = emqx_persistent_session_ds_state:del_subscription(TopicFilter, S0), - {ok, S, SharedSubS, Subscription} - end. - --spec renew_streams(emqx_persistent_session_ds_state:t(), t()) -> - {emqx_persistent_session_ds_state:t(), t()}. -renew_streams(S0, #{agent := Agent0} = SharedSubS0) -> - {StreamLeaseEvents, Agent1} = emqx_persistent_session_ds_shared_subs_agent:renew_streams( - Agent0 - ), - ?tp(info, shared_subs_new_stream_lease_events, #{stream_lease_events => StreamLeaseEvents}), - S1 = lists:foldl( - fun - (#{type := lease} = Event, S) -> accept_stream(Event, S); - (#{type := revoke} = Event, S) -> revoke_stream(Event, S) - end, - S0, - StreamLeaseEvents - ), - SharedSubS1 = SharedSubS0#{agent => Agent1}, - {S1, SharedSubS1}. - --spec on_streams_replayed( - emqx_persistent_session_ds_state:t(), - t() -) -> {emqx_persistent_session_ds_state:t(), t()}. -on_streams_replayed(S, #{agent := Agent0} = SharedSubS0) -> - %% TODO - %% Is it sufficient for a report? - Progress = fold_shared_stream_states( - fun(TopicFilter, Stream, SRS, Acc) -> - #srs{it_begin = BeginIt} = SRS, - StreamProgress = #{ - topic_filter => TopicFilter, - stream => Stream, - iterator => BeginIt - }, - [StreamProgress | Acc] - end, - [], - S - ), - Agent1 = emqx_persistent_session_ds_shared_subs_agent:on_stream_progress( - Agent0, Progress - ), - SharedSubS1 = SharedSubS0#{agent => Agent1}, - {S, SharedSubS1}. - --spec on_info(emqx_persistent_session_ds_state:t(), t(), term()) -> - {emqx_persistent_session_ds_state:t(), t()}. -on_info(S, #{agent := Agent0} = SharedSubS0, Info) -> - Agent1 = emqx_persistent_session_ds_shared_subs_agent:on_info(Agent0, Info), - SharedSubS1 = SharedSubS0#{agent => Agent1}, - {S, SharedSubS1}. - --spec to_map(emqx_persistent_session_ds_state:t(), t()) -> map(). -to_map(_S, _SharedSubS) -> - %% TODO - #{}. +on_subscribe(#share{} = ShareTopicFilter, SubOpts, #{s := S} = Session) -> + Subscription = emqx_persistent_session_ds_state:get_subscription(ShareTopicFilter, S), + on_subscribe(Subscription, ShareTopicFilter, SubOpts, Session). %%-------------------------------------------------------------------- -%% Internal functions -%%-------------------------------------------------------------------- +%% on_subscribe internal functions -fold_shared_subs(Fun, Acc, S) -> - emqx_persistent_session_ds_state:fold_subscriptions( - fun - (#share{} = TopicFilter, Sub, Acc0) -> Fun(TopicFilter, Sub, Acc0); - (_, _Sub, Acc0) -> Acc0 - end, - Acc, - S - ). - -fold_shared_stream_states(Fun, Acc, S) -> - %% TODO - %% Optimize or cache - TopicFilters = fold_shared_subs( - fun - (#share{} = TopicFilter, #{id := Id} = _Sub, Acc0) -> - Acc0#{Id => TopicFilter}; - (_, _, Acc0) -> - Acc0 - end, - #{}, - S - ), - emqx_persistent_session_ds_state:fold_streams( - fun({SubId, Stream}, SRS, Acc0) -> - case TopicFilters of - #{SubId := TopicFilter} -> - Fun(TopicFilter, Stream, SRS, Acc0); - _ -> - Acc0 - end - end, - Acc, - S - ). - -on_subscribe(undefined, TopicFilter, SubOpts, #{props := Props, s := S} = Session) -> +on_subscribe(undefined, ShareTopicFilter, SubOpts, #{props := Props, s := S} = Session) -> #{max_subscriptions := MaxSubscriptions} = Props, case emqx_persistent_session_ds_state:n_subscriptions(S) < MaxSubscriptions of true -> - create_new_subscription(TopicFilter, SubOpts, Session); + create_new_subscription(ShareTopicFilter, SubOpts, Session); false -> {error, ?RC_QUOTA_EXCEEDED} end; -on_subscribe(Subscription, TopicFilter, SubOpts, Session) -> - update_subscription(Subscription, TopicFilter, SubOpts, Session). +on_subscribe(Subscription, ShareTopicFilter, SubOpts, Session) -> + update_subscription(Subscription, ShareTopicFilter, SubOpts, Session). -dialyzer({nowarn_function, create_new_subscription/3}). -create_new_subscription(TopicFilter, SubOpts, #{ - id := SessionId, s := S0, shared_sub_s := #{agent := Agent0} = SharedSubS0, props := Props +create_new_subscription(#share{topic = TopicFilter, group = Group} = ShareTopicFilter, SubOpts, #{ + id := SessionId, + s := S0, + shared_sub_s := #{agent := Agent} = SharedSubS0, + props := Props }) -> case - emqx_persistent_session_ds_shared_subs_agent:on_subscribe( - Agent0, TopicFilter, SubOpts + emqx_persistent_session_ds_shared_subs_agent:can_subscribe( + Agent, ShareTopicFilter, SubOpts ) of - {ok, Agent1} -> + ok -> + ok = emqx_persistent_session_ds_router:do_add_route(TopicFilter, #share_dest{ + session_id = SessionId, group = Group + }), + _ = emqx_external_broker:add_persistent_shared_route(TopicFilter, Group, SessionId), #{upgrade_qos := UpgradeQoS} = Props, {SubId, S1} = emqx_persistent_session_ds_state:new_id(S0), {SStateId, S2} = emqx_persistent_session_ds_state:new_id(S1), @@ -227,20 +194,20 @@ create_new_subscription(TopicFilter, SubOpts, #{ start_time => now_ms() }, S = emqx_persistent_session_ds_state:put_subscription( - TopicFilter, Subscription, S3 + ShareTopicFilter, Subscription, S3 ), - SharedSubS = SharedSubS0#{agent => Agent1}, - ?tp(persistent_session_ds_shared_subscription_added, #{ - topic_filter => TopicFilter, session => SessionId - }), + + SharedSubS = schedule_subscribe(SharedSubS0, ShareTopicFilter, SubOpts), {ok, S, SharedSubS}; {error, _} = Error -> Error end. -update_subscription(#{current_state := SStateId0, id := SubId} = Sub0, TopicFilter, SubOpts, #{ - s := S0, shared_sub_s := SharedSubS, props := Props -}) -> +update_subscription( + #{current_state := SStateId0, id := SubId} = Sub0, ShareTopicFilter, SubOpts, #{ + s := S0, shared_sub_s := SharedSubS, props := Props + } +) -> #{upgrade_qos := UpgradeQoS} = Props, SState = #{parent_subscription => SubId, upgrade_qos => UpgradeQoS, subopts => SubOpts}, case emqx_persistent_session_ds_state:get_subscription_state(SStateId0, S0) of @@ -254,36 +221,173 @@ update_subscription(#{current_state := SStateId0, id := SubId} = Sub0, TopicFilt SStateId, SState, S1 ), Sub = Sub0#{current_state => SStateId}, - S = emqx_persistent_session_ds_state:put_subscription(TopicFilter, Sub, S2), + S = emqx_persistent_session_ds_state:put_subscription(ShareTopicFilter, Sub, S2), {ok, S, SharedSubS} end. -lookup(TopicFilter, S) -> - case emqx_persistent_session_ds_state:get_subscription(TopicFilter, S) of - Sub = #{current_state := SStateId} -> - case emqx_persistent_session_ds_state:get_subscription_state(SStateId, S) of - #{subopts := SubOpts} -> - Sub#{subopts => SubOpts}; - undefined -> - undefined - end; +-dialyzer({nowarn_function, schedule_subscribe/3}). +schedule_subscribe( + #{agent := Agent0, scheduled_actions := ScheduledActions0} = SharedSubS0, + ShareTopicFilter, + SubOpts +) -> + case ScheduledActions0 of + #{ShareTopicFilter := ScheduledAction} -> + ScheduledActions1 = ScheduledActions0#{ + ShareTopicFilter => ScheduledAction#{type => {?schedule_subscribe, SubOpts}} + }, + ?tp(warning, shared_subs_schedule_subscribe_override, #{ + share_topic_filter => ShareTopicFilter, + new_type => {?schedule_subscribe, SubOpts}, + old_action => format_schedule_action(ScheduledAction) + }), + SharedSubS0#{scheduled_actions := ScheduledActions1}; + _ -> + ?tp(warning, shared_subs_schedule_subscribe_new, #{ + share_topic_filter => ShareTopicFilter, subopts => SubOpts + }), + Agent1 = emqx_persistent_session_ds_shared_subs_agent:on_subscribe( + Agent0, ShareTopicFilter, SubOpts + ), + SharedSubS0#{agent => Agent1} + end. + +%%-------------------------------------------------------------------- +%% on_unsubscribe + +-spec on_unsubscribe( + emqx_persistent_session_ds:id(), + share_topic_filter(), + emqx_persistent_session_ds_state:t(), + t() +) -> + {ok, emqx_persistent_session_ds_state:t(), t(), emqx_persistent_session_ds:subscription()} + | {error, emqx_types:reason_code()}. +on_unsubscribe( + SessionId, #share{topic = TopicFilter, group = Group} = ShareTopicFilter, S0, SharedSubS0 +) -> + case lookup(ShareTopicFilter, S0) of undefined -> - undefined + {error, ?RC_NO_SUBSCRIPTION_EXISTED}; + #{id := SubId} = Subscription -> + ?tp(persistent_session_ds_subscription_delete, #{ + session_id => SessionId, share_topic_filter => ShareTopicFilter + }), + _ = emqx_external_broker:delete_persistent_shared_route(TopicFilter, Group, SessionId), + ok = emqx_persistent_session_ds_router:do_delete_route(TopicFilter, #share_dest{ + session_id = SessionId, group = Group + }), + S = emqx_persistent_session_ds_state:del_subscription(ShareTopicFilter, S0), + SharedSubS = schedule_unsubscribe(S, SharedSubS0, SubId, ShareTopicFilter), + {ok, S, SharedSubS, Subscription} + end. + +%%-------------------------------------------------------------------- +%% on_unsubscribe internal functions + +schedule_unsubscribe( + S, #{scheduled_actions := ScheduledActions0} = SharedSubS0, UnsubscridedSubId, ShareTopicFilter +) -> + case ScheduledActions0 of + #{ShareTopicFilter := ScheduledAction0} -> + ScheduledAction1 = ScheduledAction0#{type => ?schedule_unsubscribe}, + ScheduledActions1 = ScheduledActions0#{ + ShareTopicFilter => ScheduledAction1 + }, + ?tp(warning, shared_subs_schedule_unsubscribe_override, #{ + share_topic_filter => ShareTopicFilter, + new_type => ?schedule_unsubscribe, + old_action => format_schedule_action(ScheduledAction0) + }), + SharedSubS0#{scheduled_actions := ScheduledActions1}; + _ -> + StreamKeys = stream_keys_by_sub_id(S, UnsubscridedSubId), + ScheduledActions1 = ScheduledActions0#{ + ShareTopicFilter => #{ + type => ?schedule_unsubscribe, + stream_keys_to_wait => StreamKeys, + progresses => [] + } + }, + ?tp(warning, shared_subs_schedule_unsubscribe_new, #{ + share_topic_filter => ShareTopicFilter, + stream_keys => format_stream_keys(StreamKeys) + }), + SharedSubS0#{scheduled_actions := ScheduledActions1} + end. + +%%-------------------------------------------------------------------- +%% pre_renew_streams + +-spec pre_renew_streams(emqx_persistent_session_ds_state:t(), t()) -> + {emqx_persistent_session_ds_state:t(), t()}. +pre_renew_streams(S, SharedSubS) -> + on_streams_replay(S, SharedSubS). + +%%-------------------------------------------------------------------- +%% renew_streams + +-spec renew_streams(emqx_persistent_session_ds_state:t(), t()) -> + {emqx_persistent_session_ds_state:t(), t()}. +renew_streams(S0, #{agent := Agent0, scheduled_actions := ScheduledActions} = SharedSubS0) -> + {StreamLeaseEvents, Agent1} = emqx_persistent_session_ds_shared_subs_agent:renew_streams( + Agent0 + ), + StreamLeaseEvents =/= [] andalso + ?tp(warning, shared_subs_new_stream_lease_events, #{ + stream_lease_events => format_lease_events(StreamLeaseEvents) + }), + S1 = lists:foldl( + fun + (#{type := lease} = Event, S) -> accept_stream(Event, S, ScheduledActions); + (#{type := revoke} = Event, S) -> revoke_stream(Event, S) + end, + S0, + StreamLeaseEvents + ), + SharedSubS1 = SharedSubS0#{agent => Agent1}, + {S1, SharedSubS1}. + +%%-------------------------------------------------------------------- +%% renew_streams internal functions + +accept_stream(#{share_topic_filter := ShareTopicFilter} = Event, S, ScheduledActions) -> + %% If we have a pending action (subscribe or unsubscribe) for this topic filter, + %% we should not accept a stream and start replaying it. We won't use it anyway: + %% * if subscribe is pending, we will reset agent obtain a new lease + %% * if unsubscribe is pending, we will drop connection + case ScheduledActions of + #{ShareTopicFilter := _Action} -> + S; + _ -> + accept_stream(Event, S) end. accept_stream( - #{topic_filter := TopicFilter, stream := Stream, iterator := Iterator}, S0 + #{ + share_topic_filter := ShareTopicFilter, + stream := Stream, + progress := #{iterator := Iterator} = _Progress + } = _Event, + S0 ) -> - case emqx_persistent_session_ds_state:get_subscription(TopicFilter, S0) of + case emqx_persistent_session_ds_state:get_subscription(ShareTopicFilter, S0) of undefined -> - %% This should not happen. - %% Agent should have received unsubscribe callback - %% and should not have passed this stream as a new one - error(new_stream_without_sub); + %% We unsubscribed + S0; #{id := SubId, current_state := SStateId} -> Key = {SubId, Stream}, - case emqx_persistent_session_ds_state:get_stream(Key, S0) of - undefined -> + NeedCreateStream = + case emqx_persistent_session_ds_state:get_stream(Key, S0) of + undefined -> + true; + #srs{unsubscribed = true} -> + true; + _SRS -> + false + end, + case NeedCreateStream of + true -> NewSRS = #srs{ rank_x = ?rank_x, @@ -294,15 +398,15 @@ accept_stream( }, S1 = emqx_persistent_session_ds_state:put_stream(Key, NewSRS, S0), S1; - _SRS -> + false -> S0 end end. revoke_stream( - #{topic_filter := TopicFilter, stream := Stream}, S0 + #{share_topic_filter := ShareTopicFilter, stream := Stream}, S0 ) -> - case emqx_persistent_session_ds_state:get_subscription(TopicFilter, S0) of + case emqx_persistent_session_ds_state:get_subscription(ShareTopicFilter, S0) of undefined -> %% This should not happen. %% Agent should have received unsubscribe callback @@ -320,19 +424,363 @@ revoke_stream( end end. --spec to_agent_subscription( - emqx_persistent_session_ds_state:t(), emqx_persistent_session_ds:subscription() +%%-------------------------------------------------------------------- +%% on_streams_replay + +-spec on_streams_replay( + emqx_persistent_session_ds_state:t(), + t() +) -> {emqx_persistent_session_ds_state:t(), t()}. +on_streams_replay(S0, SharedSubS0) -> + {S1, #{agent := Agent0, scheduled_actions := ScheduledActions0} = SharedSubS1} = + renew_streams(S0, SharedSubS0), + + Progresses = all_stream_progresses(S1, Agent0), + Agent1 = emqx_persistent_session_ds_shared_subs_agent:on_stream_progress( + Agent0, Progresses + ), + {Agent2, ScheduledActions1} = run_scheduled_actions(S1, Agent1, ScheduledActions0), + SharedSubS2 = SharedSubS1#{ + agent => Agent2, + scheduled_actions => ScheduledActions1 + }, + {S1, SharedSubS2}. + +%%-------------------------------------------------------------------- +%% on_streams_replay internal functions + +all_stream_progresses(S, Agent) -> + all_stream_progresses(S, Agent, _NeedUnacked = false). + +all_stream_progresses(S, _Agent, NeedUnacked) -> + CommQos1 = emqx_persistent_session_ds_state:get_seqno(?committed(?QOS_1), S), + CommQos2 = emqx_persistent_session_ds_state:get_seqno(?committed(?QOS_2), S), + fold_shared_stream_states( + fun(ShareTopicFilter, Stream, SRS, ProgressesAcc0) -> + case + is_stream_started(CommQos1, CommQos2, SRS) and + (NeedUnacked or is_stream_fully_acked(CommQos1, CommQos2, SRS)) + of + true -> + StreamProgress = stream_progress(CommQos1, CommQos2, Stream, SRS), + maps:update_with( + ShareTopicFilter, + fun(Progresses) -> [StreamProgress | Progresses] end, + [StreamProgress], + ProgressesAcc0 + ); + false -> + ProgressesAcc0 + end + end, + #{}, + S + ). + +run_scheduled_actions(S, Agent, ScheduledActions) -> + maps:fold( + fun(ShareTopicFilter, Action0, {AgentAcc0, ScheduledActionsAcc}) -> + case run_scheduled_action(S, AgentAcc0, ShareTopicFilter, Action0) of + {ok, AgentAcc1} -> + {AgentAcc1, maps:remove(ShareTopicFilter, ScheduledActionsAcc)}; + {continue, Action1} -> + {AgentAcc0, ScheduledActionsAcc#{ShareTopicFilter => Action1}} + end + end, + {Agent, ScheduledActions}, + ScheduledActions + ). + +run_scheduled_action( + S, + Agent0, + ShareTopicFilter, + #{type := Type, stream_keys_to_wait := StreamKeysToWait0, progresses := Progresses0} = Action ) -> - emqx_persistent_session_ds_shared_subs_agent:subscription(). -to_agent_subscription(_S, Subscription) -> + StreamKeysToWait1 = filter_unfinished_streams(S, StreamKeysToWait0), + Progresses1 = stream_progresses(S, StreamKeysToWait0 -- StreamKeysToWait1) ++ Progresses0, + case StreamKeysToWait1 of + [] -> + ?tp(warning, shared_subs_schedule_action_complete, #{ + share_topic_filter => ShareTopicFilter, + progresses => format_stream_progresses(Progresses1), + type => Type + }), + %% Regular progress won't se unsubscribed streams, so we need to + %% send the progress explicitly. + Agent1 = emqx_persistent_session_ds_shared_subs_agent:on_stream_progress( + Agent0, #{ShareTopicFilter => Progresses1} + ), + case Type of + {?schedule_subscribe, SubOpts} -> + {ok, + emqx_persistent_session_ds_shared_subs_agent:on_subscribe( + Agent1, ShareTopicFilter, SubOpts + )}; + ?schedule_unsubscribe -> + {ok, + emqx_persistent_session_ds_shared_subs_agent:on_unsubscribe( + Agent1, ShareTopicFilter, Progresses1 + )} + end; + _ -> + Action1 = Action#{stream_keys_to_wait => StreamKeysToWait1, progresses => Progresses1}, + ?tp(warning, shared_subs_schedule_action_continue, #{ + share_topic_filter => ShareTopicFilter, + new_action => format_schedule_action(Action1) + }), + {continue, Action1} + end. + +filter_unfinished_streams(S, StreamKeysToWait) -> + CommQos1 = emqx_persistent_session_ds_state:get_seqno(?committed(?QOS_1), S), + CommQos2 = emqx_persistent_session_ds_state:get_seqno(?committed(?QOS_2), S), + lists:filter( + fun(Key) -> + case emqx_persistent_session_ds_state:get_stream(Key, S) of + undefined -> + %% This should not happen: we should see any stream + %% in completed state before deletion + true; + SRS -> + not is_stream_fully_acked(CommQos1, CommQos2, SRS) + end + end, + StreamKeysToWait + ). + +stream_progresses(S, StreamKeys) -> + CommQos1 = emqx_persistent_session_ds_state:get_seqno(?committed(?QOS_1), S), + CommQos2 = emqx_persistent_session_ds_state:get_seqno(?committed(?QOS_2), S), + lists:map( + fun({_SubId, Stream} = Key) -> + SRS = emqx_persistent_session_ds_state:get_stream(Key, S), + stream_progress(CommQos1, CommQos2, Stream, SRS) + end, + StreamKeys + ). + +%%-------------------------------------------------------------------- +%% on_disconnect + +on_disconnect(S0, #{agent := Agent0} = SharedSubS0) -> + S1 = revoke_all_streams(S0), + Progresses = all_stream_progresses(S1, Agent0, _NeedUnacked = true), + Agent1 = emqx_persistent_session_ds_shared_subs_agent:on_disconnect(Agent0, Progresses), + SharedSubS1 = SharedSubS0#{agent => Agent1, scheduled_actions => #{}}, + {S1, SharedSubS1}. + +%%-------------------------------------------------------------------- +%% on_disconnect helpers + +revoke_all_streams(S0) -> + fold_shared_stream_states( + fun(ShareTopicFilter, Stream, _SRS, S) -> + revoke_stream(#{share_topic_filter => ShareTopicFilter, stream => Stream}, S) + end, + S0, + S0 + ). + +%%-------------------------------------------------------------------- +%% on_info + +-spec on_info(emqx_persistent_session_ds_state:t(), t(), term()) -> + {emqx_persistent_session_ds_state:t(), t()}. +on_info(S, #{agent := Agent0} = SharedSubS0, Info) -> + Agent1 = emqx_persistent_session_ds_shared_subs_agent:on_info(Agent0, Info), + SharedSubS1 = SharedSubS0#{agent => Agent1}, + {S, SharedSubS1}. + +%%-------------------------------------------------------------------- +%% to_map + +-spec to_map(emqx_persistent_session_ds_state:t(), t()) -> map(). +to_map(S, _SharedSubS) -> + fold_shared_subs( + fun(ShareTopicFilter, _, Acc) -> Acc#{ShareTopicFilter => lookup(ShareTopicFilter, S)} end, + #{}, + S + ). + +%%-------------------------------------------------------------------- +%% cold_get_subscription + +-spec cold_get_subscription(emqx_persistent_session_ds:id(), share_topic_filter()) -> + emqx_persistent_session_ds:subscription() | undefined. +cold_get_subscription(SessionId, ShareTopicFilter) -> + case emqx_persistent_session_ds_state:cold_get_subscription(SessionId, ShareTopicFilter) of + [Sub = #{current_state := SStateId}] -> + case + emqx_persistent_session_ds_state:cold_get_subscription_state(SessionId, SStateId) + of + [#{subopts := Subopts}] -> + Sub#{subopts => Subopts}; + _ -> + undefined + end; + _ -> + undefined + end. + +%%-------------------------------------------------------------------- +%% Generic helpers +%%-------------------------------------------------------------------- + +lookup(ShareTopicFilter, S) -> + case emqx_persistent_session_ds_state:get_subscription(ShareTopicFilter, S) of + Sub = #{current_state := SStateId} -> + case emqx_persistent_session_ds_state:get_subscription_state(SStateId, S) of + #{subopts := SubOpts} -> + Sub#{subopts => SubOpts}; + undefined -> + undefined + end; + undefined -> + undefined + end. + +stream_keys_by_sub_id(S, MatchSubId) -> + emqx_persistent_session_ds_state:fold_streams( + fun({SubId, _Stream} = StreamKey, _SRS, StreamKeys) -> + case SubId of + MatchSubId -> + [StreamKey | StreamKeys]; + _ -> + StreamKeys + end + end, + [], + S + ). + +stream_progress( + CommQos1, + CommQos2, + Stream, + #srs{ + it_end = EndIt, + it_begin = BeginIt + } = SRS +) -> + Iterator = + case is_stream_fully_acked(CommQos1, CommQos2, SRS) of + true -> EndIt; + false -> BeginIt + end, + #{ + stream => Stream, + progress => #{ + iterator => Iterator + }, + use_finished => is_use_finished(SRS) + }. + +fold_shared_subs(Fun, Acc, S) -> + emqx_persistent_session_ds_state:fold_subscriptions( + fun + (#share{} = ShareTopicFilter, Sub, Acc0) -> Fun(ShareTopicFilter, Sub, Acc0); + (_, _Sub, Acc0) -> Acc0 + end, + Acc, + S + ). + +fold_shared_stream_states(Fun, Acc, S) -> %% TODO - %% do we need anything from sub state? + %% Optimize or cache + ShareTopicFilters = fold_shared_subs( + fun + (#share{} = ShareTopicFilter, #{id := Id} = _Sub, Acc0) -> + Acc0#{Id => ShareTopicFilter}; + (_, _, Acc0) -> + Acc0 + end, + #{}, + S + ), + emqx_persistent_session_ds_state:fold_streams( + fun({SubId, Stream}, SRS, Acc0) -> + case ShareTopicFilters of + #{SubId := ShareTopicFilter} -> + Fun(ShareTopicFilter, Stream, SRS, Acc0); + _ -> + Acc0 + end + end, + Acc, + S + ). + +to_agent_subscription(_S, Subscription) -> maps:with([start_time], Subscription). --spec agent_opts(opts()) -> emqx_persistent_session_ds_shared_subs_agent:opts(). agent_opts(#{session_id := SessionId}) -> #{session_id => SessionId}. -dialyzer({nowarn_function, now_ms/0}). now_ms() -> erlang:system_time(millisecond). + +is_use_finished(#srs{unsubscribed = Unsubscribed}) -> + Unsubscribed. + +is_stream_started(CommQos1, CommQos2, #srs{first_seqno_qos1 = Q1, last_seqno_qos1 = Q2}) -> + (CommQos1 >= Q1) or (CommQos2 >= Q2). + +is_stream_fully_acked(_, _, #srs{ + first_seqno_qos1 = Q1, last_seqno_qos1 = Q1, first_seqno_qos2 = Q2, last_seqno_qos2 = Q2 +}) -> + %% Streams where the last chunk doesn't contain any QoS1 and 2 + %% messages are considered fully acked: + true; +is_stream_fully_acked(Comm1, Comm2, #srs{last_seqno_qos1 = S1, last_seqno_qos2 = S2}) -> + (Comm1 >= S1) andalso (Comm2 >= S2). + +%%-------------------------------------------------------------------- +%% Formatters +%%-------------------------------------------------------------------- + +format_schedule_action(#{ + type := Type, progresses := Progresses, stream_keys_to_wait := StreamKeysToWait +}) -> + #{ + type => Type, + progresses => format_stream_progresses(Progresses), + stream_keys_to_wait => format_stream_keys(StreamKeysToWait) + }. + +format_stream_progresses(Streams) -> + lists:map( + fun format_stream_progress/1, + Streams + ). + +format_stream_progress(#{stream := Stream, progress := Progress} = Value) -> + Value#{stream => format_opaque(Stream), progress => format_progress(Progress)}. + +format_progress(#{iterator := Iterator} = Progress) -> + Progress#{iterator => format_opaque(Iterator)}. + +format_stream_key(beginning) -> beginning; +format_stream_key({SubId, Stream}) -> {SubId, format_opaque(Stream)}. + +format_stream_keys(StreamKeys) -> + lists:map( + fun format_stream_key/1, + StreamKeys + ). + +format_lease_events(Events) -> + lists:map( + fun format_lease_event/1, + Events + ). + +format_lease_event(#{stream := Stream, progress := Progress} = Event) -> + Event#{stream => format_opaque(Stream), progress => format_progress(Progress)}; +format_lease_event(#{stream := Stream} = Event) -> + Event#{stream => format_opaque(Stream)}. + +format_opaque(Opaque) -> + erlang:phash2(Opaque). diff --git a/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs_agent.erl b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs_agent.erl index 97b38d0f2..dff66de0f 100644 --- a/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs_agent.erl +++ b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs_agent.erl @@ -15,7 +15,7 @@ }. -type t() :: term(). --type topic_filter() :: emqx_persistent_session_ds:share_topic_filter(). +-type share_topic_filter() :: emqx_persistent_session_ds:share_topic_filter(). -type opts() :: #{ session_id := session_id() @@ -28,41 +28,44 @@ -type stream_lease() :: #{ type => lease, %% Used as "external" subscription_id - topic_filter := topic_filter(), + share_topic_filter := share_topic_filter(), stream := emqx_ds:stream(), iterator := emqx_ds:iterator() }. -type stream_revoke() :: #{ type => revoke, - topic_filter := topic_filter(), + share_topic_filter := share_topic_filter(), stream := emqx_ds:stream() }. -type stream_lease_event() :: stream_lease() | stream_revoke(). -type stream_progress() :: #{ - topic_filter := topic_filter(), + share_topic_filter := share_topic_filter(), stream := emqx_ds:stream(), - iterator := emqx_ds:iterator() + iterator := emqx_ds:iterator(), + use_finished := boolean() }. -export_type([ t/0, subscription/0, session_id/0, - stream_lease/0, + stream_lease_event/0, opts/0 ]). -export([ new/1, open/2, + can_subscribe/3, on_subscribe/3, - on_unsubscribe/2, + on_unsubscribe/3, on_stream_progress/2, on_info/2, + on_disconnect/2, renew_streams/1 ]). @@ -77,12 +80,13 @@ %%-------------------------------------------------------------------- -callback new(opts()) -> t(). --callback open([{topic_filter(), subscription()}], opts()) -> t(). --callback on_subscribe(t(), topic_filter(), emqx_types:subopts()) -> - {ok, t()} | {error, term()}. --callback on_unsubscribe(t(), topic_filter()) -> t(). +-callback open([{share_topic_filter(), subscription()}], opts()) -> t(). +-callback can_subscribe(t(), share_topic_filter(), emqx_types:subopts()) -> ok | {error, term()}. +-callback on_subscribe(t(), share_topic_filter(), emqx_types:subopts()) -> t(). +-callback on_unsubscribe(t(), share_topic_filter(), [stream_progress()]) -> t(). +-callback on_disconnect(t(), [stream_progress()]) -> t(). -callback renew_streams(t()) -> {[stream_lease_event()], t()}. --callback on_stream_progress(t(), [stream_progress()]) -> t(). +-callback on_stream_progress(t(), #{share_topic_filter() => [stream_progress()]}) -> t(). -callback on_info(t(), term()) -> t(). %%-------------------------------------------------------------------- @@ -93,24 +97,31 @@ new(Opts) -> ?shared_subs_agent:new(Opts). --spec open([{topic_filter(), subscription()}], opts()) -> t(). +-spec open([{share_topic_filter(), subscription()}], opts()) -> t(). open(Topics, Opts) -> ?shared_subs_agent:open(Topics, Opts). --spec on_subscribe(t(), topic_filter(), emqx_types:subopts()) -> - {ok, t()} | {error, emqx_types:reason_code()}. -on_subscribe(Agent, TopicFilter, SubOpts) -> - ?shared_subs_agent:on_subscribe(Agent, TopicFilter, SubOpts). +-spec can_subscribe(t(), share_topic_filter(), emqx_types:subopts()) -> ok | {error, term()}. +can_subscribe(Agent, ShareTopicFilter, SubOpts) -> + ?shared_subs_agent:can_subscribe(Agent, ShareTopicFilter, SubOpts). --spec on_unsubscribe(t(), topic_filter()) -> t(). -on_unsubscribe(Agent, TopicFilter) -> - ?shared_subs_agent:on_unsubscribe(Agent, TopicFilter). +-spec on_subscribe(t(), share_topic_filter(), emqx_types:subopts()) -> t(). +on_subscribe(Agent, ShareTopicFilter, SubOpts) -> + ?shared_subs_agent:on_subscribe(Agent, ShareTopicFilter, SubOpts). + +-spec on_unsubscribe(t(), share_topic_filter(), [stream_progress()]) -> t(). +on_unsubscribe(Agent, ShareTopicFilter, StreamProgresses) -> + ?shared_subs_agent:on_unsubscribe(Agent, ShareTopicFilter, StreamProgresses). + +-spec on_disconnect(t(), #{share_topic_filter() => [stream_progress()]}) -> t(). +on_disconnect(Agent, StreamProgresses) -> + ?shared_subs_agent:on_disconnect(Agent, StreamProgresses). -spec renew_streams(t()) -> {[stream_lease_event()], t()}. renew_streams(Agent) -> ?shared_subs_agent:renew_streams(Agent). --spec on_stream_progress(t(), [stream_progress()]) -> t(). +-spec on_stream_progress(t(), #{share_topic_filter() => [stream_progress()]}) -> t(). on_stream_progress(Agent, StreamProgress) -> ?shared_subs_agent:on_stream_progress(Agent, StreamProgress). diff --git a/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs_null_agent.erl b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs_null_agent.erl index e158c19e2..8156db76d 100644 --- a/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs_null_agent.erl +++ b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_shared_subs_null_agent.erl @@ -9,11 +9,13 @@ -export([ new/1, open/2, + can_subscribe/3, on_subscribe/3, - on_unsubscribe/2, + on_unsubscribe/3, on_stream_progress/2, on_info/2, + on_disconnect/2, renew_streams/1 ]). @@ -30,10 +32,16 @@ new(_Opts) -> open(_Topics, _Opts) -> undefined. -on_subscribe(_Agent, _TopicFilter, _SubOpts) -> +can_subscribe(_Agent, _TopicFilter, _SubOpts) -> {error, ?RC_SHARED_SUBSCRIPTIONS_NOT_SUPPORTED}. -on_unsubscribe(Agent, _TopicFilter) -> +on_subscribe(Agent, _TopicFilter, _SubOpts) -> + Agent. + +on_unsubscribe(Agent, _TopicFilter, _Progresses) -> + Agent. + +on_disconnect(Agent, _) -> Agent. renew_streams(Agent) -> diff --git a/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_state.erl b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_state.erl index 1d60250ea..3d3840307 100644 --- a/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_state.erl +++ b/apps/emqx/src/emqx_persistent_session_ds/emqx_persistent_session_ds_state.erl @@ -399,7 +399,9 @@ new_id(Rec) -> get_subscription(TopicFilter, Rec) -> gen_get(?subscriptions, TopicFilter, Rec). --spec cold_get_subscription(emqx_persistent_session_ds:id(), emqx_types:topic()) -> +-spec cold_get_subscription( + emqx_persistent_session_ds:id(), emqx_types:topic() | emqx_types:share() +) -> [emqx_persistent_session_ds_subs:subscription()]. cold_get_subscription(SessionId, Topic) -> kv_pmap_read(?subscription_tab, SessionId, Topic). diff --git a/apps/emqx/src/emqx_persistent_session_ds/emqx_ps_ds_int.hrl b/apps/emqx/src/emqx_persistent_session_ds/emqx_ps_ds_int.hrl index dc487376b..e533cfcb9 100644 --- a/apps/emqx/src/emqx_persistent_session_ds/emqx_ps_ds_int.hrl +++ b/apps/emqx/src/emqx_persistent_session_ds/emqx_ps_ds_int.hrl @@ -21,7 +21,7 @@ -record(ps_route, { topic :: binary(), - dest :: emqx_persistent_session_ds:id() | '_' + dest :: emqx_persistent_session_ds_router:dest() | '_' }). -record(ps_routeidx, { diff --git a/apps/emqx/src/emqx_persistent_session_ds/shared_subs_agent.hrl b/apps/emqx/src/emqx_persistent_session_ds/shared_subs_agent.hrl index 4fcd43e8a..ea2d41def 100644 --- a/apps/emqx/src/emqx_persistent_session_ds/shared_subs_agent.hrl +++ b/apps/emqx/src/emqx_persistent_session_ds/shared_subs_agent.hrl @@ -21,6 +21,7 @@ %% Till full implementation we need to dispach to the null agent. %% It will report "not implemented" error for attempts to use shared subscriptions. -define(shared_subs_agent, emqx_persistent_session_ds_shared_subs_null_agent). +% -define(shared_subs_agent, emqx_ds_shared_sub_agent). %% end of -ifdef(TEST). -endif. diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index db1d5350f..827836540 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -351,6 +351,7 @@ fields("authz_cache") -> #{ default => true, required => true, + importance => ?IMPORTANCE_NO_DOC, desc => ?DESC(fields_cache_enable) } )}, @@ -387,6 +388,7 @@ fields("flapping_detect") -> boolean(), #{ default => false, + %% importance => ?IMPORTANCE_NO_DOC, desc => ?DESC(flapping_detect_enable) } )}, @@ -423,6 +425,7 @@ fields("force_shutdown") -> boolean(), #{ default => true, + importance => ?IMPORTANCE_NO_DOC, desc => ?DESC(force_shutdown_enable) } )}, @@ -452,6 +455,7 @@ fields("overload_protection") -> boolean(), #{ desc => ?DESC(overload_protection_enable), + %% importance => ?IMPORTANCE_NO_DOC, default => false } )}, @@ -512,7 +516,11 @@ fields("force_gc") -> {"enable", sc( boolean(), - #{default => true, desc => ?DESC(force_gc_enable)} + #{ + default => true, + importance => ?IMPORTANCE_NO_DOC, + desc => ?DESC(force_gc_enable) + } )}, {"count", sc( @@ -1665,6 +1673,7 @@ fields("durable_sessions") -> sc( boolean(), #{ desc => ?DESC(durable_sessions_enable), + %% importance => ?IMPORTANCE_NO_DOC, default => false } )}, @@ -1888,6 +1897,7 @@ base_listener(Bind) -> #{ default => true, aliases => [enabled], + importance => ?IMPORTANCE_NO_DOC, desc => ?DESC(fields_listener_enabled) } )}, @@ -2416,6 +2426,7 @@ client_ssl_opts_schema(Defaults) -> boolean(), #{ default => false, + %% importance => ?IMPORTANCE_NO_DOC, desc => ?DESC(client_ssl_opts_schema_enable) } )}, diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index ce3c2543c..a33178d0a 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -78,6 +78,7 @@ start_epmd/0, start_peer/2, stop_peer/1, + ebin_path/0, listener_port/2 ]). diff --git a/apps/emqx/test/emqx_cth_suite.erl b/apps/emqx/test/emqx_cth_suite.erl index 5fe4dce66..c0e3430db 100644 --- a/apps/emqx/test/emqx_cth_suite.erl +++ b/apps/emqx/test/emqx_cth_suite.erl @@ -79,6 +79,8 @@ %% "Unofficial" `emqx_config_handler' and `emqx_conf' APIs -export([schema_module/0, upgrade_raw_conf/1]). +-export([skip_if_oss/0]). + -export_type([appspec/0]). -export_type([appspec_opts/0]). @@ -389,6 +391,8 @@ default_appspec(emqx_schema_validation, _SuiteOpts) -> #{schema_mod => emqx_schema_validation_schema, config => #{}}; default_appspec(emqx_message_transformation, _SuiteOpts) -> #{schema_mod => emqx_message_transformation_schema, config => #{}}; +default_appspec(emqx_ds_shared_sub, _SuiteOpts) -> + #{schema_mod => emqx_ds_shared_sub_schema, config => #{}}; default_appspec(_, _) -> #{}. @@ -519,3 +523,14 @@ upgrade_raw_conf(Conf) -> ce -> emqx_conf_schema:upgrade_raw_conf(Conf) end. + +skip_if_oss() -> + try emqx_release:edition() of + ee -> + false; + _ -> + {skip, not_supported_in_oss} + catch + error:undef -> + {skip, standalone_not_supported} + end. diff --git a/apps/emqx/test/emqx_exclusive_sub_SUITE.erl b/apps/emqx/test/emqx_exclusive_sub_SUITE.erl index abbdb5f44..a859612b2 100644 --- a/apps/emqx/test/emqx_exclusive_sub_SUITE.erl +++ b/apps/emqx/test/emqx_exclusive_sub_SUITE.erl @@ -56,6 +56,8 @@ t_exclusive_sub(_) -> {ok, _} = emqtt:connect(C1), ?CHECK_SUB(C1, 0), + ?CHECK_SUB(C1, 0), + {ok, C2} = emqtt:start_link([ {clientid, <<"client2">>}, {clean_start, false}, diff --git a/apps/emqx/test/emqx_log_throttler_SUITE.erl b/apps/emqx/test/emqx_log_throttler_SUITE.erl index 8b3ac0207..f95d62969 100644 --- a/apps/emqx/test/emqx_log_throttler_SUITE.erl +++ b/apps/emqx/test/emqx_log_throttler_SUITE.erl @@ -26,6 +26,7 @@ %% Have to use real msgs, as the schema is guarded by enum. -define(THROTTLE_MSG, authorization_permission_denied). -define(THROTTLE_MSG1, cannot_publish_to_topic_due_to_not_authorized). +-define(THROTTLE_UNRECOVERABLE_MSG, unrecoverable_resource_error). -define(TIME_WINDOW, <<"1s">>). all() -> emqx_common_test_helpers:all(?MODULE). @@ -59,6 +60,11 @@ end_per_suite(Config) -> emqx_cth_suite:stop(?config(suite_apps, Config)), emqx_config:delete_override_conf_files(). +init_per_testcase(t_throttle_recoverable_msg, Config) -> + ok = snabbkaffe:start_trace(), + [?THROTTLE_MSG] = Conf = emqx:get_config([log, throttling, msgs]), + {ok, _} = emqx_conf:update([log, throttling, msgs], [?THROTTLE_UNRECOVERABLE_MSG | Conf], #{}), + Config; init_per_testcase(t_throttle_add_new_msg, Config) -> ok = snabbkaffe:start_trace(), [?THROTTLE_MSG] = Conf = emqx:get_config([log, throttling, msgs]), @@ -72,6 +78,10 @@ init_per_testcase(_TC, Config) -> ok = snabbkaffe:start_trace(), Config. +end_per_testcase(t_throttle_recoverable_msg, _Config) -> + ok = snabbkaffe:stop(), + {ok, _} = emqx_conf:update([log, throttling, msgs], [?THROTTLE_MSG], #{}), + ok; end_per_testcase(t_throttle_add_new_msg, _Config) -> ok = snabbkaffe:stop(), {ok, _} = emqx_conf:update([log, throttling, msgs], [?THROTTLE_MSG], #{}), @@ -101,8 +111,8 @@ t_throttle(_Config) -> 5000 ), - ?assert(emqx_log_throttler:allow(?THROTTLE_MSG)), - ?assertNot(emqx_log_throttler:allow(?THROTTLE_MSG)), + ?assert(emqx_log_throttler:allow(?THROTTLE_MSG, undefined)), + ?assertNot(emqx_log_throttler:allow(?THROTTLE_MSG, undefined)), {ok, _} = ?block_until( #{ ?snk_kind := log_throttler_dropped, @@ -115,14 +125,48 @@ t_throttle(_Config) -> [] ). +t_throttle_recoverable_msg(_Config) -> + ResourceId = <<"resource_id">>, + ThrottledMsg = iolist_to_binary([atom_to_list(?THROTTLE_UNRECOVERABLE_MSG), ":", ResourceId]), + ?check_trace( + begin + %% Warm-up and block to increase the probability that next events + %% will be in the same throttling time window. + {ok, _} = ?block_until( + #{?snk_kind := log_throttler_new_msg, throttled_msg := ?THROTTLE_UNRECOVERABLE_MSG}, + 5000 + ), + {_, {ok, _}} = ?wait_async_action( + events(?THROTTLE_UNRECOVERABLE_MSG, ResourceId), + #{ + ?snk_kind := log_throttler_dropped, + throttled_msg := ThrottledMsg + }, + 5000 + ), + + ?assert(emqx_log_throttler:allow(?THROTTLE_UNRECOVERABLE_MSG, ResourceId)), + ?assertNot(emqx_log_throttler:allow(?THROTTLE_UNRECOVERABLE_MSG, ResourceId)), + {ok, _} = ?block_until( + #{ + ?snk_kind := log_throttler_dropped, + throttled_msg := ThrottledMsg, + dropped_count := 1 + }, + 3000 + ) + end, + [] + ). + t_throttle_add_new_msg(_Config) -> ?check_trace( begin {ok, _} = ?block_until( #{?snk_kind := log_throttler_new_msg, throttled_msg := ?THROTTLE_MSG1}, 5000 ), - ?assert(emqx_log_throttler:allow(?THROTTLE_MSG1)), - ?assertNot(emqx_log_throttler:allow(?THROTTLE_MSG1)), + ?assert(emqx_log_throttler:allow(?THROTTLE_MSG1, undefined)), + ?assertNot(emqx_log_throttler:allow(?THROTTLE_MSG1, undefined)), {ok, _} = ?block_until( #{ ?snk_kind := log_throttler_dropped, @@ -137,10 +181,15 @@ t_throttle_add_new_msg(_Config) -> t_throttle_no_msg(_Config) -> %% Must simply pass with no crashes - ?assert(emqx_log_throttler:allow(no_test_throttle_msg)), - ?assert(emqx_log_throttler:allow(no_test_throttle_msg)), - timer:sleep(10), - ?assert(erlang:is_process_alive(erlang:whereis(emqx_log_throttler))). + Pid = erlang:whereis(emqx_log_throttler), + ?assert(emqx_log_throttler:allow(no_test_throttle_msg, undefined)), + ?assert(emqx_log_throttler:allow(no_test_throttle_msg, undefined)), + %% assert process is not restarted + ?assertEqual(Pid, erlang:whereis(emqx_log_throttler)), + %% make a gen_call to ensure the process is alive + %% note: this call result in an 'unexpected_call' error log. + ?assertEqual(ignored, gen_server:call(Pid, probe)), + ok. t_update_time_window(_Config) -> ?check_trace( @@ -168,8 +217,8 @@ t_throttle_debug_primary_level(_Config) -> #{?snk_kind := log_throttler_dropped, throttled_msg := ?THROTTLE_MSG}, 5000 ), - ?assert(emqx_log_throttler:allow(?THROTTLE_MSG)), - ?assertNot(emqx_log_throttler:allow(?THROTTLE_MSG)), + ?assert(emqx_log_throttler:allow(?THROTTLE_MSG, undefined)), + ?assertNot(emqx_log_throttler:allow(?THROTTLE_MSG, undefined)), {ok, _} = ?block_until( #{ ?snk_kind := log_throttler_dropped, @@ -187,10 +236,13 @@ t_throttle_debug_primary_level(_Config) -> %%-------------------------------------------------------------------- events(Msg) -> - events(100, Msg). + events(100, Msg, undefined). -events(N, Msg) -> - [emqx_log_throttler:allow(Msg) || _ <- lists:seq(1, N)]. +events(Msg, Id) -> + events(100, Msg, Id). + +events(N, Msg, Id) -> + [emqx_log_throttler:allow(Msg, Id) || _ <- lists:seq(1, N)]. module_exists(Mod) -> case erlang:module_loaded(Mod) of diff --git a/apps/emqx/test/emqx_persistent_messages_SUITE.erl b/apps/emqx/test/emqx_persistent_messages_SUITE.erl index f225ba43d..19dac4575 100644 --- a/apps/emqx/test/emqx_persistent_messages_SUITE.erl +++ b/apps/emqx/test/emqx_persistent_messages_SUITE.erl @@ -573,7 +573,7 @@ app_specs(Opts) -> cluster() -> ExtraConf = "\n durable_storage.messages.n_sites = 2", - Spec = #{role => core, apps => app_specs(#{extra_emqx_conf => ExtraConf})}, + Spec = #{apps => app_specs(#{extra_emqx_conf => ExtraConf})}, [ {persistent_messages_SUITE1, Spec}, {persistent_messages_SUITE2, Spec} diff --git a/apps/emqx/test/emqx_routing_SUITE.erl b/apps/emqx/test/emqx_routing_SUITE.erl index 5112059ca..1e66a6ef7 100644 --- a/apps/emqx/test/emqx_routing_SUITE.erl +++ b/apps/emqx/test/emqx_routing_SUITE.erl @@ -64,18 +64,28 @@ init_per_group(routing_schema_v2, Config) -> init_per_group(batch_sync_on, Config) -> [{emqx_config, "broker.routing.batch_sync.enable_on = all"} | Config]; init_per_group(batch_sync_replicants, Config) -> - [{emqx_config, "broker.routing.batch_sync.enable_on = replicant"} | Config]; + case emqx_cth_suite:skip_if_oss() of + false -> + [{emqx_config, "broker.routing.batch_sync.enable_on = replicant"} | Config]; + True -> + True + end; init_per_group(batch_sync_off, Config) -> [{emqx_config, "broker.routing.batch_sync.enable_on = none"} | Config]; init_per_group(cluster, Config) -> - WorkDir = emqx_cth_suite:work_dir(Config), - NodeSpecs = [ - {emqx_routing_SUITE1, #{apps => [mk_emqx_appspec(1, Config)], role => core}}, - {emqx_routing_SUITE2, #{apps => [mk_emqx_appspec(2, Config)], role => core}}, - {emqx_routing_SUITE3, #{apps => [mk_emqx_appspec(3, Config)], role => replicant}} - ], - Nodes = emqx_cth_cluster:start(NodeSpecs, #{work_dir => WorkDir}), - [{cluster, Nodes} | Config]; + case emqx_cth_suite:skip_if_oss() of + false -> + WorkDir = emqx_cth_suite:work_dir(Config), + NodeSpecs = [ + {emqx_routing_SUITE1, #{apps => [mk_emqx_appspec(1, Config)], role => core}}, + {emqx_routing_SUITE2, #{apps => [mk_emqx_appspec(2, Config)], role => core}}, + {emqx_routing_SUITE3, #{apps => [mk_emqx_appspec(3, Config)], role => replicant}} + ], + Nodes = emqx_cth_cluster:start(NodeSpecs, #{work_dir => WorkDir}), + [{cluster, Nodes} | Config]; + True -> + True + end; init_per_group(GroupName, Config) when GroupName =:= single_batch_on; GroupName =:= single diff --git a/apps/emqx/test/emqx_shared_sub_SUITE.erl b/apps/emqx/test/emqx_shared_sub_SUITE.erl index 040b3d295..e15aca6a1 100644 --- a/apps/emqx/test/emqx_shared_sub_SUITE.erl +++ b/apps/emqx/test/emqx_shared_sub_SUITE.erl @@ -1247,7 +1247,7 @@ recv_msgs(Count, Msgs) -> start_peer(Name, Port) -> {ok, Node} = emqx_cth_peer:start_link( Name, - ebin_path() + emqx_common_test_helpers:ebin_path() ), pong = net_adm:ping(Node), setup_node(Node, Port), @@ -1261,9 +1261,6 @@ host() -> [_, Host] = string:tokens(atom_to_list(node()), "@"), Host. -ebin_path() -> - ["-pa" | code:get_path()]. - setup_node(Node, Port) -> EnvHandler = fun(_) -> diff --git a/apps/emqx_auth/include/emqx_authn.hrl b/apps/emqx_auth/include/emqx_authn.hrl index 782bfb9ca..d7ea32a92 100644 --- a/apps/emqx_auth/include/emqx_authn.hrl +++ b/apps/emqx_auth/include/emqx_authn.hrl @@ -28,7 +28,7 @@ -type authenticator_id() :: binary(). --define(AUTHN_RESOURCE_GROUP, <<"emqx_authn">>). +-define(AUTHN_RESOURCE_GROUP, <<"authn">>). %% VAR_NS_CLIENT_ATTRS is added here because it can be initialized before authn. %% NOTE: authn return may add more to (or even overwrite) client_attrs. diff --git a/apps/emqx_auth/include/emqx_authz.hrl b/apps/emqx_auth/include/emqx_authz.hrl index c638646f8..a7bef4b28 100644 --- a/apps/emqx_auth/include/emqx_authz.hrl +++ b/apps/emqx_auth/include/emqx_authz.hrl @@ -156,7 +156,7 @@ count => 1 }). --define(AUTHZ_RESOURCE_GROUP, <<"emqx_authz">>). +-define(AUTHZ_RESOURCE_GROUP, <<"authz">>). -define(AUTHZ_FEATURES, [rich_actions]). diff --git a/apps/emqx_auth/src/emqx_authn/emqx_authn_schema.erl b/apps/emqx_auth/src/emqx_authn/emqx_authn_schema.erl index 57f524a0c..75612550b 100644 --- a/apps/emqx_auth/src/emqx_authn/emqx_authn_schema.erl +++ b/apps/emqx_auth/src/emqx_authn/emqx_authn_schema.erl @@ -203,6 +203,7 @@ common_fields() -> enable(type) -> boolean(); enable(default) -> true; +enable(importance) -> ?IMPORTANCE_NO_DOC; enable(desc) -> ?DESC(?FUNCTION_NAME); enable(_) -> undefined. diff --git a/apps/emqx_auth/src/emqx_authz/emqx_authz_rule.erl b/apps/emqx_auth/src/emqx_authz/emqx_authz_rule.erl index 35f585e44..2c42c00e2 100644 --- a/apps/emqx_auth/src/emqx_authz/emqx_authz_rule.erl +++ b/apps/emqx_auth/src/emqx_authz/emqx_authz_rule.erl @@ -198,7 +198,7 @@ qos_from_opts(Opts) -> ) end catch - {bad_qos, QoS} -> + throw:{bad_qos, QoS} -> throw(#{ reason => invalid_authorization_qos, qos => QoS diff --git a/apps/emqx_auth/src/emqx_authz/emqx_authz_schema.erl b/apps/emqx_auth/src/emqx_authz/emqx_authz_schema.erl index 24deb0161..4095767b3 100644 --- a/apps/emqx_auth/src/emqx_authz/emqx_authz_schema.erl +++ b/apps/emqx_auth/src/emqx_authz/emqx_authz_schema.erl @@ -170,7 +170,12 @@ api_authz_refs() -> authz_common_fields(Type) -> [ {type, ?HOCON(Type, #{required => true, desc => ?DESC(type)})}, - {enable, ?HOCON(boolean(), #{default => true, desc => ?DESC(enable)})} + {enable, + ?HOCON(boolean(), #{ + default => true, + importance => ?IMPORTANCE_NO_DOC, + desc => ?DESC(enable) + })} ]. source_types() -> diff --git a/apps/emqx_auth/src/emqx_authz/emqx_authz_utils.erl b/apps/emqx_auth/src/emqx_authz/emqx_authz_utils.erl index 90c4e4fec..6e179b02c 100644 --- a/apps/emqx_auth/src/emqx_authz/emqx_authz_utils.erl +++ b/apps/emqx_auth/src/emqx_authz/emqx_authz_utils.erl @@ -16,6 +16,9 @@ -module(emqx_authz_utils). +-feature(maybe_expr, enable). + +-include_lib("emqx/include/emqx_placeholder.hrl"). -include_lib("emqx_authz.hrl"). -include_lib("snabbkaffe/include/trace.hrl"). @@ -28,7 +31,7 @@ remove_resource/1, update_config/2, vars_for_rule_query/2, - parse_rule_from_row/2 + do_authorize/6 ]). -export([ @@ -133,14 +136,18 @@ content_type(Headers) when is_list(Headers) -> -define(RAW_RULE_KEYS, [<<"permission">>, <<"action">>, <<"topic">>, <<"qos">>, <<"retain">>]). -parse_rule_from_row(ColumnNames, Row) -> - RuleRaw = maps:with(?RAW_RULE_KEYS, maps:from_list(lists:zip(ColumnNames, to_list(Row)))), - case emqx_authz_rule_raw:parse_rule(RuleRaw) of +-spec parse_rule_from_row([binary()], [binary()] | map()) -> + {ok, emqx_authz_rule:rule()} | {error, term()}. +parse_rule_from_row(_ColumnNames, RuleMap = #{}) -> + case emqx_authz_rule_raw:parse_rule(RuleMap) of {ok, {Permission, Action, Topics}} -> - emqx_authz_rule:compile({Permission, all, Action, Topics}); + {ok, emqx_authz_rule:compile({Permission, all, Action, Topics})}; {error, Reason} -> - error(Reason) - end. + {error, Reason} + end; +parse_rule_from_row(ColumnNames, Row) -> + RuleMap = maps:with(?RAW_RULE_KEYS, maps:from_list(lists:zip(ColumnNames, to_list(Row)))), + parse_rule_from_row(ColumnNames, RuleMap). vars_for_rule_query(Client, ?authz_action(PubSub, Qos) = Action) -> Client#{ @@ -157,3 +164,39 @@ to_list(Tuple) when is_tuple(Tuple) -> tuple_to_list(Tuple); to_list(List) when is_list(List) -> List. + +do_authorize(Type, Client, Action, Topic, ColumnNames, Row) -> + try + maybe + {ok, Rule} ?= parse_rule_from_row(ColumnNames, Row), + {matched, Permission} ?= emqx_authz_rule:match(Client, Action, Topic, Rule), + {matched, Permission} + else + nomatch -> + nomatch; + {error, Reason0} -> + log_match_rule_error(Type, Row, Reason0), + nomatch + end + catch + throw:Reason1 -> + log_match_rule_error(Type, Row, Reason1), + nomatch + end. + +log_match_rule_error(Type, Row, Reason0) -> + Msg0 = #{ + msg => "match_rule_error", + rule => Row, + type => Type + }, + Msg1 = + case is_map(Reason0) of + true -> maps:merge(Msg0, Reason0); + false -> Msg0#{reason => Reason0} + end, + ?SLOG( + error, + Msg1, + #{tag => "AUTHZ"} + ). diff --git a/apps/emqx_auth/test/emqx_authn/emqx_authn_schema_SUITE.erl b/apps/emqx_auth/test/emqx_authn/emqx_authn_schema_SUITE.erl index f2688fff9..46eb18b82 100644 --- a/apps/emqx_auth/test/emqx_authn/emqx_authn_schema_SUITE.erl +++ b/apps/emqx_auth/test/emqx_authn/emqx_authn_schema_SUITE.erl @@ -122,14 +122,6 @@ t_union_member_selector(_) -> }, check(BadMechanism) ), - BadCombination = Base#{<<"mechanism">> => <<"scram">>, <<"backend">> => <<"http">>}, - ?assertThrow( - #{ - reason := "unknown_mechanism", - expected := "password_based" - }, - check(BadCombination) - ), ok. t_http_auth_selector(_) -> diff --git a/apps/emqx_auth/test/emqx_authz/emqx_authz_api_cluster_SUITE.erl b/apps/emqx_auth/test/emqx_authz/emqx_authz_api_cluster_SUITE.erl index a77b3df51..3884dd052 100644 --- a/apps/emqx_auth/test/emqx_authz/emqx_authz_api_cluster_SUITE.erl +++ b/apps/emqx_auth/test/emqx_authz/emqx_authz_api_cluster_SUITE.erl @@ -118,8 +118,8 @@ mk_cluster_spec(Opts) -> Node1Apps = Apps ++ [{emqx_dashboard, "dashboard.listeners.http {enable=true,bind=18083}"}], Node2Apps = Apps, [ - {emqx_authz_api_cluster_SUITE1, Opts#{role => core, apps => Node1Apps}}, - {emqx_authz_api_cluster_SUITE2, Opts#{role => core, apps => Node2Apps}} + {emqx_authz_api_cluster_SUITE1, Opts#{apps => Node1Apps}}, + {emqx_authz_api_cluster_SUITE2, Opts#{apps => Node2Apps}} ]. request(Method, URL, Body, Config) -> diff --git a/apps/emqx_auth_http/include/emqx_auth_http.hrl b/apps/emqx_auth_http/include/emqx_auth_http.hrl index 9fc3b029e..439087e9c 100644 --- a/apps/emqx_auth_http/include/emqx_auth_http.hrl +++ b/apps/emqx_auth_http/include/emqx_auth_http.hrl @@ -22,8 +22,15 @@ -define(AUTHN_MECHANISM, password_based). -define(AUTHN_MECHANISM_BIN, <<"password_based">>). + +-define(AUTHN_MECHANISM_SCRAM, scram). +-define(AUTHN_MECHANISM_SCRAM_BIN, <<"scram">>). + -define(AUTHN_BACKEND, http). -define(AUTHN_BACKEND_BIN, <<"http">>). -define(AUTHN_TYPE, {?AUTHN_MECHANISM, ?AUTHN_BACKEND}). +-define(AUTHN_TYPE_SCRAM, {?AUTHN_MECHANISM_SCRAM, ?AUTHN_BACKEND}). + +-define(AUTHN_DATA_FIELDS, [is_superuser, client_attrs, expire_at, acl]). -endif. diff --git a/apps/emqx_auth_http/src/emqx_auth_http_app.erl b/apps/emqx_auth_http/src/emqx_auth_http_app.erl index b97743b41..8b7d08c4e 100644 --- a/apps/emqx_auth_http/src/emqx_auth_http_app.erl +++ b/apps/emqx_auth_http/src/emqx_auth_http_app.erl @@ -25,10 +25,12 @@ start(_StartType, _StartArgs) -> ok = emqx_authz:register_source(?AUTHZ_TYPE, emqx_authz_http), ok = emqx_authn:register_provider(?AUTHN_TYPE, emqx_authn_http), + ok = emqx_authn:register_provider(?AUTHN_TYPE_SCRAM, emqx_authn_scram_restapi), {ok, Sup} = emqx_auth_http_sup:start_link(), {ok, Sup}. stop(_State) -> ok = emqx_authn:deregister_provider(?AUTHN_TYPE), + ok = emqx_authn:deregister_provider(?AUTHN_TYPE_SCRAM), ok = emqx_authz:unregister_source(?AUTHZ_TYPE), ok. diff --git a/apps/emqx_auth_http/src/emqx_authn_http.erl b/apps/emqx_auth_http/src/emqx_authn_http.erl index 818e355e5..67d7403ea 100644 --- a/apps/emqx_auth_http/src/emqx_authn_http.erl +++ b/apps/emqx_auth_http/src/emqx_authn_http.erl @@ -28,6 +28,15 @@ destroy/1 ]). +-export([ + with_validated_config/2, + generate_request/2, + request_for_log/2, + response_for_log/1, + extract_auth_data/2, + safely_parse_body/2 +]). + -define(DEFAULT_CONTENT_TYPE, <<"application/json">>). %%------------------------------------------------------------------------------ @@ -187,34 +196,14 @@ handle_response(Headers, Body) -> case safely_parse_body(ContentType, Body) of {ok, NBody} -> body_to_auth_data(NBody); - {error, Reason} -> - ?TRACE_AUTHN_PROVIDER( - error, - "parse_http_response_failed", - #{content_type => ContentType, body => Body, reason => Reason} - ), + {error, _Reason} -> ignore end. body_to_auth_data(Body) -> case maps:get(<<"result">>, Body, <<"ignore">>) of <<"allow">> -> - IsSuperuser = emqx_authn_utils:is_superuser(Body), - Attrs = emqx_authn_utils:client_attrs(Body), - try - ExpireAt = expire_at(Body), - ACL = acl(ExpireAt, Body), - Result = merge_maps([ExpireAt, IsSuperuser, ACL, Attrs]), - {ok, Result} - catch - throw:{bad_acl_rule, Reason} -> - %% it's a invalid token, so ok to log - ?TRACE_AUTHN_PROVIDER("bad_acl_rule", Reason#{http_body => Body}), - {error, bad_username_or_password}; - throw:Reason -> - ?TRACE_AUTHN_PROVIDER("bad_response_body", Reason#{http_body => Body}), - {error, bad_username_or_password} - end; + extract_auth_data(http, Body); <<"deny">> -> {error, not_authorized}; <<"ignore">> -> @@ -223,6 +212,24 @@ body_to_auth_data(Body) -> ignore end. +extract_auth_data(Source, Body) -> + IsSuperuser = emqx_authn_utils:is_superuser(Body), + Attrs = emqx_authn_utils:client_attrs(Body), + try + ExpireAt = expire_at(Body), + ACL = acl(ExpireAt, Source, Body), + Result = merge_maps([ExpireAt, IsSuperuser, ACL, Attrs]), + {ok, Result} + catch + throw:{bad_acl_rule, Reason} -> + %% it's a invalid token, so ok to log + ?TRACE_AUTHN_PROVIDER("bad_acl_rule", Reason#{http_body => Body}), + {error, bad_username_or_password}; + throw:Reason -> + ?TRACE_AUTHN_PROVIDER("bad_response_body", Reason#{http_body => Body}), + {error, bad_username_or_password} + end. + merge_maps([]) -> #{}; merge_maps([Map | Maps]) -> maps:merge(Map, merge_maps(Maps)). @@ -261,40 +268,43 @@ expire_sec(#{<<"expire_at">> := _}) -> expire_sec(_) -> undefined. -acl(#{expire_at := ExpireTimeMs}, #{<<"acl">> := Rules}) -> +acl(#{expire_at := ExpireTimeMs}, Source, #{<<"acl">> := Rules}) -> #{ acl => #{ - source_for_logging => http, + source_for_logging => Source, rules => emqx_authz_rule_raw:parse_and_compile_rules(Rules), %% It's seconds level precision (like JWT) for authz %% see emqx_authz_client_info:check/1 expire => erlang:convert_time_unit(ExpireTimeMs, millisecond, second) } }; -acl(_NoExpire, #{<<"acl">> := Rules}) -> +acl(_NoExpire, Source, #{<<"acl">> := Rules}) -> #{ acl => #{ - source_for_logging => http, + source_for_logging => Source, rules => emqx_authz_rule_raw:parse_and_compile_rules(Rules) } }; -acl(_, _) -> +acl(_, _, _) -> #{}. safely_parse_body(ContentType, Body) -> try parse_body(ContentType, Body) catch - _Class:_Reason -> + _Class:Reason -> + ?TRACE_AUTHN_PROVIDER( + error, + "parse_http_response_failed", + #{content_type => ContentType, body => Body, reason => Reason} + ), {error, invalid_body} end. parse_body(<<"application/json", _/binary>>, Body) -> {ok, emqx_utils_json:decode(Body, [return_maps])}; parse_body(<<"application/x-www-form-urlencoded", _/binary>>, Body) -> - Flags = [<<"result">>, <<"is_superuser">>], - RawMap = maps:from_list(cow_qs:parse_qs(Body)), - NBody = maps:with(Flags, RawMap), + NBody = maps:from_list(cow_qs:parse_qs(Body)), {ok, NBody}; parse_body(ContentType, _) -> {error, {unsupported_content_type, ContentType}}. diff --git a/apps/emqx_auth_http/src/emqx_authn_http_schema.erl b/apps/emqx_auth_http/src/emqx_authn_http_schema.erl index aff16b824..0167571c0 100644 --- a/apps/emqx_auth_http/src/emqx_authn_http_schema.erl +++ b/apps/emqx_auth_http/src/emqx_authn_http_schema.erl @@ -27,6 +27,8 @@ namespace/0 ]). +-export([url/1, headers/1, headers_no_content_type/1, request_timeout/1]). + -include("emqx_auth_http.hrl"). -include_lib("emqx_auth/include/emqx_authn.hrl"). -include_lib("hocon/include/hoconsc.hrl"). @@ -61,12 +63,6 @@ select_union_member( got => Else }) end; -select_union_member(#{<<"backend">> := ?AUTHN_BACKEND_BIN}) -> - throw(#{ - reason => "unknown_mechanism", - expected => "password_based", - got => undefined - }); select_union_member(_Value) -> undefined. diff --git a/apps/emqx_auth_http/src/emqx_authn_scram_restapi.erl b/apps/emqx_auth_http/src/emqx_authn_scram_restapi.erl new file mode 100644 index 000000000..abb91f130 --- /dev/null +++ b/apps/emqx_auth_http/src/emqx_authn_scram_restapi.erl @@ -0,0 +1,161 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +%% Note: +%% This is not an implementation of the RFC 7804: +%% Salted Challenge Response HTTP Authentication Mechanism. +%% This backend is an implementation of scram, +%% which uses an external web resource as a source of user information. + +-module(emqx_authn_scram_restapi). + +-feature(maybe_expr, enable). + +-include("emqx_auth_http.hrl"). +-include_lib("emqx/include/logger.hrl"). +-include_lib("emqx_auth/include/emqx_authn.hrl"). + +-behaviour(emqx_authn_provider). + +-export([ + create/2, + update/2, + authenticate/2, + destroy/1 +]). + +-define(REQUIRED_USER_INFO_KEYS, [ + <<"stored_key">>, + <<"server_key">>, + <<"salt">> +]). + +%%------------------------------------------------------------------------------ +%% APIs +%%------------------------------------------------------------------------------ + +create(_AuthenticatorID, Config) -> + create(Config). + +create(Config0) -> + emqx_authn_http:with_validated_config(Config0, fun(Config, State) -> + ResourceId = emqx_authn_utils:make_resource_id(?MODULE), + % {Config, State} = parse_config(Config0), + {ok, _Data} = emqx_authn_utils:create_resource( + ResourceId, + emqx_bridge_http_connector, + Config + ), + {ok, merge_scram_conf(Config, State#{resource_id => ResourceId})} + end). + +update(Config0, #{resource_id := ResourceId} = _State) -> + emqx_authn_http:with_validated_config(Config0, fun(Config, NState) -> + % {Config, NState} = parse_config(Config0), + case emqx_authn_utils:update_resource(emqx_bridge_http_connector, Config, ResourceId) of + {error, Reason} -> + error({load_config_error, Reason}); + {ok, _} -> + {ok, merge_scram_conf(Config, NState#{resource_id => ResourceId})} + end + end). + +authenticate( + #{ + auth_method := AuthMethod, + auth_data := AuthData, + auth_cache := AuthCache + } = Credential, + State +) -> + RetrieveFun = fun(Username) -> + retrieve(Username, Credential, State) + end, + OnErrFun = fun(Msg, Reason) -> + ?TRACE_AUTHN_PROVIDER(Msg, #{ + reason => Reason + }) + end, + emqx_utils_scram:authenticate( + AuthMethod, AuthData, AuthCache, State, RetrieveFun, OnErrFun, ?AUTHN_DATA_FIELDS + ); +authenticate(_Credential, _State) -> + ignore. + +destroy(#{resource_id := ResourceId}) -> + _ = emqx_resource:remove_local(ResourceId), + ok. + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- + +retrieve( + Username, + Credential, + #{ + resource_id := ResourceId, + method := Method, + request_timeout := RequestTimeout + } = State +) -> + Request = emqx_authn_http:generate_request(Credential#{username := Username}, State), + Response = emqx_resource:simple_sync_query(ResourceId, {Method, Request, RequestTimeout}), + ?TRACE_AUTHN_PROVIDER("scram_restapi_response", #{ + request => emqx_authn_http:request_for_log(Credential, State), + response => emqx_authn_http:response_for_log(Response), + resource => ResourceId + }), + case Response of + {ok, 200, Headers, Body} -> + handle_response(Headers, Body); + {ok, _StatusCode, _Headers} -> + {error, bad_response}; + {ok, _StatusCode, _Headers, _Body} -> + {error, bad_response}; + {error, _Reason} = Error -> + Error + end. + +handle_response(Headers, Body) -> + ContentType = proplists:get_value(<<"content-type">>, Headers), + maybe + {ok, NBody} ?= emqx_authn_http:safely_parse_body(ContentType, Body), + {ok, UserInfo} ?= body_to_user_info(NBody), + {ok, AuthData} ?= emqx_authn_http:extract_auth_data(scram_restapi, NBody), + {ok, maps:merge(AuthData, UserInfo)} + end. + +body_to_user_info(Body) -> + Required0 = maps:with(?REQUIRED_USER_INFO_KEYS, Body), + case maps:size(Required0) =:= erlang:length(?REQUIRED_USER_INFO_KEYS) of + true -> + case safely_convert_hex(Required0) of + {ok, Required} -> + {ok, emqx_utils_maps:safe_atom_key_map(Required)}; + Error -> + ?TRACE_AUTHN_PROVIDER("decode_keys_failed", #{http_body => Body}), + Error + end; + _ -> + ?TRACE_AUTHN_PROVIDER("missing_requried_keys", #{http_body => Body}), + {error, bad_response} + end. + +safely_convert_hex(Required) -> + try + {ok, + maps:map( + fun(_Key, Hex) -> + binary:decode_hex(Hex) + end, + Required + )} + catch + _Class:Reason -> + {error, Reason} + end. + +merge_scram_conf(Conf, State) -> + maps:merge(maps:with([algorithm, iteration_count], Conf), State). diff --git a/apps/emqx_auth_http/src/emqx_authn_scram_restapi_schema.erl b/apps/emqx_auth_http/src/emqx_authn_scram_restapi_schema.erl new file mode 100644 index 000000000..bf3398abb --- /dev/null +++ b/apps/emqx_auth_http/src/emqx_authn_scram_restapi_schema.erl @@ -0,0 +1,81 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_authn_scram_restapi_schema). + +-behaviour(emqx_authn_schema). + +-export([ + fields/1, + validations/0, + desc/1, + refs/0, + select_union_member/1, + namespace/0 +]). + +-include("emqx_auth_http.hrl"). +-include_lib("emqx_auth/include/emqx_authn.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). + +namespace() -> "authn". + +refs() -> + [?R_REF(scram_restapi_get), ?R_REF(scram_restapi_post)]. + +select_union_member( + #{<<"mechanism">> := ?AUTHN_MECHANISM_SCRAM_BIN, <<"backend">> := ?AUTHN_BACKEND_BIN} = Value +) -> + case maps:get(<<"method">>, Value, undefined) of + <<"get">> -> + [?R_REF(scram_restapi_get)]; + <<"post">> -> + [?R_REF(scram_restapi_post)]; + Else -> + throw(#{ + reason => "unknown_http_method", + expected => "get | post", + field_name => method, + got => Else + }) + end; +select_union_member(_Value) -> + undefined. + +fields(scram_restapi_get) -> + [ + {method, #{type => get, required => true, desc => ?DESC(emqx_authn_http_schema, method)}}, + {headers, fun emqx_authn_http_schema:headers_no_content_type/1} + ] ++ common_fields(); +fields(scram_restapi_post) -> + [ + {method, #{type => post, required => true, desc => ?DESC(emqx_authn_http_schema, method)}}, + {headers, fun emqx_authn_http_schema:headers/1} + ] ++ common_fields(). + +desc(scram_restapi_get) -> + ?DESC(emqx_authn_http_schema, get); +desc(scram_restapi_post) -> + ?DESC(emqx_authn_http_schema, post); +desc(_) -> + undefined. + +validations() -> + emqx_authn_http_schema:validations(). + +common_fields() -> + emqx_authn_schema:common_fields() ++ + [ + {mechanism, emqx_authn_schema:mechanism(?AUTHN_MECHANISM_SCRAM)}, + {backend, emqx_authn_schema:backend(?AUTHN_BACKEND)}, + {algorithm, fun emqx_authn_scram_mnesia_schema:algorithm/1}, + {iteration_count, fun emqx_authn_scram_mnesia_schema:iteration_count/1}, + {url, fun emqx_authn_http_schema:url/1}, + {body, + hoconsc:mk(typerefl:alias("map", map([{fuzzy, term(), binary()}])), #{ + required => false, desc => ?DESC(emqx_authn_http_schema, body) + })}, + {request_timeout, fun emqx_authn_http_schema:request_timeout/1} + ] ++ + proplists:delete(pool_type, emqx_bridge_http_connector:fields(config)). diff --git a/apps/emqx_auth_http/src/emqx_authz_http.erl b/apps/emqx_auth_http/src/emqx_authz_http.erl index b1c2ef0ab..6d9ffff68 100644 --- a/apps/emqx_auth_http/src/emqx_authz_http.erl +++ b/apps/emqx_auth_http/src/emqx_authz_http.erl @@ -67,7 +67,11 @@ description() -> create(Config) -> NConfig = parse_config(Config), ResourceId = emqx_authn_utils:make_resource_id(?MODULE), - {ok, _Data} = emqx_authz_utils:create_resource(ResourceId, emqx_bridge_http_connector, NConfig), + {ok, _Data} = emqx_authz_utils:create_resource( + ResourceId, + emqx_bridge_http_connector, + NConfig + ), NConfig#{annotations => #{id => ResourceId}}. update(Config) -> diff --git a/apps/emqx_auth_http/test/emqx_authn_scram_restapi_SUITE.erl b/apps/emqx_auth_http/test/emqx_authn_scram_restapi_SUITE.erl new file mode 100644 index 000000000..7963cf1e3 --- /dev/null +++ b/apps/emqx_auth_http/test/emqx_authn_scram_restapi_SUITE.erl @@ -0,0 +1,509 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_authn_scram_restapi_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +-include_lib("emqx/include/emqx_mqtt.hrl"). +-include_lib("emqx_auth/include/emqx_authn.hrl"). + +-define(PATH, [authentication]). + +-define(HTTP_PORT, 34333). +-define(HTTP_PATH, "/user/[...]"). +-define(ALGORITHM, sha512). +-define(ALGORITHM_STR, <<"sha512">>). +-define(ITERATION_COUNT, 4096). + +-define(T_ACL_USERNAME, <<"username">>). +-define(T_ACL_PASSWORD, <<"password">>). + +-include_lib("emqx/include/emqx_placeholder.hrl"). + +all() -> + case emqx_release:edition() of + ce -> + []; + _ -> + emqx_common_test_helpers:all(?MODULE) + end. + +init_per_suite(Config) -> + Apps = emqx_cth_suite:start([cowboy, emqx, emqx_conf, emqx_auth, emqx_auth_http], #{ + work_dir => ?config(priv_dir, Config) + }), + + IdleTimeout = emqx_config:get([mqtt, idle_timeout]), + [{apps, Apps}, {idle_timeout, IdleTimeout} | Config]. + +end_per_suite(Config) -> + ok = emqx_config:put([mqtt, idle_timeout], ?config(idle_timeout, Config)), + emqx_authn_test_lib:delete_authenticators( + [authentication], + ?GLOBAL + ), + ok = emqx_cth_suite:stop(?config(apps, Config)), + ok. + +init_per_testcase(_Case, Config) -> + {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000), + emqx_authn_test_lib:delete_authenticators( + [authentication], + ?GLOBAL + ), + {ok, _} = emqx_authn_scram_restapi_test_server:start_link(?HTTP_PORT, ?HTTP_PATH), + Config. + +end_per_testcase(_Case, _Config) -> + ok = emqx_authn_scram_restapi_test_server:stop(). + +%%------------------------------------------------------------------------------ +%% Tests +%%------------------------------------------------------------------------------ + +t_create(_Config) -> + AuthConfig = raw_config(), + + {ok, _} = emqx:update_config( + ?PATH, + {create_authenticator, ?GLOBAL, AuthConfig} + ), + + {ok, [#{provider := emqx_authn_scram_restapi}]} = emqx_authn_chains:list_authenticators( + ?GLOBAL + ). + +t_create_invalid(_Config) -> + AuthConfig = raw_config(), + + InvalidConfigs = + [ + AuthConfig#{<<"headers">> => []}, + AuthConfig#{<<"method">> => <<"delete">>}, + AuthConfig#{<<"url">> => <<"localhost">>}, + AuthConfig#{<<"url">> => <<"http://foo.com/xxx#fragment">>}, + AuthConfig#{<<"url">> => <<"http://${foo}.com/xxx">>}, + AuthConfig#{<<"url">> => <<"//foo.com/xxx">>}, + AuthConfig#{<<"algorithm">> => <<"sha128">>} + ], + + lists:foreach( + fun(Config) -> + ct:pal("creating authenticator with invalid config: ~p", [Config]), + {error, _} = + try + emqx:update_config( + ?PATH, + {create_authenticator, ?GLOBAL, Config} + ) + catch + throw:Error -> + {error, Error} + end, + ?assertEqual( + {error, {not_found, {chain, ?GLOBAL}}}, + emqx_authn_chains:list_authenticators(?GLOBAL) + ) + end, + InvalidConfigs + ). + +t_authenticate(_Config) -> + Username = <<"u">>, + Password = <<"p">>, + + set_user_handler(Username, Password), + init_auth(), + + ok = emqx_config:put([mqtt, idle_timeout], 500), + + {ok, Pid} = create_connection(Username, Password), + emqx_authn_mqtt_test_client:stop(Pid). + +t_authenticate_bad_props(_Config) -> + Username = <<"u">>, + Password = <<"p">>, + + set_user_handler(Username, Password), + init_auth(), + + {ok, Pid} = emqx_authn_mqtt_test_client:start_link("127.0.0.1", 1883), + + ConnectPacket = ?CONNECT_PACKET( + #mqtt_packet_connect{ + proto_ver = ?MQTT_PROTO_V5, + properties = #{ + 'Authentication-Method' => <<"SCRAM-SHA-512">> + } + } + ), + + ok = emqx_authn_mqtt_test_client:send(Pid, ConnectPacket), + + ?CONNACK_PACKET(?RC_NOT_AUTHORIZED) = receive_packet(). + +t_authenticate_bad_username(_Config) -> + Username = <<"u">>, + Password = <<"p">>, + + set_user_handler(Username, Password), + init_auth(), + + {ok, Pid} = emqx_authn_mqtt_test_client:start_link("127.0.0.1", 1883), + + ClientFirstMessage = esasl_scram:client_first_message(<<"badusername">>), + + ConnectPacket = ?CONNECT_PACKET( + #mqtt_packet_connect{ + proto_ver = ?MQTT_PROTO_V5, + properties = #{ + 'Authentication-Method' => <<"SCRAM-SHA-512">>, + 'Authentication-Data' => ClientFirstMessage + } + } + ), + + ok = emqx_authn_mqtt_test_client:send(Pid, ConnectPacket), + + ?CONNACK_PACKET(?RC_NOT_AUTHORIZED) = receive_packet(). + +t_authenticate_bad_password(_Config) -> + Username = <<"u">>, + Password = <<"p">>, + + set_user_handler(Username, Password), + init_auth(), + + {ok, Pid} = emqx_authn_mqtt_test_client:start_link("127.0.0.1", 1883), + + ClientFirstMessage = esasl_scram:client_first_message(Username), + + ConnectPacket = ?CONNECT_PACKET( + #mqtt_packet_connect{ + proto_ver = ?MQTT_PROTO_V5, + properties = #{ + 'Authentication-Method' => <<"SCRAM-SHA-512">>, + 'Authentication-Data' => ClientFirstMessage + } + } + ), + + ok = emqx_authn_mqtt_test_client:send(Pid, ConnectPacket), + + ?AUTH_PACKET( + ?RC_CONTINUE_AUTHENTICATION, + #{'Authentication-Data' := ServerFirstMessage} + ) = receive_packet(), + + {continue, ClientFinalMessage, _ClientCache} = + esasl_scram:check_server_first_message( + ServerFirstMessage, + #{ + client_first_message => ClientFirstMessage, + password => <<"badpassword">>, + algorithm => ?ALGORITHM + } + ), + + AuthContinuePacket = ?AUTH_PACKET( + ?RC_CONTINUE_AUTHENTICATION, + #{ + 'Authentication-Method' => <<"SCRAM-SHA-512">>, + 'Authentication-Data' => ClientFinalMessage + } + ), + + ok = emqx_authn_mqtt_test_client:send(Pid, AuthContinuePacket), + + ?CONNACK_PACKET(?RC_NOT_AUTHORIZED) = receive_packet(). + +t_destroy(_Config) -> + Username = <<"u">>, + Password = <<"p">>, + + set_user_handler(Username, Password), + init_auth(), + + ok = emqx_config:put([mqtt, idle_timeout], 500), + + {ok, Pid} = emqx_authn_mqtt_test_client:start_link("127.0.0.1", 1883), + + ConnectPacket = ?CONNECT_PACKET( + #mqtt_packet_connect{ + proto_ver = ?MQTT_PROTO_V5, + properties = #{ + 'Authentication-Method' => <<"SCRAM-SHA-512">> + } + } + ), + + ok = emqx_authn_mqtt_test_client:send(Pid, ConnectPacket), + + ok = ct:sleep(1000), + + ?CONNACK_PACKET(?RC_NOT_AUTHORIZED) = receive_packet(), + + %% emqx_authn_mqtt_test_client:stop(Pid), + + emqx_authn_test_lib:delete_authenticators( + [authentication], + ?GLOBAL + ), + + {ok, Pid2} = emqx_authn_mqtt_test_client:start_link("127.0.0.1", 1883), + + ok = emqx_authn_mqtt_test_client:send(Pid2, ConnectPacket), + + ok = ct:sleep(1000), + + ?CONNACK_PACKET( + ?RC_SUCCESS, + _, + _ + ) = receive_packet(). + +t_acl(_Config) -> + init_auth(), + + ACL = emqx_authn_http_SUITE:acl_rules(), + set_user_handler(?T_ACL_USERNAME, ?T_ACL_PASSWORD, #{acl => ACL}), + {ok, Pid} = create_connection(?T_ACL_USERNAME, ?T_ACL_PASSWORD), + + Cases = [ + {allow, <<"http-authn-acl/#">>}, + {deny, <<"http-authn-acl/1">>}, + {deny, <<"t/#">>} + ], + + try + lists:foreach( + fun(Case) -> + test_acl(Case, Pid) + end, + Cases + ) + after + ok = emqx_authn_mqtt_test_client:stop(Pid) + end. + +t_auth_expire(_Config) -> + init_auth(), + + ExpireSec = 3, + WaitTime = timer:seconds(ExpireSec + 1), + ACL = emqx_authn_http_SUITE:acl_rules(), + + set_user_handler(?T_ACL_USERNAME, ?T_ACL_PASSWORD, #{ + acl => ACL, + expire_at => + erlang:system_time(second) + ExpireSec + }), + {ok, Pid} = create_connection(?T_ACL_USERNAME, ?T_ACL_PASSWORD), + + timer:sleep(WaitTime), + ?assertEqual(false, erlang:is_process_alive(Pid)). + +t_is_superuser() -> + State = init_auth(), + ok = test_is_superuser(State, false), + ok = test_is_superuser(State, true), + ok = test_is_superuser(State, false). + +test_is_superuser(State, ExpectedIsSuperuser) -> + Username = <<"u">>, + Password = <<"p">>, + + set_user_handler(Username, Password, #{is_superuser => ExpectedIsSuperuser}), + + ClientFirstMessage = esasl_scram:client_first_message(Username), + + {continue, ServerFirstMessage, ServerCache} = + emqx_authn_scram_restapi:authenticate( + #{ + auth_method => <<"SCRAM-SHA-512">>, + auth_data => ClientFirstMessage, + auth_cache => #{} + }, + State + ), + + {continue, ClientFinalMessage, ClientCache} = + esasl_scram:check_server_first_message( + ServerFirstMessage, + #{ + client_first_message => ClientFirstMessage, + password => Password, + algorithm => ?ALGORITHM + } + ), + + {ok, UserInfo1, ServerFinalMessage} = + emqx_authn_scram_restapi:authenticate( + #{ + auth_method => <<"SCRAM-SHA-512">>, + auth_data => ClientFinalMessage, + auth_cache => ServerCache + }, + State + ), + + ok = esasl_scram:check_server_final_message( + ServerFinalMessage, ClientCache#{algorithm => ?ALGORITHM} + ), + + ?assertMatch(#{is_superuser := ExpectedIsSuperuser}, UserInfo1). + +%%------------------------------------------------------------------------------ +%% Helpers +%%------------------------------------------------------------------------------ + +raw_config() -> + #{ + <<"mechanism">> => <<"scram">>, + <<"backend">> => <<"http">>, + <<"enable">> => <<"true">>, + <<"method">> => <<"get">>, + <<"url">> => <<"http://127.0.0.1:34333/user">>, + <<"body">> => #{<<"username">> => ?PH_USERNAME}, + <<"headers">> => #{<<"X-Test-Header">> => <<"Test Value">>}, + <<"algorithm">> => ?ALGORITHM_STR, + <<"iteration_count">> => ?ITERATION_COUNT + }. + +set_user_handler(Username, Password) -> + set_user_handler(Username, Password, #{is_superuser => false}). +set_user_handler(Username, Password, Extra0) -> + %% HTTP Server + Handler = fun(Req0, State) -> + #{ + username := Username + } = cowboy_req:match_qs([username], Req0), + + UserInfo = make_user_info(Password, ?ALGORITHM, ?ITERATION_COUNT), + Extra = maps:merge(#{is_superuser => false}, Extra0), + Req = cowboy_req:reply( + 200, + #{<<"content-type">> => <<"application/json">>}, + emqx_utils_json:encode(maps:merge(Extra, UserInfo)), + Req0 + ), + {ok, Req, State} + end, + ok = emqx_authn_scram_restapi_test_server:set_handler(Handler). + +init_auth() -> + init_auth(raw_config()). + +init_auth(Config) -> + {ok, _} = emqx:update_config( + ?PATH, + {create_authenticator, ?GLOBAL, Config} + ), + + {ok, [#{state := State}]} = emqx_authn_chains:list_authenticators(?GLOBAL), + State. + +make_user_info(Password, Algorithm, IterationCount) -> + {StoredKey, ServerKey, Salt} = esasl_scram:generate_authentication_info( + Password, + #{ + algorithm => Algorithm, + iteration_count => IterationCount + } + ), + #{ + stored_key => binary:encode_hex(StoredKey), + server_key => binary:encode_hex(ServerKey), + salt => binary:encode_hex(Salt) + }. + +receive_packet() -> + receive + {packet, Packet} -> + ct:pal("Delivered packet: ~p", [Packet]), + Packet + after 1000 -> + ct:fail("Deliver timeout") + end. + +create_connection(Username, Password) -> + {ok, Pid} = emqx_authn_mqtt_test_client:start_link("127.0.0.1", 1883), + + ClientFirstMessage = esasl_scram:client_first_message(Username), + + ConnectPacket = ?CONNECT_PACKET( + #mqtt_packet_connect{ + proto_ver = ?MQTT_PROTO_V5, + properties = #{ + 'Authentication-Method' => <<"SCRAM-SHA-512">>, + 'Authentication-Data' => ClientFirstMessage + } + } + ), + + ok = emqx_authn_mqtt_test_client:send(Pid, ConnectPacket), + + %% Intentional sleep to trigger idle timeout for the connection not yet authenticated + ok = ct:sleep(1000), + + ?AUTH_PACKET( + ?RC_CONTINUE_AUTHENTICATION, + #{'Authentication-Data' := ServerFirstMessage} + ) = receive_packet(), + + {continue, ClientFinalMessage, ClientCache} = + esasl_scram:check_server_first_message( + ServerFirstMessage, + #{ + client_first_message => ClientFirstMessage, + password => Password, + algorithm => ?ALGORITHM + } + ), + + AuthContinuePacket = ?AUTH_PACKET( + ?RC_CONTINUE_AUTHENTICATION, + #{ + 'Authentication-Method' => <<"SCRAM-SHA-512">>, + 'Authentication-Data' => ClientFinalMessage + } + ), + + ok = emqx_authn_mqtt_test_client:send(Pid, AuthContinuePacket), + + ?CONNACK_PACKET( + ?RC_SUCCESS, + _, + #{'Authentication-Data' := ServerFinalMessage} + ) = receive_packet(), + + ok = esasl_scram:check_server_final_message( + ServerFinalMessage, ClientCache#{algorithm => ?ALGORITHM} + ), + {ok, Pid}. + +test_acl({allow, Topic}, C) -> + ?assertMatch( + [0], + send_subscribe(C, Topic) + ); +test_acl({deny, Topic}, C) -> + ?assertMatch( + [?RC_NOT_AUTHORIZED], + send_subscribe(C, Topic) + ). + +send_subscribe(Client, Topic) -> + TopicOpts = #{nl => 0, rap => 0, rh => 0, qos => 0}, + Packet = ?SUBSCRIBE_PACKET(1, [{Topic, TopicOpts}]), + emqx_authn_mqtt_test_client:send(Client, Packet), + timer:sleep(200), + + ?SUBACK_PACKET(1, ReasonCode) = receive_packet(), + ReasonCode. diff --git a/apps/emqx_auth_http/test/emqx_authn_scram_restapi_test_server.erl b/apps/emqx_auth_http/test/emqx_authn_scram_restapi_test_server.erl new file mode 100644 index 000000000..1e1432e0b --- /dev/null +++ b/apps/emqx_auth_http/test/emqx_authn_scram_restapi_test_server.erl @@ -0,0 +1,115 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_authn_scram_restapi_test_server). + +-behaviour(supervisor). +-behaviour(cowboy_handler). + +% cowboy_server callbacks +-export([init/2]). + +% supervisor callbacks +-export([init/1]). + +% API +-export([ + start_link/2, + start_link/3, + stop/0, + set_handler/1 +]). + +%%------------------------------------------------------------------------------ +%% API +%%------------------------------------------------------------------------------ + +start_link(Port, Path) -> + start_link(Port, Path, false). + +start_link(Port, Path, SSLOpts) -> + supervisor:start_link({local, ?MODULE}, ?MODULE, [Port, Path, SSLOpts]). + +stop() -> + gen_server:stop(?MODULE). + +set_handler(F) when is_function(F, 2) -> + true = ets:insert(?MODULE, {handler, F}), + ok. + +%%------------------------------------------------------------------------------ +%% supervisor API +%%------------------------------------------------------------------------------ + +init([Port, Path, SSLOpts]) -> + Dispatch = cowboy_router:compile( + [ + {'_', [{Path, ?MODULE, []}]} + ] + ), + + ProtoOpts = #{env => #{dispatch => Dispatch}}, + + Tab = ets:new(?MODULE, [set, named_table, public]), + ets:insert(Tab, {handler, fun default_handler/2}), + + {Transport, TransOpts, CowboyModule} = transport_settings(Port, SSLOpts), + + ChildSpec = ranch:child_spec(?MODULE, Transport, TransOpts, CowboyModule, ProtoOpts), + + {ok, {#{}, [ChildSpec]}}. + +%%------------------------------------------------------------------------------ +%% cowboy_server API +%%------------------------------------------------------------------------------ + +init(Req, State) -> + [{handler, Handler}] = ets:lookup(?MODULE, handler), + Handler(Req, State). + +%%------------------------------------------------------------------------------ +%% Internal functions +%%------------------------------------------------------------------------------ + +transport_settings(Port, false) -> + TransOpts = #{ + socket_opts => [{port, Port}], + connection_type => supervisor + }, + {ranch_tcp, TransOpts, cowboy_clear}; +transport_settings(Port, SSLOpts) -> + TransOpts = #{ + socket_opts => [ + {port, Port}, + {next_protocols_advertised, [<<"h2">>, <<"http/1.1">>]}, + {alpn_preferred_protocols, [<<"h2">>, <<"http/1.1">>]} + | SSLOpts + ], + connection_type => supervisor + }, + {ranch_ssl, TransOpts, cowboy_tls}. + +default_handler(Req0, State) -> + Req = cowboy_req:reply( + 400, + #{<<"content-type">> => <<"text/plain">>}, + <<"">>, + Req0 + ), + {ok, Req, State}. + +make_user_info(Password, Algorithm, IterationCount) -> + {StoredKey, ServerKey, Salt} = esasl_scram:generate_authentication_info( + Password, + #{ + algorithm => Algorithm, + iteration_count => IterationCount + } + ), + #{ + stored_key => StoredKey, + server_key => ServerKey, + salt => Salt, + is_superuser => false + }. diff --git a/apps/emqx_auth_jwt/src/emqx_authn_jwks_connector.erl b/apps/emqx_auth_jwt/src/emqx_authn_jwks_connector.erl index ffa8175b7..22aed8e57 100644 --- a/apps/emqx_auth_jwt/src/emqx_authn_jwks_connector.erl +++ b/apps/emqx_auth_jwt/src/emqx_authn_jwks_connector.erl @@ -22,6 +22,7 @@ %% callbacks of behaviour emqx_resource -export([ + resource_type/0, callback_mode/0, on_start/2, on_stop/2, @@ -32,6 +33,8 @@ -define(DEFAULT_POOL_SIZE, 8). +resource_type() -> jwks. + callback_mode() -> always_sync. on_start(InstId, Opts) -> diff --git a/apps/emqx_auth_jwt/src/emqx_authn_jwt.erl b/apps/emqx_auth_jwt/src/emqx_authn_jwt.erl index fff7d056e..0d3d1a8c2 100644 --- a/apps/emqx_auth_jwt/src/emqx_authn_jwt.erl +++ b/apps/emqx_auth_jwt/src/emqx_authn_jwt.erl @@ -188,7 +188,8 @@ do_create( ResourceId, ?AUTHN_RESOURCE_GROUP, emqx_authn_jwks_connector, - connector_opts(Config) + connector_opts(Config), + #{} ), {ok, #{ jwk_resource => ResourceId, diff --git a/apps/emqx_auth_ldap/src/emqx_auth_ldap.app.src b/apps/emqx_auth_ldap/src/emqx_auth_ldap.app.src index d84d6ff81..a58117356 100644 --- a/apps/emqx_auth_ldap/src/emqx_auth_ldap.app.src +++ b/apps/emqx_auth_ldap/src/emqx_auth_ldap.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_auth_ldap, [ {description, "EMQX LDAP Authentication and Authorization"}, - {vsn, "0.1.2"}, + {vsn, "0.1.3"}, {registered, []}, {mod, {emqx_auth_ldap_app, []}}, {applications, [ diff --git a/apps/emqx_auth_ldap/test/emqx_authn_ldap_SUITE.erl b/apps/emqx_auth_ldap/test/emqx_authn_ldap_SUITE.erl index ac941f268..af8100c23 100644 --- a/apps/emqx_auth_ldap/test/emqx_authn_ldap_SUITE.erl +++ b/apps/emqx_auth_ldap/test/emqx_authn_ldap_SUITE.erl @@ -21,6 +21,7 @@ -include_lib("emqx_auth/include/emqx_authn.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). -define(LDAP_HOST, "ldap"). -define(LDAP_DEFAULT_PORT, 389). @@ -46,13 +47,6 @@ init_per_suite(Config) -> Apps = emqx_cth_suite:start([emqx, emqx_conf, emqx_auth, emqx_auth_ldap], #{ work_dir => ?config(priv_dir, Config) }), - {ok, _} = emqx_resource:create_local( - ?LDAP_RESOURCE, - ?AUTHN_RESOURCE_GROUP, - emqx_ldap, - ldap_config(), - #{} - ), [{apps, Apps} | Config]; false -> {skip, no_ldap} @@ -63,7 +57,6 @@ end_per_suite(Config) -> [authentication], ?GLOBAL ), - ok = emqx_resource:remove_local(?LDAP_RESOURCE), ok = emqx_cth_suite:stop(?config(apps, Config)). %%------------------------------------------------------------------------------ @@ -128,6 +121,87 @@ t_create_invalid(_Config) -> InvalidConfigs ). +t_authenticate_timeout_cause_reconnect(_Config) -> + TestPid = self(), + meck:new(eldap, [non_strict, no_link, passthrough]), + try + %% cause eldap process to be killed + meck:expect( + eldap, + search, + fun + (Pid, [{base, <<"uid=mqttuser0007", _/binary>>} | _]) -> + TestPid ! {eldap_pid, Pid}, + {error, {gen_tcp_error, timeout}}; + (Pid, Args) -> + meck:passthrough([Pid, Args]) + end + ), + + Credentials = fun(Username) -> + #{ + username => Username, + password => Username, + listener => 'tcp:default', + protocol => mqtt + } + end, + + SpecificConfigParams = #{}, + Result = {ok, #{is_superuser => true}}, + + Timeout = 1000, + Config0 = raw_ldap_auth_config(), + Config = Config0#{ + <<"pool_size">> => 1, + <<"request_timeout">> => Timeout + }, + AuthConfig = maps:merge(Config, SpecificConfigParams), + {ok, _} = emqx:update_config( + ?PATH, + {create_authenticator, ?GLOBAL, AuthConfig} + ), + + %% 0006 is a disabled user + ?assertEqual( + {error, user_disabled}, + emqx_access_control:authenticate(Credentials(<<"mqttuser0006">>)) + ), + ?assertEqual( + {error, not_authorized}, + emqx_access_control:authenticate(Credentials(<<"mqttuser0007">>)) + ), + ok = wait_for_ldap_pid(1000), + [#{id := ResourceID}] = emqx_resource_manager:list_all(), + ?retry(1_000, 10, {ok, connected} = emqx_resource_manager:health_check(ResourceID)), + %% turn back to normal + meck:expect( + eldap, + search, + 2, + fun(Pid2, Query) -> + meck:passthrough([Pid2, Query]) + end + ), + %% expect eldap process to be restarted + ?assertEqual(Result, emqx_access_control:authenticate(Credentials(<<"mqttuser0007">>))), + emqx_authn_test_lib:delete_authenticators( + [authentication], + ?GLOBAL + ) + after + meck:unload(eldap) + end. + +wait_for_ldap_pid(After) -> + receive + {eldap_pid, Pid} -> + ?assertNot(is_process_alive(Pid)), + ok + after After -> + error(timeout) + end. + t_authenticate(_Config) -> ok = lists:foreach( fun(Sample) -> @@ -300,6 +374,3 @@ user_seeds() -> ldap_server() -> iolist_to_binary(io_lib:format("~s:~B", [?LDAP_HOST, ?LDAP_DEFAULT_PORT])). - -ldap_config() -> - emqx_ldap_SUITE:ldap_config([]). diff --git a/apps/emqx_auth_ldap/test/emqx_authz_ldap_SUITE.erl b/apps/emqx_auth_ldap/test/emqx_authz_ldap_SUITE.erl index 09875a3fa..7ff6fdebe 100644 --- a/apps/emqx_auth_ldap/test/emqx_authz_ldap_SUITE.erl +++ b/apps/emqx_auth_ldap/test/emqx_authz_ldap_SUITE.erl @@ -44,7 +44,6 @@ init_per_suite(Config) -> ], #{work_dir => emqx_cth_suite:work_dir(Config)} ), - ok = create_ldap_resource(), [{apps, Apps} | Config]; false -> {skip, no_ldap} @@ -167,21 +166,8 @@ setup_config(SpecialParams) -> ldap_server() -> iolist_to_binary(io_lib:format("~s:~B", [?LDAP_HOST, ?LDAP_DEFAULT_PORT])). -ldap_config() -> - emqx_ldap_SUITE:ldap_config([]). - start_apps(Apps) -> lists:foreach(fun application:ensure_all_started/1, Apps). stop_apps(Apps) -> lists:foreach(fun application:stop/1, Apps). - -create_ldap_resource() -> - {ok, _} = emqx_resource:create_local( - ?LDAP_RESOURCE, - ?AUTHZ_RESOURCE_GROUP, - emqx_ldap, - ldap_config(), - #{} - ), - ok. diff --git a/apps/emqx_auth_mnesia/src/emqx_authn_scram_mnesia.erl b/apps/emqx_auth_mnesia/src/emqx_authn_scram_mnesia.erl index 611469c5b..9880b71ee 100644 --- a/apps/emqx_auth_mnesia/src/emqx_authn_scram_mnesia.erl +++ b/apps/emqx_auth_mnesia/src/emqx_authn_scram_mnesia.erl @@ -133,17 +133,17 @@ authenticate( }, State ) -> - case ensure_auth_method(AuthMethod, AuthData, State) of - true -> - case AuthCache of - #{next_step := client_final} -> - check_client_final_message(AuthData, AuthCache, State); - _ -> - check_client_first_message(AuthData, AuthCache, State) - end; - false -> - ignore - end; + RetrieveFun = fun(Username) -> + retrieve(Username, State) + end, + OnErrFun = fun(Msg, Reason) -> + ?TRACE_AUTHN_PROVIDER(Msg, #{ + reason => Reason + }) + end, + emqx_utils_scram:authenticate( + AuthMethod, AuthData, AuthCache, State, RetrieveFun, OnErrFun, [is_superuser] + ); authenticate(_Credential, _State) -> ignore. @@ -257,55 +257,6 @@ run_fuzzy_filter( %% Internal functions %%------------------------------------------------------------------------------ -ensure_auth_method(_AuthMethod, undefined, _State) -> - false; -ensure_auth_method(<<"SCRAM-SHA-256">>, _AuthData, #{algorithm := sha256}) -> - true; -ensure_auth_method(<<"SCRAM-SHA-512">>, _AuthData, #{algorithm := sha512}) -> - true; -ensure_auth_method(_AuthMethod, _AuthData, _State) -> - false. - -check_client_first_message(Bin, _Cache, #{iteration_count := IterationCount} = State) -> - RetrieveFun = fun(Username) -> - retrieve(Username, State) - end, - case - esasl_scram:check_client_first_message( - Bin, - #{ - iteration_count => IterationCount, - retrieve => RetrieveFun - } - ) - of - {continue, ServerFirstMessage, Cache} -> - {continue, ServerFirstMessage, Cache}; - ignore -> - ignore; - {error, Reason} -> - ?TRACE_AUTHN_PROVIDER("check_client_first_message_error", #{ - reason => Reason - }), - {error, not_authorized} - end. - -check_client_final_message(Bin, #{is_superuser := IsSuperuser} = Cache, #{algorithm := Alg}) -> - case - esasl_scram:check_client_final_message( - Bin, - Cache#{algorithm => Alg} - ) - of - {ok, ServerFinalMessage} -> - {ok, #{is_superuser => IsSuperuser}, ServerFinalMessage}; - {error, Reason} -> - ?TRACE_AUTHN_PROVIDER("check_client_final_message_error", #{ - reason => Reason - }), - {error, not_authorized} - end. - user_info_record( #{ user_id := UserID, diff --git a/apps/emqx_auth_mnesia/src/emqx_authn_scram_mnesia_schema.erl b/apps/emqx_auth_mnesia/src/emqx_authn_scram_mnesia_schema.erl index 5d442cd57..dbad2118f 100644 --- a/apps/emqx_auth_mnesia/src/emqx_authn_scram_mnesia_schema.erl +++ b/apps/emqx_auth_mnesia/src/emqx_authn_scram_mnesia_schema.erl @@ -29,6 +29,8 @@ select_union_member/1 ]). +-export([algorithm/1, iteration_count/1]). + namespace() -> "authn". refs() -> @@ -38,11 +40,6 @@ select_union_member(#{ <<"mechanism">> := ?AUTHN_MECHANISM_SCRAM_BIN, <<"backend">> := ?AUTHN_BACKEND_BIN }) -> refs(); -select_union_member(#{<<"mechanism">> := ?AUTHN_MECHANISM_SCRAM_BIN}) -> - throw(#{ - reason => "unknown_backend", - expected => ?AUTHN_BACKEND - }); select_union_member(_) -> undefined. diff --git a/apps/emqx_auth_mysql/src/emqx_authz_mysql.erl b/apps/emqx_auth_mysql/src/emqx_authz_mysql.erl index 6a58c6c16..731c64af6 100644 --- a/apps/emqx_auth_mysql/src/emqx_authz_mysql.erl +++ b/apps/emqx_auth_mysql/src/emqx_authz_mysql.erl @@ -101,19 +101,9 @@ authorize( do_authorize(_Client, _Action, _Topic, _ColumnNames, []) -> nomatch; do_authorize(Client, Action, Topic, ColumnNames, [Row | Tail]) -> - try - emqx_authz_rule:match( - Client, Action, Topic, emqx_authz_utils:parse_rule_from_row(ColumnNames, Row) - ) - of - {matched, Permission} -> {matched, Permission}; - nomatch -> do_authorize(Client, Action, Topic, ColumnNames, Tail) - catch - error:Reason -> - ?SLOG(error, #{ - msg => "match_rule_error", - reason => Reason, - rule => Row - }), - do_authorize(Client, Action, Topic, ColumnNames, Tail) + case emqx_authz_utils:do_authorize(mysql, Client, Action, Topic, ColumnNames, Row) of + nomatch -> + do_authorize(Client, Action, Topic, ColumnNames, Tail); + {matched, Permission} -> + {matched, Permission} end. diff --git a/apps/emqx_auth_postgresql/src/emqx_authz_postgresql.erl b/apps/emqx_auth_postgresql/src/emqx_authz_postgresql.erl index b09f6e009..31bd968d0 100644 --- a/apps/emqx_auth_postgresql/src/emqx_authz_postgresql.erl +++ b/apps/emqx_auth_postgresql/src/emqx_authz_postgresql.erl @@ -107,22 +107,11 @@ authorize( do_authorize(_Client, _Action, _Topic, _ColumnNames, []) -> nomatch; do_authorize(Client, Action, Topic, ColumnNames, [Row | Tail]) -> - try - emqx_authz_rule:match( - Client, Action, Topic, emqx_authz_utils:parse_rule_from_row(ColumnNames, Row) - ) - of - {matched, Permission} -> {matched, Permission}; - nomatch -> do_authorize(Client, Action, Topic, ColumnNames, Tail) - catch - error:Reason:Stack -> - ?SLOG(error, #{ - msg => "match_rule_error", - reason => Reason, - rule => Row, - stack => Stack - }), - do_authorize(Client, Action, Topic, ColumnNames, Tail) + case emqx_authz_utils:do_authorize(postgresql, Client, Action, Topic, ColumnNames, Row) of + nomatch -> + do_authorize(Client, Action, Topic, ColumnNames, Tail); + {matched, Permission} -> + {matched, Permission} end. column_names(Columns) -> diff --git a/apps/emqx_auth_postgresql/test/emqx_authn_postgresql_SUITE.erl b/apps/emqx_auth_postgresql/test/emqx_authn_postgresql_SUITE.erl index 50bff634d..1dfd30899 100644 --- a/apps/emqx_auth_postgresql/test/emqx_authn_postgresql_SUITE.erl +++ b/apps/emqx_auth_postgresql/test/emqx_authn_postgresql_SUITE.erl @@ -198,9 +198,9 @@ test_user_auth(#{ t_authenticate_disabled_prepared_statements(_Config) -> ResConfig = maps:merge(pgsql_config(), #{disable_prepared_statements => true}), - {ok, _} = emqx_resource:recreate_local(?PGSQL_RESOURCE, emqx_postgresql, ResConfig), + {ok, _} = emqx_resource:recreate_local(?PGSQL_RESOURCE, emqx_postgresql, ResConfig, #{}), on_exit(fun() -> - emqx_resource:recreate_local(?PGSQL_RESOURCE, emqx_postgresql, pgsql_config()) + emqx_resource:recreate_local(?PGSQL_RESOURCE, emqx_postgresql, pgsql_config(), #{}) end), ok = lists:foreach( fun(Sample0) -> diff --git a/apps/emqx_auth_redis/src/emqx_authz_redis.erl b/apps/emqx_auth_redis/src/emqx_authz_redis.erl index e1c675a61..669eb081c 100644 --- a/apps/emqx_auth_redis/src/emqx_authz_redis.erl +++ b/apps/emqx_auth_redis/src/emqx_authz_redis.erl @@ -92,44 +92,30 @@ authorize( do_authorize(_Client, _Action, _Topic, []) -> nomatch; do_authorize(Client, Action, Topic, [TopicFilterRaw, RuleEncoded | Tail]) -> - try - emqx_authz_rule:match( - Client, - Action, - Topic, - compile_rule(RuleEncoded, TopicFilterRaw) - ) - of - {matched, Permission} -> {matched, Permission}; - nomatch -> do_authorize(Client, Action, Topic, Tail) - catch - error:Reason:Stack -> - ?SLOG(error, #{ - msg => "match_rule_error", - reason => Reason, - rule_encoded => RuleEncoded, - topic_filter_raw => TopicFilterRaw, - stacktrace => Stack + case parse_rule(RuleEncoded) of + {ok, RuleMap0} -> + RuleMap = + maps:merge( + #{ + <<"permission">> => <<"allow">>, + <<"topic">> => TopicFilterRaw + }, + RuleMap0 + ), + case emqx_authz_utils:do_authorize(redis, Client, Action, Topic, undefined, RuleMap) of + nomatch -> + do_authorize(Client, Action, Topic, Tail); + {matched, Permission} -> + {matched, Permission} + end; + {error, Reason} -> + ?SLOG(error, Reason#{ + msg => "parse_rule_error", + rule => RuleEncoded }), do_authorize(Client, Action, Topic, Tail) end. -compile_rule(RuleBin, TopicFilterRaw) -> - RuleRaw = - maps:merge( - #{ - <<"permission">> => <<"allow">>, - <<"topic">> => TopicFilterRaw - }, - parse_rule(RuleBin) - ), - case emqx_authz_rule_raw:parse_rule(RuleRaw) of - {ok, {Permission, Action, Topics}} -> - emqx_authz_rule:compile({Permission, all, Action, Topics}); - {error, Reason} -> - error(Reason) - end. - parse_cmd(Query) -> case emqx_redis_command:split(Query) of {ok, Cmd} -> @@ -154,17 +140,17 @@ validate_cmd(Cmd) -> end. parse_rule(<<"publish">>) -> - #{<<"action">> => <<"publish">>}; + {ok, #{<<"action">> => <<"publish">>}}; parse_rule(<<"subscribe">>) -> - #{<<"action">> => <<"subscribe">>}; + {ok, #{<<"action">> => <<"subscribe">>}}; parse_rule(<<"all">>) -> - #{<<"action">> => <<"all">>}; + {ok, #{<<"action">> => <<"all">>}}; parse_rule(Bin) when is_binary(Bin) -> case emqx_utils_json:safe_decode(Bin, [return_maps]) of {ok, Map} when is_map(Map) -> - maps:with([<<"qos">>, <<"action">>, <<"retain">>], Map); + {ok, maps:with([<<"qos">>, <<"action">>, <<"retain">>], Map)}; {ok, _} -> - error({invalid_topic_rule, Bin, notamap}); - {error, Error} -> - error({invalid_topic_rule, Bin, Error}) + {error, #{reason => invalid_topic_rule_not_map, value => Bin}}; + {error, _Error} -> + {error, #{reason => invalid_topic_rule_not_json, value => Bin}} end. diff --git a/apps/emqx_bridge/src/emqx_bridge_resource.erl b/apps/emqx_bridge/src/emqx_bridge_resource.erl index 9215e0787..9ac9a27a8 100644 --- a/apps/emqx_bridge/src/emqx_bridge_resource.erl +++ b/apps/emqx_bridge/src/emqx_bridge_resource.erl @@ -198,7 +198,7 @@ create(Type, Name, Conf0, Opts) -> Conf = Conf0#{bridge_type => TypeBin, bridge_name => Name}, {ok, _Data} = emqx_resource:create_local( resource_id(Type, Name), - <<"emqx_bridge">>, + <<"bridge">>, bridge_to_resource_type(Type), parse_confs(TypeBin, Name, Conf), parse_opts(Conf, Opts) @@ -284,7 +284,7 @@ create_dry_run(Type0, Conf0) -> create_dry_run_bridge_v1(Type, Conf0) -> TmpName = iolist_to_binary([?TEST_ID_PREFIX, emqx_utils:gen_id(8)]), TmpPath = emqx_utils:safe_filename(TmpName), - %% Already typechecked, no need to catch errors + %% Already type checked, no need to catch errors TypeBin = bin(Type), TypeAtom = safe_atom(Type), Conf1 = maps:without([<<"name">>], Conf0), diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl b/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl index aea9f8a86..1c9d861a1 100644 --- a/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl +++ b/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl @@ -123,6 +123,7 @@ common_bridge_fields() -> boolean(), #{ desc => ?DESC("desc_enable"), + importance => ?IMPORTANCE_NO_DOC, default => true } )}, diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl b/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl index 6dbad456b..e5a3465ff 100644 --- a/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl +++ b/apps/emqx_bridge/src/schema/emqx_bridge_v2_schema.erl @@ -65,6 +65,7 @@ -export([ make_producer_action_schema/1, make_producer_action_schema/2, make_consumer_action_schema/1, make_consumer_action_schema/2, + common_fields/0, top_level_common_action_keys/0, top_level_common_source_keys/0, project_to_actions_resource_opts/1, @@ -507,16 +508,26 @@ make_consumer_action_schema(ParametersRef, Opts) -> })} ]. -common_schema(ParametersRef, _Opts) -> +common_fields() -> [ - {enable, mk(boolean(), #{desc => ?DESC("config_enable"), default => true})}, + {enable, + mk(boolean(), #{ + desc => ?DESC("config_enable"), + importance => ?IMPORTANCE_NO_DOC, + default => true + })}, {connector, mk(binary(), #{ desc => ?DESC(emqx_connector_schema, "connector_field"), required => true })}, {tags, emqx_schema:tags_schema()}, - {description, emqx_schema:description_schema()}, + {description, emqx_schema:description_schema()} + ]. + +common_schema(ParametersRef, _Opts) -> + [ {parameters, ParametersRef} + | common_fields() ]. project_to_actions_resource_opts(OldResourceOpts) -> diff --git a/apps/emqx_bridge/test/emqx_bridge_v2_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_v2_SUITE.erl index dc2e8f275..d81d710ff 100644 --- a/apps/emqx_bridge/test/emqx_bridge_v2_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_v2_SUITE.erl @@ -1110,6 +1110,7 @@ t_query_uses_action_query_mode(_Config) -> %% ... now we use a quite different query mode for the action meck:expect(con_mod(), query_mode, 1, simple_async_internal_buffer), + meck:expect(con_mod(), resource_type, 0, dummy), meck:expect(con_mod(), callback_mode, 0, async_if_possible), {ok, _} = emqx_bridge_v2:create(bridge_type(), ActionName, ActionConfig), diff --git a/apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl index 27db7486f..ae1eb58fc 100644 --- a/apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl @@ -302,6 +302,7 @@ init_mocks() -> meck:new(emqx_connector_resource, [passthrough, no_link]), meck:expect(emqx_connector_resource, connector_to_resource_type, 1, ?CONNECTOR_IMPL), meck:new(?CONNECTOR_IMPL, [non_strict, no_link]), + meck:expect(?CONNECTOR_IMPL, resource_type, 0, dummy), meck:expect(?CONNECTOR_IMPL, callback_mode, 0, async_if_possible), meck:expect( ?CONNECTOR_IMPL, diff --git a/apps/emqx_bridge/test/emqx_bridge_v2_dummy_connector.erl b/apps/emqx_bridge/test/emqx_bridge_v2_dummy_connector.erl index 6b4001b6b..1bb7fd37f 100644 --- a/apps/emqx_bridge/test/emqx_bridge_v2_dummy_connector.erl +++ b/apps/emqx_bridge/test/emqx_bridge_v2_dummy_connector.erl @@ -15,15 +15,17 @@ %% this module is only intended to be mocked -module(emqx_bridge_v2_dummy_connector). +-behavior(emqx_resource). -export([ + resource_type/0, callback_mode/0, on_start/2, on_stop/2, on_add_channel/4, on_get_channel_status/3 ]). - +resource_type() -> dummy. callback_mode() -> error(unexpected). on_start(_, _) -> error(unexpected). on_stop(_, _) -> error(unexpected). diff --git a/apps/emqx_bridge/test/emqx_bridge_v2_test_connector.erl b/apps/emqx_bridge/test/emqx_bridge_v2_test_connector.erl index c528d097c..7daedf19a 100644 --- a/apps/emqx_bridge/test/emqx_bridge_v2_test_connector.erl +++ b/apps/emqx_bridge/test/emqx_bridge_v2_test_connector.erl @@ -19,6 +19,7 @@ -export([ query_mode/1, + resource_type/0, callback_mode/0, on_start/2, on_stop/2, @@ -34,6 +35,8 @@ query_mode(_Config) -> sync. +resource_type() -> test_connector. + callback_mode() -> always_sync. diff --git a/apps/emqx_bridge_azure_blob_storage/src/emqx_bridge_azure_blob_storage_connector.erl b/apps/emqx_bridge_azure_blob_storage/src/emqx_bridge_azure_blob_storage_connector.erl index cdf16a8cb..740493f49 100644 --- a/apps/emqx_bridge_azure_blob_storage/src/emqx_bridge_azure_blob_storage_connector.erl +++ b/apps/emqx_bridge_azure_blob_storage/src/emqx_bridge_azure_blob_storage_connector.erl @@ -18,6 +18,7 @@ %% `emqx_resource' API -export([ callback_mode/0, + resource_type/0, on_start/2, on_stop/2, @@ -148,6 +149,10 @@ callback_mode() -> always_sync. +-spec resource_type() -> atom(). +resource_type() -> + azure_blob_storage. + -spec on_start(connector_resource_id(), connector_config()) -> {ok, connector_state()} | {error, _Reason}. on_start(_ConnResId, ConnConfig) -> diff --git a/apps/emqx_bridge_azure_event_hub/mix.exs b/apps/emqx_bridge_azure_event_hub/mix.exs index 42edddbbe..8f5068d0e 100644 --- a/apps/emqx_bridge_azure_event_hub/mix.exs +++ b/apps/emqx_bridge_azure_event_hub/mix.exs @@ -23,7 +23,7 @@ defmodule EMQXBridgeAzureEventHub.MixProject do def deps() do [ - {:wolff, github: "kafka4beam/wolff", tag: "2.0.0"}, + {:wolff, github: "kafka4beam/wolff", tag: "3.0.2"}, {:kafka_protocol, github: "kafka4beam/kafka_protocol", tag: "4.1.5", override: true}, {:brod_gssapi, github: "kafka4beam/brod_gssapi", tag: "v0.1.1"}, {:brod, github: "kafka4beam/brod", tag: "3.18.0"}, diff --git a/apps/emqx_bridge_azure_event_hub/rebar.config b/apps/emqx_bridge_azure_event_hub/rebar.config index 76ea7fa6c..c8be2a6a3 100644 --- a/apps/emqx_bridge_azure_event_hub/rebar.config +++ b/apps/emqx_bridge_azure_event_hub/rebar.config @@ -2,7 +2,7 @@ {erl_opts, [debug_info]}. {deps, [ - {wolff, {git, "https://github.com/kafka4beam/wolff.git", {tag, "2.0.0"}}}, + {wolff, {git, "https://github.com/kafka4beam/wolff.git", {tag, "3.0.2"}}}, {kafka_protocol, {git, "https://github.com/kafka4beam/kafka_protocol.git", {tag, "4.1.5"}}}, {brod_gssapi, {git, "https://github.com/kafka4beam/brod_gssapi.git", {tag, "v0.1.1"}}}, {brod, {git, "https://github.com/kafka4beam/brod.git", {tag, "3.18.0"}}}, diff --git a/apps/emqx_bridge_azure_event_hub/src/emqx_bridge_azure_event_hub.app.src b/apps/emqx_bridge_azure_event_hub/src/emqx_bridge_azure_event_hub.app.src index 7e70fffff..69348b60a 100644 --- a/apps/emqx_bridge_azure_event_hub/src/emqx_bridge_azure_event_hub.app.src +++ b/apps/emqx_bridge_azure_event_hub/src/emqx_bridge_azure_event_hub.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_azure_event_hub, [ {description, "EMQX Enterprise Azure Event Hub Bridge"}, - {vsn, "0.1.7"}, + {vsn, "0.1.8"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_azure_event_hub/src/emqx_bridge_azure_event_hub.erl b/apps/emqx_bridge_azure_event_hub/src/emqx_bridge_azure_event_hub.erl index 213c2331c..197bdd0a7 100644 --- a/apps/emqx_bridge_azure_event_hub/src/emqx_bridge_azure_event_hub.erl +++ b/apps/emqx_bridge_azure_event_hub/src/emqx_bridge_azure_event_hub.erl @@ -129,16 +129,7 @@ fields(actions) -> override( emqx_bridge_kafka:producer_opts(action), bridge_v2_overrides() - ) ++ - [ - {enable, mk(boolean(), #{desc => ?DESC("config_enable"), default => true})}, - {connector, - mk(binary(), #{ - desc => ?DESC(emqx_connector_schema, "connector_field"), required => true - })}, - {tags, emqx_schema:tags_schema()}, - {description, emqx_schema:description_schema()} - ], + ) ++ emqx_bridge_v2_schema:common_fields(), override_documentations(Fields); fields(Method) -> Fields = emqx_bridge_kafka:fields(Method), diff --git a/apps/emqx_bridge_azure_event_hub/test/emqx_bridge_azure_event_hub_v2_SUITE.erl b/apps/emqx_bridge_azure_event_hub/test/emqx_bridge_azure_event_hub_v2_SUITE.erl index 661b8819c..f2a06cf65 100644 --- a/apps/emqx_bridge_azure_event_hub/test/emqx_bridge_azure_event_hub_v2_SUITE.erl +++ b/apps/emqx_bridge_azure_event_hub/test/emqx_bridge_azure_event_hub_v2_SUITE.erl @@ -382,12 +382,31 @@ t_multiple_actions_sharing_topic(Config) -> ActionConfig0, #{<<"parameters">> => #{<<"query_mode">> => <<"sync">>}} ), - ok = emqx_bridge_v2_kafka_producer_SUITE:t_multiple_actions_sharing_topic( - [ - {type, ?BRIDGE_TYPE_BIN}, - {connector_name, ?config(connector_name, Config)}, - {connector_config, ?config(connector_config, Config)}, - {action_config, ActionConfig} - ] - ), + ok = + emqx_bridge_v2_kafka_producer_SUITE:?FUNCTION_NAME( + [ + {type, ?BRIDGE_TYPE_BIN}, + {connector_name, ?config(connector_name, Config)}, + {connector_config, ?config(connector_config, Config)}, + {action_config, ActionConfig} + ] + ), + ok. + +t_dynamic_topics(Config) -> + ActionConfig0 = ?config(action_config, Config), + ActionConfig = + emqx_utils_maps:deep_merge( + ActionConfig0, + #{<<"parameters">> => #{<<"query_mode">> => <<"sync">>}} + ), + ok = + emqx_bridge_v2_kafka_producer_SUITE:?FUNCTION_NAME( + [ + {type, ?BRIDGE_TYPE_BIN}, + {connector_name, ?config(connector_name, Config)}, + {connector_config, ?config(connector_config, Config)}, + {action_config, ActionConfig} + ] + ), ok. diff --git a/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.app.src b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.app.src index 946ca591a..3e7422112 100644 --- a/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.app.src +++ b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_cassandra, [ {description, "EMQX Enterprise Cassandra Bridge"}, - {vsn, "0.3.1"}, + {vsn, "0.3.2"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_connector.erl b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_connector.erl index df278b791..9f830eb69 100644 --- a/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_connector.erl +++ b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_connector.erl @@ -19,6 +19,7 @@ %% callbacks of behaviour emqx_resource -export([ + resource_type/0, callback_mode/0, on_start/2, on_stop/2, @@ -94,6 +95,7 @@ desc("connector") -> %%-------------------------------------------------------------------- %% callbacks for emqx_resource +resource_type() -> cassandra. callback_mode() -> async_if_possible. diff --git a/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse.app.src b/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse.app.src index f38036b83..794d067bd 100644 --- a/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse.app.src +++ b/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_clickhouse, [ {description, "EMQX Enterprise ClickHouse Bridge"}, - {vsn, "0.4.1"}, + {vsn, "0.4.2"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse_connector.erl b/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse_connector.erl index f6888cad5..c5b82122a 100644 --- a/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse_connector.erl +++ b/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse_connector.erl @@ -29,6 +29,7 @@ %% callbacks for behaviour emqx_resource -export([ + resource_type/0, callback_mode/0, on_start/2, on_stop/2, @@ -128,6 +129,7 @@ values(_) -> %% =================================================================== %% Callbacks defined in emqx_resource %% =================================================================== +resource_type() -> clickhouse. callback_mode() -> always_sync. diff --git a/apps/emqx_bridge_confluent/mix.exs b/apps/emqx_bridge_confluent/mix.exs index 46cbe9a02..134e924fc 100644 --- a/apps/emqx_bridge_confluent/mix.exs +++ b/apps/emqx_bridge_confluent/mix.exs @@ -23,7 +23,7 @@ defmodule EMQXBridgeConfluent.MixProject do def deps() do [ - {:wolff, github: "kafka4beam/wolff", tag: "2.0.0"}, + {:wolff, github: "kafka4beam/wolff", tag: "3.0.2"}, {:kafka_protocol, github: "kafka4beam/kafka_protocol", tag: "4.1.5", override: true}, {:brod_gssapi, github: "kafka4beam/brod_gssapi", tag: "v0.1.1"}, {:brod, github: "kafka4beam/brod", tag: "3.18.0"}, diff --git a/apps/emqx_bridge_confluent/rebar.config b/apps/emqx_bridge_confluent/rebar.config index 1a91f501d..786b1cf82 100644 --- a/apps/emqx_bridge_confluent/rebar.config +++ b/apps/emqx_bridge_confluent/rebar.config @@ -2,7 +2,7 @@ {erl_opts, [debug_info]}. {deps, [ - {wolff, {git, "https://github.com/kafka4beam/wolff.git", {tag, "2.0.0"}}}, + {wolff, {git, "https://github.com/kafka4beam/wolff.git", {tag, "3.0.2"}}}, {kafka_protocol, {git, "https://github.com/kafka4beam/kafka_protocol.git", {tag, "4.1.5"}}}, {brod_gssapi, {git, "https://github.com/kafka4beam/brod_gssapi.git", {tag, "v0.1.1"}}}, {brod, {git, "https://github.com/kafka4beam/brod.git", {tag, "3.18.0"}}}, diff --git a/apps/emqx_bridge_confluent/src/emqx_bridge_confluent.app.src b/apps/emqx_bridge_confluent/src/emqx_bridge_confluent.app.src index 46d6617c3..de3074ae6 100644 --- a/apps/emqx_bridge_confluent/src/emqx_bridge_confluent.app.src +++ b/apps/emqx_bridge_confluent/src/emqx_bridge_confluent.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_confluent, [ {description, "EMQX Enterprise Confluent Connector and Action"}, - {vsn, "0.1.2"}, + {vsn, "0.1.3"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_confluent/src/emqx_bridge_confluent_producer.erl b/apps/emqx_bridge_confluent/src/emqx_bridge_confluent_producer.erl index bf01626e3..93d644ef7 100644 --- a/apps/emqx_bridge_confluent/src/emqx_bridge_confluent_producer.erl +++ b/apps/emqx_bridge_confluent/src/emqx_bridge_confluent_producer.erl @@ -116,16 +116,7 @@ fields(actions) -> override( emqx_bridge_kafka:producer_opts(action), bridge_v2_overrides() - ) ++ - [ - {enable, mk(boolean(), #{desc => ?DESC("config_enable"), default => true})}, - {connector, - mk(binary(), #{ - desc => ?DESC(emqx_connector_schema, "connector_field"), required => true - })}, - {tags, emqx_schema:tags_schema()}, - {description, emqx_schema:description_schema()} - ], + ) ++ emqx_bridge_v2_schema:common_fields(), override_documentations(Fields); fields(Method) -> Fields = emqx_bridge_kafka:fields(Method), diff --git a/apps/emqx_bridge_confluent/test/emqx_bridge_confluent_producer_SUITE.erl b/apps/emqx_bridge_confluent/test/emqx_bridge_confluent_producer_SUITE.erl index 0b3a22a99..f10e88463 100644 --- a/apps/emqx_bridge_confluent/test/emqx_bridge_confluent_producer_SUITE.erl +++ b/apps/emqx_bridge_confluent/test/emqx_bridge_confluent_producer_SUITE.erl @@ -391,12 +391,31 @@ t_multiple_actions_sharing_topic(Config) -> ActionConfig0, #{<<"parameters">> => #{<<"query_mode">> => <<"sync">>}} ), - ok = emqx_bridge_v2_kafka_producer_SUITE:t_multiple_actions_sharing_topic( - [ - {type, ?ACTION_TYPE_BIN}, - {connector_name, ?config(connector_name, Config)}, - {connector_config, ?config(connector_config, Config)}, - {action_config, ActionConfig} - ] - ), + ok = + emqx_bridge_v2_kafka_producer_SUITE:?FUNCTION_NAME( + [ + {type, ?ACTION_TYPE_BIN}, + {connector_name, ?config(connector_name, Config)}, + {connector_config, ?config(connector_config, Config)}, + {action_config, ActionConfig} + ] + ), + ok. + +t_dynamic_topics(Config) -> + ActionConfig0 = ?config(action_config, Config), + ActionConfig = + emqx_utils_maps:deep_merge( + ActionConfig0, + #{<<"parameters">> => #{<<"query_mode">> => <<"sync">>}} + ), + ok = + emqx_bridge_v2_kafka_producer_SUITE:?FUNCTION_NAME( + [ + {type, ?ACTION_TYPE_BIN}, + {connector_name, ?config(connector_name, Config)}, + {connector_config, ?config(connector_config, Config)}, + {action_config, ActionConfig} + ] + ), ok. diff --git a/apps/emqx_bridge_couchbase/src/emqx_bridge_couchbase_connector.erl b/apps/emqx_bridge_couchbase/src/emqx_bridge_couchbase_connector.erl index 1e7122800..2c104ee16 100644 --- a/apps/emqx_bridge_couchbase/src/emqx_bridge_couchbase_connector.erl +++ b/apps/emqx_bridge_couchbase/src/emqx_bridge_couchbase_connector.erl @@ -15,6 +15,7 @@ %% `emqx_resource' API -export([ callback_mode/0, + resource_type/0, on_start/2, on_stop/2, @@ -84,6 +85,10 @@ callback_mode() -> always_sync. +-spec resource_type() -> atom(). +resource_type() -> + couchbase. + -spec on_start(connector_resource_id(), connector_config()) -> {ok, connector_state()} | {error, _Reason}. on_start(ConnResId, ConnConfig) -> diff --git a/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo.app.src b/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo.app.src index ac71e04e7..b8fee4dee 100644 --- a/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo.app.src +++ b/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_dynamo, [ {description, "EMQX Enterprise Dynamo Bridge"}, - {vsn, "0.2.2"}, + {vsn, "0.2.3"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector.erl b/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector.erl index 82f5fb18d..181de34a8 100644 --- a/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector.erl +++ b/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector.erl @@ -17,6 +17,7 @@ %% `emqx_resource' API -export([ + resource_type/0, callback_mode/0, on_start/2, on_stop/2, @@ -68,6 +69,7 @@ fields(config) -> %%======================================================================================== %% `emqx_resource' API %%======================================================================================== +resource_type() -> dynamo. callback_mode() -> always_sync. diff --git a/apps/emqx_bridge_es/src/emqx_bridge_es.app.src b/apps/emqx_bridge_es/src/emqx_bridge_es.app.src index 262ac84bd..8f3dc3a7e 100644 --- a/apps/emqx_bridge_es/src/emqx_bridge_es.app.src +++ b/apps/emqx_bridge_es/src/emqx_bridge_es.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_bridge_es, [ {description, "EMQX Enterprise Elastic Search Bridge"}, - {vsn, "0.1.3"}, + {vsn, "0.1.4"}, {modules, [ emqx_bridge_es, emqx_bridge_es_connector diff --git a/apps/emqx_bridge_es/src/emqx_bridge_es_connector.erl b/apps/emqx_bridge_es/src/emqx_bridge_es_connector.erl index 20de92e6e..feccd42f1 100644 --- a/apps/emqx_bridge_es/src/emqx_bridge_es_connector.erl +++ b/apps/emqx_bridge_es/src/emqx_bridge_es_connector.erl @@ -14,6 +14,7 @@ %% `emqx_resource' API -export([ + resource_type/0, callback_mode/0, on_start/2, on_stop/2, @@ -207,6 +208,8 @@ base_url(#{server := Server}) -> "http://" ++ Server. %%------------------------------------------------------------------------------------- %% `emqx_resource' API %%------------------------------------------------------------------------------------- +resource_type() -> elastic_search. + callback_mode() -> async_if_possible. -spec on_start(manager_id(), config()) -> {ok, state()} | no_return(). diff --git a/apps/emqx_bridge_gcp_pubsub/mix.exs b/apps/emqx_bridge_gcp_pubsub/mix.exs index 3a9fae0a1..34e02ca9f 100644 --- a/apps/emqx_bridge_gcp_pubsub/mix.exs +++ b/apps/emqx_bridge_gcp_pubsub/mix.exs @@ -23,6 +23,7 @@ defmodule EMQXBridgeGcpPubsub.MixProject do def deps() do [ + {:emqx_connector_jwt, in_umbrella: true}, {:emqx_connector, in_umbrella: true, runtime: false}, {:emqx_resource, in_umbrella: true}, {:emqx_bridge, in_umbrella: true, runtime: false}, diff --git a/apps/emqx_bridge_gcp_pubsub/rebar.config b/apps/emqx_bridge_gcp_pubsub/rebar.config index a6a12b429..e5a65b745 100644 --- a/apps/emqx_bridge_gcp_pubsub/rebar.config +++ b/apps/emqx_bridge_gcp_pubsub/rebar.config @@ -9,6 +9,7 @@ debug_info ]}. {deps, [ + {emqx_connector_jwt, {path, "../../apps/emqx_connector_jwt"}}, {emqx_connector, {path, "../../apps/emqx_connector"}}, {emqx_resource, {path, "../../apps/emqx_resource"}}, {emqx_bridge, {path, "../../apps/emqx_bridge"}}, diff --git a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.app.src b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.app.src index eff7847f2..a39c4be99 100644 --- a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.app.src +++ b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.app.src @@ -6,7 +6,8 @@ kernel, stdlib, emqx_resource, - ehttpc + ehttpc, + emqx_connector_jwt ]}, {env, [ {emqx_action_info_modules, [ diff --git a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_client.erl b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_client.erl index 67218fcf0..ea6f67112 100644 --- a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_client.erl +++ b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_client.erl @@ -5,7 +5,7 @@ -module(emqx_bridge_gcp_pubsub_client). -include_lib("jose/include/jose_jwk.hrl"). --include_lib("emqx_connector/include/emqx_connector_tables.hrl"). +-include_lib("emqx_connector_jwt/include/emqx_connector_jwt_tables.hrl"). -include_lib("emqx_resource/include/emqx_resource.hrl"). -include_lib("typerefl/include/types.hrl"). -include_lib("emqx/include/logger.hrl"). diff --git a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_impl_consumer.erl b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_impl_consumer.erl index 5c51cd2d9..344fc05c6 100644 --- a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_impl_consumer.erl +++ b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_impl_consumer.erl @@ -8,6 +8,7 @@ %% `emqx_resource' API -export([ + resource_type/0, callback_mode/0, query_mode/1, on_start/2, @@ -84,6 +85,8 @@ %%------------------------------------------------------------------------------------------------- %% `emqx_resource' API %%------------------------------------------------------------------------------------------------- +-spec resource_type() -> resource_type(). +resource_type() -> gcp_pubsub_consumer. -spec callback_mode() -> callback_mode(). callback_mode() -> async_if_possible. diff --git a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_impl_producer.erl b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_impl_producer.erl index 0c668cb95..a6a65ab97 100644 --- a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_impl_producer.erl +++ b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_impl_producer.erl @@ -41,6 +41,7 @@ %% `emqx_resource' API -export([ + resource_type/0, callback_mode/0, query_mode/1, on_start/2, @@ -62,6 +63,7 @@ %%------------------------------------------------------------------------------------------------- %% `emqx_resource' API %%------------------------------------------------------------------------------------------------- +resource_type() -> gcp_pubsub. callback_mode() -> async_if_possible. diff --git a/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_consumer_SUITE.erl b/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_consumer_SUITE.erl index 0e6956d58..656413225 100644 --- a/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_consumer_SUITE.erl +++ b/apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_consumer_SUITE.erl @@ -594,7 +594,7 @@ cluster(Config) -> Cluster = emqx_common_test_helpers:emqx_cluster( [core, core], [ - {apps, [emqx_conf, emqx_rule_engine, emqx_bridge]}, + {apps, [emqx_conf, emqx_rule_engine, emqx_bridge_gcp_pubsub, emqx_bridge]}, {listener_ports, []}, {priv_data_dir, PrivDataDir}, {load_schema, true}, diff --git a/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector.erl b/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector.erl index e4cc0aa31..a2ffd6219 100644 --- a/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector.erl +++ b/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector.erl @@ -16,6 +16,7 @@ %% callbacks of behaviour emqx_resource -export([ + resource_type/0, callback_mode/0, on_start/2, on_stop/2, @@ -67,6 +68,8 @@ %% ------------------------------------------------------------------------------------------------- %% resource callback +resource_type() -> greptimedb. + callback_mode() -> async_if_possible. on_add_channel( diff --git a/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb.app.src b/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb.app.src index af232accc..7ae86bba0 100644 --- a/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb.app.src +++ b/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_hstreamdb, [ {description, "EMQX Enterprise HStreamDB Bridge"}, - {vsn, "0.2.1"}, + {vsn, "0.2.2"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb_connector.erl b/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb_connector.erl index cf53291b2..2d061e455 100644 --- a/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb_connector.erl +++ b/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb_connector.erl @@ -16,6 +16,7 @@ %% callbacks of behaviour emqx_resource -export([ + resource_type/0, callback_mode/0, on_start/2, on_stop/2, @@ -44,6 +45,8 @@ %% ------------------------------------------------------------------------------------------------- %% resource callback +resource_type() -> hstreamdb. + callback_mode() -> always_sync. on_start(InstId, Config) -> diff --git a/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl b/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl index 2b125c1db..0d848a064 100644 --- a/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl +++ b/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl @@ -26,6 +26,7 @@ %% callbacks of behaviour emqx_resource -export([ + resource_type/0, callback_mode/0, on_start/2, on_stop/2, @@ -183,6 +184,7 @@ sc(Type, Meta) -> hoconsc:mk(Type, Meta). ref(Field) -> hoconsc:ref(?MODULE, Field). %% =================================================================== +resource_type() -> webhook. callback_mode() -> async_if_possible. diff --git a/apps/emqx_bridge_http/src/emqx_bridge_http_schema.erl b/apps/emqx_bridge_http/src/emqx_bridge_http_schema.erl index cadbcf0d2..7868d6694 100644 --- a/apps/emqx_bridge_http/src/emqx_bridge_http_schema.erl +++ b/apps/emqx_bridge_http/src/emqx_bridge_http_schema.erl @@ -72,35 +72,29 @@ fields(action) -> } )}; fields("http_action") -> - [ - {enable, mk(boolean(), #{desc => ?DESC("config_enable_bridge"), default => true})}, - {connector, - mk(binary(), #{ - desc => ?DESC(emqx_connector_schema, "connector_field"), required => true - })}, - {tags, emqx_schema:tags_schema()}, - {description, emqx_schema:description_schema()}, - %% Note: there's an implicit convention in `emqx_bridge' that, - %% for egress bridges with this config, the published messages - %% will be forwarded to such bridges. - {local_topic, - mk( - binary(), - #{ - required => false, - desc => ?DESC("config_local_topic"), - importance => ?IMPORTANCE_HIDDEN - } - )}, - %% Since e5.3.2, we split the http bridge to two parts: a) connector. b) actions. - %% some fields are moved to connector, some fields are moved to actions and composed into the - %% `parameters` field. - {parameters, - mk(ref("parameters_opts"), #{ - required => true, - desc => ?DESC("config_parameters_opts") - })} - ] ++ + emqx_bridge_v2_schema:common_fields() ++ + [ + %% Note: there's an implicit convention in `emqx_bridge' that, + %% for egress bridges with this config, the published messages + %% will be forwarded to such bridges. + {local_topic, + mk( + binary(), + #{ + required => false, + desc => ?DESC("config_local_topic"), + importance => ?IMPORTANCE_HIDDEN + } + )}, + %% Since e5.3.2, we split the http bridge to two parts: a) connector. b) actions. + %% some fields are moved to connector, some fields are moved to actions and composed into the + %% `parameters` field. + {parameters, + mk(ref("parameters_opts"), #{ + required => true, + desc => ?DESC("config_parameters_opts") + })} + ] ++ emqx_connector_schema:resource_opts_ref( ?MODULE, action_resource_opts, fun legacy_action_resource_opts_converter/2 ); diff --git a/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.app.src b/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.app.src index a8314541a..eae5028c6 100644 --- a/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.app.src +++ b/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_influxdb, [ {description, "EMQX Enterprise InfluxDB Bridge"}, - {vsn, "0.2.3"}, + {vsn, "0.2.4"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb_connector.erl b/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb_connector.erl index 852f78485..bf93309f8 100644 --- a/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb_connector.erl +++ b/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb_connector.erl @@ -16,6 +16,7 @@ %% callbacks of behaviour emqx_resource -export([ + resource_type/0, callback_mode/0, on_start/2, on_stop/2, @@ -70,6 +71,8 @@ %% ------------------------------------------------------------------------------------------------- %% resource callback +resource_type() -> influxdb. + callback_mode() -> async_if_possible. on_add_channel( diff --git a/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.app.src b/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.app.src index 691778cfd..88ac09da9 100644 --- a/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.app.src +++ b/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_bridge_iotdb, [ {description, "EMQX Enterprise Apache IoTDB Bridge"}, - {vsn, "0.2.2"}, + {vsn, "0.2.3"}, {modules, [ emqx_bridge_iotdb, emqx_bridge_iotdb_connector diff --git a/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb_connector.erl b/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb_connector.erl index 78866ef79..ec880e785 100644 --- a/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb_connector.erl +++ b/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb_connector.erl @@ -15,6 +15,7 @@ %% `emqx_resource' API -export([ + resource_type/0, callback_mode/0, on_start/2, on_stop/2, @@ -206,6 +207,8 @@ proplists_without(Keys, List) -> %%------------------------------------------------------------------------------------- %% `emqx_resource' API %%------------------------------------------------------------------------------------- +resource_type() -> iotdb. + callback_mode() -> async_if_possible. -spec on_start(manager_id(), config()) -> {ok, state()} | no_return(). diff --git a/apps/emqx_bridge_kafka/mix.exs b/apps/emqx_bridge_kafka/mix.exs index b74b1fdd0..a1a59cb08 100644 --- a/apps/emqx_bridge_kafka/mix.exs +++ b/apps/emqx_bridge_kafka/mix.exs @@ -23,7 +23,7 @@ defmodule EMQXBridgeKafka.MixProject do def deps() do [ - {:wolff, github: "kafka4beam/wolff", tag: "2.0.0"}, + {:wolff, github: "kafka4beam/wolff", tag: "3.0.2"}, {:kafka_protocol, github: "kafka4beam/kafka_protocol", tag: "4.1.5", override: true}, {:brod_gssapi, github: "kafka4beam/brod_gssapi", tag: "v0.1.1"}, {:brod, github: "kafka4beam/brod", tag: "3.18.0"}, diff --git a/apps/emqx_bridge_kafka/rebar.config b/apps/emqx_bridge_kafka/rebar.config index b89c9190f..77d9b95ef 100644 --- a/apps/emqx_bridge_kafka/rebar.config +++ b/apps/emqx_bridge_kafka/rebar.config @@ -2,7 +2,7 @@ {erl_opts, [debug_info]}. {deps, [ - {wolff, {git, "https://github.com/kafka4beam/wolff.git", {tag, "2.0.0"}}}, + {wolff, {git, "https://github.com/kafka4beam/wolff.git", {tag, "3.0.2"}}}, {kafka_protocol, {git, "https://github.com/kafka4beam/kafka_protocol.git", {tag, "4.1.5"}}}, {brod_gssapi, {git, "https://github.com/kafka4beam/brod_gssapi.git", {tag, "v0.1.1"}}}, {brod, {git, "https://github.com/kafka4beam/brod.git", {tag, "3.18.0"}}}, diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl index 9d15a26ee..377f07c59 100644 --- a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl +++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl @@ -295,17 +295,10 @@ fields("config_producer") -> fields("config_consumer") -> fields(kafka_consumer); fields(kafka_producer) -> + %% Schema used by bridges V1. connector_config_fields() ++ producer_opts(v1); fields(kafka_producer_action) -> - [ - {enable, mk(boolean(), #{desc => ?DESC("config_enable"), default => true})}, - {connector, - mk(binary(), #{ - desc => ?DESC(emqx_connector_schema, "connector_field"), required => true - })}, - {tags, emqx_schema:tags_schema()}, - {description, emqx_schema:description_schema()} - ] ++ producer_opts(action); + emqx_bridge_v2_schema:common_fields() ++ producer_opts(action); fields(kafka_consumer) -> connector_config_fields() ++ fields(consumer_opts); fields(ssl_client_opts) -> @@ -364,9 +357,33 @@ fields(socket_opts) -> validator => fun emqx_schema:validate_tcp_keepalive/1 })} ]; +fields(v1_producer_kafka_opts) -> + OldSchemaFields = + [ + topic, + message, + max_batch_bytes, + compression, + partition_strategy, + required_acks, + kafka_headers, + kafka_ext_headers, + kafka_header_value_encode_mode, + partition_count_refresh_interval, + partitions_limit, + max_inflight, + buffer, + query_mode, + sync_query_timeout + ], + Fields = fields(producer_kafka_opts), + lists:filter( + fun({K, _V}) -> lists:member(K, OldSchemaFields) end, + Fields + ); fields(producer_kafka_opts) -> [ - {topic, mk(string(), #{required => true, desc => ?DESC(kafka_topic)})}, + {topic, mk(emqx_schema:template(), #{required => true, desc => ?DESC(kafka_topic)})}, {message, mk(ref(kafka_message), #{required => false, desc => ?DESC(kafka_message)})}, {max_batch_bytes, mk(emqx_schema:bytesize(), #{default => <<"896KB">>, desc => ?DESC(max_batch_bytes)})}, @@ -680,15 +697,15 @@ resource_opts() -> %% However we need to keep it backward compatible for generated schema json (version 0.1.0) %% since schema is data for the 'schemas' API. parameters_field(ActionOrBridgeV1) -> - {Name, Alias} = + {Name, Alias, Ref} = case ActionOrBridgeV1 of v1 -> - {kafka, parameters}; + {kafka, parameters, v1_producer_kafka_opts}; action -> - {parameters, kafka} + {parameters, kafka, producer_kafka_opts} end, {Name, - mk(ref(producer_kafka_opts), #{ + mk(ref(Ref), #{ required => true, aliases => [Alias], desc => ?DESC(producer_kafka_opts), diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_consumer.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_consumer.erl index 2f0aff68d..8c7c5ffa1 100644 --- a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_consumer.erl +++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_consumer.erl @@ -7,6 +7,7 @@ %% `emqx_resource' API -export([ + resource_type/0, callback_mode/0, query_mode/1, on_start/2, @@ -126,6 +127,7 @@ %%------------------------------------------------------------------------------------- %% `emqx_resource' API %%------------------------------------------------------------------------------------- +resource_type() -> kafka_consumer. callback_mode() -> async_if_possible. @@ -631,16 +633,6 @@ consumer_group_id(_ConsumerParams, BridgeName0) -> BridgeName = to_bin(BridgeName0), <<"emqx-kafka-consumer-", BridgeName/binary>>. --spec is_dry_run(connector_resource_id()) -> boolean(). -is_dry_run(ConnectorResId) -> - TestIdStart = string:find(ConnectorResId, ?TEST_ID_PREFIX), - case TestIdStart of - nomatch -> - false; - _ -> - string:equal(TestIdStart, ConnectorResId) - end. - -spec check_client_connectivity(pid()) -> ?status_connected | ?status_disconnected @@ -676,7 +668,7 @@ maybe_clean_error(Reason) -> -spec make_client_id(connector_resource_id(), binary(), atom() | binary()) -> atom(). make_client_id(ConnectorResId, BridgeType, BridgeName) -> - case is_dry_run(ConnectorResId) of + case emqx_resource:is_dry_run(ConnectorResId) of false -> ClientID0 = emqx_bridge_kafka_impl:make_client_id(BridgeType, BridgeName), binary_to_atom(ClientID0); diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl index 8395251e8..1b18a1767 100644 --- a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl +++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl @@ -3,6 +3,8 @@ %%-------------------------------------------------------------------- -module(emqx_bridge_kafka_impl_producer). +-feature(maybe_expr, enable). + -behaviour(emqx_resource). -include_lib("emqx_resource/include/emqx_resource.hrl"). @@ -10,6 +12,7 @@ %% callbacks of behaviour emqx_resource -export([ + resource_type/0, query_mode/1, callback_mode/0, on_start/2, @@ -35,6 +38,8 @@ -define(kafka_client_id, kafka_client_id). -define(kafka_producers, kafka_producers). +resource_type() -> kafka_producer. + query_mode(#{parameters := #{query_mode := sync}}) -> simple_sync_internal_buffer; query_mode(_) -> @@ -122,8 +127,8 @@ on_add_channel( {ok, NewState}. create_producers_for_bridge_v2( - InstId, - BridgeV2Id, + ConnResId, + ActionResId, ClientId, #{ bridge_type := BridgeType, @@ -132,40 +137,42 @@ create_producers_for_bridge_v2( ) -> #{ message := MessageTemplate, - topic := KafkaTopic, + topic := KafkaTopic0, sync_query_timeout := SyncQueryTimeout } = KafkaConfig, + TopicTemplate = {TopicType, TopicOrTemplate} = maybe_preproc_topic(KafkaTopic0), + MKafkaTopic = + case TopicType of + fixed -> TopicOrTemplate; + dynamic -> dynamic + end, KafkaHeadersTokens = preproc_kafka_headers(maps:get(kafka_headers, KafkaConfig, undefined)), KafkaExtHeadersTokens = preproc_ext_headers(maps:get(kafka_ext_headers, KafkaConfig, [])), KafkaHeadersValEncodeMode = maps:get(kafka_header_value_encode_mode, KafkaConfig, none), MaxPartitions = maps:get(partitions_limit, KafkaConfig, all_partitions), - #{name := BridgeName} = emqx_bridge_v2:parse_id(BridgeV2Id), - TestIdStart = string:find(BridgeV2Id, ?TEST_ID_PREFIX), - IsDryRun = - case TestIdStart of - nomatch -> - false; - _ -> - string:equal(TestIdStart, InstId) - end, - ok = check_topic_and_leader_connections(ClientId, KafkaTopic, MaxPartitions), + #{name := BridgeName} = emqx_bridge_v2:parse_id(ActionResId), + IsDryRun = emqx_resource:is_dry_run(ActionResId), + ok = check_topic_and_leader_connections(ActionResId, ClientId, MKafkaTopic, MaxPartitions), WolffProducerConfig = producers_config( - BridgeType, BridgeName, KafkaConfig, IsDryRun, BridgeV2Id + BridgeType, BridgeName, KafkaConfig, IsDryRun, ActionResId ), - case wolff:ensure_supervised_producers(ClientId, KafkaTopic, WolffProducerConfig) of + case wolff:ensure_supervised_dynamic_producers(ClientId, WolffProducerConfig) of {ok, Producers} -> - ok = emqx_resource:allocate_resource(InstId, {?kafka_producers, BridgeV2Id}, Producers), ok = emqx_resource:allocate_resource( - InstId, {?kafka_telemetry_id, BridgeV2Id}, BridgeV2Id + ConnResId, {?kafka_producers, ActionResId}, Producers ), - _ = maybe_install_wolff_telemetry_handlers(BridgeV2Id), + ok = emqx_resource:allocate_resource( + ConnResId, {?kafka_telemetry_id, ActionResId}, ActionResId + ), + _ = maybe_install_wolff_telemetry_handlers(ActionResId), {ok, #{ message_template => compile_message_template(MessageTemplate), kafka_client_id => ClientId, - kafka_topic => KafkaTopic, + topic_template => TopicTemplate, + topic => MKafkaTopic, producers => Producers, - resource_id => BridgeV2Id, - connector_resource_id => InstId, + resource_id => ActionResId, + connector_resource_id => ConnResId, sync_query_timeout => SyncQueryTimeout, kafka_config => KafkaConfig, headers_tokens => KafkaHeadersTokens, @@ -176,9 +183,9 @@ create_producers_for_bridge_v2( {error, Reason2} -> ?SLOG(error, #{ msg => "failed_to_start_kafka_producer", - instance_id => InstId, + instance_id => ConnResId, kafka_client_id => ClientId, - kafka_topic => KafkaTopic, + kafka_topic => MKafkaTopic, reason => Reason2 }), throw( @@ -271,7 +278,9 @@ remove_producers_for_bridge_v2( ClientId = maps:get(?kafka_client_id, AllocatedResources, no_client_id), maps:foreach( fun - ({?kafka_producers, BridgeV2IdCheck}, Producers) when BridgeV2IdCheck =:= BridgeV2Id -> + ({?kafka_producers, BridgeV2IdCheck}, Producers) when + BridgeV2IdCheck =:= BridgeV2Id + -> deallocate_producers(ClientId, Producers); ({?kafka_telemetry_id, BridgeV2IdCheck}, TelemetryId) when BridgeV2IdCheck =:= BridgeV2Id @@ -304,7 +313,8 @@ on_query( #{installed_bridge_v2s := BridgeV2Configs} = _ConnectorState ) -> #{ - message_template := Template, + message_template := MessageTemplate, + topic_template := TopicTemplate, producers := Producers, sync_query_timeout := SyncTimeout, headers_tokens := KafkaHeadersTokens, @@ -317,7 +327,8 @@ on_query( headers_val_encode_mode => KafkaHeadersValEncodeMode }, try - KafkaMessage = render_message(Template, KafkaHeaders, Message), + KafkaTopic = render_topic(TopicTemplate, Message), + KafkaMessage = render_message(MessageTemplate, KafkaHeaders, Message), ?tp( emqx_bridge_kafka_impl_producer_sync_query, #{headers_config => KafkaHeaders, instance_id => InstId} @@ -325,9 +336,15 @@ on_query( emqx_trace:rendered_action_template(MessageTag, #{ message => KafkaMessage }), - do_send_msg(sync, KafkaMessage, Producers, SyncTimeout) + do_send_msg(sync, KafkaTopic, KafkaMessage, Producers, SyncTimeout) catch - error:{invalid_partition_count, Count, _Partitioner} -> + throw:bad_topic -> + ?tp("kafka_producer_failed_to_render_topic", #{}), + {error, {unrecoverable_error, failed_to_render_topic}}; + throw:#{cause := unknown_topic_or_partition, topic := Topic} -> + ?tp("kafka_producer_resolved_to_unknown_topic", #{}), + {error, {unrecoverable_error, {resolved_to_unknown_topic, Topic}}}; + throw:#{cause := invalid_partition_count, count := Count} -> ?tp("kafka_producer_invalid_partition_count", #{ action_id => MessageTag, query_mode => sync @@ -372,6 +389,7 @@ on_query_async( ) -> #{ message_template := Template, + topic_template := TopicTemplate, producers := Producers, headers_tokens := KafkaHeadersTokens, ext_headers_tokens := KafkaExtHeadersTokens, @@ -383,6 +401,7 @@ on_query_async( headers_val_encode_mode => KafkaHeadersValEncodeMode }, try + KafkaTopic = render_topic(TopicTemplate, Message), KafkaMessage = render_message(Template, KafkaHeaders, Message), ?tp( emqx_bridge_kafka_impl_producer_async_query, @@ -391,9 +410,15 @@ on_query_async( emqx_trace:rendered_action_template(MessageTag, #{ message => KafkaMessage }), - do_send_msg(async, KafkaMessage, Producers, AsyncReplyFn) + do_send_msg(async, KafkaTopic, KafkaMessage, Producers, AsyncReplyFn) catch - error:{invalid_partition_count, Count, _Partitioner} -> + throw:bad_topic -> + ?tp("kafka_producer_failed_to_render_topic", #{}), + {error, {unrecoverable_error, failed_to_render_topic}}; + throw:#{cause := unknown_topic_or_partition, topic := Topic} -> + ?tp("kafka_producer_resolved_to_unknown_topic", #{}), + {error, {unrecoverable_error, {resolved_to_unknown_topic, Topic}}}; + throw:#{cause := invalid_partition_count, count := Count} -> ?tp("kafka_producer_invalid_partition_count", #{ action_id => MessageTag, query_mode => async @@ -431,9 +456,28 @@ compile_message_template(T) -> timestamp => preproc_tmpl(TimestampTemplate) }. +maybe_preproc_topic(Topic) -> + Template = emqx_template:parse(Topic), + case emqx_template:placeholders(Template) of + [] -> + {fixed, bin(Topic)}; + [_ | _] -> + {dynamic, Template} + end. + preproc_tmpl(Tmpl) -> emqx_placeholder:preproc_tmpl(Tmpl). +render_topic({fixed, KafkaTopic}, _Message) -> + KafkaTopic; +render_topic({dynamic, Template}, Message) -> + try + iolist_to_binary(emqx_template:render_strict(Template, {emqx_jsonish, Message})) + catch + error:_Errors -> + throw(bad_topic) + end. + render_message( #{key := KeyTemplate, value := ValueTemplate, timestamp := TimestampTemplate}, #{ @@ -475,9 +519,11 @@ render_timestamp(Template, Message) -> erlang:system_time(millisecond) end. -do_send_msg(sync, KafkaMessage, Producers, SyncTimeout) -> +do_send_msg(sync, KafkaTopic, KafkaMessage, Producers, SyncTimeout) -> try - {_Partition, _Offset} = wolff:send_sync(Producers, [KafkaMessage], SyncTimeout), + {_Partition, _Offset} = wolff:send_sync2( + Producers, KafkaTopic, [KafkaMessage], SyncTimeout + ), ok catch error:{producer_down, _} = Reason -> @@ -485,7 +531,7 @@ do_send_msg(sync, KafkaMessage, Producers, SyncTimeout) -> error:timeout -> {error, timeout} end; -do_send_msg(async, KafkaMessage, Producers, AsyncReplyFn) -> +do_send_msg(async, KafkaTopic, KafkaMessage, Producers, AsyncReplyFn) -> %% * Must be a batch because wolff:send and wolff:send_sync are batch APIs %% * Must be a single element batch because wolff books calls, but not batch sizes %% for counters and gauges. @@ -493,7 +539,9 @@ do_send_msg(async, KafkaMessage, Producers, AsyncReplyFn) -> %% The retuned information is discarded here. %% If the producer process is down when sending, this function would %% raise an error exception which is to be caught by the caller of this callback - {_Partition, Pid} = wolff:send(Producers, Batch, {fun ?MODULE:on_kafka_ack/3, [AsyncReplyFn]}), + {_Partition, Pid} = wolff:send2( + Producers, KafkaTopic, Batch, {fun ?MODULE:on_kafka_ack/3, [AsyncReplyFn]} + ), %% this Pid is so far never used because Kafka producer is by-passing the buffer worker {ok, Pid}. @@ -516,7 +564,7 @@ on_kafka_ack(_Partition, message_too_large, {ReplyFn, Args}) -> %% `emqx_resource_buffer_worker', we must avoid returning `disconnected' here. Otherwise, %% `emqx_resource_manager' will kill the wolff producers and messages might be lost. on_get_status( - _InstId, + ConnResId, #{client_id := ClientId} = State ) -> %% Note: we must avoid returning `?status_disconnected' here if the connector ever was @@ -526,7 +574,7 @@ on_get_status( %% held in wolff producer's replayq. case check_client_connectivity(ClientId) of ok -> - maybe_check_health_check_topic(State); + maybe_check_health_check_topic(ConnResId, State); {error, {find_client, _Error}} -> ?status_connecting; {error, {connectivity, Error}} -> @@ -534,20 +582,23 @@ on_get_status( end. on_get_channel_status( - _ResId, - ChannelId, + _ConnResId, + ActionResId, #{ client_id := ClientId, installed_bridge_v2s := Channels - } = _State + } = _ConnState ) -> %% Note: we must avoid returning `?status_disconnected' here. Returning %% `?status_disconnected' will make resource manager try to restart the producers / %% connector, thus potentially dropping data held in wolff producer's replayq. The %% only exception is if the topic does not exist ("unhealthy target"). - #{kafka_topic := KafkaTopic, partitions_limit := MaxPartitions} = maps:get(ChannelId, Channels), + #{ + topic := MKafkaTopic, + partitions_limit := MaxPartitions + } = maps:get(ActionResId, Channels), try - ok = check_topic_and_leader_connections(ClientId, KafkaTopic, MaxPartitions), + ok = check_topic_and_leader_connections(ActionResId, ClientId, MKafkaTopic, MaxPartitions), ?status_connected catch throw:{unhealthy_target, Msg} -> @@ -556,22 +607,29 @@ on_get_channel_status( {?status_connecting, {K, E}} end. -check_topic_and_leader_connections(ClientId, KafkaTopic, MaxPartitions) -> +check_topic_and_leader_connections(ActionResId, ClientId, MKafkaTopic, MaxPartitions) -> case wolff_client_sup:find_client(ClientId) of {ok, Pid} -> - ok = check_topic_status(ClientId, Pid, KafkaTopic), - ok = check_if_healthy_leaders(ClientId, Pid, KafkaTopic, MaxPartitions); + maybe + true ?= is_binary(MKafkaTopic), + ok = check_topic_status(ClientId, Pid, MKafkaTopic), + ok = check_if_healthy_leaders( + ActionResId, ClientId, Pid, MKafkaTopic, MaxPartitions + ) + else + false -> ok + end; {error, #{reason := no_such_client}} -> throw(#{ reason => cannot_find_kafka_client, kafka_client => ClientId, - kafka_topic => KafkaTopic + kafka_topic => MKafkaTopic }); {error, #{reason := client_supervisor_not_initialized}} -> throw(#{ reason => restarting, kafka_client => ClientId, - kafka_topic => KafkaTopic + kafka_topic => MKafkaTopic }) end. @@ -590,21 +648,23 @@ check_client_connectivity(ClientId) -> {error, {find_client, Reason}} end. -maybe_check_health_check_topic(#{health_check_topic := Topic} = ConnectorState) when +maybe_check_health_check_topic(ConnResId, #{health_check_topic := Topic} = ConnectorState) when is_binary(Topic) -> #{client_id := ClientId} = ConnectorState, MaxPartitions = all_partitions, - try check_topic_and_leader_connections(ClientId, Topic, MaxPartitions) of + try check_topic_and_leader_connections(ConnResId, ClientId, Topic, MaxPartitions) of ok -> ?status_connected catch + throw:{unhealthy_target, Msg} -> + {?status_disconnected, ConnectorState, Msg}; throw:#{reason := {connection_down, _} = Reason} -> {?status_disconnected, ConnectorState, Reason}; throw:#{reason := Reason} -> {?status_connecting, ConnectorState, Reason} end; -maybe_check_health_check_topic(_) -> +maybe_check_health_check_topic(_ConnResId, _ConnState) -> %% Cannot infer further information. Maybe upgraded from older version. ?status_connected. @@ -616,8 +676,10 @@ error_summary(Map, [Error]) -> error_summary(Map, [Error | More]) -> Map#{first_error => Error, total_errors => length(More) + 1}. -check_if_healthy_leaders(ClientId, ClientPid, KafkaTopic, MaxPartitions) when is_pid(ClientPid) -> - case wolff_client:get_leader_connections(ClientPid, KafkaTopic, MaxPartitions) of +check_if_healthy_leaders(ActionResId, ClientId, ClientPid, KafkaTopic, MaxPartitions) when + is_pid(ClientPid) +-> + case wolff_client:get_leader_connections(ClientPid, ActionResId, KafkaTopic, MaxPartitions) of {ok, Leaders} -> %% Kafka is considered healthy as long as any of the partition leader is reachable. case lists:partition(fun({_Partition, Pid}) -> is_alive(Pid) end, Leaders) of @@ -679,7 +741,7 @@ ssl(#{enable := true} = SSL) -> ssl(_) -> false. -producers_config(BridgeType, BridgeName, Input, IsDryRun, BridgeV2Id) -> +producers_config(BridgeType, BridgeName, Input, IsDryRun, ActionResId) -> #{ max_batch_bytes := MaxBatchBytes, compression := Compression, @@ -721,8 +783,8 @@ producers_config(BridgeType, BridgeName, Input, IsDryRun, BridgeV2Id) -> max_batch_bytes => MaxBatchBytes, max_send_ahead => MaxInflight - 1, compression => Compression, - alias => BridgeV2Id, - telemetry_meta_data => #{bridge_id => BridgeV2Id}, + group => ActionResId, + telemetry_meta_data => #{bridge_id => ActionResId}, max_partitions => MaxPartitions }. @@ -798,20 +860,19 @@ handle_telemetry_event(_EventId, _Metrics, _MetaData, _HandlerConfig) -> %% Note: don't use the instance/manager ID, as that changes everytime %% the bridge is recreated, and will lead to multiplication of %% metrics. --spec telemetry_handler_id(resource_id()) -> binary(). -telemetry_handler_id(ResourceID) -> - <<"emqx-bridge-kafka-producer-", ResourceID/binary>>. +-spec telemetry_handler_id(action_resource_id()) -> binary(). +telemetry_handler_id(ActionResId) -> + ActionResId. -uninstall_telemetry_handlers(ResourceID) -> - HandlerID = telemetry_handler_id(ResourceID), - telemetry:detach(HandlerID). +uninstall_telemetry_handlers(TelemetryId) -> + telemetry:detach(TelemetryId). -maybe_install_wolff_telemetry_handlers(ResourceID) -> +maybe_install_wolff_telemetry_handlers(TelemetryId) -> %% Attach event handlers for Kafka telemetry events. If a handler with the %% handler id already exists, the attach_many function does nothing telemetry:attach_many( %% unique handler id - telemetry_handler_id(ResourceID), + telemetry_handler_id(TelemetryId), [ [wolff, dropped_queue_full], [wolff, queuing], @@ -823,7 +884,7 @@ maybe_install_wolff_telemetry_handlers(ResourceID) -> %% wolff producers; otherwise, multiple kafka producer bridges %% will install multiple handlers to the same wolff events, %% multiplying the metric counts... - #{bridge_id => ResourceID} + #{bridge_id => TelemetryId} ). preproc_kafka_headers(HeadersTmpl) when HeadersTmpl =:= <<>>; HeadersTmpl =:= undefined -> diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_producer_action_info.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_producer_action_info.erl index d97e68ba6..b9e13e717 100644 --- a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_producer_action_info.erl +++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_producer_action_info.erl @@ -26,7 +26,12 @@ schema_module() -> emqx_bridge_kafka. connector_action_config_to_bridge_v1_config(ConnectorConfig, ActionConfig) -> BridgeV1Config1 = maps:remove(<<"connector">>, ActionConfig), BridgeV1Config2 = emqx_utils_maps:deep_merge(ConnectorConfig, BridgeV1Config1), - emqx_utils_maps:rename(<<"parameters">>, <<"kafka">>, BridgeV1Config2). + BridgeV1Config = emqx_utils_maps:rename(<<"parameters">>, <<"kafka">>, BridgeV1Config2), + maps:update_with( + <<"kafka">>, + fun(Params) -> maps:with(v1_parameters(), Params) end, + BridgeV1Config + ). bridge_v1_config_to_action_config(BridgeV1Conf0 = #{<<"producer">> := _}, ConnectorName) -> %% Ancient v1 config, when `kafka' key was wrapped by `producer' @@ -51,6 +56,12 @@ bridge_v1_config_to_action_config(BridgeV1Conf, ConnectorName) -> %% Internal helper functions %%------------------------------------------------------------------------------------------ +v1_parameters() -> + [ + to_bin(K) + || {K, _} <- emqx_bridge_kafka:fields(v1_producer_kafka_opts) + ]. + producer_action_field_keys() -> [ to_bin(K) diff --git a/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl index 24405eb9e..74d3a5f54 100644 --- a/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl +++ b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl @@ -477,7 +477,7 @@ do_start_producer(KafkaClientId, KafkaTopic) -> ProducerConfig = #{ name => Name, - partitioner => roundrobin, + partitioner => random, partition_count_refresh_interval_seconds => 1_000, replayq_max_total_bytes => 10_000, replayq_seg_bytes => 9_000, @@ -1520,7 +1520,7 @@ t_receive_after_recovery(Config) -> key => <<"commit", (integer_to_binary(N))/binary>>, value => <<"commit", (integer_to_binary(N))/binary>> } - || N <- lists:seq(1, NPartitions) + || N <- lists:seq(1, NPartitions * 10) ], %% we do distinct passes over this producing part so that %% wolff won't batch everything together. @@ -1933,7 +1933,7 @@ t_node_joins_existing_cluster(Config) -> Val = <<"v", (integer_to_binary(N))/binary>>, publish(Config, KafkaTopic, [#{key => Key, value => Val}]) end, - lists:seq(1, NPartitions) + lists:seq(1, 10 * NPartitions) ), {ok, _} = snabbkaffe:receive_events(SRef1), diff --git a/apps/emqx_bridge_kafka/test/emqx_bridge_v2_kafka_producer_SUITE.erl b/apps/emqx_bridge_kafka/test/emqx_bridge_v2_kafka_producer_SUITE.erl index 98260a2c2..1db3c1725 100644 --- a/apps/emqx_bridge_kafka/test/emqx_bridge_v2_kafka_producer_SUITE.erl +++ b/apps/emqx_bridge_kafka/test/emqx_bridge_v2_kafka_producer_SUITE.erl @@ -23,6 +23,7 @@ -include_lib("snabbkaffe/include/snabbkaffe.hrl"). -include_lib("brod/include/brod.hrl"). -include_lib("emqx_resource/include/emqx_resource.hrl"). +-include_lib("emqx/include/asserts.hrl"). -import(emqx_common_test_helpers, [on_exit/1]). @@ -165,6 +166,9 @@ send_message(Type, ActionName) -> resolve_kafka_offset() -> KafkaTopic = emqx_bridge_kafka_impl_producer_SUITE:test_topic_one_partition(), + resolve_kafka_offset(KafkaTopic). + +resolve_kafka_offset(KafkaTopic) -> Partition = 0, Hosts = emqx_bridge_kafka_impl_producer_SUITE:kafka_hosts(), {ok, Offset0} = emqx_bridge_kafka_impl_producer_SUITE:resolve_kafka_offset( @@ -174,11 +178,32 @@ resolve_kafka_offset() -> check_kafka_message_payload(Offset, ExpectedPayload) -> KafkaTopic = emqx_bridge_kafka_impl_producer_SUITE:test_topic_one_partition(), + check_kafka_message_payload(KafkaTopic, Offset, ExpectedPayload). + +check_kafka_message_payload(KafkaTopic, Offset, ExpectedPayload) -> Partition = 0, Hosts = emqx_bridge_kafka_impl_producer_SUITE:kafka_hosts(), {ok, {_, [KafkaMsg0]}} = brod:fetch(Hosts, KafkaTopic, Partition, Offset), ?assertMatch(#kafka_message{value = ExpectedPayload}, KafkaMsg0). +ensure_kafka_topic(KafkaTopic) -> + TopicConfigs = [ + #{ + name => KafkaTopic, + num_partitions => 1, + replication_factor => 1, + assignments => [], + configs => [] + } + ], + RequestConfig = #{timeout => 5_000}, + ConnConfig = #{}, + Endpoints = emqx_bridge_kafka_impl_producer_SUITE:kafka_hosts(), + case brod:create_topics(Endpoints, TopicConfigs, RequestConfig, ConnConfig) of + ok -> ok; + {error, topic_already_exists} -> ok + end. + action_config(ConnectorName) -> action_config(ConnectorName, _Overrides = #{}). @@ -715,6 +740,21 @@ t_connector_health_check_topic(_Config) -> emqx_bridge_v2_testlib:update_connector_api(Name, Type, ConnectorConfig1) ), + %% By providing an inexistent health check topic, we should detect it's + %% disconnected without the need for an action. + ConnectorConfig2 = connector_config(#{ + <<"bootstrap_hosts">> => iolist_to_binary(kafka_hosts_string()), + <<"health_check_topic">> => <<"i-dont-exist-999">> + }), + ?assertMatch( + {ok, + {{_, 200, _}, _, #{ + <<"status">> := <<"disconnected">>, + <<"status_reason">> := <<"Unknown topic or partition", _/binary>> + }}}, + emqx_bridge_v2_testlib:update_connector_api(Name, Type, ConnectorConfig2) + ), + ok end, [] @@ -768,9 +808,13 @@ t_invalid_partition_count_metrics(Config) -> %% Simulate `invalid_partition_count' emqx_common_test_helpers:with_mock( wolff, - send_sync, - fun(_Producers, _Msgs, _Timeout) -> - error({invalid_partition_count, 0, partitioner}) + send_sync2, + fun(_Producers, _Topic, _Msgs, _Timeout) -> + throw(#{ + cause => invalid_partition_count, + count => 0, + partitioner => partitioner + }) end, fun() -> {{ok, _}, {ok, _}} = @@ -813,9 +857,13 @@ t_invalid_partition_count_metrics(Config) -> %% Simulate `invalid_partition_count' emqx_common_test_helpers:with_mock( wolff, - send, - fun(_Producers, _Msgs, _Timeout) -> - error({invalid_partition_count, 0, partitioner}) + send2, + fun(_Producers, _Topic, _Msgs, _AckCallback) -> + throw(#{ + cause => invalid_partition_count, + count => 0, + partitioner => partitioner + }) end, fun() -> {{ok, _}, {ok, _}} = @@ -921,3 +969,126 @@ t_multiple_actions_sharing_topic(Config) -> end ), ok. + +%% Smoke tests for using a templated topic and adynamic kafka topics. +t_dynamic_topics(Config) -> + Type = proplists:get_value(type, Config, ?TYPE), + ConnectorName = proplists:get_value(connector_name, Config, <<"c">>), + ConnectorConfig = proplists:get_value(connector_config, Config, connector_config()), + ActionName = <<"dynamic_topics">>, + ActionConfig1 = proplists:get_value(action_config, Config, action_config(ConnectorName)), + PreConfiguredTopic1 = <<"pct1">>, + PreConfiguredTopic2 = <<"pct2">>, + ensure_kafka_topic(PreConfiguredTopic1), + ensure_kafka_topic(PreConfiguredTopic2), + ActionConfig = emqx_bridge_v2_testlib:parse_and_check( + action, + Type, + ActionName, + emqx_utils_maps:deep_merge( + ActionConfig1, + #{ + <<"parameters">> => #{ + <<"topic">> => <<"pct${.payload.n}">>, + <<"message">> => #{ + <<"key">> => <<"${.clientid}">>, + <<"value">> => <<"${.payload.p}">> + } + } + } + ) + ), + ?check_trace( + #{timetrap => 7_000}, + begin + ConnectorParams = [ + {connector_config, ConnectorConfig}, + {connector_name, ConnectorName}, + {connector_type, Type} + ], + ActionParams = [ + {action_config, ActionConfig}, + {action_name, ActionName}, + {action_type, Type} + ], + {ok, {{_, 201, _}, _, #{}}} = + emqx_bridge_v2_testlib:create_connector_api(ConnectorParams), + + {ok, {{_, 201, _}, _, #{}}} = + emqx_bridge_v2_testlib:create_action_api(ActionParams), + RuleTopic = <<"pct">>, + {ok, _} = emqx_bridge_v2_testlib:create_rule_and_action_http( + Type, + RuleTopic, + [ + {bridge_name, ActionName} + ] + ), + ?assertStatusAPI(Type, ActionName, <<"connected">>), + + HandlerId = ?FUNCTION_NAME, + TestPid = self(), + telemetry:attach_many( + HandlerId, + emqx_resource_metrics:events(), + fun(EventName, Measurements, Metadata, _Config) -> + Data = #{ + name => EventName, + measurements => Measurements, + metadata => Metadata + }, + TestPid ! {telemetry, Data}, + ok + end, + unused_config + ), + on_exit(fun() -> telemetry:detach(HandlerId) end), + + {ok, C} = emqtt:start_link(#{}), + {ok, _} = emqtt:connect(C), + Payload = fun(Map) -> emqx_utils_json:encode(Map) end, + Offset1 = resolve_kafka_offset(PreConfiguredTopic1), + Offset2 = resolve_kafka_offset(PreConfiguredTopic2), + {ok, _} = emqtt:publish(C, RuleTopic, Payload(#{n => 1, p => <<"p1">>}), [{qos, 1}]), + {ok, _} = emqtt:publish(C, RuleTopic, Payload(#{n => 2, p => <<"p2">>}), [{qos, 1}]), + + check_kafka_message_payload(PreConfiguredTopic1, Offset1, <<"p1">>), + check_kafka_message_payload(PreConfiguredTopic2, Offset2, <<"p2">>), + + ActionId = emqx_bridge_v2:id(Type, ActionName), + ?assertEqual(2, emqx_resource_metrics:matched_get(ActionId)), + ?assertEqual(2, emqx_resource_metrics:success_get(ActionId)), + ?assertEqual(0, emqx_resource_metrics:queuing_get(ActionId)), + + ?assertReceive( + {telemetry, #{ + measurements := #{gauge_set := _}, + metadata := #{worker_id := _, resource_id := ActionId} + }} + ), + + %% If there isn't enough information in the context to resolve to a topic, it + %% should be an unrecoverable error. + ?assertMatch( + {_, {ok, _}}, + ?wait_async_action( + emqtt:publish(C, RuleTopic, Payload(#{not_enough => <<"info">>}), [{qos, 1}]), + #{?snk_kind := "kafka_producer_failed_to_render_topic"} + ) + ), + + %% If it's possible to render the topic, but it isn't in the pre-configured + %% list, it should be an unrecoverable error. + ?assertMatch( + {_, {ok, _}}, + ?wait_async_action( + emqtt:publish(C, RuleTopic, Payload(#{n => 99}), [{qos, 1}]), + #{?snk_kind := "kafka_producer_resolved_to_unknown_topic"} + ) + ), + + ok + end, + [] + ), + ok. diff --git a/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis.app.src b/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis.app.src index f411b95fb..a85121905 100644 --- a/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis.app.src +++ b/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_kinesis, [ {description, "EMQX Enterprise Amazon Kinesis Bridge"}, - {vsn, "0.2.1"}, + {vsn, "0.2.2"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis_impl_producer.erl b/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis_impl_producer.erl index 95d193d92..3143cf904 100644 --- a/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis_impl_producer.erl +++ b/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis_impl_producer.erl @@ -30,6 +30,7 @@ %% `emqx_resource' API -export([ + resource_type/0, callback_mode/0, on_start/2, on_stop/2, @@ -50,6 +51,7 @@ %%------------------------------------------------------------------------------------------------- %% `emqx_resource' API %%------------------------------------------------------------------------------------------------- +resource_type() -> kinesis_producer. callback_mode() -> always_sync. diff --git a/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.app.src b/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.app.src index df9935dbb..5bb7e396d 100644 --- a/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.app.src +++ b/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_mongodb, [ {description, "EMQX Enterprise MongoDB Bridge"}, - {vsn, "0.3.2"}, + {vsn, "0.3.3"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb_connector.erl b/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb_connector.erl index 6b6db358a..dac9bef57 100644 --- a/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb_connector.erl +++ b/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb_connector.erl @@ -11,6 +11,7 @@ %% `emqx_resource' API -export([ on_remove_channel/3, + resource_type/0, callback_mode/0, on_add_channel/4, on_get_channel_status/3, @@ -25,6 +26,7 @@ %%======================================================================================== %% `emqx_resource' API %%======================================================================================== +resource_type() -> emqx_mongodb:resource_type(). callback_mode() -> emqx_mongodb:callback_mode(). diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src index 26b8967f0..d43ec5591 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_bridge_mqtt, [ {description, "EMQX MQTT Broker Bridge"}, - {vsn, "0.2.2"}, + {vsn, "0.2.3"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl index d507d11b8..118542356 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl @@ -30,6 +30,7 @@ %% callbacks of behaviour emqx_resource -export([ + resource_type/0, callback_mode/0, on_start/2, on_stop/2, @@ -76,6 +77,8 @@ on_message_received(Msg, HookPoints, ResId) -> ok. %% =================================================================== +resource_type() -> mqtt. + callback_mode() -> async_if_possible. on_start(ResourceId, #{server := Server} = Conf) -> @@ -207,7 +210,7 @@ start_mqtt_clients(ResourceId, Conf) -> start_mqtt_clients(ResourceId, Conf, ClientOpts). start_mqtt_clients(ResourceId, StartConf, ClientOpts) -> - PoolName = <>, + PoolName = ResourceId, #{ pool_size := PoolSize } = StartConf, @@ -227,7 +230,7 @@ start_mqtt_clients(ResourceId, StartConf, ClientOpts) -> on_stop(ResourceId, State) -> ?SLOG(info, #{ msg => "stopping_mqtt_connector", - connector => ResourceId + resource_id => ResourceId }), %% on_stop can be called with State = undefined StateMap = @@ -271,7 +274,7 @@ on_query( on_query(ResourceId, {_ChannelId, Msg}, #{}) -> ?SLOG(error, #{ msg => "forwarding_unavailable", - connector => ResourceId, + resource_id => ResourceId, message => Msg, reason => "Egress is not configured" }). @@ -298,7 +301,7 @@ on_query_async( on_query_async(ResourceId, {_ChannelId, Msg}, _Callback, #{}) -> ?SLOG(error, #{ msg => "forwarding_unavailable", - connector => ResourceId, + resource_id => ResourceId, message => Msg, reason => "Egress is not configured" }). @@ -463,8 +466,10 @@ connect(Options) -> {ok, Pid} -> connect(Pid, Name); {error, Reason} = Error -> - ?SLOG(error, #{ + IsDryRun = emqx_resource:is_dry_run(Name), + ?SLOG(?LOG_LEVEL(IsDryRun), #{ msg => "client_start_failed", + resource_id => Name, config => emqx_utils:redact(ClientOpts), reason => Reason }), @@ -508,10 +513,11 @@ connect(Pid, Name) -> {ok, _Props} -> {ok, Pid}; {error, Reason} = Error -> - ?SLOG(warning, #{ + IsDryRun = emqx_resource:is_dry_run(Name), + ?SLOG(?LOG_LEVEL(IsDryRun), #{ msg => "ingress_client_connect_failed", reason => Reason, - name => Name + resource_id => Name }), _ = catch emqtt:stop(Pid), Error diff --git a/apps/emqx_bridge_mysql/src/emqx_bridge_mysql.app.src b/apps/emqx_bridge_mysql/src/emqx_bridge_mysql.app.src index 63bc61e62..fb670e072 100644 --- a/apps/emqx_bridge_mysql/src/emqx_bridge_mysql.app.src +++ b/apps/emqx_bridge_mysql/src/emqx_bridge_mysql.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_mysql, [ {description, "EMQX Enterprise MySQL Bridge"}, - {vsn, "0.1.7"}, + {vsn, "0.1.8"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_mysql/src/emqx_bridge_mysql_connector.erl b/apps/emqx_bridge_mysql/src/emqx_bridge_mysql_connector.erl index da9377814..6905c86eb 100644 --- a/apps/emqx_bridge_mysql/src/emqx_bridge_mysql_connector.erl +++ b/apps/emqx_bridge_mysql/src/emqx_bridge_mysql_connector.erl @@ -10,6 +10,7 @@ %% `emqx_resource' API -export([ on_remove_channel/3, + resource_type/0, callback_mode/0, on_add_channel/4, on_batch_query/3, @@ -24,6 +25,7 @@ %%======================================================================================== %% `emqx_resource' API %%======================================================================================== +resource_type() -> emqx_mysql:resource_type(). callback_mode() -> emqx_mysql:callback_mode(). diff --git a/apps/emqx_bridge_opents/src/emqx_bridge_opents.app.src b/apps/emqx_bridge_opents/src/emqx_bridge_opents.app.src index 65cc97e4c..a27791853 100644 --- a/apps/emqx_bridge_opents/src/emqx_bridge_opents.app.src +++ b/apps/emqx_bridge_opents/src/emqx_bridge_opents.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_opents, [ {description, "EMQX Enterprise OpenTSDB Bridge"}, - {vsn, "0.2.1"}, + {vsn, "0.2.2"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_opents/src/emqx_bridge_opents_connector.erl b/apps/emqx_bridge_opents/src/emqx_bridge_opents_connector.erl index 19e117a0d..a970bb374 100644 --- a/apps/emqx_bridge_opents/src/emqx_bridge_opents_connector.erl +++ b/apps/emqx_bridge_opents/src/emqx_bridge_opents_connector.erl @@ -18,6 +18,7 @@ %% `emqx_resource' API -export([ + resource_type/0, callback_mode/0, on_start/2, on_stop/2, @@ -114,6 +115,8 @@ connector_example_values() -> -define(HTTP_CONNECT_TIMEOUT, 1000). +resource_type() -> opents. + callback_mode() -> always_sync. on_start( diff --git a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_connector.erl b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_connector.erl index c98dd19ed..470f7f832 100644 --- a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_connector.erl +++ b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_connector.erl @@ -10,6 +10,7 @@ %% `emqx_resource' API -export([ + resource_type/0, callback_mode/0, query_mode/1, on_start/2, @@ -55,6 +56,7 @@ %%------------------------------------------------------------------------------------- %% `emqx_resource' API %%------------------------------------------------------------------------------------- +resource_type() -> pulsar. callback_mode() -> async_if_possible. @@ -263,7 +265,7 @@ format_servers(Servers0) -> -spec make_client_id(resource_id()) -> pulsar_client_id(). make_client_id(InstanceId) -> - case is_dry_run(InstanceId) of + case emqx_resource:is_dry_run(InstanceId) of true -> pulsar_producer_probe; false -> @@ -277,14 +279,6 @@ make_client_id(InstanceId) -> binary_to_atom(ClientIdBin) end. --spec is_dry_run(resource_id()) -> boolean(). -is_dry_run(InstanceId) -> - TestIdStart = string:find(InstanceId, ?TEST_ID_PREFIX), - case TestIdStart of - nomatch -> false; - _ -> string:equal(TestIdStart, InstanceId) - end. - conn_opts(#{authentication := none}) -> #{}; conn_opts(#{authentication := #{username := Username, password := Password}}) -> @@ -305,7 +299,7 @@ replayq_dir(ClientId) -> filename:join([emqx:data_dir(), "pulsar", emqx_utils_conv:bin(ClientId)]). producer_name(InstanceId, ChannelId) -> - case is_dry_run(InstanceId) of + case emqx_resource:is_dry_run(InstanceId) of %% do not create more atom true -> pulsar_producer_probe_worker; diff --git a/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq_connector.erl b/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq_connector.erl index ab7c0e331..3e75138d5 100644 --- a/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq_connector.erl +++ b/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq_connector.erl @@ -31,6 +31,7 @@ on_remove_channel/3, on_get_channels/1, on_stop/2, + resource_type/0, callback_mode/0, on_get_status/2, on_get_channel_status/3, @@ -60,6 +61,7 @@ fields(config) -> %% =================================================================== %% emqx_resource callback +resource_type() -> rabbitmq. callback_mode() -> always_sync. diff --git a/apps/emqx_bridge_redis/src/emqx_bridge_redis.app.src b/apps/emqx_bridge_redis/src/emqx_bridge_redis.app.src index 2cd037ed5..61cd837bb 100644 --- a/apps/emqx_bridge_redis/src/emqx_bridge_redis.app.src +++ b/apps/emqx_bridge_redis/src/emqx_bridge_redis.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_redis, [ {description, "EMQX Enterprise Redis Bridge"}, - {vsn, "0.1.8"}, + {vsn, "0.1.9"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_redis/src/emqx_bridge_redis_connector.erl b/apps/emqx_bridge_redis/src/emqx_bridge_redis_connector.erl index f117c4e7a..162c38368 100644 --- a/apps/emqx_bridge_redis/src/emqx_bridge_redis_connector.erl +++ b/apps/emqx_bridge_redis/src/emqx_bridge_redis_connector.erl @@ -12,6 +12,7 @@ %% callbacks of behaviour emqx_resource -export([ + resource_type/0, callback_mode/0, on_add_channel/4, on_remove_channel/3, @@ -29,7 +30,9 @@ %% resource callbacks %% ------------------------------------------------------------------------------------------------- -callback_mode() -> always_sync. +resource_type() -> emqx_redis:resource_type(). + +callback_mode() -> emqx_redis:callback_mode(). on_add_channel( _InstanceId, diff --git a/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq.app.src b/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq.app.src index fc59aeeca..9657ac115 100644 --- a/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq.app.src +++ b/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_rocketmq, [ {description, "EMQX Enterprise RocketMQ Bridge"}, - {vsn, "0.2.2"}, + {vsn, "0.2.3"}, {registered, []}, {applications, [kernel, stdlib, emqx_resource, rocketmq]}, {env, [ diff --git a/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq_connector.erl b/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq_connector.erl index 4fe3ea4c4..b03602bd2 100644 --- a/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq_connector.erl +++ b/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq_connector.erl @@ -16,6 +16,7 @@ %% `emqx_resource' API -export([ + resource_type/0, callback_mode/0, on_start/2, on_stop/2, @@ -90,6 +91,8 @@ servers() -> %% `emqx_resource' API %%======================================================================================== +resource_type() -> rocketmq. + callback_mode() -> always_sync. on_start( diff --git a/apps/emqx_bridge_s3/src/emqx_bridge_s3_connector.erl b/apps/emqx_bridge_s3/src/emqx_bridge_s3_connector.erl index e90016927..f6260ea1d 100644 --- a/apps/emqx_bridge_s3/src/emqx_bridge_s3_connector.erl +++ b/apps/emqx_bridge_s3/src/emqx_bridge_s3_connector.erl @@ -13,6 +13,7 @@ -behaviour(emqx_resource). -export([ + resource_type/0, callback_mode/0, on_start/2, on_stop/2, @@ -92,6 +93,8 @@ -define(AGGREG_SUP, emqx_bridge_s3_sup). %% +-spec resource_type() -> resource_type(). +resource_type() -> s3. -spec callback_mode() -> callback_mode(). callback_mode() -> diff --git a/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver_connector.erl b/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver_connector.erl index 521a215a5..4b1033ca9 100644 --- a/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver_connector.erl +++ b/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver_connector.erl @@ -30,6 +30,7 @@ %% callbacks for behaviour emqx_resource -export([ + resource_type/0, callback_mode/0, on_start/2, on_stop/2, @@ -173,6 +174,7 @@ server() -> %%==================================================================== %% Callbacks defined in emqx_resource %%==================================================================== +resource_type() -> sqlserver. callback_mode() -> always_sync. diff --git a/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper.app.src b/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper.app.src index 5ae95ca67..cd1d51b01 100644 --- a/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper.app.src +++ b/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_syskeeper, [ {description, "EMQX Enterprise Data bridge for Syskeeper"}, - {vsn, "0.1.3"}, + {vsn, "0.1.4"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper.erl b/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper.erl index 547562f26..16f28e40d 100644 --- a/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper.erl +++ b/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper.erl @@ -84,30 +84,16 @@ fields(action) -> } )}; fields(config) -> - [ - {enable, mk(boolean(), #{desc => ?DESC("config_enable"), default => true})}, - {tags, emqx_schema:tags_schema()}, - {description, emqx_schema:description_schema()}, - {connector, - mk(binary(), #{ - desc => ?DESC(emqx_connector_schema, "connector_field"), required => true - })}, - {parameters, - mk( - ref(?MODULE, "parameters"), - #{required => true, desc => ?DESC("parameters")} - )}, - {local_topic, mk(binary(), #{required => false, desc => ?DESC(mqtt_topic)})}, - {resource_opts, - mk( - ref(?MODULE, "creation_opts"), - #{ - required => false, - default => #{}, - desc => ?DESC(emqx_resource_schema, <<"resource_opts">>) - } - )} - ]; + emqx_bridge_v2_schema:make_producer_action_schema( + mk( + ref(?MODULE, "parameters"), + #{ + required => true, + desc => ?DESC("parameters") + } + ), + #{resource_opts_ref => ref(?MODULE, "creation_opts")} + ); fields("parameters") -> [ {target_topic, diff --git a/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_connector.erl b/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_connector.erl index 898915f56..c277faa4f 100644 --- a/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_connector.erl +++ b/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_connector.erl @@ -18,6 +18,7 @@ %% `emqx_resource' API -export([ + resource_type/0, callback_mode/0, query_mode/1, on_start/2, @@ -147,6 +148,7 @@ server() -> %% ------------------------------------------------------------------------------------------------- %% `emqx_resource' API +resource_type() -> syskeeper. callback_mode() -> always_sync. diff --git a/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_proxy_server.erl b/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_proxy_server.erl index e26ae43c1..d0aa44cd3 100644 --- a/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_proxy_server.erl +++ b/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_proxy_server.erl @@ -12,6 +12,7 @@ %% `emqx_resource' API -export([ + resource_type/0, query_mode/1, on_start/2, on_stop/2, @@ -40,6 +41,8 @@ %% ------------------------------------------------------------------------------------------------- %% emqx_resource +resource_type() -> + syskeeper_proxy_server. query_mode(_) -> no_queries. diff --git a/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine.app.src b/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine.app.src index d358ba8fa..5fe325b38 100644 --- a/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine.app.src +++ b/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_tdengine, [ {description, "EMQX Enterprise TDEngine Bridge"}, - {vsn, "0.2.1"}, + {vsn, "0.2.2"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine_connector.erl b/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine_connector.erl index 324694edc..46980e768 100644 --- a/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine_connector.erl +++ b/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine_connector.erl @@ -19,6 +19,7 @@ %% `emqx_resource' API -export([ + resource_type/0, callback_mode/0, on_start/2, on_stop/2, @@ -140,6 +141,7 @@ connector_example_values() -> %%======================================================================================== %% `emqx_resource' API %%======================================================================================== +resource_type() -> tdengine. callback_mode() -> always_sync. diff --git a/apps/emqx_cluster_link/include/emqx_cluster_link.hrl b/apps/emqx_cluster_link/include/emqx_cluster_link.hrl index 08dc7f4ad..8a0c374ed 100644 --- a/apps/emqx_cluster_link/include/emqx_cluster_link.hrl +++ b/apps/emqx_cluster_link/include/emqx_cluster_link.hrl @@ -17,3 +17,10 @@ %% Fairly compact text encoding. -define(SHARED_ROUTE_ID(Topic, Group), <<"$s/", Group/binary, "/", Topic/binary>>). -define(PERSISTENT_ROUTE_ID(Topic, ID), <<"$p/", ID/binary, "/", Topic/binary>>). + +-define(METRIC_NAME, cluster_link). + +-define(route_metric, 'routes'). +-define(PERSISTENT_SHARED_ROUTE_ID(Topic, Group, ID), + <<"$sp/", Group/binary, "/", ID/binary, "/", Topic/binary>> +). diff --git a/apps/emqx_cluster_link/src/emqx_cluster_link.erl b/apps/emqx_cluster_link/src/emqx_cluster_link.erl index 76228c052..d68ffb4be 100644 --- a/apps/emqx_cluster_link/src/emqx_cluster_link.erl +++ b/apps/emqx_cluster_link/src/emqx_cluster_link.erl @@ -16,6 +16,8 @@ delete_shared_route/2, add_persistent_route/2, delete_persistent_route/2, + add_persistent_shared_route/3, + delete_persistent_shared_route/3, forward/1 ]). @@ -71,6 +73,16 @@ add_persistent_route(Topic, ID) -> delete_persistent_route(Topic, ID) -> maybe_push_route_op(delete, Topic, ?PERSISTENT_ROUTE_ID(Topic, ID), push_persistent_route). +add_persistent_shared_route(Topic, Group, ID) -> + maybe_push_route_op( + add, Topic, ?PERSISTENT_SHARED_ROUTE_ID(Topic, Group, ID), push_persistent_route + ). + +delete_persistent_shared_route(Topic, Group, ID) -> + maybe_push_route_op( + delete, Topic, ?PERSISTENT_SHARED_ROUTE_ID(Topic, Group, ID), push_persistent_route + ). + forward(#delivery{message = #message{extra = #{link_origin := _}}}) -> %% Do not forward any external messages to other links. %% Only forward locally originated messages to all the relevant links, i.e. no gossip diff --git a/apps/emqx_cluster_link/src/emqx_cluster_link_api.erl b/apps/emqx_cluster_link/src/emqx_cluster_link_api.erl index 33634607e..77f613e45 100644 --- a/apps/emqx_cluster_link/src/emqx_cluster_link_api.erl +++ b/apps/emqx_cluster_link/src/emqx_cluster_link_api.erl @@ -7,84 +7,523 @@ -include_lib("hocon/include/hoconsc.hrl"). -include_lib("emqx/include/http_api.hrl"). +-include_lib("emqx_utils/include/emqx_utils_api.hrl"). +-include_lib("emqx_resource/include/emqx_resource.hrl"). +-include_lib("emqx/include/logger.hrl"). +-include("emqx_cluster_link.hrl"). -export([ api_spec/0, paths/0, + namespace/0, + fields/1, schema/1 ]). --export([config/2]). +-export([ + '/cluster/links'/2, + '/cluster/links/link/:name'/2, + '/cluster/links/link/:name/metrics'/2 +]). -define(CONF_PATH, [cluster, links]). -define(TAGS, [<<"Cluster">>]). +-type cluster_name() :: binary(). + +namespace() -> "cluster_link". + api_spec() -> emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}). paths() -> [ - "/cluster/links" + "/cluster/links", + "/cluster/links/link/:name", + "/cluster/links/link/:name/metrics" ]. schema("/cluster/links") -> #{ - 'operationId' => config, + 'operationId' => '/cluster/links', get => #{ description => "Get cluster links configuration", tags => ?TAGS, responses => - #{200 => links_config_schema()} + #{200 => links_config_schema_response()} + }, + post => + #{ + description => "Create a cluster link", + tags => ?TAGS, + 'requestBody' => link_config_schema(), + responses => + #{ + 200 => link_config_schema_response(), + 400 => + emqx_dashboard_swagger:error_codes( + [?BAD_REQUEST, ?ALREADY_EXISTS], + <<"Update Config Failed">> + ) + } + } + }; +schema("/cluster/links/link/:name") -> + #{ + 'operationId' => '/cluster/links/link/:name', + get => + #{ + description => "Get a cluster link configuration", + tags => ?TAGS, + parameters => [param_path_name()], + responses => + #{ + 200 => link_config_schema_response(), + 404 => emqx_dashboard_swagger:error_codes( + [?NOT_FOUND], <<"Cluster link not found">> + ) + } + }, + delete => + #{ + description => "Delete a cluster link", + tags => ?TAGS, + parameters => [param_path_name()], + responses => + #{ + 204 => <<"Link deleted">>, + 404 => emqx_dashboard_swagger:error_codes( + [?NOT_FOUND], <<"Cluster link not found">> + ) + } }, put => #{ - description => "Update cluster links configuration", + description => "Update a cluster link configuration", tags => ?TAGS, - 'requestBody' => links_config_schema(), + parameters => [param_path_name()], + 'requestBody' => update_link_config_schema(), responses => #{ - 200 => links_config_schema(), + 200 => link_config_schema_response(), + 404 => emqx_dashboard_swagger:error_codes( + [?NOT_FOUND], <<"Cluster link not found">> + ), 400 => emqx_dashboard_swagger:error_codes( [?BAD_REQUEST], <<"Update Config Failed">> ) } } + }; +schema("/cluster/links/link/:name/metrics") -> + #{ + 'operationId' => '/cluster/links/link/:name/metrics', + get => + #{ + description => "Get a cluster link metrics", + tags => ?TAGS, + parameters => [param_path_name()], + responses => + #{ + 200 => link_metrics_schema_response(), + 404 => emqx_dashboard_swagger:error_codes( + [?NOT_FOUND], <<"Cluster link not found">> + ) + } + } }. +fields(link_config_response) -> + [ + {node, hoconsc:mk(binary(), #{desc => ?DESC("node")})}, + {status, hoconsc:mk(status(), #{desc => ?DESC("status")})} + | emqx_cluster_link_schema:fields("link") + ]; +fields(metrics) -> + [ + {metrics, hoconsc:mk(map(), #{desc => ?DESC("metrics")})} + ]; +fields(link_metrics_response) -> + [ + {node_metrics, + hoconsc:mk( + hoconsc:array(hoconsc:ref(?MODULE, node_metrics)), + #{desc => ?DESC("node_metrics")} + )} + | fields(metrics) + ]; +fields(node_metrics) -> + [ + {node, hoconsc:mk(atom(), #{desc => ?DESC("node")})} + | fields(metrics) + ]. + %%-------------------------------------------------------------------- %% API Handler funcs %%-------------------------------------------------------------------- -config(get, _Params) -> - {200, get_raw()}; -config(put, #{body := Body}) -> - case emqx_cluster_link_config:update(Body) of - {ok, NewConfig} -> - {200, NewConfig}; - {error, Reason} -> - Message = list_to_binary(io_lib:format("Update config failed ~p", [Reason])), - {400, ?BAD_REQUEST, Message} - end. +'/cluster/links'(get, _Params) -> + handle_list(); +'/cluster/links'(post, #{body := Body = #{<<"name">> := Name}}) -> + with_link( + Name, + return(?BAD_REQUEST('ALREADY_EXISTS', <<"Cluster link already exists">>)), + fun() -> handle_create(Name, Body) end + ). + +'/cluster/links/link/:name'(get, #{bindings := #{name := Name}}) -> + with_link(Name, fun(Link) -> handle_lookup(Name, Link) end, not_found()); +'/cluster/links/link/:name'(put, #{bindings := #{name := Name}, body := Params0}) -> + with_link(Name, fun() -> handle_update(Name, Params0) end, not_found()); +'/cluster/links/link/:name'(delete, #{bindings := #{name := Name}}) -> + with_link( + Name, + fun() -> + case emqx_cluster_link_config:delete_link(Name) of + ok -> + ?NO_CONTENT; + {error, Reason} -> + Message = list_to_binary(io_lib:format("Delete link failed ~p", [Reason])), + ?BAD_REQUEST(Message) + end + end, + not_found() + ). + +'/cluster/links/link/:name/metrics'(get, #{bindings := #{name := Name}}) -> + with_link(Name, fun() -> handle_metrics(Name) end, not_found()). %%-------------------------------------------------------------------- %% Internal funcs %%-------------------------------------------------------------------- +handle_list() -> + Links = get_raw(), + NodeRPCResults = emqx_cluster_link_mqtt:get_all_resources_cluster(), + {NameToStatus, Errors} = collect_all_status(NodeRPCResults), + NodeErrors = lists:map( + fun({Node, Error}) -> + #{node => Node, status => inconsistent, reason => Error} + end, + Errors + ), + EmptyStatus = #{status => inconsistent, node_status => NodeErrors}, + Response = + lists:map( + fun(#{<<"name">> := Name} = Link) -> + Status = maps:get(Name, NameToStatus, EmptyStatus), + maps:merge(Link, Status) + end, + Links + ), + ?OK(Response). + +handle_create(Name, Params) -> + case emqx_cluster_link_config:create_link(Params) of + {ok, Link} -> + ?CREATED(add_status(Name, Link)); + {error, Reason} -> + Message = list_to_binary(io_lib:format("Create link failed ~p", [Reason])), + ?BAD_REQUEST(Message) + end. + +handle_lookup(Name, Link) -> + ?OK(add_status(Name, Link)). + +handle_metrics(Name) -> + Results = emqx_cluster_link_metrics:get_metrics(Name), + {NodeMetrics0, NodeErrors} = + lists:foldl( + fun({Node, RouterMetrics0, ResourceMetrics0}, {OkAccIn, ErrAccIn}) -> + {RouterMetrics, RouterError} = get_metrics_or_errors(RouterMetrics0), + {ResourceMetrics, ResourceError} = get_metrics_or_errors(ResourceMetrics0), + ErrAcc = append_errors(RouterError, ResourceError, Node, ErrAccIn), + {[format_metrics(Node, RouterMetrics, ResourceMetrics) | OkAccIn], ErrAcc} + end, + {[], []}, + Results + ), + case NodeErrors of + [] -> + ok; + [_ | _] -> + ?SLOG(warning, #{ + msg => "cluster_link_api_metrics_bad_erpc_results", + errors => maps:from_list(NodeErrors) + }) + end, + NodeMetrics1 = lists:map(fun({Node, _Error}) -> format_metrics(Node, #{}, #{}) end, NodeErrors), + NodeMetrics = NodeMetrics1 ++ NodeMetrics0, + AggregatedMetrics = aggregate_metrics(NodeMetrics), + Response = #{metrics => AggregatedMetrics, node_metrics => NodeMetrics}, + ?OK(Response). + +get_metrics_or_errors({ok, Metrics}) -> + {Metrics, undefined}; +get_metrics_or_errors(Error) -> + {#{}, Error}. + +append_errors(undefined, undefined, _Node, Acc) -> + Acc; +append_errors(RouterError, ResourceError, Node, Acc) -> + Err0 = emqx_utils_maps:put_if(#{}, router, RouterError, RouterError =/= undefined), + Err = emqx_utils_maps:put_if(Err0, resource, ResourceError, ResourceError =/= undefined), + [{Node, Err} | Acc]. + +aggregate_metrics(NodeMetrics) -> + ErrorLogger = fun(_) -> ok end, + #{metrics := #{router := EmptyRouterMetrics}} = format_metrics(node(), #{}, #{}), + {RouterMetrics, ResourceMetrics} = lists:foldl( + fun( + #{metrics := #{router := RMetrics, forwarding := FMetrics}}, + {RouterAccIn, ResourceAccIn} + ) -> + ResourceAcc = + emqx_utils_maps:best_effort_recursive_sum(FMetrics, ResourceAccIn, ErrorLogger), + RouterAcc = merge_cluster_wide_metrics(RMetrics, RouterAccIn), + {RouterAcc, ResourceAcc} + end, + {EmptyRouterMetrics, #{}}, + NodeMetrics + ), + #{router => RouterMetrics, forwarding => ResourceMetrics}. + +merge_cluster_wide_metrics(Metrics, Acc) -> + %% For cluster-wide metrics, all nodes should report the same values, except if the + %% RPC to fetch a node's metrics failed, in which case all values will be 0. + F = + fun(_Key, V1, V2) -> + case {erlang:is_map(V1), erlang:is_map(V2)} of + {true, true} -> + merge_cluster_wide_metrics(V1, V2); + {true, false} -> + merge_cluster_wide_metrics(V1, #{}); + {false, true} -> + merge_cluster_wide_metrics(V2, #{}); + {false, false} -> + true = is_number(V1), + true = is_number(V2), + max(V1, V2) + end + end, + maps:merge_with(F, Acc, Metrics). + +format_metrics(Node, RouterMetrics, ResourceMetrics) -> + Get = fun(Path, Map) -> emqx_utils_maps:deep_get(Path, Map, 0) end, + Routes = Get([gauges, ?route_metric], RouterMetrics), + #{ + node => Node, + metrics => #{ + router => #{ + ?route_metric => Routes + }, + forwarding => #{ + 'matched' => Get([counters, 'matched'], ResourceMetrics), + 'success' => Get([counters, 'success'], ResourceMetrics), + 'failed' => Get([counters, 'failed'], ResourceMetrics), + 'dropped' => Get([counters, 'dropped'], ResourceMetrics), + 'retried' => Get([counters, 'retried'], ResourceMetrics), + 'received' => Get([counters, 'received'], ResourceMetrics), + + 'queuing' => Get([gauges, 'queuing'], ResourceMetrics), + 'inflight' => Get([gauges, 'inflight'], ResourceMetrics), + + 'rate' => Get([rate, 'matched', current], ResourceMetrics), + 'rate_last5m' => Get([rate, 'matched', last5m], ResourceMetrics), + 'rate_max' => Get([rate, 'matched', max], ResourceMetrics) + } + } + }. + +add_status(Name, Link) -> + NodeRPCResults = emqx_cluster_link_mqtt:get_resource_cluster(Name), + Status = collect_single_status(NodeRPCResults), + maps:merge(Link, Status). + +handle_update(Name, Params0) -> + Params = Params0#{<<"name">> => Name}, + case emqx_cluster_link_config:update_link(Params) of + {ok, Link} -> + ?OK(add_status(Name, Link)); + {error, Reason} -> + Message = list_to_binary(io_lib:format("Update link failed ~p", [Reason])), + ?BAD_REQUEST(Message) + end. + get_raw() -> - #{<<"links">> := Conf} = + #{<<"cluster">> := #{<<"links">> := Links}} = emqx_config:fill_defaults( - #{<<"links">> => emqx_conf:get_raw(?CONF_PATH)}, + #{<<"cluster">> => #{<<"links">> => emqx_conf:get_raw(?CONF_PATH)}}, #{obfuscate_sensitive_values => true} ), - Conf. + Links. -links_config_schema() -> - emqx_cluster_link_schema:links_schema( - #{ - examples => #{<<"example">> => links_config_example()} +-spec collect_all_status([{node(), {ok, #{cluster_name() => _}} | _Error}]) -> + {ClusterToStatus, Errors} +when + ClusterToStatus :: #{ + cluster_name() => #{ + node := node(), + status := emqx_resource:resource_status() | inconsistent } + }, + Errors :: [{node(), term()}]. +collect_all_status(NodeResults) -> + {Reindexed, Errors} = lists:foldl( + fun + ({Node, {ok, AllLinkData}}, {OkAccIn, ErrAccIn}) -> + OkAcc = maps:fold( + fun(Name, Data, AccIn) -> + collect_all_status1(Node, Name, Data, AccIn) + end, + OkAccIn, + AllLinkData + ), + {OkAcc, ErrAccIn}; + ({Node, Error}, {OkAccIn, ErrAccIn}) -> + {OkAccIn, [{Node, Error} | ErrAccIn]} + end, + {#{}, []}, + NodeResults + ), + NoErrors = + case Errors of + [] -> + true; + [_ | _] -> + ?SLOG(warning, #{ + msg => "cluster_link_api_lookup_status_bad_erpc_results", + errors => Errors + }), + false + end, + ClusterToStatus = maps:fold( + fun(Name, NodeToData, Acc) -> + OnlyStatus = [S || #{status := S} <- maps:values(NodeToData)], + SummaryStatus = + case lists:usort(OnlyStatus) of + [SameStatus] when NoErrors -> SameStatus; + _ -> inconsistent + end, + NodeStatus = lists:map( + fun + ({Node, #{status := S}}) -> + #{node => Node, status => S}; + ({Node, Error0}) -> + Error = emqx_logger_jsonfmt:best_effort_json(Error0), + #{node => Node, status => inconsistent, reason => Error} + end, + maps:to_list(NodeToData) ++ Errors + ), + Acc#{ + Name => #{ + status => SummaryStatus, + node_status => NodeStatus + } + } + end, + #{}, + Reindexed + ), + {ClusterToStatus, Errors}. + +collect_all_status1(Node, Name, Data, Acc) -> + maps:update_with( + Name, + fun(Old) -> Old#{Node => Data} end, + #{Node => Data}, + Acc + ). + +collect_single_status(NodeResults) -> + NodeStatus = + lists:map( + fun + ({Node, {ok, {ok, #{status := S}}}}) -> + #{node => Node, status => S}; + ({Node, {ok, {error, _}}}) -> + #{node => Node, status => ?status_disconnected}; + ({Node, Error0}) -> + Error = emqx_logger_jsonfmt:best_effort_json(Error0), + #{node => Node, status => inconsistent, reason => Error} + end, + NodeResults + ), + OnlyStatus = [S || #{status := S} <- NodeStatus], + SummaryStatus = + case lists:usort(OnlyStatus) of + [SameStatus] -> SameStatus; + _ -> inconsistent + end, + #{ + status => SummaryStatus, + node_status => NodeStatus + }. + +links_config_schema_response() -> + hoconsc:mk(hoconsc:array(hoconsc:ref(?MODULE, link_config_response)), #{ + examples => #{<<"example">> => links_config_response_example()} + }). + +link_config_schema() -> + hoconsc:mk(emqx_cluster_link_schema:link_schema(), #{ + examples => #{<<"example">> => hd(links_config_example())} + }). + +link_config_schema_response() -> + hoconsc:mk( + hoconsc:ref(?MODULE, link_config_response), + #{ + examples => #{ + <<"example">> => hd(links_config_response_example()) + } + } + ). + +link_metrics_schema_response() -> + hoconsc:mk( + hoconsc:ref(?MODULE, link_metrics_response), + #{ + examples => #{ + <<"example">> => link_metrics_response_example() + } + } + ). + +status() -> + hoconsc:enum([?status_connected, ?status_disconnected, ?status_connecting, inconsistent]). + +param_path_name() -> + {name, + hoconsc:mk( + binary(), + #{ + in => path, + required => true, + example => <<"my_link">>, + desc => ?DESC("param_path_name") + } + )}. + +update_link_config_schema() -> + proplists:delete(name, emqx_cluster_link_schema:fields("link")). + +links_config_response_example() -> + lists:map( + fun(LinkEx) -> + LinkEx#{ + <<"status">> => <<"connected">>, + <<"node_status">> => [ + #{ + <<"node">> => <<"emqx1@emqx.net">>, + <<"status">> => <<"connected">> + } + ] + } + end, + links_config_example() ). links_config_example() -> @@ -114,3 +553,39 @@ links_config_example() -> <<"name">> => <<"emqxcl_c">> } ]. + +link_metrics_response_example() -> + #{ + <<"metrics">> => #{<<"routes">> => 10240}, + <<"node_metrics">> => [ + #{ + <<"node">> => <<"emqx1@emqx.net">>, + <<"metrics">> => #{<<"routes">> => 10240} + } + ] + }. + +with_link(Name, FoundFn, NotFoundFn) -> + case emqx_cluster_link_config:link_raw(Name) of + undefined -> + NotFoundFn(); + Link0 = #{} when is_function(FoundFn, 1) -> + Link = fill_defaults_single(Link0), + FoundFn(Link); + _Link = #{} when is_function(FoundFn, 0) -> + FoundFn() + end. + +fill_defaults_single(Link0) -> + #{<<"cluster">> := #{<<"links">> := [Link]}} = + emqx_config:fill_defaults( + #{<<"cluster">> => #{<<"links">> => [Link0]}}, + #{obfuscate_sensitive_values => true} + ), + Link. + +return(Response) -> + fun() -> Response end. + +not_found() -> + return(?NOT_FOUND(<<"Cluster link not found">>)). diff --git a/apps/emqx_cluster_link/src/emqx_cluster_link_app.erl b/apps/emqx_cluster_link/src/emqx_cluster_link_app.erl index 41f1a0a77..9502ad1c3 100644 --- a/apps/emqx_cluster_link/src/emqx_cluster_link_app.erl +++ b/apps/emqx_cluster_link/src/emqx_cluster_link_app.erl @@ -22,7 +22,9 @@ start(_StartType, _StartArgs) -> _ -> ok end, - emqx_cluster_link_sup:start_link(LinksConf). + {ok, Sup} = emqx_cluster_link_sup:start_link(LinksConf), + ok = create_metrics(LinksConf), + {ok, Sup}. prep_stop(State) -> emqx_cluster_link_config:remove_handler(), @@ -53,3 +55,11 @@ remove_msg_fwd_resources(LinksConf) -> end, LinksConf ). + +create_metrics(LinksConf) -> + lists:foreach( + fun(#{name := ClusterName}) -> + ok = emqx_cluster_link_metrics:maybe_create_metrics(ClusterName) + end, + LinksConf + ). diff --git a/apps/emqx_cluster_link/src/emqx_cluster_link_bookkeeper.erl b/apps/emqx_cluster_link/src/emqx_cluster_link_bookkeeper.erl new file mode 100644 index 000000000..826d4f0db --- /dev/null +++ b/apps/emqx_cluster_link/src/emqx_cluster_link_bookkeeper.erl @@ -0,0 +1,90 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_cluster_link_bookkeeper). + +%% API +-export([ + start_link/0 +]). + +%% `gen_server' API +-export([ + init/1, + handle_continue/2, + handle_call/3, + handle_cast/2, + handle_info/2 +]). + +%%------------------------------------------------------------------------------ +%% Type declarations +%%------------------------------------------------------------------------------ + +-ifdef(TEST). +%% ms +-define(TALLY_ROUTES_INTERVAL, 300). +-else. +%% ms +-define(TALLY_ROUTES_INTERVAL, 15_000). +-endif. + +%% call/cast/info events +-record(tally_routes, {}). + +%%------------------------------------------------------------------------------ +%% API +%%------------------------------------------------------------------------------ + +-spec start_link() -> gen_server:start_ret(). +start_link() -> + gen_server:start_link({local, ?MODULE}, ?MODULE, _InitOpts = #{}, _Opts = []). + +%%------------------------------------------------------------------------------ +%% `gen_server' API +%%------------------------------------------------------------------------------ + +init(_Opts) -> + State = #{}, + {ok, State, {continue, #tally_routes{}}}. + +handle_continue(#tally_routes{}, State) -> + handle_tally_routes(), + {noreply, State}. + +handle_call(_Call, _From, State) -> + {reply, {error, bad_call}, State}. + +handle_cast(_Cast, State) -> + {noreply, State}. + +handle_info(#tally_routes{}, State) -> + handle_tally_routes(), + {noreply, State}; +handle_info(_Info, State) -> + {noreply, State}. + +%%------------------------------------------------------------------------------ +%% Internal fns +%%------------------------------------------------------------------------------ + +cluster_names() -> + Links = emqx_cluster_link_config:links(), + lists:map(fun(#{name := Name}) -> Name end, Links). + +ensure_timer(Event, Timeout) -> + _ = erlang:send_after(Timeout, self(), Event), + ok. + +handle_tally_routes() -> + ClusterNames = cluster_names(), + tally_routes(ClusterNames), + ensure_timer(#tally_routes{}, ?TALLY_ROUTES_INTERVAL), + ok. + +tally_routes([ClusterName | ClusterNames]) -> + NumRoutes = emqx_cluster_link_extrouter:count(ClusterName), + emqx_cluster_link_metrics:routes_set(ClusterName, NumRoutes), + tally_routes(ClusterNames); +tally_routes([]) -> + ok. diff --git a/apps/emqx_cluster_link/src/emqx_cluster_link_config.erl b/apps/emqx_cluster_link/src/emqx_cluster_link_config.erl index f27c7702e..2a97f2d69 100644 --- a/apps/emqx_cluster_link/src/emqx_cluster_link_config.erl +++ b/apps/emqx_cluster_link/src/emqx_cluster_link_config.erl @@ -4,6 +4,8 @@ -module(emqx_cluster_link_config). +-feature(maybe_expr, enable). + -behaviour(emqx_config_handler). -include_lib("emqx/include/logger.hrl"). @@ -28,11 +30,15 @@ -export([ %% General + create_link/1, + delete_link/1, + update_link/1, update/1, cluster/0, enabled_links/0, links/0, link/1, + link_raw/1, topic_filters/1, %% Connections emqtt_options/1, @@ -55,6 +61,52 @@ %% +create_link(LinkConfig) -> + #{<<"name">> := Name} = LinkConfig, + case + emqx_conf:update( + ?LINKS_PATH, + {create, LinkConfig}, + #{rawconf_with_defaults => true, override_to => cluster} + ) + of + {ok, #{raw_config := NewConfigRows}} -> + NewLinkConfig = find_link(Name, NewConfigRows), + {ok, NewLinkConfig}; + {error, Reason} -> + {error, Reason} + end. + +delete_link(Name) -> + case + emqx_conf:update( + ?LINKS_PATH, + {delete, Name}, + #{rawconf_with_defaults => true, override_to => cluster} + ) + of + {ok, _} -> + ok; + {error, Reason} -> + {error, Reason} + end. + +update_link(LinkConfig) -> + #{<<"name">> := Name} = LinkConfig, + case + emqx_conf:update( + ?LINKS_PATH, + {update, LinkConfig}, + #{rawconf_with_defaults => true, override_to => cluster} + ) + of + {ok, #{raw_config := NewConfigRows}} -> + NewLinkConfig = find_link(Name, NewConfigRows), + {ok, NewLinkConfig}; + {error, Reason} -> + {error, Reason} + end. + update(Config) -> case emqx_conf:update( @@ -75,11 +127,20 @@ cluster() -> links() -> emqx:get_config(?LINKS_PATH, []). +links_raw() -> + emqx:get_raw_config(?LINKS_PATH, []). + enabled_links() -> [L || L = #{enable := true} <- links()]. link(Name) -> - case lists:dropwhile(fun(L) -> Name =/= upstream_name(L) end, links()) of + find_link(Name, links()). + +link_raw(Name) -> + find_link(Name, links_raw()). + +find_link(Name, Links) -> + case lists:dropwhile(fun(L) -> Name =/= upstream_name(L) end, Links) of [LinkConf | _] -> LinkConf; [] -> undefined end. @@ -133,6 +194,37 @@ remove_handler() -> pre_config_update(?LINKS_PATH, RawConf, RawConf) -> {ok, RawConf}; +pre_config_update(?LINKS_PATH, {create, LinkRawConf}, OldRawConf) -> + #{<<"name">> := Name} = LinkRawConf, + maybe + undefined ?= find_link(Name, OldRawConf), + NewRawConf0 = OldRawConf ++ [LinkRawConf], + NewRawConf = convert_certs(maybe_increment_ps_actor_incr(NewRawConf0, OldRawConf)), + {ok, NewRawConf} + else + _ -> + {error, already_exists} + end; +pre_config_update(?LINKS_PATH, {update, LinkRawConf}, OldRawConf) -> + #{<<"name">> := Name} = LinkRawConf, + maybe + {_Found, Front, Rear} ?= safe_take(Name, OldRawConf), + NewRawConf0 = Front ++ [LinkRawConf] ++ Rear, + NewRawConf = convert_certs(maybe_increment_ps_actor_incr(NewRawConf0, OldRawConf)), + {ok, NewRawConf} + else + not_found -> + {error, not_found} + end; +pre_config_update(?LINKS_PATH, {delete, Name}, OldRawConf) -> + maybe + {_Found, Front, Rear} ?= safe_take(Name, OldRawConf), + NewRawConf = Front ++ Rear, + {ok, NewRawConf} + else + _ -> + {error, not_found} + end; pre_config_update(?LINKS_PATH, NewRawConf, OldRawConf) -> {ok, convert_certs(maybe_increment_ps_actor_incr(NewRawConf, OldRawConf))}. @@ -185,9 +277,10 @@ all_ok(Results) -> add_links(LinksConf) -> [add_link(Link) || Link <- LinksConf]. -add_link(#{enable := true} = LinkConf) -> +add_link(#{name := ClusterName, enable := true} = LinkConf) -> {ok, _Pid} = emqx_cluster_link_sup:ensure_actor(LinkConf), {ok, _} = emqx_cluster_link_mqtt:ensure_msg_fwd_resource(LinkConf), + ok = emqx_cluster_link_metrics:maybe_create_metrics(ClusterName), ok; add_link(_DisabledLinkConf) -> ok. @@ -197,12 +290,13 @@ remove_links(LinksConf) -> remove_link(Name) -> _ = emqx_cluster_link_mqtt:remove_msg_fwd_resource(Name), - ensure_actor_stopped(Name). + _ = ensure_actor_stopped(Name), + emqx_cluster_link_metrics:drop_metrics(Name). update_links(LinksConf) -> - [update_link(Link) || Link <- LinksConf]. + [do_update_link(Link) || Link <- LinksConf]. -update_link({OldLinkConf, #{enable := true, name := Name} = NewLinkConf}) -> +do_update_link({OldLinkConf, #{enable := true, name := Name} = NewLinkConf}) -> case what_is_changed(OldLinkConf, NewLinkConf) of both -> _ = ensure_actor_stopped(Name), @@ -215,7 +309,7 @@ update_link({OldLinkConf, #{enable := true, name := Name} = NewLinkConf}) -> msg_resource -> ok = update_msg_fwd_resource(OldLinkConf, NewLinkConf) end; -update_link({_OldLinkConf, #{enable := false, name := Name} = _NewLinkConf}) -> +do_update_link({_OldLinkConf, #{enable := false, name := Name} = _NewLinkConf}) -> _ = emqx_cluster_link_mqtt:remove_msg_fwd_resource(Name), ensure_actor_stopped(Name). @@ -320,3 +414,11 @@ do_convert_certs(LinkName, SSLOpts) -> ), throw({bad_ssl_config, Reason}) end. + +safe_take(Name, Transformations) -> + case lists:splitwith(fun(#{<<"name">> := N}) -> N =/= Name end, Transformations) of + {_Front, []} -> + not_found; + {Front, [Found | Rear]} -> + {Found, Front, Rear} + end. diff --git a/apps/emqx_cluster_link/src/emqx_cluster_link_extrouter.erl b/apps/emqx_cluster_link/src/emqx_cluster_link_extrouter.erl index 79d96e207..44b147454 100644 --- a/apps/emqx_cluster_link/src/emqx_cluster_link_extrouter.erl +++ b/apps/emqx_cluster_link/src/emqx_cluster_link_extrouter.erl @@ -34,6 +34,9 @@ apply_actor_operation/5 ]). +%% Internal export for bookkeeping +-export([count/1]). + %% Strictly monotonically increasing integer. -type smint() :: integer(). @@ -147,6 +150,16 @@ make_extroute_rec_pat(Entry) -> [{1, extroute}, {#extroute.entry, Entry}] ). +%% Internal exports for bookkeeping +count(ClusterName) -> + TopicPat = '_', + RouteIDPat = '_', + Pat = make_extroute_rec_pat( + emqx_trie_search:make_pat(TopicPat, ?ROUTE_ID(ClusterName, RouteIDPat)) + ), + MS = [{Pat, [], [true]}], + ets:select_count(?EXTROUTE_TAB, MS). + %% -record(state, { @@ -280,7 +293,9 @@ apply_operation(Entry, MCounter, OpName, Lane) -> Marker = 1 bsl Lane, case MCounter band Marker of 0 when OpName =:= add -> - mria:dirty_update_counter(?EXTROUTE_TAB, Entry, Marker); + Res = mria:dirty_update_counter(?EXTROUTE_TAB, Entry, Marker), + ?tp("cluster_link_extrouter_route_added", #{}), + Res; Marker when OpName =:= add -> %% Already added. MCounter; @@ -289,6 +304,7 @@ apply_operation(Entry, MCounter, OpName, Lane) -> 0 -> Record = #extroute{entry = Entry, mcounter = 0}, ok = mria:dirty_delete_object(?EXTROUTE_TAB, Record), + ?tp("cluster_link_extrouter_route_deleted", #{}), 0; C -> C diff --git a/apps/emqx_cluster_link/src/emqx_cluster_link_metrics.erl b/apps/emqx_cluster_link/src/emqx_cluster_link_metrics.erl new file mode 100644 index 000000000..36c5e791d --- /dev/null +++ b/apps/emqx_cluster_link/src/emqx_cluster_link_metrics.erl @@ -0,0 +1,60 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_cluster_link_metrics). + +-include("emqx_cluster_link.hrl"). + +%% API +-export([ + maybe_create_metrics/1, + drop_metrics/1, + + get_metrics/1, + routes_set/2 +]). + +%%-------------------------------------------------------------------- +%% Type definitions +%%-------------------------------------------------------------------- + +-define(METRICS, [ + ?route_metric +]). +-define(RATE_METRICS, []). + +%%-------------------------------------------------------------------- +%% metrics API +%%-------------------------------------------------------------------- + +get_metrics(ClusterName) -> + Nodes = emqx:running_nodes(), + Timeout = 15_000, + RouterResults = emqx_metrics_proto_v2:get_metrics(Nodes, ?METRIC_NAME, ClusterName, Timeout), + ResourceId = emqx_cluster_link_mqtt:resource_id(ClusterName), + ResourceResults = emqx_metrics_proto_v2:get_metrics( + Nodes, resource_metrics, ResourceId, Timeout + ), + lists:zip3(Nodes, RouterResults, ResourceResults). + +maybe_create_metrics(ClusterName) -> + case emqx_metrics_worker:has_metrics(?METRIC_NAME, ClusterName) of + true -> + ok = emqx_metrics_worker:reset_metrics(?METRIC_NAME, ClusterName); + false -> + ok = emqx_metrics_worker:create_metrics( + ?METRIC_NAME, ClusterName, ?METRICS, ?RATE_METRICS + ) + end. + +drop_metrics(ClusterName) -> + ok = emqx_metrics_worker:clear_metrics(?METRIC_NAME, ClusterName). + +routes_set(ClusterName, Val) -> + catch emqx_metrics_worker:set_gauge( + ?METRIC_NAME, ClusterName, <<"singleton">>, ?route_metric, Val + ). + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- diff --git a/apps/emqx_cluster_link/src/emqx_cluster_link_mqtt.erl b/apps/emqx_cluster_link/src/emqx_cluster_link_mqtt.erl index 5185803b6..cb9955863 100644 --- a/apps/emqx_cluster_link/src/emqx_cluster_link_mqtt.erl +++ b/apps/emqx_cluster_link/src/emqx_cluster_link_mqtt.erl @@ -9,6 +9,7 @@ -include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("emqx/include/logger.hrl"). -include_lib("snabbkaffe/include/trace.hrl"). +-include_lib("emqx_resource/include/emqx_resource.hrl"). -behaviour(emqx_resource). -behaviour(ecpool_worker). @@ -19,6 +20,7 @@ %% callbacks of behaviour emqx_resource -export([ callback_mode/0, + resource_type/0, on_start/2, on_stop/2, on_query/3, @@ -27,6 +29,7 @@ ]). -export([ + resource_id/1, ensure_msg_fwd_resource/1, remove_msg_fwd_resource/1, decode_route_op/1, @@ -46,6 +49,16 @@ forward/2 ]). +-export([ + get_all_resources_cluster/0, + get_resource_cluster/1 +]). +%% BpAPI / RPC Targets +-export([ + get_resource_local_v1/1, + get_all_resources_local_v1/0 +]). + -define(MSG_CLIENTID_SUFFIX, ":msg:"). -define(MQTT_HOST_OPTS, #{default_port => 1883}). @@ -80,6 +93,12 @@ -define(PUB_TIMEOUT, 10_000). +-type cluster_name() :: binary(). + +-spec resource_id(cluster_name()) -> resource_id(). +resource_id(ClusterName) -> + ?MSG_RES_ID(ClusterName). + -spec ensure_msg_fwd_resource(map()) -> {ok, emqx_resource:resource_data() | already_started} | {error, Reason :: term()}. ensure_msg_fwd_resource(#{name := Name, resource_opts := ResOpts} = ClusterConf) -> @@ -89,16 +108,65 @@ ensure_msg_fwd_resource(#{name := Name, resource_opts := ResOpts} = ClusterConf) }, emqx_resource:create_local(?MSG_RES_ID(Name), ?RES_GROUP, ?MODULE, ClusterConf, ResOpts1). --spec remove_msg_fwd_resource(binary() | map()) -> ok | {error, Reason :: term()}. +-spec remove_msg_fwd_resource(cluster_name()) -> ok | {error, Reason :: term()}. remove_msg_fwd_resource(ClusterName) -> emqx_resource:remove_local(?MSG_RES_ID(ClusterName)). +-spec get_all_resources_cluster() -> + [{node(), emqx_rpc:erpc(#{cluster_name() => emqx_resource:resource_data()})}]. +get_all_resources_cluster() -> + Nodes = emqx:running_nodes(), + Results = emqx_cluster_link_proto_v1:get_all_resources(Nodes), + lists:zip(Nodes, Results). + +-spec get_resource_cluster(cluster_name()) -> + [{node(), {ok, {ok, emqx_resource:resource_data()} | {error, not_found}} | _Error}]. +get_resource_cluster(ClusterName) -> + Nodes = emqx:running_nodes(), + Results = emqx_cluster_link_proto_v1:get_resource(Nodes, ClusterName), + lists:zip(Nodes, Results). + +%% RPC Target in `emqx_cluster_link_proto_v1'. +-spec get_resource_local_v1(cluster_name()) -> + {ok, emqx_resource:resource_data()} | {error, not_found}. +get_resource_local_v1(ClusterName) -> + case emqx_resource:get_instance(?MSG_RES_ID(ClusterName)) of + {ok, _ResourceGroup, ResourceData} -> + {ok, ResourceData}; + {error, not_found} -> + {error, not_found} + end. + +%% RPC Target in `emqx_cluster_link_proto_v1'. +-spec get_all_resources_local_v1() -> #{cluster_name() => emqx_resource:resource_data()}. +get_all_resources_local_v1() -> + lists:foldl( + fun + (?MSG_RES_ID(Name) = Id, Acc) -> + case emqx_resource:get_instance(Id) of + {ok, ?RES_GROUP, ResourceData} -> + Acc#{Name => ResourceData}; + _ -> + Acc + end; + (_Id, Acc) -> + %% Doesn't follow the naming pattern; manually crafted? + Acc + end, + #{}, + emqx_resource:list_group_instances(?RES_GROUP) + ). + %%-------------------------------------------------------------------- %% emqx_resource callbacks (message forwarding) %%-------------------------------------------------------------------- callback_mode() -> async_if_possible. +-spec resource_type() -> atom(). +resource_type() -> + cluster_link_mqtt. + on_start(ResourceId, #{pool_size := PoolSize} = ClusterConf) -> PoolName = ResourceId, Options = [ diff --git a/apps/emqx_cluster_link/src/emqx_cluster_link_router_bootstrap.erl b/apps/emqx_cluster_link/src/emqx_cluster_link_router_bootstrap.erl index 1670c2ab4..6656c8c89 100644 --- a/apps/emqx_cluster_link/src/emqx_cluster_link_router_bootstrap.erl +++ b/apps/emqx_cluster_link/src/emqx_cluster_link_router_bootstrap.erl @@ -3,6 +3,7 @@ %%-------------------------------------------------------------------- -module(emqx_cluster_link_router_bootstrap). +-include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/emqx_router.hrl"). -include_lib("emqx/include/emqx_shared_sub.hrl"). -include_lib("emqx/src/emqx_persistent_session_ds/emqx_ps_ds_int.hrl"). @@ -67,7 +68,7 @@ routes_by_topic(Topics, _IsPersistentRoute = true) -> lists:foldl( fun(T, Acc) -> Routes = emqx_persistent_session_ds_router:lookup_routes(T), - [encode_route(T, ?PERSISTENT_ROUTE_ID(T, D)) || #ps_route{dest = D} <- Routes] ++ Acc + [encode_route(T, ps_route_id(PSRoute)) || #ps_route{} = PSRoute <- Routes] ++ Acc end, [], Topics @@ -79,17 +80,22 @@ routes_by_wildcards(Wildcards, _IsPersistentRoute = false) -> Routes ++ SharedRoutes; routes_by_wildcards(Wildcards, _IsPersistentRoute = true) -> emqx_persistent_session_ds_router:foldl_routes( - fun(#ps_route{dest = D, topic = T}, Acc) -> + fun(#ps_route{topic = T} = PSRoute, Acc) -> case topic_intersect_any(T, Wildcards) of false -> Acc; Intersec -> - [encode_route(Intersec, ?PERSISTENT_ROUTE_ID(T, D)) | Acc] + [encode_route(Intersec, ps_route_id(PSRoute)) | Acc] end end, [] ). +ps_route_id(#ps_route{topic = T, dest = #share_dest{group = Group, session_id = SessionId}}) -> + ?PERSISTENT_SHARED_ROUTE_ID(T, Group, SessionId); +ps_route_id(#ps_route{topic = T, dest = SessionId}) -> + ?PERSISTENT_ROUTE_ID(T, SessionId). + select_routes_by_topics(Topics) -> [encode_route(Topic, Topic) || Topic <- Topics, emqx_broker:subscribers(Topic) =/= []]. diff --git a/apps/emqx_cluster_link/src/emqx_cluster_link_schema.erl b/apps/emqx_cluster_link/src/emqx_cluster_link_schema.erl index f46249a4f..9bbdde2ac 100644 --- a/apps/emqx_cluster_link/src/emqx_cluster_link_schema.erl +++ b/apps/emqx_cluster_link/src/emqx_cluster_link_schema.erl @@ -12,7 +12,7 @@ -export([injected_fields/0]). %% Used in emqx_cluster_link_api --export([links_schema/1]). +-export([links_schema/1, link_schema/0]). -export([ roots/0, @@ -30,22 +30,34 @@ namespace() -> "cluster". roots() -> []. injected_fields() -> - #{cluster => [{links, links_schema(#{})}]}. + #{ + cluster => [ + {links, links_schema(#{})} + ] + }. links_schema(Meta) -> ?HOCON(?ARRAY(?R_REF("link")), Meta#{ default => [], validator => fun links_validator/1, desc => ?DESC("links") }). +link_schema() -> + hoconsc:ref(?MODULE, "link"). + fields("link") -> [ - {enable, ?HOCON(boolean(), #{default => true, desc => ?DESC(enable)})}, + {enable, + ?HOCON(boolean(), #{ + default => true, + importance => ?IMPORTANCE_NO_DOC, + desc => ?DESC(enable) + })}, {name, ?HOCON(binary(), #{required => true, desc => ?DESC(link_name)})}, {server, emqx_schema:servers_sc(#{required => true, desc => ?DESC(server)}, ?MQTT_HOST_OPTS)}, {clientid, ?HOCON(binary(), #{desc => ?DESC(clientid)})}, - {username, ?HOCON(binary(), #{desc => ?DESC(username)})}, - {password, emqx_schema_secret:mk(#{desc => ?DESC(password)})}, + {username, ?HOCON(binary(), #{required => false, desc => ?DESC(username)})}, + {password, emqx_schema_secret:mk(#{required => false, desc => ?DESC(password)})}, {ssl, #{ type => ?R_REF(emqx_schema, "ssl_client_opts"), default => #{<<"enable">> => false}, diff --git a/apps/emqx_cluster_link/src/emqx_cluster_link_sup.erl b/apps/emqx_cluster_link/src/emqx_cluster_link_sup.erl index 2025510fc..42f195cf7 100644 --- a/apps/emqx_cluster_link/src/emqx_cluster_link_sup.erl +++ b/apps/emqx_cluster_link/src/emqx_cluster_link_sup.erl @@ -6,6 +6,8 @@ -behaviour(supervisor). +-include("emqx_cluster_link.hrl"). + -export([start_link/1]). -export([ @@ -27,12 +29,14 @@ init(LinksConf) -> intensity => 10, period => 5 }, + Metrics = emqx_metrics_worker:child_spec(metrics, ?METRIC_NAME), + BookKeeper = bookkeeper_spec(), ExtrouterGC = extrouter_gc_spec(), RouteActors = [ sup_spec(Name, ?ACTOR_MODULE, [LinkConf]) || #{name := Name} = LinkConf <- LinksConf ], - {ok, {SupFlags, [ExtrouterGC | RouteActors]}}. + {ok, {SupFlags, [Metrics, BookKeeper, ExtrouterGC | RouteActors]}}. extrouter_gc_spec() -> %% NOTE: This one is currently global, not per-link. @@ -53,6 +57,15 @@ sup_spec(Id, Mod, Args) -> modules => [Mod] }. +bookkeeper_spec() -> + #{ + id => bookkeeper, + start => {emqx_cluster_link_bookkeeper, start_link, []}, + restart => permanent, + type => worker, + shutdown => 5_000 + }. + ensure_actor(#{name := Name} = LinkConf) -> case supervisor:start_child(?SERVER, sup_spec(Name, ?ACTOR_MODULE, [LinkConf])) of {ok, Pid} -> diff --git a/apps/emqx_cluster_link/src/proto/emqx_cluster_link_proto_v1.erl b/apps/emqx_cluster_link/src/proto/emqx_cluster_link_proto_v1.erl new file mode 100644 index 000000000..725bb8afc --- /dev/null +++ b/apps/emqx_cluster_link/src/proto/emqx_cluster_link_proto_v1.erl @@ -0,0 +1,31 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_cluster_link_proto_v1). + +-behaviour(emqx_bpapi). + +-export([ + introduced_in/0, + + get_resource/2, + get_all_resources/1 +]). + +-include_lib("emqx/include/bpapi.hrl"). + +-define(TIMEOUT, 15000). + +introduced_in() -> + "5.7.2". + +-spec get_resource([node()], binary()) -> + emqx_rpc:erpc_multicall({ok, emqx_resource:resource_data()} | {error, not_found}). +get_resource(Nodes, ClusterName) -> + erpc:multicall(Nodes, emqx_cluster_link_mqtt, get_resource_local_v1, [ClusterName], ?TIMEOUT). + +-spec get_all_resources([node()]) -> + emqx_rpc:erpc_multicall(#{binary() => emqx_resource:resource_data()}). +get_all_resources(Nodes) -> + erpc:multicall(Nodes, emqx_cluster_link_mqtt, get_all_resources_local_v1, [], ?TIMEOUT). diff --git a/apps/emqx_cluster_link/test/emqx_cluster_link_api_SUITE.erl b/apps/emqx_cluster_link/test/emqx_cluster_link_api_SUITE.erl index c5ec8da6c..8157c86d6 100644 --- a/apps/emqx_cluster_link/test/emqx_cluster_link_api_SUITE.erl +++ b/apps/emqx_cluster_link/test/emqx_cluster_link_api_SUITE.erl @@ -11,6 +11,8 @@ -include_lib("common_test/include/ct.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). +-import(emqx_common_test_helpers, [on_exit/1]). + -define(API_PATH, emqx_mgmt_api_test_util:api_path(["cluster", "links"])). -define(CONF_PATH, [cluster, links]). @@ -37,8 +39,28 @@ "-----END CERTIFICATE-----" >>). +-define(ON(NODE, BODY), erpc:call(NODE, fun() -> BODY end)). + +%%------------------------------------------------------------------------------ +%% CT boilerplate +%%------------------------------------------------------------------------------ + all() -> - emqx_common_test_helpers:all(?MODULE). + AllTCs = emqx_common_test_helpers:all(?MODULE), + OtherTCs = AllTCs -- cluster_test_cases(), + [ + {group, cluster} + | OtherTCs + ]. + +groups() -> + [{cluster, cluster_test_cases()}]. + +cluster_test_cases() -> + [ + t_status, + t_metrics + ]. init_per_suite(Config) -> %% This is called by emqx_machine in EMQX release @@ -47,7 +69,7 @@ init_per_suite(Config) -> [ emqx_conf, emqx_management, - {emqx_dashboard, "dashboard.listeners.http { enable = true, bind = 18083 }"}, + emqx_mgmt_api_test_util:emqx_dashboard(), emqx_cluster_link ], #{work_dir => emqx_cth_suite:work_dir(Config)} @@ -60,73 +82,641 @@ end_per_suite(Config) -> emqx_config:delete_override_conf_files(), ok. +init_per_group(cluster = Group, Config) -> + ok = emqx_cth_suite:stop_apps([emqx_dashboard]), + SourceClusterSpec = emqx_cluster_link_SUITE:mk_source_cluster(Group, Config), + TargetClusterSpec = emqx_cluster_link_SUITE:mk_target_cluster(Group, Config), + SourceNodes = [SN1 | _] = emqx_cth_cluster:start(SourceClusterSpec), + TargetNodes = [TN1 | _] = emqx_cth_cluster:start(TargetClusterSpec), + emqx_cluster_link_SUITE:start_cluster_link(SourceNodes ++ TargetNodes, Config), + erpc:call(SN1, emqx_cth_suite, start_apps, [ + [emqx_management, emqx_mgmt_api_test_util:emqx_dashboard()], + #{work_dir => emqx_cth_suite:work_dir(Group, Config)} + ]), + erpc:call(TN1, emqx_cth_suite, start_apps, [ + [ + emqx_management, + emqx_mgmt_api_test_util:emqx_dashboard( + "dashboard.listeners.http { enable = true, bind = 28083 }" + ) + ], + #{work_dir => emqx_cth_suite:work_dir(Group, Config)} + ]), + [ + {source_nodes, SourceNodes}, + {target_nodes, TargetNodes} + | Config + ]; +init_per_group(_Group, Config) -> + Config. + +end_per_group(cluster, Config) -> + SourceNodes = ?config(source_nodes, Config), + TargetNodes = ?config(target_nodes, Config), + ok = emqx_cth_cluster:stop(SourceNodes), + ok = emqx_cth_cluster:stop(TargetNodes), + _ = emqx_cth_suite:start_apps( + [emqx_mgmt_api_test_util:emqx_dashboard()], + #{work_dir => emqx_cth_suite:work_dir(Config)} + ), + ok; +end_per_group(_Group, _Config) -> + ok. + auth_header() -> - {ok, API} = emqx_common_test_http:create_default_app(), - emqx_common_test_http:auth_header(API). + emqx_mgmt_api_test_util:auth_header_(). init_per_testcase(_TC, Config) -> {ok, _} = emqx_cluster_link_config:update([]), + snabbkaffe:start_trace(), Config. end_per_testcase(_TC, _Config) -> + snabbkaffe:stop(), + emqx_common_test_helpers:call_janitor(), ok. -t_put_get_valid(Config) -> - Auth = ?config(auth, Config), - Path = ?API_PATH, - {ok, Resp} = emqx_mgmt_api_test_util:request_api(get, Path, Auth), - ?assertMatch([], emqx_utils_json:decode(Resp)), +%%------------------------------------------------------------------------------ +%% Helper fns +%%------------------------------------------------------------------------------ - Link1 = #{ +api_root() -> + <<"cluster/links">>. + +list() -> + Path = emqx_mgmt_api_test_util:api_path([api_root()]), + emqx_mgmt_api_test_util:simple_request(get, Path, _Params = ""). + +get_link(Name) -> + get_link(source, Name). + +get_link(SourceOrTargetCluster, Name) -> + Host = host(SourceOrTargetCluster), + Path = emqx_mgmt_api_test_util:api_path(Host, [api_root(), "link", Name]), + emqx_mgmt_api_test_util:simple_request(get, Path, _Params = ""). + +delete_link(Name) -> + Path = emqx_mgmt_api_test_util:api_path([api_root(), "link", Name]), + emqx_mgmt_api_test_util:simple_request(delete, Path, _Params = ""). + +update_link(Name, Params) -> + update_link(source, Name, Params). + +update_link(SourceOrTargetCluster, Name, Params) -> + Host = host(SourceOrTargetCluster), + Path = emqx_mgmt_api_test_util:api_path(Host, [api_root(), "link", Name]), + emqx_mgmt_api_test_util:simple_request(put, Path, Params). + +create_link(Name, Params0) -> + Params = Params0#{<<"name">> => Name}, + Path = emqx_mgmt_api_test_util:api_path([api_root()]), + emqx_mgmt_api_test_util:simple_request(post, Path, Params). + +get_metrics(Name) -> + get_metrics(source, Name). + +get_metrics(SourceOrTargetCluster, Name) -> + Host = host(SourceOrTargetCluster), + Path = emqx_mgmt_api_test_util:api_path(Host, [api_root(), "link", Name, "metrics"]), + emqx_mgmt_api_test_util:simple_request(get, Path, _Params = []). + +host(source) -> "http://127.0.0.1:18083"; +host(target) -> "http://127.0.0.1:28083". + +link_params() -> + link_params(_Overrides = #{}). + +link_params(Overrides) -> + Default = #{ + <<"clientid">> => <<"linkclientid">>, <<"pool_size">> => 1, <<"server">> => <<"emqxcl_2.nohost:31883">>, - <<"topics">> => [<<"t/test-topic">>, <<"t/test/#">>], - <<"name">> => <<"emqcl_1">> + <<"topics">> => [<<"t/test-topic">>, <<"t/test/#">>] }, - Link2 = #{ - <<"pool_size">> => 1, - <<"server">> => <<"emqxcl_2.nohost:41883">>, - <<"topics">> => [<<"t/test-topic">>, <<"t/test/#">>], - <<"name">> => <<"emqcl_2">> - }, - ?assertMatch({ok, _}, emqx_mgmt_api_test_util:request_api(put, Path, "", Auth, [Link1, Link2])), + emqx_utils_maps:deep_merge(Default, Overrides). - {ok, Resp1} = emqx_mgmt_api_test_util:request_api(get, Path, Auth), - ?assertMatch([Link1, Link2], emqx_utils_json:decode(Resp1)), +%%------------------------------------------------------------------------------ +%% Test cases +%%------------------------------------------------------------------------------ + +t_put_get_valid(_Config) -> + ?assertMatch({200, []}, list()), + + Name1 = <<"emqcl_1">>, + Link1 = link_params(#{ + <<"server">> => <<"emqxcl_2.nohost:31883">>, + <<"name">> => Name1 + }), + Name2 = <<"emqcl_2">>, + Link2 = link_params(#{ + <<"server">> => <<"emqxcl_2.nohost:41883">>, + <<"name">> => Name2 + }), + ?assertMatch({201, _}, create_link(Name1, Link1)), + ?assertMatch({201, _}, create_link(Name2, Link2)), + ?assertMatch({200, [_, _]}, list()), DisabledLink1 = Link1#{<<"enable">> => false}, - ?assertMatch( - {ok, _}, emqx_mgmt_api_test_util:request_api(put, Path, "", Auth, [DisabledLink1, Link2]) - ), - - {ok, Resp2} = emqx_mgmt_api_test_util:request_api(get, Path, Auth), - ?assertMatch([DisabledLink1, Link2], emqx_utils_json:decode(Resp2)), + ?assertMatch({200, _}, update_link(Name1, maps:remove(<<"name">>, DisabledLink1))), + ?assertMatch({200, #{<<"enable">> := false}}, get_link(Name1)), + ?assertMatch({200, #{<<"enable">> := true}}, get_link(Name2)), SSL = #{<<"enable">> => true, <<"cacertfile">> => ?CACERT}, SSLLink1 = Link1#{<<"ssl">> => SSL}, + ?assertMatch({200, _}, update_link(Name1, maps:remove(<<"name">>, SSLLink1))), ?assertMatch( - {ok, _}, emqx_mgmt_api_test_util:request_api(put, Path, "", Auth, [Link2, SSLLink1]) + {200, #{<<"ssl">> := #{<<"enable">> := true, <<"cacertfile">> := _Path}}}, + get_link(Name1) ), - {ok, Resp3} = emqx_mgmt_api_test_util:request_api(get, Path, Auth), + ok. +t_put_invalid(_Config) -> + Name = <<"l1">>, + {201, _} = create_link(Name, link_params()), ?assertMatch( - [Link2, #{<<"ssl">> := #{<<"enable">> := true, <<"cacertfile">> := _Path}}], - emqx_utils_json:decode(Resp3) + {400, _}, + update_link(Name, maps:remove(<<"server">>, link_params())) ). -t_put_invalid(Config) -> - Auth = ?config(auth, Config), - Path = ?API_PATH, - Link = #{ - <<"pool_size">> => 1, - <<"server">> => <<"emqxcl_2.nohost:31883">>, - <<"topics">> => [<<"t/test-topic">>, <<"t/test/#">>], - <<"name">> => <<"emqcl_1">> - }, +%% Tests a sequence of CRUD operations and their expected responses, for common use cases +%% and configuration states. +t_crud(_Config) -> + %% No links initially. + ?assertMatch({200, []}, list()), + NameA = <<"a">>, + ?assertMatch({404, _}, get_link(NameA)), + ?assertMatch({404, _}, delete_link(NameA)), + ?assertMatch({404, _}, update_link(NameA, link_params())), + ?assertMatch({404, _}, get_metrics(NameA)), + + Params1 = link_params(), ?assertMatch( - {error, {_, 400, _}}, emqx_mgmt_api_test_util:request_api(put, Path, "", Auth, [Link, Link]) + {201, #{ + <<"name">> := NameA, + <<"status">> := _, + <<"node_status">> := [#{<<"node">> := _, <<"status">> := _} | _] + }}, + create_link(NameA, Params1) + ), + ?assertMatch({400, #{<<"code">> := <<"ALREADY_EXISTS">>}}, create_link(NameA, Params1)), + ?assertMatch( + {200, [ + #{ + <<"name">> := NameA, + <<"status">> := _, + <<"node_status">> := [#{<<"node">> := _, <<"status">> := _} | _] + } + ]}, + list() ), ?assertMatch( - {error, {_, 400, _}}, - emqx_mgmt_api_test_util:request_api(put, Path, "", Auth, [maps:remove(<<"name">>, Link)]) - ). + {200, #{ + <<"name">> := NameA, + <<"status">> := _, + <<"node_status">> := [#{<<"node">> := _, <<"status">> := _} | _] + }}, + get_link(NameA) + ), + ?assertMatch({200, _}, get_metrics(NameA)), + + Params2 = Params1#{<<"pool_size">> := 2}, + ?assertMatch( + {200, #{ + <<"name">> := NameA, + <<"status">> := _, + <<"node_status">> := [#{<<"node">> := _, <<"status">> := _} | _] + }}, + update_link(NameA, Params2) + ), + + ?assertMatch({204, _}, delete_link(NameA)), + ?assertMatch({404, _}, delete_link(NameA)), + ?assertMatch({404, _}, get_link(NameA)), + ?assertMatch({404, _}, update_link(NameA, Params1)), + ?assertMatch({404, _}, get_metrics(NameA)), + ?assertMatch({200, []}, list()), + + ok. + +%% Verifies the behavior of reported status under different conditions when listing all +%% links and when fetching a specific link. +t_status(Config) -> + [SN1 | _] = ?config(source_nodes, Config), + Name = <<"cl.target">>, + ?retry( + 100, + 10, + ?assertMatch( + {200, [ + #{ + <<"status">> := <<"connected">>, + <<"node_status">> := [ + #{ + <<"node">> := _, + <<"status">> := <<"connected">> + }, + #{ + <<"node">> := _, + <<"status">> := <<"connected">> + } + ] + } + ]}, + list() + ) + ), + ?assertMatch( + {200, #{ + <<"status">> := <<"connected">>, + <<"node_status">> := [ + #{ + <<"node">> := _, + <<"status">> := <<"connected">> + }, + #{ + <<"node">> := _, + <<"status">> := <<"connected">> + } + ] + }}, + get_link(Name) + ), + + %% If one of the nodes reports a different status, the cluster is inconsistent. + ProtoMod = emqx_cluster_link_proto_v1, + ?ON(SN1, begin + ok = meck:new(ProtoMod, [no_link, passthrough, no_history]), + meck:expect(ProtoMod, get_all_resources, fun(Nodes) -> + [Res1, {ok, Res2A} | Rest] = meck:passthrough([Nodes]), + %% Res2A :: #{cluster_name() => emqx_resource:resource_data()} + Res2B = maps:map(fun(_, Data) -> Data#{status := disconnected} end, Res2A), + [Res1, {ok, Res2B} | Rest] + end), + meck:expect(ProtoMod, get_resource, fun(Nodes, LinkName) -> + [Res1, {ok, {ok, Res2A}} | Rest] = meck:passthrough([Nodes, LinkName]), + Res2B = Res2A#{status := disconnected}, + [Res1, {ok, {ok, Res2B}} | Rest] + end) + end), + on_exit(fun() -> catch ?ON(SN1, meck:unload()) end), + ?assertMatch( + {200, [ + #{ + <<"status">> := <<"inconsistent">>, + <<"node_status">> := [ + #{ + <<"node">> := _, + <<"status">> := <<"connected">> + }, + #{ + <<"node">> := _, + <<"status">> := <<"disconnected">> + } + ] + } + ]}, + list() + ), + ?assertMatch( + {200, #{ + <<"status">> := <<"inconsistent">>, + <<"node_status">> := [ + #{ + <<"node">> := _, + <<"status">> := <<"connected">> + }, + #{ + <<"node">> := _, + <<"status">> := <<"disconnected">> + } + ] + }}, + get_link(Name) + ), + + %% Simulating erpc failures + ?ON(SN1, begin + meck:expect(ProtoMod, get_all_resources, fun(Nodes) -> + [Res1, _ | Rest] = meck:passthrough([Nodes]), + [Res1, {error, {erpc, noconnection}} | Rest] + end), + meck:expect(ProtoMod, get_resource, fun(Nodes, LinkName) -> + [Res1, _ | Rest] = meck:passthrough([Nodes, LinkName]), + [Res1, {error, {erpc, noconnection}} | Rest] + end) + end), + ?assertMatch( + {200, [ + #{ + <<"status">> := <<"inconsistent">>, + <<"node_status">> := [ + #{ + <<"node">> := _, + <<"status">> := <<"connected">> + }, + #{ + <<"node">> := _, + <<"status">> := <<"inconsistent">>, + <<"reason">> := _ + } + ] + } + ]}, + list() + ), + ?assertMatch( + {200, #{ + <<"status">> := <<"inconsistent">>, + <<"node_status">> := [ + #{ + <<"node">> := _, + <<"status">> := <<"connected">> + }, + #{ + <<"node">> := _, + <<"status">> := <<"inconsistent">>, + <<"reason">> := _ + } + ] + }}, + get_link(Name) + ), + %% Simulate another inconsistency + ?ON(SN1, begin + meck:expect(ProtoMod, get_resource, fun(Nodes, LinkName) -> + [Res1, _ | Rest] = meck:passthrough([Nodes, LinkName]), + [Res1, {ok, {error, not_found}} | Rest] + end) + end), + ?assertMatch( + {200, #{ + <<"status">> := <<"inconsistent">>, + <<"node_status">> := [ + #{ + <<"node">> := _, + <<"status">> := <<"connected">> + }, + #{ + <<"node">> := _, + <<"status">> := <<"disconnected">> + } + ] + }}, + get_link(Name) + ), + + ok. + +t_metrics(Config) -> + ct:timetrap({seconds, 10}), + [SN1, SN2] = ?config(source_nodes, Config), + [TN1, TN2] = ?config(target_nodes, Config), + %% N.B. Link names on each cluster, so they are switched. + SourceName = <<"cl.target">>, + TargetName = <<"cl.source">>, + + ?assertMatch( + {200, #{ + <<"metrics">> := #{ + <<"router">> := #{ + <<"routes">> := 0 + }, + <<"forwarding">> := #{ + <<"matched">> := _, + <<"success">> := _, + <<"failed">> := _, + <<"dropped">> := _, + <<"retried">> := _, + <<"received">> := _, + <<"queuing">> := _, + <<"inflight">> := _, + <<"rate">> := _, + <<"rate_last5m">> := _, + <<"rate_max">> := _ + } + }, + <<"node_metrics">> := [ + #{ + <<"node">> := _, + <<"metrics">> := #{ + <<"router">> := #{ + <<"routes">> := 0 + }, + <<"forwarding">> := #{ + <<"matched">> := _, + <<"success">> := _, + <<"failed">> := _, + <<"dropped">> := _, + <<"retried">> := _, + <<"received">> := _, + <<"queuing">> := _, + <<"inflight">> := _, + <<"rate">> := _, + <<"rate_last5m">> := _, + <<"rate_max">> := _ + } + } + }, + #{ + <<"node">> := _, + <<"metrics">> := #{ + <<"router">> := #{ + <<"routes">> := 0 + }, + <<"forwarding">> := #{ + <<"matched">> := _, + <<"success">> := _, + <<"failed">> := _, + <<"dropped">> := _, + <<"retried">> := _, + <<"received">> := _, + <<"queuing">> := _, + <<"inflight">> := _, + <<"rate">> := _, + <<"rate_last5m">> := _, + <<"rate_max">> := _ + } + } + } + ] + }}, + get_metrics(source, SourceName) + ), + ?assertMatch( + {200, #{ + <<"metrics">> := #{<<"router">> := #{<<"routes">> := 0}}, + <<"node_metrics">> := [ + #{ + <<"node">> := _, + <<"metrics">> := #{<<"router">> := #{<<"routes">> := 0}} + }, + #{ + <<"node">> := _, + <<"metrics">> := #{<<"router">> := #{<<"routes">> := 0}} + } + ] + }}, + get_metrics(target, TargetName) + ), + + SourceC1 = emqx_cluster_link_SUITE:start_client(<<"sc1">>, SN1), + SourceC2 = emqx_cluster_link_SUITE:start_client(<<"sc2">>, SN2), + {ok, _, _} = emqtt:subscribe(SourceC1, <<"t/sc1">>), + {ok, _, _} = emqtt:subscribe(SourceC2, <<"t/sc2">>), + + %% Still no routes, as routes in the source cluster are replicated to the target + %% cluster. + ?assertMatch( + {200, #{ + <<"metrics">> := #{<<"router">> := #{<<"routes">> := 0}}, + <<"node_metrics">> := [ + #{ + <<"node">> := _, + <<"metrics">> := #{<<"router">> := #{<<"routes">> := 0}} + }, + #{ + <<"node">> := _, + <<"metrics">> := #{<<"router">> := #{<<"routes">> := 0}} + } + ] + }}, + get_metrics(source, SourceName) + ), + ?assertMatch( + {200, #{ + <<"metrics">> := #{<<"router">> := #{<<"routes">> := 0}}, + <<"node_metrics">> := [ + #{ + <<"node">> := _, + <<"metrics">> := #{<<"router">> := #{<<"routes">> := 0}} + }, + #{ + <<"node">> := _, + <<"metrics">> := #{<<"router">> := #{<<"routes">> := 0}} + } + ] + }}, + get_metrics(target, TargetName) + ), + + TargetC1 = emqx_cluster_link_SUITE:start_client(<<"tc1">>, TN1), + TargetC2 = emqx_cluster_link_SUITE:start_client(<<"tc2">>, TN2), + {ok, _, _} = emqtt:subscribe(TargetC1, <<"t/tc1">>), + {ok, _, _} = emqtt:subscribe(TargetC2, <<"t/tc2">>), + {_, {ok, _}} = + ?wait_async_action( + begin + {ok, _, _} = emqtt:subscribe(TargetC1, <<"t/tc1">>), + {ok, _, _} = emqtt:subscribe(TargetC2, <<"t/tc2">>) + end, + #{?snk_kind := clink_route_sync_complete} + ), + + %% Routes = 2 in source cluster, because the target cluster has some topic filters + %% configured and subscribers to them, which were replicated to the source cluster. + %% This metric is global (cluster-wide). + ?retry( + 300, + 10, + ?assertMatch( + {200, #{ + <<"metrics">> := #{<<"router">> := #{<<"routes">> := 2}}, + <<"node_metrics">> := [ + #{<<"metrics">> := #{<<"router">> := #{<<"routes">> := 2}}}, + #{<<"metrics">> := #{<<"router">> := #{<<"routes">> := 2}}} + ] + }}, + get_metrics(source, SourceName) + ) + ), + ?assertMatch( + {200, #{ + <<"metrics">> := #{<<"router">> := #{<<"routes">> := 0}}, + <<"node_metrics">> := _ + }}, + get_metrics(target, TargetName) + ), + + %% Unsubscribe and remove route. + ct:pal("unsubscribing"), + {_, {ok, _}} = + ?wait_async_action( + begin + {ok, _, _} = emqtt:unsubscribe(TargetC1, <<"t/tc1">>) + end, + #{?snk_kind := clink_route_sync_complete} + ), + + ?retry( + 300, + 10, + ?assertMatch( + {200, #{ + <<"metrics">> := #{<<"router">> := #{<<"routes">> := 1}}, + <<"node_metrics">> := [ + #{<<"metrics">> := #{<<"router">> := #{<<"routes">> := 1}}}, + #{<<"metrics">> := #{<<"router">> := #{<<"routes">> := 1}}} + ] + }}, + get_metrics(source, SourceName) + ) + ), + + %% Disabling the link should remove the routes. + ct:pal("disabling"), + {200, TargetLink0} = get_link(target, TargetName), + TargetLink1 = maps:without([<<"status">>, <<"node_status">>], TargetLink0), + TargetLink2 = TargetLink1#{<<"enable">> := false}, + {_, {ok, _}} = + ?wait_async_action( + begin + {200, _} = update_link(target, TargetName, TargetLink2), + %% Note that only when the GC runs and collects the stopped actor it'll actually + %% remove the routes + NowMS = erlang:system_time(millisecond), + TTL = emqx_cluster_link_config:actor_ttl(), + ct:pal("gc"), + %% 2 Actors: one for normal routes, one for PS routes + 1 = ?ON(SN1, emqx_cluster_link_extrouter:actor_gc(#{timestamp => NowMS + TTL * 3})), + 1 = ?ON(SN1, emqx_cluster_link_extrouter:actor_gc(#{timestamp => NowMS + TTL * 3})), + ct:pal("gc done"), + ok + end, + #{?snk_kind := "cluster_link_extrouter_route_deleted"} + ), + + ?retry( + 300, + 10, + ?assertMatch( + {200, #{ + <<"metrics">> := #{<<"router">> := #{<<"routes">> := 0}}, + <<"node_metrics">> := _ + }}, + get_metrics(source, SourceName) + ) + ), + + %% Enabling again + TargetLink3 = TargetLink2#{<<"enable">> := true}, + {_, {ok, _}} = + ?wait_async_action( + begin + {200, _} = update_link(target, TargetName, TargetLink3) + end, + #{?snk_kind := "cluster_link_extrouter_route_added"} + ), + + ?retry( + 300, + 10, + ?assertMatch( + {200, #{ + <<"metrics">> := #{<<"router">> := #{<<"routes">> := 1}}, + <<"node_metrics">> := _ + }}, + get_metrics(source, SourceName) + ) + ), + + ok. diff --git a/apps/emqx_conf/src/emqx_conf_cli.erl b/apps/emqx_conf/src/emqx_conf_cli.erl index 3c0e98096..cb0c0b7d8 100644 --- a/apps/emqx_conf/src/emqx_conf_cli.erl +++ b/apps/emqx_conf/src/emqx_conf_cli.erl @@ -302,8 +302,6 @@ hidden_roots() -> <<"trace">>, <<"stats">>, <<"broker">>, - <<"persistent_session_store">>, - <<"durable_sessions">>, <<"plugins">>, <<"zones">> ]. diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index df906911e..4d0b8c1fa 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -83,7 +83,8 @@ dropped_msg_due_to_mqueue_is_full, socket_receive_paused_by_rate_limit, data_bridge_buffer_overflow, - external_broker_crashed + external_broker_crashed, + unrecoverable_resource_error ]). -define(DEFAULT_RPC_PORT, 5369). @@ -194,18 +195,6 @@ fields("cluster") -> 'readOnly' => true } )}, - {"core_nodes", - sc( - node_array(), - #{ - %% This config is nerver needed (since 5.0.0) - importance => ?IMPORTANCE_HIDDEN, - mapping => "mria.core_nodes", - default => [], - 'readOnly' => true, - desc => ?DESC(db_core_nodes) - } - )}, {"autoclean", sc( emqx_schema:duration(), @@ -600,7 +589,7 @@ fields("node") -> )}, {"role", sc( - hoconsc:enum([core, replicant]), + hoconsc:enum([core] ++ emqx_schema_hooks:injection_point('node.role')), #{ mapping => "mria.node_role", default => core, @@ -997,6 +986,7 @@ fields("log_overload_kill") -> boolean(), #{ default => true, + importance => ?IMPORTANCE_NO_DOC, desc => ?DESC("log_overload_kill_enable") } )}, @@ -1032,6 +1022,7 @@ fields("log_burst_limit") -> boolean(), #{ default => true, + importance => ?IMPORTANCE_NO_DOC, desc => ?DESC("log_burst_limit_enable") } )}, @@ -1269,6 +1260,11 @@ log_handler_common_confs(Handler, Default) -> EnvValue = os:getenv("EMQX_DEFAULT_LOG_HANDLER"), Enable = lists:member(EnvValue, EnableValues), LevelDesc = maps:get(level_desc, Default, "common_handler_level"), + EnableImportance = + case Enable of + true -> ?IMPORTANCE_NO_DOC; + false -> ?IMPORTANCE_MEDIUM + end, [ {"level", sc( @@ -1285,7 +1281,7 @@ log_handler_common_confs(Handler, Default) -> #{ default => Enable, desc => ?DESC("common_handler_enable"), - importance => ?IMPORTANCE_MEDIUM + importance => EnableImportance } )}, {"formatter", diff --git a/apps/emqx_conf/src/emqx_conf_schema_inject.erl b/apps/emqx_conf/src/emqx_conf_schema_inject.erl index 0e0f36401..9564a5915 100644 --- a/apps/emqx_conf/src/emqx_conf_schema_inject.erl +++ b/apps/emqx_conf/src/emqx_conf_schema_inject.erl @@ -22,12 +22,18 @@ schemas() -> schemas(emqx_release:edition()). schemas(Edition) -> - auth_ext(Edition) ++ + mria(Edition) ++ + auth_ext(Edition) ++ cluster_linking(Edition) ++ authn(Edition) ++ authz() ++ customized(Edition). +mria(ce) -> + []; +mria(ee) -> + [emqx_enterprise_schema]. + auth_ext(ce) -> []; auth_ext(ee) -> @@ -55,7 +61,10 @@ authn_mods(ce) -> ]; authn_mods(ee) -> authn_mods(ce) ++ - [emqx_gcp_device_authn_schema]. + [ + emqx_gcp_device_authn_schema, + emqx_authn_scram_restapi_schema + ]. authz() -> [{emqx_authz_schema, authz_mods()}]. diff --git a/apps/emqx_conf/test/emqx_conf_app_SUITE.erl b/apps/emqx_conf/test/emqx_conf_app_SUITE.erl index 5ba3f0b49..711274aaa 100644 --- a/apps/emqx_conf/test/emqx_conf_app_SUITE.erl +++ b/apps/emqx_conf/test/emqx_conf_app_SUITE.erl @@ -30,7 +30,7 @@ t_copy_conf_override_on_restarts(Config) -> ct:timetrap({seconds, 120}), Cluster = cluster( ?FUNCTION_NAME, - [cluster_spec({core, 1}), cluster_spec({core, 2}), cluster_spec({core, 3})], + [cluster_spec(1), cluster_spec(2), cluster_spec(3)], Config ), @@ -59,7 +59,7 @@ t_copy_new_data_dir(Config) -> ct:timetrap({seconds, 120}), Cluster = cluster( ?FUNCTION_NAME, - [cluster_spec({core, 4}), cluster_spec({core, 5}), cluster_spec({core, 6})], + [cluster_spec(4), cluster_spec(5), cluster_spec(6)], Config ), @@ -84,7 +84,7 @@ t_copy_deprecated_data_dir(Config) -> ct:timetrap({seconds, 120}), Cluster = cluster( ?FUNCTION_NAME, - [cluster_spec({core, 7}), cluster_spec({core, 8}), cluster_spec({core, 9})], + [cluster_spec(7), cluster_spec(8), cluster_spec(9)], Config ), @@ -109,7 +109,7 @@ t_no_copy_from_newer_version_node(Config) -> ct:timetrap({seconds, 120}), Cluster = cluster( ?FUNCTION_NAME, - [cluster_spec({core, 10}), cluster_spec({core, 11}), cluster_spec({core, 12})], + [cluster_spec(10), cluster_spec(11), cluster_spec(12)], Config ), OKs = [ok, ok, ok], @@ -242,12 +242,12 @@ cluster(TC, Specs, Config) -> {emqx_conf, #{}} ], emqx_cth_cluster:mk_nodespecs( - [{Name, #{role => Role, apps => Apps}} || {Role, Name} <- Specs], + [{Name, #{apps => Apps}} || Name <- Specs], #{work_dir => emqx_cth_suite:work_dir(TC, Config)} ). -cluster_spec({Type, Num}) -> - {Type, list_to_atom(atom_to_list(?MODULE) ++ integer_to_list(Num))}. +cluster_spec(Num) -> + list_to_atom(atom_to_list(?MODULE) ++ integer_to_list(Num)). sort_highest_uptime(Nodes) -> Ranking = lists:sort([{-get_node_uptime(N), N} || N <- Nodes]), diff --git a/apps/emqx_conf/test/emqx_conf_schema_tests.erl b/apps/emqx_conf/test/emqx_conf_schema_tests.erl index 72834f6d2..2bb56e5cc 100644 --- a/apps/emqx_conf/test/emqx_conf_schema_tests.erl +++ b/apps/emqx_conf/test/emqx_conf_schema_tests.erl @@ -32,7 +32,6 @@ name = emqxcl discovery_strategy = static static.seeds = ~p - core_nodes = ~p } "). @@ -41,7 +40,7 @@ array_nodes_test() -> ExpectNodes = ['emqx1@127.0.0.1', 'emqx2@127.0.0.1'], lists:foreach( fun(Nodes) -> - ConfFile = to_bin(?BASE_CONF, [Nodes, Nodes]), + ConfFile = to_bin(?BASE_CONF, [Nodes]), {ok, Conf} = hocon:binary(ConfFile, #{format => richmap}), ConfList = hocon_tconf:generate(emqx_conf_schema, Conf), VMArgs = proplists:get_value(vm_args, ConfList), @@ -57,11 +56,6 @@ array_nodes_test() -> {static, [{seeds, ExpectNodes}]}, ClusterDiscovery, Nodes - ), - ?assertEqual( - ExpectNodes, - proplists:get_value(core_nodes, proplists:get_value(mria, ConfList)), - Nodes ) end, [["emqx1@127.0.0.1", "emqx2@127.0.0.1"], "emqx1@127.0.0.1, emqx2@127.0.0.1"] @@ -158,7 +152,7 @@ outdated_log_test() -> validate_log(Conf) -> ensure_acl_conf(), - BaseConf = to_bin(?BASE_CONF, ["emqx1@127.0.0.1", "emqx1@127.0.0.1"]), + BaseConf = to_bin(?BASE_CONF, ["emqx1@127.0.0.1"]), Conf0 = <>, {ok, ConfMap0} = hocon:binary(Conf0, #{format => richmap}), ConfList = hocon_tconf:generate(emqx_conf_schema, ConfMap0), @@ -214,7 +208,7 @@ validate_log(Conf) -> file_log_infinity_rotation_size_test_() -> ensure_acl_conf(), - BaseConf = to_bin(?BASE_CONF, ["emqx1@127.0.0.1", "emqx1@127.0.0.1"]), + BaseConf = to_bin(?BASE_CONF, ["emqx1@127.0.0.1"]), Gen = fun(#{count := Count, size := Size}) -> Conf0 = to_bin(?FILE_LOG_BASE_CONF, [Count, Size]), Conf1 = [BaseConf, Conf0], @@ -292,7 +286,7 @@ log_rotation_count_limit_test() -> rotation_size = \"1024MB\" } ", - BaseConf = to_bin(?BASE_CONF, ["emqx1@127.0.0.1", "emqx1@127.0.0.1"]), + BaseConf = to_bin(?BASE_CONF, ["emqx1@127.0.0.1"]), lists:foreach(fun({Conf, Count}) -> Conf0 = <>, {ok, ConfMap0} = hocon:binary(Conf0, #{format => richmap}), @@ -352,7 +346,7 @@ log_rotation_count_limit_test() -> authn_validations_test() -> ensure_acl_conf(), - BaseConf = to_bin(?BASE_CONF, ["emqx1@127.0.0.1", "emqx1@127.0.0.1"]), + BaseConf = to_bin(?BASE_CONF, ["emqx1@127.0.0.1"]), OKHttps = to_bin(?BASE_AUTHN_ARRAY, [post, true, <<"https://127.0.0.1:8080">>]), Conf0 = <>, @@ -410,7 +404,7 @@ authn_validations_test() -> listeners_test() -> ensure_acl_conf(), - BaseConf = to_bin(?BASE_CONF, ["emqx1@127.0.0.1", "emqx1@127.0.0.1"]), + BaseConf = to_bin(?BASE_CONF, ["emqx1@127.0.0.1"]), Conf = <>, {ok, ConfMap0} = hocon:binary(Conf, #{format => richmap}), diff --git a/apps/emqx_connector/include/emqx_connector.hrl b/apps/emqx_connector/include/emqx_connector.hrl index 4b29dd5ce..0004cd72c 100644 --- a/apps/emqx_connector/include/emqx_connector.hrl +++ b/apps/emqx_connector/include/emqx_connector.hrl @@ -37,4 +37,4 @@ "The " ++ TYPE ++ " default port " ++ DEFAULT_PORT ++ " is used if `[:Port]` is not specified." ). --define(CONNECTOR_RESOURCE_GROUP, <<"emqx_connector">>). +-define(CONNECTOR_RESOURCE_GROUP, <<"connector">>). diff --git a/apps/emqx_connector/mix.exs b/apps/emqx_connector/mix.exs index a641c27fe..a818d8072 100644 --- a/apps/emqx_connector/mix.exs +++ b/apps/emqx_connector/mix.exs @@ -33,6 +33,7 @@ defmodule EMQXConnector.MixProject do [ {:emqx, in_umbrella: true}, {:emqx_resource, in_umbrella: true}, + {:emqx_connector_jwt, in_umbrella: true}, UMP.common_dep(:jose), UMP.common_dep(:ecpool), {:eredis_cluster, github: "emqx/eredis_cluster", tag: "0.8.4"}, diff --git a/apps/emqx_connector/rebar.config b/apps/emqx_connector/rebar.config index 94da3c580..7e0ff4ea3 100644 --- a/apps/emqx_connector/rebar.config +++ b/apps/emqx_connector/rebar.config @@ -8,7 +8,8 @@ {deps, [ {emqx, {path, "../emqx"}}, {emqx_utils, {path, "../emqx_utils"}}, - {emqx_resource, {path, "../emqx_resource"}} + {emqx_resource, {path, "../emqx_resource"}}, + {emqx_connector_jwt, {path, "../emqx_connector_jwt"}} ]}. {shell, [ diff --git a/apps/emqx_connector/src/emqx_connector_resource.erl b/apps/emqx_connector/src/emqx_connector_resource.erl index be8d3a32d..8cb61793d 100644 --- a/apps/emqx_connector/src/emqx_connector_resource.erl +++ b/apps/emqx_connector/src/emqx_connector_resource.erl @@ -18,6 +18,7 @@ -include("../../emqx_bridge/include/emqx_bridge_resource.hrl"). -include_lib("emqx/include/logger.hrl"). -include_lib("emqx_resource/include/emqx_resource.hrl"). +-include("emqx_connector.hrl"). -export([ connector_to_resource_type/1, @@ -126,7 +127,7 @@ create(Type, Name, Conf0, Opts) -> Conf = Conf0#{connector_type => TypeBin, connector_name => Name}, {ok, _Data} = emqx_resource:create_local( ResourceId, - <<"emqx_connector">>, + ?CONNECTOR_RESOURCE_GROUP, ?MODULE:connector_to_resource_type(Type), parse_confs(TypeBin, Name, Conf), parse_opts(Conf, Opts) @@ -208,7 +209,7 @@ create_dry_run(Type, Conf) -> create_dry_run(Type, Conf, fun(_) -> ok end). create_dry_run(Type, Conf0, Callback) -> - %% Already typechecked, no need to catch errors + %% Already type checked, no need to catch errors TypeBin = bin(Type), TypeAtom = safe_atom(Type), %% We use a fixed name here to avoid creating an atom @@ -351,8 +352,10 @@ safe_atom(Bin) when is_binary(Bin) -> binary_to_existing_atom(Bin, utf8); safe_atom(Atom) when is_atom(Atom) -> Atom. parse_opts(Conf, Opts0) -> - Opts1 = override_start_after_created(Conf, Opts0), - set_no_buffer_workers(Opts1). + Opts1 = emqx_resource:fetch_creation_opts(Conf), + Opts2 = maps:merge(Opts1, Opts0), + Opts = override_start_after_created(Conf, Opts2), + set_no_buffer_workers(Opts). override_start_after_created(Config, Opts) -> Enabled = maps:get(enable, Config, true), diff --git a/apps/emqx_connector/src/emqx_connector_sup.erl b/apps/emqx_connector/src/emqx_connector_sup.erl index 0d2f337a3..09cd8ea68 100644 --- a/apps/emqx_connector/src/emqx_connector_sup.erl +++ b/apps/emqx_connector/src/emqx_connector_sup.erl @@ -32,17 +32,5 @@ init([]) -> intensity => 5, period => 20 }, - ChildSpecs = [ - child_spec(emqx_connector_jwt_sup) - ], + ChildSpecs = [], {ok, {SupFlags, ChildSpecs}}. - -child_spec(Mod) -> - #{ - id => Mod, - start => {Mod, start_link, []}, - restart => permanent, - shutdown => 3000, - type => supervisor, - modules => [Mod] - }. diff --git a/apps/emqx_connector/src/schema/emqx_connector_schema.erl b/apps/emqx_connector/src/schema/emqx_connector_schema.erl index 060f3ba83..872bbd2c7 100644 --- a/apps/emqx_connector/src/schema/emqx_connector_schema.erl +++ b/apps/emqx_connector/src/schema/emqx_connector_schema.erl @@ -489,7 +489,12 @@ api_fields("put_connector", _Type, Fields) -> common_fields() -> [ - {enable, mk(boolean(), #{desc => ?DESC("config_enable"), default => true})}, + {enable, + mk(boolean(), #{ + desc => ?DESC("config_enable"), + importance => ?IMPORTANCE_NO_DOC, + default => true + })}, {tags, emqx_schema:tags_schema()}, {description, emqx_schema:description_schema()} ]. diff --git a/apps/emqx_connector/test/emqx_connector_SUITE.erl b/apps/emqx_connector/test/emqx_connector_SUITE.erl index fbdece6ff..8e5a6d288 100644 --- a/apps/emqx_connector/test/emqx_connector_SUITE.erl +++ b/apps/emqx_connector/test/emqx_connector_SUITE.erl @@ -50,6 +50,7 @@ t_connector_lifecycle({init, Config}) -> meck:new(emqx_connector_resource, [passthrough]), meck:expect(emqx_connector_resource, connector_to_resource_type, 1, ?CONNECTOR), meck:new(?CONNECTOR, [non_strict]), + meck:expect(?CONNECTOR, resource_type, 0, dummy), meck:expect(?CONNECTOR, callback_mode, 0, async_if_possible), meck:expect(?CONNECTOR, on_start, 2, {ok, connector_state}), meck:expect(?CONNECTOR, on_stop, 2, ok), @@ -171,6 +172,7 @@ t_remove_fail({'init', Config}) -> meck:expect(emqx_connector_resource, connector_to_resource_type, 1, ?CONNECTOR), meck:new(?CONNECTOR, [non_strict]), meck:expect(?CONNECTOR, callback_mode, 0, async_if_possible), + meck:expect(?CONNECTOR, resource_type, 0, dummy), meck:expect(?CONNECTOR, on_start, 2, {ok, connector_state}), meck:expect(?CONNECTOR, on_get_channels, 1, [{<<"my_channel">>, #{enable => true}}]), meck:expect(?CONNECTOR, on_add_channel, 4, {ok, connector_state}), @@ -234,6 +236,7 @@ t_create_with_bad_name_direct_path({init, Config}) -> meck:new(emqx_connector_resource, [passthrough]), meck:expect(emqx_connector_resource, connector_to_resource_type, 1, ?CONNECTOR), meck:new(?CONNECTOR, [non_strict]), + meck:expect(?CONNECTOR, resource_type, 0, dummy), meck:expect(?CONNECTOR, callback_mode, 0, async_if_possible), meck:expect(?CONNECTOR, on_start, 2, {ok, connector_state}), meck:expect(?CONNECTOR, on_stop, 2, ok), @@ -265,6 +268,7 @@ t_create_with_bad_name_root_path({init, Config}) -> meck:new(emqx_connector_resource, [passthrough]), meck:expect(emqx_connector_resource, connector_to_resource_type, 1, ?CONNECTOR), meck:new(?CONNECTOR, [non_strict]), + meck:expect(?CONNECTOR, resource_type, 0, dummy), meck:expect(?CONNECTOR, callback_mode, 0, async_if_possible), meck:expect(?CONNECTOR, on_start, 2, {ok, connector_state}), meck:expect(?CONNECTOR, on_stop, 2, ok), @@ -299,6 +303,7 @@ t_no_buffer_workers({'init', Config}) -> meck:new(emqx_connector_resource, [passthrough]), meck:expect(emqx_connector_resource, connector_to_resource_type, 1, ?CONNECTOR), meck:new(?CONNECTOR, [non_strict]), + meck:expect(?CONNECTOR, resource_type, 0, dummy), meck:expect(?CONNECTOR, callback_mode, 0, async_if_possible), meck:expect(?CONNECTOR, on_start, 2, {ok, connector_state}), meck:expect(?CONNECTOR, on_get_channels, 1, []), diff --git a/apps/emqx_connector/test/emqx_connector_api_SUITE.erl b/apps/emqx_connector/test/emqx_connector_api_SUITE.erl index 31069b075..6d119473a 100644 --- a/apps/emqx_connector/test/emqx_connector_api_SUITE.erl +++ b/apps/emqx_connector/test/emqx_connector_api_SUITE.erl @@ -225,6 +225,7 @@ init_mocks(_TestCase) -> meck:new(emqx_connector_resource, [passthrough, no_link]), meck:expect(emqx_connector_resource, connector_to_resource_type, 1, ?CONNECTOR_IMPL), meck:new(?CONNECTOR_IMPL, [non_strict, no_link]), + meck:expect(?CONNECTOR_IMPL, resource_type, 0, dummy), meck:expect(?CONNECTOR_IMPL, callback_mode, 0, async_if_possible), meck:expect( ?CONNECTOR_IMPL, diff --git a/apps/emqx_connector/test/emqx_connector_dummy_impl.erl b/apps/emqx_connector/test/emqx_connector_dummy_impl.erl index c5d9e4f83..d506c9633 100644 --- a/apps/emqx_connector/test/emqx_connector_dummy_impl.erl +++ b/apps/emqx_connector/test/emqx_connector_dummy_impl.erl @@ -15,8 +15,10 @@ %% this module is only intended to be mocked -module(emqx_connector_dummy_impl). +-behavior(emqx_resource). -export([ + resource_type/0, query_mode/1, callback_mode/0, on_start/2, @@ -25,6 +27,7 @@ on_get_channel_status/3 ]). +resource_type() -> dummy. query_mode(_) -> error(unexpected). callback_mode() -> error(unexpected). on_start(_, _) -> error(unexpected). diff --git a/apps/emqx_connector_jwt/README.md b/apps/emqx_connector_jwt/README.md new file mode 100644 index 000000000..868d7b08f --- /dev/null +++ b/apps/emqx_connector_jwt/README.md @@ -0,0 +1,3 @@ +# emqx_connector_jwt + +This is a small helper application for connectors, actions and sources to generate JWTs. diff --git a/apps/emqx_connector/include/emqx_connector_tables.hrl b/apps/emqx_connector_jwt/include/emqx_connector_jwt_tables.hrl similarity index 100% rename from apps/emqx_connector/include/emqx_connector_tables.hrl rename to apps/emqx_connector_jwt/include/emqx_connector_jwt_tables.hrl diff --git a/apps/emqx_connector_jwt/mix.exs b/apps/emqx_connector_jwt/mix.exs new file mode 100644 index 000000000..752275a56 --- /dev/null +++ b/apps/emqx_connector_jwt/mix.exs @@ -0,0 +1,34 @@ +defmodule EMQXConnectorJWT.MixProject do + use Mix.Project + alias EMQXUmbrella.MixProject, as: UMP + + def project do + [ + app: :emqx_connector_jwt, + version: "0.1.0", + build_path: "../../_build", + erlc_options: UMP.erlc_options(), + erlc_paths: UMP.erlc_paths(), + deps_path: "../../deps", + lockfile: "../../mix.lock", + elixir: "~> 1.14", + start_permanent: Mix.env() == :prod, + deps: deps() + ] + end + + # Run "mix help compile.app" to learn about applications + def application do + [ + extra_applications: UMP.extra_applications(), + mod: {:emqx_connector_jwt_app, []} + ] + end + + def deps() do + [ + {:emqx_resource, in_umbrella: true}, + UMP.common_dep(:jose), + ] + end +end diff --git a/apps/emqx_connector_jwt/rebar.config b/apps/emqx_connector_jwt/rebar.config new file mode 100644 index 000000000..252534c46 --- /dev/null +++ b/apps/emqx_connector_jwt/rebar.config @@ -0,0 +1,10 @@ +%% -*- mode: erlang -*- + +{erl_opts, [ + nowarn_unused_import, + debug_info +]}. + +{deps, [ + {emqx_resource, {path, "../emqx_resource"}} +]}. diff --git a/apps/emqx_connector_jwt/src/emqx_connector_jwt.app.src b/apps/emqx_connector_jwt/src/emqx_connector_jwt.app.src new file mode 100644 index 000000000..e284d7471 --- /dev/null +++ b/apps/emqx_connector_jwt/src/emqx_connector_jwt.app.src @@ -0,0 +1,16 @@ +%% -*- mode: erlang -*- +{application, emqx_connector_jwt, [ + {description, "EMQX JWT Connector Utility"}, + {vsn, "0.1.0"}, + {registered, []}, + {applications, [ + kernel, + stdlib + ]}, + {env, []}, + {modules, []}, + {mod, {emqx_connector_jwt_app, []}}, + + {licenses, ["Apache 2.0"]}, + {links, []} +]}. diff --git a/apps/emqx_connector/src/emqx_connector_jwt.erl b/apps/emqx_connector_jwt/src/emqx_connector_jwt.erl similarity index 92% rename from apps/emqx_connector/src/emqx_connector_jwt.erl rename to apps/emqx_connector_jwt/src/emqx_connector_jwt.erl index dd74754ba..933a259de 100644 --- a/apps/emqx_connector/src/emqx_connector_jwt.erl +++ b/apps/emqx_connector_jwt/src/emqx_connector_jwt.erl @@ -16,7 +16,7 @@ -module(emqx_connector_jwt). --include_lib("emqx_connector/include/emqx_connector_tables.hrl"). +-include("emqx_connector_jwt_tables.hrl"). -include_lib("emqx_resource/include/emqx_resource.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). -include_lib("jose/include/jose_jwt.hrl"). @@ -37,7 +37,7 @@ -type jwt_config() :: #{ expiration := duration(), resource_id := resource_id(), - table := ets:table(), + table => ets:table(), jwk := wrapped_jwk() | jwk(), iss := binary(), sub := binary(), @@ -82,7 +82,8 @@ delete_jwt(TId, ResourceId) -> %% one. -spec ensure_jwt(jwt_config()) -> jwt(). ensure_jwt(JWTConfig) -> - #{resource_id := ResourceId, table := Table} = JWTConfig, + #{resource_id := ResourceId} = JWTConfig, + Table = maps:get(table, JWTConfig, ?JWT_TABLE), case lookup_jwt(Table, ResourceId) of {error, not_found} -> JWT = do_generate_jwt(JWTConfig), @@ -132,8 +133,9 @@ do_generate_jwt(#{ JWT. -spec store_jwt(jwt_config(), jwt()) -> ok. -store_jwt(#{resource_id := ResourceId, table := TId}, JWT) -> - true = ets:insert(TId, {{ResourceId, jwt}, JWT}), +store_jwt(#{resource_id := ResourceId} = JWTConfig, JWT) -> + Table = maps:get(table, JWTConfig, ?JWT_TABLE), + true = ets:insert(Table, {{ResourceId, jwt}, JWT}), ?tp(emqx_connector_jwt_token_stored, #{resource_id => ResourceId}), ok. @@ -141,5 +143,5 @@ store_jwt(#{resource_id := ResourceId, table := TId}, JWT) -> is_about_to_expire(JWT) -> #jose_jwt{fields = #{<<"exp">> := Exp}} = jose_jwt:peek(JWT), Now = erlang:system_time(seconds), - GraceExp = Exp - timer:seconds(5), + GraceExp = Exp - 5, Now >= GraceExp. diff --git a/apps/emqx_connector_jwt/src/emqx_connector_jwt_app.erl b/apps/emqx_connector_jwt/src/emqx_connector_jwt_app.erl new file mode 100644 index 000000000..9c9c134de --- /dev/null +++ b/apps/emqx_connector_jwt/src/emqx_connector_jwt_app.erl @@ -0,0 +1,39 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_connector_jwt_app). + +-behaviour(application). + +%% `application' API +-export([start/2, stop/1]). + +%%------------------------------------------------------------------------------ +%% Type declarations +%%------------------------------------------------------------------------------ + +%%------------------------------------------------------------------------------ +%% `application' API +%%------------------------------------------------------------------------------ + +start(_StartType, _StartArgs) -> + emqx_connector_jwt_sup:start_link(). + +stop(_State) -> + ok. + +%%------------------------------------------------------------------------------ +%% Internal fns +%%------------------------------------------------------------------------------ diff --git a/apps/emqx_connector/src/emqx_connector_jwt_sup.erl b/apps/emqx_connector_jwt/src/emqx_connector_jwt_sup.erl similarity index 97% rename from apps/emqx_connector/src/emqx_connector_jwt_sup.erl rename to apps/emqx_connector_jwt/src/emqx_connector_jwt_sup.erl index d50be6395..4579b221c 100644 --- a/apps/emqx_connector/src/emqx_connector_jwt_sup.erl +++ b/apps/emqx_connector_jwt/src/emqx_connector_jwt_sup.erl @@ -18,7 +18,7 @@ -behaviour(supervisor). --include_lib("emqx_connector/include/emqx_connector_tables.hrl"). +-include("emqx_connector_jwt_tables.hrl"). -export([ start_link/0, diff --git a/apps/emqx_connector/src/emqx_connector_jwt_worker.erl b/apps/emqx_connector_jwt/src/emqx_connector_jwt_worker.erl similarity index 100% rename from apps/emqx_connector/src/emqx_connector_jwt_worker.erl rename to apps/emqx_connector_jwt/src/emqx_connector_jwt_worker.erl diff --git a/apps/emqx_connector/test/emqx_connector_jwt_SUITE.erl b/apps/emqx_connector_jwt/test/emqx_connector_jwt_SUITE.erl similarity index 94% rename from apps/emqx_connector/test/emqx_connector_jwt_SUITE.erl rename to apps/emqx_connector_jwt/test/emqx_connector_jwt_SUITE.erl index 6469614f8..a0416b9d5 100644 --- a/apps/emqx_connector/test/emqx_connector_jwt_SUITE.erl +++ b/apps/emqx_connector_jwt/test/emqx_connector_jwt_SUITE.erl @@ -20,7 +20,7 @@ -include_lib("common_test/include/ct.hrl"). -include_lib("jose/include/jose_jwt.hrl"). -include_lib("jose/include/jose_jws.hrl"). --include("emqx_connector_tables.hrl"). +-include("emqx_connector_jwt_tables.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). -compile([export_all, nowarn_export_all]). @@ -33,11 +33,12 @@ all() -> emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> - emqx_common_test_helpers:start_apps([emqx_connector]), - Config. + Apps = emqx_cth_suite:start([emqx_connector_jwt], #{work_dir => emqx_cth_suite:work_dir(Config)}), + [{apps, Apps} | Config]. -end_per_suite(_Config) -> - emqx_common_test_helpers:stop_apps([emqx_connector]), +end_per_suite(Config) -> + Apps = ?config(apps, Config), + emqx_cth_suite:stop(Apps), ok. init_per_testcase(_TestCase, Config) -> @@ -125,7 +126,7 @@ t_ensure_jwt(_Config) -> JWT0 = emqx_connector_jwt:ensure_jwt(JWTConfig), ?assertNot(is_expired(JWT0)), %% should refresh 5 s before expiration - ct:sleep(Expiration - 5500), + ct:sleep(Expiration - 3000), JWT1 = emqx_connector_jwt:ensure_jwt(JWTConfig), ?assertNot(is_expired(JWT1)), %% fully expired diff --git a/apps/emqx_connector/test/emqx_connector_jwt_worker_SUITE.erl b/apps/emqx_connector_jwt/test/emqx_connector_jwt_worker_SUITE.erl similarity index 100% rename from apps/emqx_connector/test/emqx_connector_jwt_worker_SUITE.erl rename to apps/emqx_connector_jwt/test/emqx_connector_jwt_worker_SUITE.erl diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl index e8cea1db5..c73a06a73 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl @@ -1008,7 +1008,7 @@ parse_object_loop([{Name, Hocon} | Rest], Module, Options, Props, Required, Refs %% return true if the field has 'importance' set to 'hidden' is_hidden(Hocon) -> - hocon_schema:is_hidden(Hocon, #{include_importance_up_from => ?IMPORTANCE_LOW}). + hocon_schema:is_hidden(Hocon, #{include_importance_up_from => ?IMPORTANCE_NO_DOC}). is_required(Hocon) -> hocon_schema:field_schema(Hocon, required) =:= true. diff --git a/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_manager.erl b/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_manager.erl index 6834da9e9..60fec5171 100644 --- a/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_manager.erl +++ b/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_manager.erl @@ -45,7 +45,7 @@ -define(MOD_TAB, emqx_dashboard_sso). -define(MOD_KEY_PATH, [dashboard, sso]). -define(MOD_KEY_PATH(Sub), [dashboard, sso, Sub]). --define(RESOURCE_GROUP, <<"emqx_dashboard_sso">>). +-define(RESOURCE_GROUP, <<"dashboard_sso">>). -define(NO_ERROR, <<>>). -define(DEFAULT_RESOURCE_OPTS, #{ start_after_created => false diff --git a/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_oidc.erl b/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_oidc.erl index 1d2520d0f..4d1fa9439 100644 --- a/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_oidc.erl +++ b/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_oidc.erl @@ -260,7 +260,15 @@ convert_certs(_Dir, Conf) -> %%------------------------------------------------------------------------------ save_jwks_file(Dir, Content) -> - Path = filename:join([emqx_tls_lib:pem_dir(Dir), "client_jwks"]), + case filelib:is_file(Content) of + true -> + {ok, Content}; + _ -> + Path = filename:join([emqx_tls_lib:pem_dir(Dir), "client_jwks"]), + write_jwks_file(Path, Content) + end. + +write_jwks_file(Path, Content) -> case filelib:ensure_dir(Path) of ok -> case file:write_file(Path, Content) of @@ -288,11 +296,18 @@ maybe_require_pkce(true, Opts) -> }. init_client_jwks(#{client_jwks := #{type := file, file := File}}) -> - case jose_jwk:from_file(File) of - {error, _} -> - none; - Jwks -> - Jwks + try + case jose_jwk:from_file(File) of + {error, Reason} -> + ?SLOG(error, #{msg => "failed_to_initialize_jwks", reason => Reason}), + none; + Jwks -> + Jwks + end + catch + _:CReason -> + ?SLOG(error, #{msg => "failed_to_initialize_jwks", reason => CReason}), + none end; init_client_jwks(_) -> none. diff --git a/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_oidc_api.erl b/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_oidc_api.erl index 3514b4fbb..eb887cce3 100644 --- a/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_oidc_api.erl +++ b/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_oidc_api.erl @@ -28,6 +28,7 @@ -export([code_callback/2, make_callback_url/1]). +-define(BAD_REQUEST, 'BAD_REQUEST'). -define(BAD_USERNAME_OR_PWD, 'BAD_USERNAME_OR_PWD'). -define(BACKEND_NOT_FOUND, 'BACKEND_NOT_FOUND'). @@ -62,6 +63,7 @@ schema("/sso/oidc/callback") -> desc => ?DESC(code_callback), responses => #{ 200 => emqx_dashboard_api:fields([token, version, license]), + 400 => response_schema(400), 401 => response_schema(401), 404 => response_schema(404) }, @@ -78,8 +80,9 @@ code_callback(get, #{query_string := QS}) -> ?SLOG(info, #{ msg => "dashboard_sso_login_successful" }), - {302, ?RESPHEADERS#{<<"location">> => Target}, ?REDIRECT_BODY}; + {error, invalid_query_string_param} -> + {400, #{code => ?BAD_REQUEST, message => <<"Invalid query string">>}}; {error, invalid_backend} -> {404, #{code => ?BACKEND_NOT_FOUND, message => <<"Backend not found">>}}; {error, Reason} -> @@ -93,11 +96,14 @@ code_callback(get, #{query_string := QS}) -> %%-------------------------------------------------------------------- %% internal %%-------------------------------------------------------------------- - +response_schema(400) -> + emqx_dashboard_swagger:error_codes([?BAD_REQUEST], <<"Bad Request">>); response_schema(401) -> - emqx_dashboard_swagger:error_codes([?BAD_USERNAME_OR_PWD], ?DESC(login_failed401)); + emqx_dashboard_swagger:error_codes( + [?BAD_USERNAME_OR_PWD], ?DESC(emqx_dashboard_api, login_failed401) + ); response_schema(404) -> - emqx_dashboard_swagger:error_codes([?BACKEND_NOT_FOUND], ?DESC(backend_not_found)). + emqx_dashboard_swagger:error_codes([?BACKEND_NOT_FOUND], <<"Backend not found">>). reason_to_message(Bin) when is_binary(Bin) -> Bin; @@ -119,7 +125,9 @@ ensure_oidc_state(#{<<"state">> := State} = QS, Cfg) -> retrieve_token(QS, Cfg, Data); _ -> {error, session_not_exists} - end. + end; +ensure_oidc_state(_, _Cfg) -> + {error, invalid_query_string_param}. retrieve_token( #{<<"code">> := Code}, diff --git a/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_schema.erl b/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_schema.erl index d6184b42a..f64c7a7df 100644 --- a/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_schema.erl +++ b/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_schema.erl @@ -47,6 +47,7 @@ common_backend_schema(Backend) -> mk( boolean(), #{ desc => ?DESC(backend_enable), + %% importance => ?IMPORTANCE_NO_DOC, required => false, default => false } diff --git a/apps/emqx_dashboard_sso/test/emqx_dashboard_sso_ldap_SUITE.erl b/apps/emqx_dashboard_sso/test/emqx_dashboard_sso_ldap_SUITE.erl index 51524f0fd..40c5de9e5 100644 --- a/apps/emqx_dashboard_sso/test/emqx_dashboard_sso_ldap_SUITE.erl +++ b/apps/emqx_dashboard_sso/test/emqx_dashboard_sso_ldap_SUITE.erl @@ -24,7 +24,7 @@ -define(MOD_TAB, emqx_dashboard_sso). -define(MOD_KEY_PATH, [dashboard, sso, ldap]). --define(RESOURCE_GROUP, <<"emqx_dashboard_sso">>). +-define(RESOURCE_GROUP, <<"dashboard_sso">>). -import(emqx_mgmt_api_test_util, [request/2, request/3, uri/1, request_api/3]). diff --git a/apps/emqx_ds_shared_sub/README.md b/apps/emqx_ds_shared_sub/README.md index 9c4c15870..6ff57b84e 100644 --- a/apps/emqx_ds_shared_sub/README.md +++ b/apps/emqx_ds_shared_sub/README.md @@ -4,10 +4,14 @@ This application makes durable session capable to cooperatively replay messages # General layout and interaction with session +The general idea is described in the [EIP 0028](https://github.com/emqx/eip/blob/main/active/0028-durable-shared-subscriptions.md). + +On the code level, the application is organized in the following way: + ![General layout](docs/images/ds_shared_subs.png) * The nesting reflects nesting/ownership of entity states. -* The bold arrow represent the [most complex interaction](https://github.com/emqx/eip/blob/main/active/0028-durable-shared-subscriptions.md#shared-subscription-session-handler), between session-side group subscription state machine and the shared subscription leader. +* The bold arrow represent the [most complex interaction](https://github.com/emqx/eip/blob/main/active/0028-durable-shared-subscriptions.md#shared-subscription-session-handler), between session-side group subscription state machine (**GroupSM**) and the shared subscription leader (**Leader**). # Contributing diff --git a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_agent.erl b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_agent.erl index 29745aa4a..a90f1286d 100644 --- a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_agent.erl +++ b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_agent.erl @@ -6,83 +6,218 @@ -include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("emqx/include/logger.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). -include("emqx_ds_shared_sub_proto.hrl"). +-include("emqx_ds_shared_sub_config.hrl"). -export([ new/1, open/2, + can_subscribe/3, on_subscribe/3, - on_unsubscribe/2, + on_unsubscribe/3, on_stream_progress/2, on_info/2, + on_disconnect/2, renew_streams/1 ]). -behaviour(emqx_persistent_session_ds_shared_subs_agent). +-type share_topic_filter() :: emqx_persistent_session_ds:share_topic_filter(). +-type group_id() :: share_topic_filter(). + +-type progress() :: emqx_persistent_session_ds_shared_subs:progress(). +-type external_lease_event() :: + #{ + type => lease, + stream => emqx_ds:stream(), + progress => progress(), + share_topic_filter => emqx_persistent_session_ds:share_topic_filter() + } + | #{ + type => revoke, + stream => emqx_ds:stream(), + share_topic_filter => emqx_persistent_session_ds:share_topic_filter() + }. + +-type options() :: #{ + session_id := emqx_persistent_session_ds:id() +}. + +-type t() :: #{ + groups := #{ + group_id() => emqx_ds_shared_sub_group_sm:t() + }, + session_id := emqx_persistent_session_ds:id() +}. + +%% We speak in the terms of share_topic_filter in the module API +%% which is consumed by persistent session. +%% +%% We speak in the terms of group_id internally: +%% * to identfy shared subscription's group_sm in the state; +%% * to addres agent's group_sm while communicating with leader. +%% * to identify the leader itself. +%% +%% share_topic_filter should be uniquely determined by group_id. See MQTT 5.0 spec: +%% +%% > Note that "$share/consumer1//finance" and "$share/consumer1/sport/tennis/+" +%% > are distinct shared subscriptions, even though they have the same ShareName. +%% > While they might be related in some way, no specific relationship between them +%% > is implied by them having the same ShareName. +%% +%% So we just use the full share_topic_filter record as group_id. + +-define(group_id(ShareTopicFilter), ShareTopicFilter). +-define(share_topic_filter(GroupId), GroupId). + -record(message_to_group_sm, { - group :: emqx_types:group(), + group_id :: group_id(), message :: term() }). +-export_type([ + t/0, + group_id/0, + options/0, + external_lease_event/0 +]). + %%-------------------------------------------------------------------- %% API %%-------------------------------------------------------------------- +-spec new(options()) -> t(). new(Opts) -> init_state(Opts). +-spec open([{share_topic_filter(), emqx_types:subopts()}], options()) -> t(). open(TopicSubscriptions, Opts) -> State0 = init_state(Opts), State1 = lists:foldl( fun({ShareTopicFilter, #{}}, State) -> - add_group_subscription(State, ShareTopicFilter) + ?tp(warning, ds_agent_open_subscription, #{ + topic_filter => ShareTopicFilter + }), + add_shared_subscription(State, ShareTopicFilter) end, State0, TopicSubscriptions ), State1. -on_subscribe(State0, TopicFilter, _SubOpts) -> - State1 = add_group_subscription(State0, TopicFilter), - {ok, State1}. +-spec can_subscribe(t(), share_topic_filter(), emqx_types:subopts()) -> + ok | {error, emqx_types:reason_code()}. +can_subscribe(_State, _ShareTopicFilter, _SubOpts) -> + case ?dq_config(enable) of + true -> ok; + false -> {error, ?RC_SHARED_SUBSCRIPTIONS_NOT_SUPPORTED} + end. -on_unsubscribe(State, TopicFilter) -> - delete_group_subscription(State, TopicFilter). +-spec on_subscribe(t(), share_topic_filter(), emqx_types:subopts()) -> t(). +on_subscribe(State0, ShareTopicFilter, _SubOpts) -> + ?tp(warning, ds_agent_on_subscribe, #{ + share_topic_filter => ShareTopicFilter + }), + add_shared_subscription(State0, ShareTopicFilter). +-spec on_unsubscribe(t(), share_topic_filter(), [ + emqx_persistent_session_ds_shared_subs:agent_stream_progress() +]) -> t(). +on_unsubscribe(State, ShareTopicFilter, GroupProgress) -> + delete_shared_subscription(State, ShareTopicFilter, GroupProgress). + +-spec renew_streams(t()) -> + {[emqx_persistent_session_ds_shared_subs_agent:stream_lease_event()], t()}. renew_streams(#{} = State) -> fetch_stream_events(State). -on_stream_progress(State, _StreamProgress) -> - %% TODO https://emqx.atlassian.net/browse/EMQX-12572 - %% Send to leader - State. +-spec on_stream_progress(t(), #{ + share_topic_filter() => [emqx_persistent_session_ds_shared_subs:agent_stream_progress()] +}) -> t(). +on_stream_progress(State, StreamProgresses) -> + maps:fold( + fun(ShareTopicFilter, GroupProgresses, StateAcc) -> + with_group_sm(StateAcc, ?group_id(ShareTopicFilter), fun(GSM) -> + emqx_ds_shared_sub_group_sm:handle_stream_progress(GSM, GroupProgresses) + end) + end, + State, + StreamProgresses + ). -on_info(State, ?leader_lease_streams_match(Group, StreamProgresses, Version)) -> +-spec on_disconnect(t(), [emqx_persistent_session_ds_shared_subs:agent_stream_progress()]) -> t(). +on_disconnect(#{groups := Groups0} = State, StreamProgresses) -> + ok = maps:foreach( + fun(GroupId, GroupSM0) -> + GroupProgresses = maps:get(?share_topic_filter(GroupId), StreamProgresses, []), + emqx_ds_shared_sub_group_sm:handle_disconnect(GroupSM0, GroupProgresses) + end, + Groups0 + ), + State#{groups => #{}}. + +-spec on_info(t(), term()) -> t(). +on_info(State, ?leader_lease_streams_match(GroupId, Leader, StreamProgresses, Version)) -> ?SLOG(info, #{ msg => leader_lease_streams, - group => Group, + group_id => GroupId, streams => StreamProgresses, - version => Version + version => Version, + leader => Leader }), - with_group_sm(State, Group, fun(GSM) -> - emqx_ds_shared_sub_group_sm:handle_leader_lease_streams(GSM, StreamProgresses, Version) + with_group_sm(State, GroupId, fun(GSM) -> + emqx_ds_shared_sub_group_sm:handle_leader_lease_streams( + GSM, Leader, StreamProgresses, Version + ) end); -on_info(State, ?leader_renew_stream_lease_match(Group, Version)) -> +on_info(State, ?leader_renew_stream_lease_match(GroupId, Version)) -> ?SLOG(info, #{ msg => leader_renew_stream_lease, - group => Group, + group_id => GroupId, version => Version }), - with_group_sm(State, Group, fun(GSM) -> + with_group_sm(State, GroupId, fun(GSM) -> emqx_ds_shared_sub_group_sm:handle_leader_renew_stream_lease(GSM, Version) end); +on_info(State, ?leader_renew_stream_lease_match(GroupId, VersionOld, VersionNew)) -> + ?SLOG(info, #{ + msg => leader_renew_stream_lease, + group_id => GroupId, + version_old => VersionOld, + version_new => VersionNew + }), + with_group_sm(State, GroupId, fun(GSM) -> + emqx_ds_shared_sub_group_sm:handle_leader_renew_stream_lease(GSM, VersionOld, VersionNew) + end); +on_info(State, ?leader_update_streams_match(GroupId, VersionOld, VersionNew, StreamsNew)) -> + ?SLOG(info, #{ + msg => leader_update_streams, + group_id => GroupId, + version_old => VersionOld, + version_new => VersionNew, + streams_new => StreamsNew + }), + with_group_sm(State, GroupId, fun(GSM) -> + emqx_ds_shared_sub_group_sm:handle_leader_update_streams( + GSM, VersionOld, VersionNew, StreamsNew + ) + end); +on_info(State, ?leader_invalidate_match(GroupId)) -> + ?SLOG(info, #{ + msg => leader_invalidate, + group_id => GroupId + }), + with_group_sm(State, GroupId, fun(GSM) -> + emqx_ds_shared_sub_group_sm:handle_leader_invalidate(GSM) + end); %% Generic messages sent by group_sm's to themselves (timeouts). -on_info(State, #message_to_group_sm{group = Group, message = Message}) -> - with_group_sm(State, Group, fun(GSM) -> +on_info(State, #message_to_group_sm{group_id = GroupId, message = Message}) -> + with_group_sm(State, GroupId, fun(GSM) -> emqx_ds_shared_sub_group_sm:handle_info(GSM, Message) end). @@ -97,23 +232,30 @@ init_state(Opts) -> groups => #{} }. -delete_group_subscription(State, _ShareTopicFilter) -> - %% TODO https://emqx.atlassian.net/browse/EMQX-12572 - State. +delete_shared_subscription(State, ShareTopicFilter, GroupProgress) -> + GroupId = ?group_id(ShareTopicFilter), + case State of + #{groups := #{GroupId := GSM} = Groups} -> + _ = emqx_ds_shared_sub_group_sm:handle_disconnect(GSM, GroupProgress), + State#{groups => maps:remove(GroupId, Groups)}; + _ -> + State + end. -add_group_subscription( - #{groups := Groups0} = State0, ShareTopicFilter +add_shared_subscription( + #{session_id := SessionId, groups := Groups0} = State0, ShareTopicFilter ) -> ?SLOG(info, #{ - msg => agent_add_group_subscription, - topic_filter => ShareTopicFilter + msg => agent_add_shared_subscription, + share_topic_filter => ShareTopicFilter }), - #share{group = Group} = ShareTopicFilter, + GroupId = ?group_id(ShareTopicFilter), Groups1 = Groups0#{ - Group => emqx_ds_shared_sub_group_sm:new(#{ - topic_filter => ShareTopicFilter, - agent => this_agent(), - send_after => send_to_subscription_after(Group) + GroupId => emqx_ds_shared_sub_group_sm:new(#{ + session_id => SessionId, + share_topic_filter => ShareTopicFilter, + agent => this_agent(SessionId), + send_after => send_to_subscription_after(GroupId) }) }, State1 = State0#{groups => Groups1}, @@ -121,9 +263,9 @@ add_group_subscription( fetch_stream_events(#{groups := Groups0} = State0) -> {Groups1, Events} = maps:fold( - fun(Group, GroupSM0, {GroupsAcc, EventsAcc}) -> + fun(GroupId, GroupSM0, {GroupsAcc, EventsAcc}) -> {GroupSM1, Events} = emqx_ds_shared_sub_group_sm:fetch_stream_events(GroupSM0), - {GroupsAcc#{Group => GroupSM1}, [Events | EventsAcc]} + {GroupsAcc#{GroupId => GroupSM1}, [Events | EventsAcc]} end, {#{}, []}, Groups0 @@ -131,26 +273,23 @@ fetch_stream_events(#{groups := Groups0} = State0) -> State1 = State0#{groups => Groups1}, {lists:concat(Events), State1}. -%%-------------------------------------------------------------------- -%% Internal functions -%%-------------------------------------------------------------------- +this_agent(Id) -> + emqx_ds_shared_sub_proto:agent(Id, self()). -this_agent() -> self(). - -send_to_subscription_after(Group) -> +send_to_subscription_after(GroupId) -> fun(Time, Msg) -> emqx_persistent_session_ds_shared_subs_agent:send_after( Time, self(), - #message_to_group_sm{group = Group, message = Msg} + #message_to_group_sm{group_id = GroupId, message = Msg} ) end. -with_group_sm(State, Group, Fun) -> +with_group_sm(State, GroupId, Fun) -> case State of - #{groups := #{Group := GSM0} = Groups} -> - GSM1 = Fun(GSM0), - State#{groups => Groups#{Group => GSM1}}; + #{groups := #{GroupId := GSM0} = Groups} -> + #{} = GSM1 = Fun(GSM0), + State#{groups => Groups#{GroupId => GSM1}}; _ -> %% TODO %% Error? diff --git a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_api.erl b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_api.erl new file mode 100644 index 000000000..0a8d41116 --- /dev/null +++ b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_api.erl @@ -0,0 +1,218 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_ds_shared_sub_api). + +-behaviour(minirest_api). + +-include_lib("typerefl/include/types.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). +-include_lib("emqx/include/logger.hrl"). + +%% Swagger specs from hocon schema +-export([ + api_spec/0, + paths/0, + schema/1, + namespace/0 +]). + +-export([ + fields/1, + roots/0 +]). + +-define(TAGS, [<<"Durable Queues">>]). + +%% API callbacks +-export([ + '/durable_queues'/2, + '/durable_queues/:id'/2 +]). + +-import(hoconsc, [mk/2, ref/1, ref/2]). +-import(emqx_dashboard_swagger, [error_codes/2]). + +namespace() -> "durable_queues". + +api_spec() -> + emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}). + +paths() -> + [ + "/durable_queues", + "/durable_queues/:id" + ]. + +-define(NOT_FOUND, 'NOT_FOUND'). + +schema("/durable_queues") -> + #{ + 'operationId' => '/durable_queues', + get => #{ + tags => ?TAGS, + summary => <<"List declared durable queues">>, + description => ?DESC("durable_queues_get"), + responses => #{ + 200 => emqx_dashboard_swagger:schema_with_example( + durable_queues_get(), + durable_queues_get_example() + ) + } + } + }; +schema("/durable_queues/:id") -> + #{ + 'operationId' => '/durable_queues/:id', + get => #{ + tags => ?TAGS, + summary => <<"Get a declared durable queue">>, + description => ?DESC("durable_queue_get"), + parameters => [param_queue_id()], + responses => #{ + 200 => emqx_dashboard_swagger:schema_with_example( + durable_queue_get(), + durable_queue_get_example() + ), + 404 => error_codes([?NOT_FOUND], <<"Queue Not Found">>) + } + }, + delete => #{ + tags => ?TAGS, + summary => <<"Delete a declared durable queue">>, + description => ?DESC("durable_queue_delete"), + parameters => [param_queue_id()], + responses => #{ + 200 => <<"Queue deleted">>, + 404 => error_codes([?NOT_FOUND], <<"Queue Not Found">>) + } + }, + put => #{ + tags => ?TAGS, + summary => <<"Declare a durable queue">>, + description => ?DESC("durable_queues_put"), + parameters => [param_queue_id()], + 'requestBody' => durable_queue_put(), + responses => #{ + 200 => emqx_dashboard_swagger:schema_with_example( + durable_queue_get(), + durable_queue_get_example() + ) + } + } + }. + +'/durable_queues'(get, _Params) -> + {200, queue_list()}. + +'/durable_queues/:id'(get, Params) -> + case queue_get(Params) of + {ok, Queue} -> {200, Queue}; + not_found -> serialize_error(not_found) + end; +'/durable_queues/:id'(delete, Params) -> + case queue_delete(Params) of + ok -> {200, <<"Queue deleted">>}; + not_found -> serialize_error(not_found) + end; +'/durable_queues/:id'(put, Params) -> + {200, queue_put(Params)}. + +%%-------------------------------------------------------------------- +%% Actual handlers: stubs +%%-------------------------------------------------------------------- + +queue_list() -> + persistent_term:get({?MODULE, queues}, []). + +queue_get(#{bindings := #{id := ReqId}}) -> + case [Q || #{id := Id} = Q <- queue_list(), Id =:= ReqId] of + [Queue] -> {ok, Queue}; + [] -> not_found + end. + +queue_delete(#{bindings := #{id := ReqId}}) -> + Queues0 = queue_list(), + Queues1 = [Q || #{id := Id} = Q <- Queues0, Id =/= ReqId], + persistent_term:put({?MODULE, queues}, Queues1), + case Queues0 =:= Queues1 of + true -> not_found; + false -> ok + end. + +queue_put(#{bindings := #{id := ReqId}}) -> + Queues0 = queue_list(), + Queues1 = [Q || #{id := Id} = Q <- Queues0, Id =/= ReqId], + NewQueue = #{ + id => ReqId + }, + Queues2 = [NewQueue | Queues1], + persistent_term:put({?MODULE, queues}, Queues2), + NewQueue. + +%%-------------------------------------------------------------------- +%% Schemas +%%-------------------------------------------------------------------- + +param_queue_id() -> + { + id, + mk(binary(), #{ + in => path, + desc => ?DESC(param_queue_id), + required => true, + validator => fun validate_queue_id/1 + }) + }. + +validate_queue_id(Id) -> + case emqx_topic:words(Id) of + [Segment] when is_binary(Segment) -> true; + _ -> {error, <<"Invalid queue id">>} + end. + +durable_queues_get() -> + hoconsc:array(ref(durable_queue_get)). + +durable_queue_get() -> + ref(durable_queue_get). + +durable_queue_put() -> + map(). + +roots() -> []. + +fields(durable_queue_get) -> + [ + {id, mk(binary(), #{})} + ]. + +%%-------------------------------------------------------------------- +%% Examples +%%-------------------------------------------------------------------- + +durable_queue_get_example() -> + #{ + id => <<"queue1">> + }. + +durable_queues_get_example() -> + [ + #{ + id => <<"queue1">> + }, + #{ + id => <<"queue2">> + } + ]. + +%%-------------------------------------------------------------------- +%% Error codes +%%-------------------------------------------------------------------- + +serialize_error(not_found) -> + {404, #{ + code => <<"NOT_FOUND">>, + message => <<"Queue Not Found">> + }}. diff --git a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_app.erl b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_app.erl index 5c2d8d964..80e728a80 100644 --- a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_app.erl +++ b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_app.erl @@ -15,9 +15,11 @@ -spec start(application:start_type(), term()) -> {ok, pid()}. start(_Type, _Args) -> + ok = emqx_ds_shared_sub_config:load(), {ok, Sup} = emqx_ds_shared_sub_sup:start_link(), {ok, Sup}. -spec stop(term()) -> ok. stop(_State) -> + ok = emqx_ds_shared_sub_config:unload(), ok. diff --git a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_config.erl b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_config.erl new file mode 100644 index 000000000..454e2b6e8 --- /dev/null +++ b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_config.erl @@ -0,0 +1,84 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_ds_shared_sub_config). + +-behaviour(emqx_config_handler). +-behaviour(emqx_config_backup). + +-type update_request() :: emqx_config:config(). + +%% callbacks for emqx_config_handler +-export([ + pre_config_update/3, + post_config_update/5 +]). + +%% callbacks for emqx_config_backup +-export([ + import_config/1 +]). + +%% API +-export([ + load/0, + unload/0, + get/1 +]). + +%%-------------------------------------------------------------------- +%% API +%%-------------------------------------------------------------------- + +-spec load() -> ok. +load() -> + emqx_conf:add_handler([durable_queues], ?MODULE). + +-spec unload() -> ok. +unload() -> + ok = emqx_conf:remove_handler([durable_queues]). + +-spec get(atom() | [atom()]) -> term(). +get(Name) when is_atom(Name) -> + emqx_config:get([durable_queues, Name]); +get(Name) when is_list(Name) -> + emqx_config:get([durable_queues | Name]). + +%%-------------------------------------------------------------------- +%% emqx_config_handler callbacks +%%-------------------------------------------------------------------- + +-spec pre_config_update(list(atom()), update_request(), emqx_config:raw_config()) -> + {ok, emqx_config:update_request()}. +pre_config_update([durable_queues | _], NewConfig, _OldConfig) -> + {ok, NewConfig}. + +-spec post_config_update( + list(atom()), + update_request(), + emqx_config:config(), + emqx_config:config(), + emqx_config:app_envs() +) -> + ok. +post_config_update([durable_queues | _], _Req, _NewConfig, _OldConfig, _AppEnvs) -> + ok. + +%%---------------------------------------------------------------------------------------- +%% Data backup +%%---------------------------------------------------------------------------------------- + +import_config(#{<<"durable_queues">> := DQConf}) -> + OldDQConf = emqx:get_raw_config([durable_queues], #{}), + NewDQConf = maps:merge(OldDQConf, DQConf), + case emqx_conf:update([durable_queues], NewDQConf, #{override_to => cluster}) of + {ok, #{raw_config := NewRawConf}} -> + Changed = maps:get(changed, emqx_utils_maps:diff_maps(NewRawConf, DQConf)), + ChangedPaths = [[durable_queues, K] || K <- maps:keys(Changed)], + {ok, #{root_key => durable_queues, changed => ChangedPaths}}; + Error -> + {error, #{root_key => durable_queues, reason => Error}} + end; +import_config(_) -> + {ok, #{root_key => durable_queues, changed => []}}. diff --git a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_config.hrl b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_config.hrl new file mode 100644 index 000000000..592a60643 --- /dev/null +++ b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_config.hrl @@ -0,0 +1,5 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-define(dq_config(Path), emqx_ds_shared_sub_config:get(Path)). diff --git a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_group_sm.erl b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_group_sm.erl index c6bdf9d93..a648bbaef 100644 --- a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_group_sm.erl +++ b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_group_sm.erl @@ -10,73 +10,113 @@ -module(emqx_ds_shared_sub_group_sm). -include_lib("emqx/include/logger.hrl"). +-include("emqx_ds_shared_sub_config.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). -export([ new/1, %% Leader messages - handle_leader_lease_streams/3, + handle_leader_lease_streams/4, handle_leader_renew_stream_lease/2, + handle_leader_renew_stream_lease/3, + handle_leader_update_streams/4, + handle_leader_invalidate/1, %% Self-initiated messages handle_info/2, %% API - fetch_stream_events/1 + fetch_stream_events/1, + handle_stream_progress/2, + handle_disconnect/2 +]). + +-export_type([ + t/0, + options/0, + state/0 ]). -type options() :: #{ + session_id := emqx_persistent_session_ds:id(), agent := emqx_ds_shared_sub_proto:agent(), - topic_filter := emqx_persistent_session_ds:share_topic_filter(), + share_topic_filter := emqx_persistent_session_ds:share_topic_filter(), send_after := fun((non_neg_integer(), term()) -> reference()) }. -%% Subscription states +-type progress() :: emqx_persistent_session_ds_shared_subs:progress(). + +-type stream_lease_event() :: + #{ + type => lease, + stream => emqx_ds:stream(), + progress => progress() + } + | #{ + type => revoke, + stream => emqx_ds:stream() + }. + +%% GroupSM States -define(connecting, connecting). -define(replaying, replaying). -define(updating, updating). +-define(disconnected, disconnected). --type state() :: ?connecting | ?replaying | ?updating. +-type state() :: ?connecting | ?replaying | ?updating | ?disconnected. --type group_sm() :: #{ - topic_filter => emqx_persistent_session_ds:share_topic_filter(), - agent => emqx_ds_shared_sub_proto:agent(), - send_after => fun((non_neg_integer(), term()) -> reference()), - - state => state(), - state_data => map(), - state_timers => map() +-type connecting_data() :: #{}. +-type replaying_data() :: #{ + leader => emqx_ds_shared_sub_proto:leader(), + streams => #{emqx_ds:stream() => progress()}, + version => emqx_ds_shared_sub_proto:version(), + prev_version => undefined }. +-type updating_data() :: #{ + leader => emqx_ds_shared_sub_proto:leader(), + streams => #{emqx_ds:stream() => progress()}, + version => emqx_ds_shared_sub_proto:version(), + prev_version => emqx_ds_shared_sub_proto:version() +}. + +-type state_data() :: connecting_data() | replaying_data() | updating_data(). -record(state_timeout, { id :: reference(), name :: atom(), message :: term() }). + -record(timer, { ref :: reference(), id :: reference() }). -%%----------------------------------------------------------------------- -%% Constants -%%----------------------------------------------------------------------- +-type timer_name() :: atom(). +-type timer() :: #timer{}. -%% TODO https://emqx.atlassian.net/browse/EMQX-12574 -%% Move to settings --define(FIND_LEADER_TIMEOUT, 1000). --define(RENEW_LEASE_TIMEOUT, 2000). +-type t() :: #{ + share_topic_filter => emqx_persistent_session_ds:share_topic_filter(), + agent => emqx_ds_shared_sub_proto:agent(), + send_after => fun((non_neg_integer(), term()) -> reference()), + stream_lease_events => list(stream_lease_event()), + + state => state(), + state_data => state_data(), + state_timers => #{timer_name() => timer()} +}. %%----------------------------------------------------------------------- %% API %%----------------------------------------------------------------------- --spec new(options()) -> group_sm(). +-spec new(options()) -> t(). new(#{ + session_id := SessionId, agent := Agent, - topic_filter := ShareTopicFilter, + share_topic_filter := ShareTopicFilter, send_after := SendAfter }) -> ?SLOG( @@ -84,37 +124,49 @@ new(#{ #{ msg => group_sm_new, agent => Agent, - topic_filter => ShareTopicFilter + share_topic_filter => ShareTopicFilter } ), GSM0 = #{ - topic_filter => ShareTopicFilter, + id => SessionId, + share_topic_filter => ShareTopicFilter, agent => Agent, send_after => SendAfter }, + ?tp(warning, group_sm_new, #{ + agent => Agent, + share_topic_filter => ShareTopicFilter + }), transition(GSM0, ?connecting, #{}). +-spec fetch_stream_events(t()) -> + {t(), [emqx_ds_shared_sub_agent:external_lease_event()]}. fetch_stream_events( #{ - state := ?replaying, - topic_filter := TopicFilter, - state_data := #{stream_lease_events := Events0} = Data + state := _State, + share_topic_filter := ShareTopicFilter, + stream_lease_events := Events0 } = GSM ) -> Events1 = lists:map( fun(Event) -> - Event#{topic_filter => TopicFilter} + Event#{share_topic_filter => ShareTopicFilter} end, Events0 ), - { - GSM#{ - state_data => Data#{stream_lease_events => []} - }, - Events1 - }; -fetch_stream_events(GSM) -> - {GSM, []}. + {GSM#{stream_lease_events => []}, Events1}. + +-spec handle_disconnect(t(), emqx_ds_shared_sub_proto:agent_stream_progress()) -> t(). +handle_disconnect(#{state := ?connecting} = GSM, _StreamProgresses) -> + transition(GSM, ?disconnected, #{}); +handle_disconnect( + #{agent := Agent, state_data := #{leader := Leader, version := Version} = StateData} = GSM, + StreamProgresses +) -> + ok = emqx_ds_shared_sub_proto:agent_disconnect( + Leader, Agent, StreamProgresses, Version + ), + transition(GSM, ?disconnected, StateData). %%----------------------------------------------------------------------- %% Event Handlers @@ -123,89 +175,282 @@ fetch_stream_events(GSM) -> %%----------------------------------------------------------------------- %% Connecting state -handle_connecting(#{agent := Agent, topic_filter := ShareTopicFilter} = GSM) -> - ok = emqx_ds_shared_sub_registry:lookup_leader(Agent, ShareTopicFilter), - ensure_state_timeout(GSM, find_leader_timeout, ?FIND_LEADER_TIMEOUT). +handle_connecting(#{agent := Agent, share_topic_filter := ShareTopicFilter} = GSM) -> + ?tp(warning, group_sm_enter_connecting, #{ + agent => Agent, + share_topic_filter => ShareTopicFilter + }), + ok = emqx_ds_shared_sub_registry:lookup_leader(Agent, agent_metadata(GSM), ShareTopicFilter), + ensure_state_timeout(GSM, find_leader_timeout, ?dq_config(session_find_leader_timeout_ms)). handle_leader_lease_streams( - #{state := ?connecting, topic_filter := TopicFilter} = GSM0, StreamProgresses, Version + #{state := ?connecting, share_topic_filter := ShareTopicFilter} = GSM0, + Leader, + StreamProgresses, + Version ) -> - ?tp(debug, leader_lease_streams, #{topic_filter => TopicFilter}), - Streams = lists:foldl( - fun(#{stream := Stream, iterator := It}, Acc) -> - Acc#{Stream => It} - end, - #{}, - StreamProgresses - ), - StreamLeaseEvents = lists:map( - fun(#{stream := Stream, iterator := It}) -> - #{ - type => lease, - stream => Stream, - iterator => It - } - end, - StreamProgresses - ), + ?tp(debug, leader_lease_streams, #{share_topic_filter => ShareTopicFilter}), + Streams = progresses_to_map(StreamProgresses), + StreamLeaseEvents = progresses_to_lease_events(StreamProgresses), transition( GSM0, ?replaying, #{ + leader => Leader, streams => Streams, - stream_lease_events => StreamLeaseEvents, prev_version => undefined, version => Version - } + }, + StreamLeaseEvents ); -handle_leader_lease_streams(GSM, _StreamProgresses, _Version) -> +handle_leader_lease_streams(GSM, _Leader, _StreamProgresses, _Version) -> GSM. -handle_find_leader_timeout(#{agent := Agent, topic_filter := TopicFilter} = GSM0) -> - ok = emqx_ds_shared_sub_registry:lookup_leader(Agent, TopicFilter), - GSM1 = ensure_state_timeout(GSM0, find_leader_timeout, ?FIND_LEADER_TIMEOUT), +handle_find_leader_timeout(#{agent := Agent, share_topic_filter := ShareTopicFilter} = GSM0) -> + ?tp(warning, group_sm_find_leader_timeout, #{ + agent => Agent, + share_topic_filter => ShareTopicFilter + }), + ok = emqx_ds_shared_sub_registry:lookup_leader(Agent, agent_metadata(GSM0), ShareTopicFilter), + GSM1 = ensure_state_timeout( + GSM0, find_leader_timeout, ?dq_config(session_find_leader_timeout_ms) + ), GSM1. %%----------------------------------------------------------------------- %% Replaying state -handle_replaying(GSM) -> - ensure_state_timeout(GSM, renew_lease_timeout, ?RENEW_LEASE_TIMEOUT). +handle_replaying(GSM0) -> + GSM1 = ensure_state_timeout( + GSM0, renew_lease_timeout, ?dq_config(session_renew_lease_timeout_ms) + ), + GSM2 = ensure_state_timeout( + GSM1, update_stream_state_timeout, ?dq_config(session_min_update_stream_state_interval_ms) + ), + GSM2. -handle_leader_renew_stream_lease( - #{state := ?replaying, state_data := #{version := Version}} = GSM, Version -) -> - ensure_state_timeout(GSM, renew_lease_timeout, ?RENEW_LEASE_TIMEOUT); -handle_leader_renew_stream_lease(GSM, _Version) -> - GSM. - -handle_renew_lease_timeout(GSM) -> - ?tp(debug, renew_lease_timeout, #{}), +handle_renew_lease_timeout(#{agent := Agent, share_topic_filter := ShareTopicFilter} = GSM) -> + ?tp(warning, renew_lease_timeout, #{agent => Agent, share_topic_filter => ShareTopicFilter}), transition(GSM, ?connecting, #{}). %%----------------------------------------------------------------------- %% Updating state -% handle_updating(GSM) -> -% GSM. +handle_updating(GSM0) -> + GSM1 = ensure_state_timeout( + GSM0, renew_lease_timeout, ?dq_config(session_renew_lease_timeout_ms) + ), + GSM2 = ensure_state_timeout( + GSM1, update_stream_state_timeout, ?dq_config(session_min_update_stream_state_interval_ms) + ), + GSM2. + +%%----------------------------------------------------------------------- +%% Disconnected state + +handle_disconnected(GSM) -> + GSM. + +%%----------------------------------------------------------------------- +%% Common handlers + +handle_leader_update_streams( + #{ + id := Id, + state := ?replaying, + state_data := #{streams := Streams0, version := VersionOld} = StateData + } = GSM, + VersionOld, + VersionNew, + StreamProgresses +) -> + ?tp(warning, shared_sub_group_sm_leader_update_streams, #{ + id => Id, + version_old => VersionOld, + version_new => VersionNew, + stream_progresses => emqx_ds_shared_sub_proto:format_stream_progresses(StreamProgresses) + }), + {AddEvents, Streams1} = lists:foldl( + fun(#{stream := Stream, progress := Progress}, {AddEventAcc, StreamsAcc}) -> + case maps:is_key(Stream, StreamsAcc) of + true -> + %% We prefer our own progress + {AddEventAcc, StreamsAcc}; + false -> + { + [#{type => lease, stream => Stream, progress => Progress} | AddEventAcc], + StreamsAcc#{Stream => Progress} + } + end + end, + {[], Streams0}, + StreamProgresses + ), + NewStreamMap = progresses_to_map(StreamProgresses), + {RevokeEvents, Streams2} = lists:foldl( + fun(Stream, {RevokeEventAcc, StreamsAcc}) -> + case maps:is_key(Stream, NewStreamMap) of + true -> + {RevokeEventAcc, StreamsAcc}; + false -> + { + [#{type => revoke, stream => Stream} | RevokeEventAcc], + maps:remove(Stream, StreamsAcc) + } + end + end, + {[], Streams1}, + maps:keys(Streams1) + ), + StreamLeaseEvents = AddEvents ++ RevokeEvents, + ?tp(warning, shared_sub_group_sm_leader_update_streams, #{ + id => Id, + stream_lease_events => emqx_ds_shared_sub_proto:format_lease_events(StreamLeaseEvents) + }), + transition( + GSM, + ?updating, + StateData#{ + streams => Streams2, + prev_version => VersionOld, + version => VersionNew + }, + StreamLeaseEvents + ); +handle_leader_update_streams( + #{ + state := ?updating, + state_data := #{version := VersionNew} = _StreamData + } = GSM, + _VersionOld, + VersionNew, + _StreamProgresses +) -> + ensure_state_timeout(GSM, renew_lease_timeout, ?dq_config(session_renew_lease_timeout_ms)); +handle_leader_update_streams( + #{state := ?disconnected} = GSM, _VersionOld, _VersionNew, _StreamProgresses +) -> + GSM; +handle_leader_update_streams(GSM, VersionOld, VersionNew, _StreamProgresses) -> + %% Unexpected versions or state + ?tp(warning, shared_sub_group_sm_unexpected_leader_update_streams, #{ + gsm => GSM, + version_old => VersionOld, + version_new => VersionNew + }), + transition(GSM, ?connecting, #{}). + +handle_leader_renew_stream_lease( + #{state := ?replaying, state_data := #{version := Version}} = GSM, Version +) -> + ensure_state_timeout(GSM, renew_lease_timeout, ?dq_config(session_renew_lease_timeout_ms)); +handle_leader_renew_stream_lease( + #{state := ?updating, state_data := #{version := Version} = StateData} = GSM, Version +) -> + transition( + GSM, + ?replaying, + StateData#{prev_version => undefined} + ); +handle_leader_renew_stream_lease(GSM, _Version) -> + GSM. + +handle_leader_renew_stream_lease( + #{state := ?replaying, state_data := #{version := Version}} = GSM, VersionOld, VersionNew +) when VersionOld =:= Version orelse VersionNew =:= Version -> + ensure_state_timeout(GSM, renew_lease_timeout, ?dq_config(session_renew_lease_timeout_ms)); +handle_leader_renew_stream_lease( + #{state := ?updating, state_data := #{version := VersionNew, prev_version := VersionOld}} = GSM, + VersionOld, + VersionNew +) -> + ensure_state_timeout(GSM, renew_lease_timeout, ?dq_config(session_renew_lease_timeout_ms)); +handle_leader_renew_stream_lease( + #{state := ?disconnected} = GSM, _VersionOld, _VersionNew +) -> + GSM; +handle_leader_renew_stream_lease(GSM, VersionOld, VersionNew) -> + %% Unexpected versions or state + ?tp(warning, shared_sub_group_sm_unexpected_leader_renew_stream_lease, #{ + gsm => GSM, + version_old => VersionOld, + version_new => VersionNew + }), + transition(GSM, ?connecting, #{}). + +-spec handle_stream_progress(t(), list(emqx_ds_shared_sub_proto:agent_stream_progress())) -> + t(). +handle_stream_progress(#{state := ?connecting} = GSM, _StreamProgresses) -> + GSM; +handle_stream_progress( + #{ + state := ?replaying, + agent := Agent, + state_data := #{ + leader := Leader, + version := Version + } + } = GSM, + StreamProgresses +) -> + ok = emqx_ds_shared_sub_proto:agent_update_stream_states( + Leader, Agent, StreamProgresses, Version + ), + ensure_state_timeout( + GSM, update_stream_state_timeout, ?dq_config(session_min_update_stream_state_interval_ms) + ); +handle_stream_progress( + #{ + state := ?updating, + agent := Agent, + state_data := #{ + leader := Leader, + version := Version, + prev_version := PrevVersion + } + } = GSM, + StreamProgresses +) -> + ok = emqx_ds_shared_sub_proto:agent_update_stream_states( + Leader, Agent, StreamProgresses, PrevVersion, Version + ), + ensure_state_timeout( + GSM, update_stream_state_timeout, ?dq_config(session_min_update_stream_state_interval_ms) + ); +handle_stream_progress(#{state := ?disconnected} = GSM, _StreamProgresses) -> + GSM. + +handle_leader_invalidate(#{agent := Agent, share_topic_filter := ShareTopicFilter} = GSM) -> + ?tp(warning, shared_sub_group_sm_leader_invalidate, #{ + agent => Agent, + share_topic_filter => ShareTopicFilter + }), + transition(GSM, ?connecting, #{}). %%----------------------------------------------------------------------- %% Internal API %%----------------------------------------------------------------------- handle_state_timeout( - #{state := ?connecting, topic_filter := TopicFilter} = GSM, + #{state := ?connecting, share_topic_filter := ShareTopicFilter} = GSM, find_leader_timeout, _Message ) -> - ?tp(debug, find_leader_timeout, #{topic_filter => TopicFilter}), + ?tp(debug, find_leader_timeout, #{share_topic_filter => ShareTopicFilter}), handle_find_leader_timeout(GSM); handle_state_timeout( #{state := ?replaying} = GSM, renew_lease_timeout, _Message ) -> - handle_renew_lease_timeout(GSM). + handle_renew_lease_timeout(GSM); +handle_state_timeout( + GSM, + update_stream_state_timeout, + _Message +) -> + ?tp(debug, update_stream_state_timeout, #{}), + handle_stream_progress(GSM, []). handle_info( #{state_timers := Timers} = GSM, #state_timeout{message = Message, name = Name, id = Id} = _Info @@ -225,6 +470,9 @@ handle_info(GSM, _Info) -> %%-------------------------------------------------------------------- transition(GSM0, NewState, NewStateData) -> + transition(GSM0, NewState, NewStateData, []). + +transition(GSM0, NewState, NewStateData, LeaseEvents) -> Timers = maps:get(state_timers, GSM0, #{}), TimerNames = maps:keys(Timers), GSM1 = lists:foldl( @@ -237,10 +485,14 @@ transition(GSM0, NewState, NewStateData) -> GSM2 = GSM1#{ state => NewState, state_data => NewStateData, - state_timers => #{} + state_timers => #{}, + stream_lease_events => LeaseEvents }, run_enter_callback(GSM2). +agent_metadata(#{id := Id} = _GSM) -> + #{id => Id}. + ensure_state_timeout(GSM0, Name, Delay) -> ensure_state_timeout(GSM0, Name, Delay, Name). @@ -277,6 +529,29 @@ cancel_timer(GSM, Name) -> run_enter_callback(#{state := ?connecting} = GSM) -> handle_connecting(GSM); run_enter_callback(#{state := ?replaying} = GSM) -> - handle_replaying(GSM). -% run_enter_callback(#{state := ?updating} = GSM) -> -% handle_updating(GSM). + handle_replaying(GSM); +run_enter_callback(#{state := ?updating} = GSM) -> + handle_updating(GSM); +run_enter_callback(#{state := ?disconnected} = GSM) -> + handle_disconnected(GSM). + +progresses_to_lease_events(StreamProgresses) -> + lists:map( + fun(#{stream := Stream, progress := Progress}) -> + #{ + type => lease, + stream => Stream, + progress => Progress + } + end, + StreamProgresses + ). + +progresses_to_map(StreamProgresses) -> + lists:foldl( + fun(#{stream := Stream, progress := Progress}, Acc) -> + Acc#{Stream => Progress} + end, + #{}, + StreamProgresses + ). diff --git a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl index 5323595cf..912253205 100644 --- a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl +++ b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl @@ -6,10 +6,13 @@ -behaviour(gen_statem). +-include("emqx_ds_shared_sub_proto.hrl"). +-include("emqx_ds_shared_sub_config.hrl"). + -include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("emqx/include/logger.hrl"). -include_lib("emqx/include/emqx_persistent_message.hrl"). --include("emqx_ds_shared_sub_proto.hrl"). +-include_lib("snabbkaffe/include/trace.hrl"). -export([ register/2, @@ -24,45 +27,75 @@ terminate/3 ]). +-type share_topic_filter() :: emqx_persistent_session_ds:share_topic_filter(). + +-type group_id() :: share_topic_filter(). + -type options() :: #{ - topic_filter := emqx_persistent_session_ds:share_topic_filter() + share_topic_filter := share_topic_filter() }. --type stream_assignment() :: #{ +%% Agent states + +-define(waiting_replaying, waiting_replaying). +-define(replaying, replaying). +-define(waiting_updating, waiting_updating). +-define(updating, updating). + +-type agent_state() :: #{ + %% Our view of group_id sm's status + %% it lags the actual state + state := ?waiting_replaying | ?replaying | ?waiting_updating | ?updating, prev_version := emqx_maybe:t(emqx_ds_shared_sub_proto:version()), version := emqx_ds_shared_sub_proto:version(), - streams := list(emqx_ds:stream()) + agent_metadata := emqx_ds_shared_sub_proto:agent_metadata(), + streams := list(emqx_ds:stream()), + revoked_streams := list(emqx_ds:stream()) }. +-type progress() :: emqx_persistent_session_ds_shared_subs:progress(). + +-type stream_state() :: #{ + progress => progress(), + rank => emqx_ds:stream_rank() +}. + +%% TODO https://emqx.atlassian.net/browse/EMQX-12307 +%% Some data should be persisted -type data() :: #{ - group := emqx_types:group(), + %% + %% Persistent data + %% + group_id := group_id(), topic := emqx_types:topic(), - %% For ds router, not an actual session_id - router_id := binary(), - %% TODO https://emqx.atlassian.net/browse/EMQX-12307 - %% Persist progress %% TODO https://emqx.atlassian.net/browse/EMQX-12575 %% Implement some stats to assign evenly? - stream_progresses := #{ - emqx_ds:stream() => emqx_ds:iterator() + stream_states := #{ + emqx_ds:stream() => stream_state() }, - agent_stream_assignments := #{ - emqx_ds_shared_sub_proto:agent() => stream_assignment() + rank_progress := emqx_ds_shared_sub_leader_rank_progress:t(), + + %% + %% Ephemeral data, should not be persisted + %% + agents := #{ + emqx_ds_shared_sub_proto:agent() => agent_state() }, - stream_assignments := #{ + stream_owners := #{ emqx_ds:stream() => emqx_ds_shared_sub_proto:agent() } }. -export_type([ options/0, - data/0 + data/0, + progress/0 ]). %% States --define(waiting_registration, waiting_registration). --define(replaying, replaying). +-define(leader_waiting_registration, leader_waiting_registration). +-define(leader_active, leader_active). %% Events @@ -71,13 +104,7 @@ }). -record(renew_streams, {}). -record(renew_leases, {}). - -%% Constants - -%% TODO https://emqx.atlassian.net/browse/EMQX-12574 -%% Move to settings --define(RENEW_LEASE_INTERVAL, 5000). --define(RENEW_STREAMS_INTERVAL, 5000). +-record(drop_timeout, {}). %%-------------------------------------------------------------------- %% API @@ -90,9 +117,9 @@ register(Pid, Fun) -> %% Internal API %%-------------------------------------------------------------------- -child_spec(#{topic_filter := TopicFilter} = Options) -> +child_spec(#{share_topic_filter := ShareTopicFilter} = Options) -> #{ - id => id(TopicFilter), + id => id(ShareTopicFilter), start => {?MODULE, start_link, [Options]}, restart => temporary, shutdown => 5000, @@ -102,8 +129,8 @@ child_spec(#{topic_filter := TopicFilter} = Options) -> start_link(Options) -> gen_statem:start_link(?MODULE, [Options], []). -id(#share{group = Group} = _TopicFilter) -> - {?MODULE, Group}. +id(ShareTopicFilter) -> + {?MODULE, ShareTopicFilter}. %%-------------------------------------------------------------------- %% gen_statem callbacks @@ -111,216 +138,878 @@ id(#share{group = Group} = _TopicFilter) -> callback_mode() -> [handle_event_function, state_enter]. -init([#{topic_filter := #share{group = Group, topic = Topic}} = _Options]) -> +init([#{share_topic_filter := #share{topic = Topic} = ShareTopicFilter} = _Options]) -> Data = #{ - group => Group, + group_id => ShareTopicFilter, topic => Topic, - router_id => router_id(), - stream_progresses => #{}, - stream_assignments => #{}, - agent_stream_assignments => #{} + start_time => now_ms(), + stream_states => #{}, + stream_owners => #{}, + agents => #{}, + rank_progress => emqx_ds_shared_sub_leader_rank_progress:init() }, - {ok, ?waiting_registration, Data}. + {ok, ?leader_waiting_registration, Data}. %%-------------------------------------------------------------------- %% waiting_registration state -handle_event({call, From}, #register{register_fun = Fun}, ?waiting_registration, Data) -> +handle_event({call, From}, #register{register_fun = Fun}, ?leader_waiting_registration, Data) -> Self = self(), case Fun() of Self -> - {next_state, ?replaying, Data, {reply, From, {ok, Self}}}; + {next_state, ?leader_active, Data, {reply, From, {ok, Self}}}; OtherPid -> {stop_and_reply, normal, {reply, From, {ok, OtherPid}}} end; %%-------------------------------------------------------------------- %% repalying state -handle_event(enter, _OldState, ?replaying, #{topic := Topic, router_id := RouterId} = _Data) -> - ok = emqx_persistent_session_ds_router:do_add_route(Topic, RouterId), +handle_event(enter, _OldState, ?leader_active, #{topic := Topic} = _Data) -> + ?tp(warning, shared_sub_leader_enter_actve, #{topic => Topic}), {keep_state_and_data, [ - {state_timeout, ?RENEW_LEASE_INTERVAL, #renew_leases{}}, - {state_timeout, 0, #renew_streams{}} + {{timeout, #renew_streams{}}, 0, #renew_streams{}}, + {{timeout, #renew_leases{}}, ?dq_config(leader_renew_lease_interval_ms), #renew_leases{}}, + {{timeout, #drop_timeout{}}, ?dq_config(leader_drop_timeout_interval_ms), #drop_timeout{}} ]}; -handle_event(state_timeout, #renew_streams{}, ?replaying, Data0) -> +%%-------------------------------------------------------------------- +%% timers +%% renew_streams timer +handle_event({timeout, #renew_streams{}}, #renew_streams{}, ?leader_active, Data0) -> + % ?tp(warning, shared_sub_leader_timeout, #{timeout => renew_streams}), Data1 = renew_streams(Data0), - {keep_state, Data1, {state_timeout, ?RENEW_STREAMS_INTERVAL, #renew_streams{}}}; -handle_event(state_timeout, #renew_leases{}, ?replaying, Data0) -> + {keep_state, Data1, + { + {timeout, #renew_streams{}}, + ?dq_config(leader_renew_streams_interval_ms), + #renew_streams{} + }}; +%% renew_leases timer +handle_event({timeout, #renew_leases{}}, #renew_leases{}, ?leader_active, Data0) -> + % ?tp(warning, shared_sub_leader_timeout, #{timeout => renew_leases}), Data1 = renew_leases(Data0), - {keep_state, Data1, {state_timeout, ?RENEW_LEASE_INTERVAL, #renew_leases{}}}; -handle_event(info, ?agent_connect_leader_match(Agent, _TopicFilter), ?replaying, Data0) -> - Data1 = connect_agent(Data0, Agent), + {keep_state, Data1, + {{timeout, #renew_leases{}}, ?dq_config(leader_renew_lease_interval_ms), #renew_leases{}}}; +%% drop_timeout timer +handle_event({timeout, #drop_timeout{}}, #drop_timeout{}, ?leader_active, Data0) -> + Data1 = drop_timeout_agents(Data0), + {keep_state, Data1, + {{timeout, #drop_timeout{}}, ?dq_config(leader_drop_timeout_interval_ms), #drop_timeout{}}}; +%%-------------------------------------------------------------------- +%% agent events +handle_event( + info, ?agent_connect_leader_match(Agent, AgentMetadata, _TopicFilter), ?leader_active, Data0 +) -> + Data1 = connect_agent(Data0, Agent, AgentMetadata), {keep_state, Data1}; handle_event( - info, ?agent_update_stream_states_match(Agent, StreamProgresses, Version), ?replaying, Data0 + info, + ?agent_update_stream_states_match(Agent, StreamProgresses, Version), + ?leader_active, + Data0 ) -> - Data1 = update_agent_stream_states(Data0, Agent, StreamProgresses, Version), + Data1 = with_agent(Data0, Agent, fun() -> + update_agent_stream_states(Data0, Agent, StreamProgresses, Version) + end), + {keep_state, Data1}; +handle_event( + info, + ?agent_update_stream_states_match(Agent, StreamProgresses, VersionOld, VersionNew), + ?leader_active, + Data0 +) -> + Data1 = with_agent(Data0, Agent, fun() -> + update_agent_stream_states(Data0, Agent, StreamProgresses, VersionOld, VersionNew) + end), + {keep_state, Data1}; +handle_event( + info, + ?agent_disconnect_match(Agent, StreamProgresses, Version), + ?leader_active, + Data0 +) -> + Data1 = with_agent(Data0, Agent, fun() -> + disconnect_agent(Data0, Agent, StreamProgresses, Version) + end), {keep_state, Data1}; %%-------------------------------------------------------------------- %% fallback handle_event(enter, _OldState, _State, _Data) -> keep_state_and_data; -handle_event(Event, _Content, State, _Data) -> +handle_event(Event, Content, State, _Data) -> ?SLOG(warning, #{ msg => unexpected_event, event => Event, + content => Content, state => State }), keep_state_and_data. -terminate(_Reason, _State, #{topic := Topic, router_id := RouterId} = _Data) -> - ok = emqx_persistent_session_ds_router:do_delete_route(Topic, RouterId), +terminate(_Reason, _State, _Data) -> ok. %%-------------------------------------------------------------------- -%% Internal functions +%% Event handlers %%-------------------------------------------------------------------- -renew_streams(#{stream_progresses := Progresses, topic := Topic} = Data0) -> +%%-------------------------------------------------------------------- +%% Renew streams + +%% * Find new streams in DS +%% * Revoke streams from agents having too many streams +%% * Assign streams to agents having too few streams + +renew_streams( + #{ + start_time := StartTime, + stream_states := StreamStates, + topic := Topic, + rank_progress := RankProgress0 + } = Data0 +) -> TopicFilter = emqx_topic:words(Topic), - StartTime = now_ms(), - {_, Streams} = lists:unzip( - emqx_ds:get_streams(?PERSISTENT_MESSAGE_DB, TopicFilter, now_ms()) + StreamsWRanks = emqx_ds:get_streams(?PERSISTENT_MESSAGE_DB, TopicFilter, StartTime), + + %% Discard streams that are already replayed and init new + {NewStreamsWRanks, RankProgress1} = emqx_ds_shared_sub_leader_rank_progress:add_streams( + StreamsWRanks, RankProgress0 ), - %% TODO https://emqx.atlassian.net/browse/EMQX-12572 - %% Handle stream removal - NewProgresses = lists:foldl( - fun(Stream, ProgressesAcc) -> - case ProgressesAcc of - #{Stream := _} -> - ProgressesAcc; + {NewStreamStates, VanishedStreamStates} = update_progresses( + StreamStates, NewStreamsWRanks, TopicFilter, StartTime + ), + Data1 = removed_vanished_streams(Data0, VanishedStreamStates), + Data2 = Data1#{stream_states => NewStreamStates, rank_progress => RankProgress1}, + Data3 = revoke_streams(Data2), + Data4 = assign_streams(Data3), + ?SLOG(info, #{ + msg => leader_renew_streams, + topic_filter => TopicFilter, + new_streams => length(NewStreamsWRanks) + }), + Data4. + +update_progresses(StreamStates, NewStreamsWRanks, TopicFilter, StartTime) -> + lists:foldl( + fun({Rank, Stream}, {NewStreamStatesAcc, OldStreamStatesAcc}) -> + case OldStreamStatesAcc of + #{Stream := StreamData} -> + { + NewStreamStatesAcc#{Stream => StreamData}, + maps:remove(Stream, OldStreamStatesAcc) + }; _ -> {ok, It} = emqx_ds:make_iterator( ?PERSISTENT_MESSAGE_DB, Stream, TopicFilter, StartTime ), - ProgressesAcc#{Stream => It} + Progress = #{ + iterator => It + }, + { + NewStreamStatesAcc#{Stream => #{progress => Progress, rank => Rank}}, + OldStreamStatesAcc + } end end, - Progresses, - Streams - ), - %% TODO https://emqx.atlassian.net/browse/EMQX-12572 - %% Initiate reassigment - ?SLOG(info, #{ - msg => leader_renew_streams, - topic_filter => TopicFilter, - streams => length(Streams) - }), - Data0#{stream_progresses => NewProgresses}. + {#{}, StreamStates}, + NewStreamsWRanks + ). -%% TODO https://emqx.atlassian.net/browse/EMQX-12572 -%% This just gives unassigned streams to the connecting agent, -%% we need to implement actual stream (re)assignment. -connect_agent( +%% We just remove disappeared streams from anywhere. +%% +%% If streams disappear from DS during leader being in replaying state +%% this is an abnormal situation (we should receive `end_of_stream` first), +%% but clients clients are unlikely to report any progress on them. +%% +%% If streams disappear after long leader sleep, it is a normal situation. +%% This removal will be a part of initialization before any agents connect. +removed_vanished_streams(Data0, VanishedStreamStates) -> + VanishedStreams = maps:keys(VanishedStreamStates), + Data1 = lists:foldl( + fun(Stream, #{stream_owners := StreamOwners0} = DataAcc) -> + case StreamOwners0 of + #{Stream := Agent} -> + #{streams := Streams0, revoked_streams := RevokedStreams0} = + AgentState0 = get_agent_state(Data0, Agent), + Streams1 = Streams0 -- [Stream], + RevokedStreams1 = RevokedStreams0 -- [Stream], + AgentState1 = AgentState0#{ + streams => Streams1, + revoked_streams => RevokedStreams1 + }, + set_agent_state(DataAcc, Agent, AgentState1); + _ -> + DataAcc + end + end, + Data0, + VanishedStreams + ), + Data2 = unassign_streams(Data1, VanishedStreams), + Data2. + +%% We revoke streams from agents that have too many streams (> desired_stream_count_per_agent). +%% We revoke only from replaying agents. +%% After revoking, no unassigned streams appear. Streams will become unassigned +%% only after agents report them as acked and unsubscribed. +revoke_streams(Data0) -> + DesiredStreamsPerAgent = desired_stream_count_per_agent(Data0), + Agents = replaying_agents(Data0), + lists:foldl( + fun(Agent, DataAcc) -> + revoke_excess_streams_from_agent(DataAcc, Agent, DesiredStreamsPerAgent) + end, + Data0, + Agents + ). + +revoke_excess_streams_from_agent(Data0, Agent, DesiredCount) -> + #{streams := Streams0, revoked_streams := []} = AgentState0 = get_agent_state(Data0, Agent), + RevokeCount = length(Streams0) - DesiredCount, + AgentState1 = + case RevokeCount > 0 of + false -> + AgentState0; + true -> + ?tp(warning, shared_sub_leader_revoke_streams, #{ + agent => Agent, + agent_stream_count => length(Streams0), + revoke_count => RevokeCount, + desired_count => DesiredCount + }), + revoke_streams_from_agent(Data0, Agent, AgentState0, RevokeCount) + end, + set_agent_state(Data0, Agent, AgentState1). + +revoke_streams_from_agent( + Data, + Agent, #{ - group := Group, - agent_stream_assignments := AgentStreamAssignments0, - stream_assignments := StreamAssignments0, - stream_progresses := StreamProgresses - } = Data0, - Agent + streams := Streams0, revoked_streams := [] + } = AgentState0, + RevokeCount +) -> + RevokedStreams = select_streams_for_revoke(Data, AgentState0, RevokeCount), + Streams = Streams0 -- RevokedStreams, + agent_transition_to_waiting_updating(Data, Agent, AgentState0, Streams, RevokedStreams). + +select_streams_for_revoke( + _Data, #{streams := Streams, revoked_streams := []} = _AgentState, RevokeCount +) -> + %% TODO + %% Some intellectual logic should be used regarding: + %% * shard ids (better do not mix shards in the same agent); + %% * stream stats (how much data was replayed from stream), + %% heavy streams should be distributed across different agents); + %% * data locality (agents better preserve streams with data available on the agent's node) + lists:sublist(shuffle(Streams), RevokeCount). + +%% We assign streams to agents that have too few streams (< desired_stream_count_per_agent). +%% We assign only to replaying agents. +assign_streams(Data0) -> + DesiredStreamsPerAgent = desired_stream_count_per_agent(Data0), + Agents = replaying_agents(Data0), + lists:foldl( + fun(Agent, DataAcc) -> + assign_lacking_streams(DataAcc, Agent, DesiredStreamsPerAgent) + end, + Data0, + Agents + ). + +assign_lacking_streams(Data0, Agent, DesiredCount) -> + #{streams := Streams0, revoked_streams := []} = get_agent_state(Data0, Agent), + AssignCount = DesiredCount - length(Streams0), + case AssignCount > 0 of + false -> + Data0; + true -> + ?tp(warning, shared_sub_leader_assign_streams, #{ + agent => Agent, + agent_stream_count => length(Streams0), + assign_count => AssignCount, + desired_count => DesiredCount + }), + assign_streams_to_agent(Data0, Agent, AssignCount) + end. + +assign_streams_to_agent(Data0, Agent, AssignCount) -> + StreamsToAssign = select_streams_for_assign(Data0, Agent, AssignCount), + Data1 = set_stream_ownership_to_agent(Data0, Agent, StreamsToAssign), + #{agents := #{Agent := AgentState0}} = Data1, + #{streams := Streams0, revoked_streams := []} = AgentState0, + Streams1 = Streams0 ++ StreamsToAssign, + AgentState1 = agent_transition_to_waiting_updating(Data0, Agent, AgentState0, Streams1, []), + set_agent_state(Data1, Agent, AgentState1). + +select_streams_for_assign(Data0, _Agent, AssignCount) -> + %% TODO + %% Some intellectual logic should be used. See `select_streams_for_revoke/3`. + UnassignedStreams = unassigned_streams(Data0), + lists:sublist(shuffle(UnassignedStreams), AssignCount). + +%%-------------------------------------------------------------------- +%% renew_leases - send lease confirmations to agents + +renew_leases(#{agents := AgentStates} = Data) -> + ?tp(warning, shared_sub_leader_renew_leases, #{agents => maps:keys(AgentStates)}), + ok = lists:foreach( + fun({Agent, AgentState}) -> + renew_lease(Data, Agent, AgentState) + end, + maps:to_list(AgentStates) + ), + Data. + +renew_lease(#{group_id := GroupId}, Agent, #{state := ?replaying, version := Version}) -> + ok = emqx_ds_shared_sub_proto:leader_renew_stream_lease(Agent, GroupId, Version); +renew_lease(#{group_id := GroupId}, Agent, #{state := ?waiting_replaying, version := Version}) -> + ok = emqx_ds_shared_sub_proto:leader_renew_stream_lease(Agent, GroupId, Version); +renew_lease(#{group_id := GroupId} = Data, Agent, #{ + streams := Streams, state := ?waiting_updating, version := Version, prev_version := PrevVersion +}) -> + StreamProgresses = stream_progresses(Data, Streams), + ok = emqx_ds_shared_sub_proto:leader_update_streams( + Agent, GroupId, PrevVersion, Version, StreamProgresses + ), + ok = emqx_ds_shared_sub_proto:leader_renew_stream_lease(Agent, GroupId, PrevVersion, Version); +renew_lease(#{group_id := GroupId}, Agent, #{ + state := ?updating, version := Version, prev_version := PrevVersion +}) -> + ok = emqx_ds_shared_sub_proto:leader_renew_stream_lease(Agent, GroupId, PrevVersion, Version). + +%%-------------------------------------------------------------------- +%% Drop agents that stopped reporting progress + +drop_timeout_agents(#{agents := Agents} = Data) -> + Now = now_ms_monotonic(), + lists:foldl( + fun( + {Agent, + #{update_deadline := UpdateDeadline, not_replaying_deadline := NoReplayingDeadline} = + _AgentState}, + DataAcc + ) -> + case + (UpdateDeadline < Now) orelse + (is_integer(NoReplayingDeadline) andalso NoReplayingDeadline < Now) + of + true -> + ?SLOG(info, #{ + msg => leader_agent_timeout, + now => Now, + update_deadline => UpdateDeadline, + not_replaying_deadline => NoReplayingDeadline, + agent => Agent + }), + drop_invalidate_agent(DataAcc, Agent); + false -> + DataAcc + end + end, + Data, + maps:to_list(Agents) + ). + +%%-------------------------------------------------------------------- +%% Handle a newly connected agent + +connect_agent( + #{group_id := GroupId, agents := Agents} = Data, + Agent, + AgentMetadata ) -> ?SLOG(info, #{ msg => leader_agent_connected, agent => Agent, - group => Group + group_id => GroupId }), - {AgentStreamAssignments, StreamAssignments} = - case AgentStreamAssignments0 of - #{Agent := _} -> - {AgentStreamAssignments0, StreamAssignments0}; - _ -> - UnassignedStreams = unassigned_streams(Data0), - Version = 0, - StreamAssignment = #{ - prev_version => undefined, - version => Version, - streams => UnassignedStreams - }, - AgentStreamAssignments1 = AgentStreamAssignments0#{Agent => StreamAssignment}, - StreamAssignments1 = lists:foldl( - fun(Stream, Acc) -> - Acc#{Stream => Agent} - end, - StreamAssignments0, - UnassignedStreams - ), - StreamLease = lists:map( - fun(Stream) -> - #{ - stream => Stream, - iterator => maps:get(Stream, StreamProgresses) - } - end, - UnassignedStreams - ), - ?SLOG(info, #{ - msg => leader_lease_streams, - agent => Agent, - group => Group, - streams => length(StreamLease), - version => Version - }), - ok = emqx_ds_shared_sub_proto:leader_lease_streams( - Agent, Group, StreamLease, Version - ), - {AgentStreamAssignments1, StreamAssignments1} + case Agents of + #{Agent := AgentState} -> + ?tp(warning, shared_sub_leader_agent_already_connected, #{ + agent => Agent + }), + reconnect_agent(Data, Agent, AgentMetadata, AgentState); + _ -> + DesiredCount = desired_stream_count_for_new_agent(Data), + assign_initial_streams_to_agent(Data, Agent, AgentMetadata, DesiredCount) + end. + +assign_initial_streams_to_agent(Data, Agent, AgentMetadata, AssignCount) -> + InitialStreamsToAssign = select_streams_for_assign(Data, Agent, AssignCount), + Data1 = set_stream_ownership_to_agent(Data, Agent, InitialStreamsToAssign), + AgentState = agent_transition_to_initial_waiting_replaying( + Data1, Agent, AgentMetadata, InitialStreamsToAssign + ), + set_agent_state(Data1, Agent, AgentState). + +reconnect_agent( + Data0, + Agent, + AgentMetadata, + #{streams := OldStreams, revoked_streams := OldRevokedStreams} = _OldAgentState +) -> + ?tp(warning, shared_sub_leader_agent_reconnect, #{ + agent => Agent, + agent_metadata => AgentMetadata, + inherited_streams => OldStreams + }), + AgentState = agent_transition_to_initial_waiting_replaying( + Data0, Agent, AgentMetadata, OldStreams + ), + Data1 = set_agent_state(Data0, Agent, AgentState), + %% If client reconnected gracefully then it either had already sent all the final progresses + %% for the revoked streams (so `OldRevokedStreams` should be empty) or it had not started + %% to replay them (if we revoked streams after it desided to reconnect). So we can safely + %% unassign them. + %% + %% If client reconnects after a crash, then we wouldn't be here (the agent identity will be new). + Data2 = unassign_streams(Data1, OldRevokedStreams), + Data2. + +%%-------------------------------------------------------------------- +%% Handle stream progress updates from agent in replaying state + +update_agent_stream_states(Data0, Agent, AgentStreamProgresses, Version) -> + #{state := State, version := AgentVersion, prev_version := AgentPrevVersion} = + AgentState0 = get_agent_state(Data0, Agent), + case {State, Version} of + {?waiting_updating, AgentPrevVersion} -> + %% Stale update, ignoring + Data0; + {?waiting_replaying, AgentVersion} -> + %% Agent finished updating, now replaying + {Data1, AgentState1} = update_stream_progresses( + Data0, Agent, AgentState0, AgentStreamProgresses + ), + AgentState2 = update_agent_timeout(AgentState1), + AgentState3 = agent_transition_to_replaying(Agent, AgentState2), + set_agent_state(Data1, Agent, AgentState3); + {?replaying, AgentVersion} -> + %% Common case, agent is replaying + {Data1, AgentState1} = update_stream_progresses( + Data0, Agent, AgentState0, AgentStreamProgresses + ), + AgentState2 = update_agent_timeout(AgentState1), + set_agent_state(Data1, Agent, AgentState2); + {OtherState, OtherVersion} -> + ?tp(warning, unexpected_update, #{ + agent => Agent, + update_version => OtherVersion, + state => OtherState, + our_agent_version => AgentVersion, + our_agent_prev_version => AgentPrevVersion + }), + drop_invalidate_agent(Data0, Agent) + end. + +update_stream_progresses( + #{stream_states := StreamStates0, stream_owners := StreamOwners} = Data0, + Agent, + AgentState0, + ReceivedStreamProgresses +) -> + {StreamStates1, ReplayedStreams} = lists:foldl( + fun(#{stream := Stream, progress := Progress}, {StreamStatesAcc, ReplayedStreamsAcc}) -> + case StreamOwners of + #{Stream := Agent} -> + StreamData0 = maps:get(Stream, StreamStatesAcc), + case Progress of + #{iterator := end_of_stream} -> + Rank = maps:get(rank, StreamData0), + {maps:remove(Stream, StreamStatesAcc), ReplayedStreamsAcc#{ + Stream => Rank + }}; + _ -> + StreamData1 = StreamData0#{progress => Progress}, + {StreamStatesAcc#{Stream => StreamData1}, ReplayedStreamsAcc} + end; + _ -> + {StreamStatesAcc, ReplayedStreamsAcc} + end end, - Data0#{ - agent_stream_assignments => AgentStreamAssignments, stream_assignments => StreamAssignments + {StreamStates0, #{}}, + ReceivedStreamProgresses + ), + Data1 = update_rank_progress(Data0, ReplayedStreams), + Data2 = Data1#{stream_states => StreamStates1}, + AgentState1 = filter_replayed_streams(AgentState0, ReplayedStreams), + {Data2, AgentState1}. + +update_rank_progress(#{rank_progress := RankProgress0} = Data, ReplayedStreams) -> + RankProgress1 = maps:fold( + fun(Stream, Rank, RankProgressAcc) -> + emqx_ds_shared_sub_leader_rank_progress:set_replayed({Rank, Stream}, RankProgressAcc) + end, + RankProgress0, + ReplayedStreams + ), + Data#{rank_progress => RankProgress1}. + +%% No need to revoke fully replayed streams. We do not assign them anymore. +%% The agent's session also will drop replayed streams itself. +filter_replayed_streams( + #{streams := Streams0, revoked_streams := RevokedStreams0} = AgentState0, + ReplayedStreams +) -> + Streams1 = lists:filter( + fun(Stream) -> not maps:is_key(Stream, ReplayedStreams) end, + Streams0 + ), + RevokedStreams1 = lists:filter( + fun(Stream) -> not maps:is_key(Stream, ReplayedStreams) end, + RevokedStreams0 + ), + AgentState0#{ + streams => Streams1, + revoked_streams => RevokedStreams1 }. -renew_leases(#{group := Group, agent_stream_assignments := AgentStreamAssignments} = Data) -> - ok = lists:foreach( - fun({Agent, #{version := Version}}) -> - ok = emqx_ds_shared_sub_proto:leader_renew_stream_lease(Agent, Group, Version) - end, - maps:to_list(AgentStreamAssignments) - ), - Data. - -update_agent_stream_states( - #{ - agent_stream_assignments := AgentStreamAssignments, - stream_assignments := StreamAssignments, - stream_progresses := StreamProgresses0 - } = Data0, - Agent, - AgentStreamProgresses, - Version +clean_revoked_streams( + Data0, _Agent, #{revoked_streams := RevokedStreams0} = AgentState0, ReceivedStreamProgresses ) -> - AgentVersion = emqx_utils_maps:deep_get([Agent, version], AgentStreamAssignments, undefined), - AgentPrevVersion = emqx_utils_maps:deep_get( - [Agent, prev_version], AgentStreamAssignments, undefined + FinishedReportedStreams = maps:from_list( + lists:filtermap( + fun + ( + #{ + stream := Stream, + use_finished := true + } + ) -> + {true, {Stream, true}}; + (_) -> + false + end, + ReceivedStreamProgresses + ) ), - case AgentVersion == Version orelse AgentPrevVersion == Version of - false -> - %% TODO https://emqx.atlassian.net/browse/EMQX-12572 - %% send invalidate to agent - Data0; - true -> - StreamProgresses1 = lists:foldl( - fun(#{stream := Stream, iterator := It}, ProgressesAcc) -> - %% Assert Stream is assigned to Agent - Agent = maps:get(Stream, StreamAssignments), - ProgressesAcc#{Stream => It} - end, - StreamProgresses0, - AgentStreamProgresses + {FinishedStreams, StillRevokingStreams} = lists:partition( + fun(Stream) -> + maps:is_key(Stream, FinishedReportedStreams) + end, + RevokedStreams0 + ), + Data1 = unassign_streams(Data0, FinishedStreams), + AgentState1 = AgentState0#{revoked_streams => StillRevokingStreams}, + {AgentState1, Data1}. + +unassign_streams(#{stream_owners := StreamOwners0} = Data, Streams) -> + StreamOwners1 = maps:without(Streams, StreamOwners0), + Data#{ + stream_owners => StreamOwners1 + }. + +%%-------------------------------------------------------------------- +%% Handle stream progress updates from agent in updating (VersionOld -> VersionNew) state + +update_agent_stream_states(Data0, Agent, AgentStreamProgresses, VersionOld, VersionNew) -> + #{state := State, version := AgentVersion, prev_version := AgentPrevVersion} = + AgentState0 = get_agent_state(Data0, Agent), + case {State, VersionOld, VersionNew} of + {?waiting_updating, AgentPrevVersion, AgentVersion} -> + %% Client started updating + {Data1, AgentState1} = update_stream_progresses( + Data0, Agent, AgentState0, AgentStreamProgresses ), - Data0#{stream_progresses => StreamProgresses1} + AgentState2 = update_agent_timeout(AgentState1), + {AgentState3, Data2} = clean_revoked_streams( + Data1, Agent, AgentState2, AgentStreamProgresses + ), + AgentState4 = + case AgentState3 of + #{revoked_streams := []} -> + agent_transition_to_waiting_replaying(Data1, Agent, AgentState3); + _ -> + agent_transition_to_updating(Agent, AgentState3) + end, + set_agent_state(Data2, Agent, AgentState4); + {?updating, AgentPrevVersion, AgentVersion} -> + {Data1, AgentState1} = update_stream_progresses( + Data0, Agent, AgentState0, AgentStreamProgresses + ), + AgentState2 = update_agent_timeout(AgentState1), + {AgentState3, Data2} = clean_revoked_streams( + Data1, Agent, AgentState2, AgentStreamProgresses + ), + AgentState4 = + case AgentState3 of + #{revoked_streams := []} -> + agent_transition_to_waiting_replaying(Data1, Agent, AgentState3); + _ -> + AgentState3 + end, + set_agent_state(Data2, Agent, AgentState4); + {?waiting_replaying, _, AgentVersion} -> + {Data1, AgentState1} = update_stream_progresses( + Data0, Agent, AgentState0, AgentStreamProgresses + ), + AgentState2 = update_agent_timeout(AgentState1), + set_agent_state(Data1, Agent, AgentState2); + {?replaying, _, AgentVersion} -> + {Data1, AgentState1} = update_stream_progresses( + Data0, Agent, AgentState0, AgentStreamProgresses + ), + AgentState2 = update_agent_timeout(AgentState1), + set_agent_state(Data1, Agent, AgentState2); + {OtherState, OtherVersionOld, OtherVersionNew} -> + ?tp(warning, unexpected_update, #{ + agent => Agent, + update_version_old => OtherVersionOld, + update_version_new => OtherVersionNew, + state => OtherState, + our_agent_version => AgentVersion, + our_agent_prev_version => AgentPrevVersion + }), + drop_invalidate_agent(Data0, Agent) end. +%%-------------------------------------------------------------------- +%% Disconnect agent gracefully + +disconnect_agent(Data0, Agent, AgentStreamProgresses, Version) -> + case get_agent_state(Data0, Agent) of + #{version := Version} -> + ?tp(warning, shared_sub_leader_disconnect_agent, #{ + agent => Agent, + version => Version + }), + Data1 = update_agent_stream_states(Data0, Agent, AgentStreamProgresses, Version), + Data2 = drop_agent(Data1, Agent), + Data2; + _ -> + ?tp(warning, shared_sub_leader_unexpected_disconnect, #{ + agent => Agent, + version => Version + }), + Data1 = drop_agent(Data0, Agent), + Data1 + end. + +%%-------------------------------------------------------------------- +%% Agent state transitions +%%-------------------------------------------------------------------- + +agent_transition_to_waiting_updating( + #{group_id := GroupId} = Data, + Agent, + #{state := OldState, version := Version, prev_version := undefined} = AgentState0, + Streams, + RevokedStreams +) -> + ?tp(warning, shared_sub_leader_agent_state_transition, #{ + agent => Agent, + old_state => OldState, + new_state => ?waiting_updating + }), + NewVersion = next_version(Version), + + AgentState1 = AgentState0#{ + state => ?waiting_updating, + streams => Streams, + revoked_streams => RevokedStreams, + prev_version => Version, + version => NewVersion + }, + AgentState2 = renew_no_replaying_deadline(AgentState1), + StreamProgresses = stream_progresses(Data, Streams), + ok = emqx_ds_shared_sub_proto:leader_update_streams( + Agent, GroupId, Version, NewVersion, StreamProgresses + ), + AgentState2. + +agent_transition_to_waiting_replaying( + #{group_id := GroupId} = _Data, Agent, #{state := OldState, version := Version} = AgentState0 +) -> + ?tp(warning, shared_sub_leader_agent_state_transition, #{ + agent => Agent, + old_state => OldState, + new_state => ?waiting_replaying + }), + ok = emqx_ds_shared_sub_proto:leader_renew_stream_lease(Agent, GroupId, Version), + AgentState1 = AgentState0#{ + state => ?waiting_replaying, + revoked_streams => [] + }, + renew_no_replaying_deadline(AgentState1). + +agent_transition_to_initial_waiting_replaying( + #{group_id := GroupId} = Data, Agent, AgentMetadata, InitialStreams +) -> + ?tp(warning, shared_sub_leader_agent_state_transition, #{ + agent => Agent, + old_state => none, + new_state => ?waiting_replaying + }), + Version = 0, + StreamProgresses = stream_progresses(Data, InitialStreams), + Leader = this_leader(Data), + ok = emqx_ds_shared_sub_proto:leader_lease_streams( + Agent, GroupId, Leader, StreamProgresses, Version + ), + AgentState = #{ + metadata => AgentMetadata, + state => ?waiting_replaying, + version => Version, + prev_version => undefined, + streams => InitialStreams, + revoked_streams => [], + update_deadline => now_ms_monotonic() + ?dq_config(leader_session_update_timeout_ms) + }, + renew_no_replaying_deadline(AgentState). + +agent_transition_to_replaying(Agent, #{state := ?waiting_replaying} = AgentState) -> + ?tp(warning, shared_sub_leader_agent_state_transition, #{ + agent => Agent, + old_state => ?waiting_replaying, + new_state => ?replaying + }), + AgentState#{ + state => ?replaying, + prev_version => undefined, + not_replaying_deadline => undefined + }. + +agent_transition_to_updating(Agent, #{state := ?waiting_updating} = AgentState0) -> + ?tp(warning, shared_sub_leader_agent_state_transition, #{ + agent => Agent, + old_state => ?waiting_updating, + new_state => ?updating + }), + AgentState1 = AgentState0#{state => ?updating}, + renew_no_replaying_deadline(AgentState1). + %%-------------------------------------------------------------------- %% Helper functions %%-------------------------------------------------------------------- -router_id() -> - emqx_guid:to_hexstr(emqx_guid:gen()). - now_ms() -> erlang:system_time(millisecond). -unassigned_streams(#{stream_progresses := StreamProgresses, stream_assignments := StreamAssignments}) -> - Streams = maps:keys(StreamProgresses), - AssignedStreams = maps:keys(StreamAssignments), +now_ms_monotonic() -> + erlang:monotonic_time(millisecond). + +renew_no_replaying_deadline(#{not_replaying_deadline := undefined} = AgentState) -> + AgentState#{ + not_replaying_deadline => now_ms_monotonic() + + ?dq_config(leader_session_not_replaying_timeout_ms) + }; +renew_no_replaying_deadline(#{not_replaying_deadline := _Deadline} = AgentState) -> + AgentState; +renew_no_replaying_deadline(#{} = AgentState) -> + AgentState#{ + not_replaying_deadline => now_ms_monotonic() + + ?dq_config(leader_session_not_replaying_timeout_ms) + }. + +unassigned_streams(#{stream_states := StreamStates, stream_owners := StreamOwners}) -> + Streams = maps:keys(StreamStates), + AssignedStreams = maps:keys(StreamOwners), Streams -- AssignedStreams. + +%% Those who are not connecting or updating, i.e. not in a transient state. +replaying_agents(#{agents := AgentStates}) -> + lists:filtermap( + fun + ({Agent, #{state := ?replaying}}) -> + {true, Agent}; + (_) -> + false + end, + maps:to_list(AgentStates) + ). + +desired_stream_count_per_agent(#{agents := AgentStates} = Data) -> + desired_stream_count_per_agent(Data, maps:size(AgentStates)). + +desired_stream_count_for_new_agent(#{agents := AgentStates} = Data) -> + desired_stream_count_per_agent(Data, maps:size(AgentStates) + 1). + +desired_stream_count_per_agent(#{stream_states := StreamStates}, AgentCount) -> + case AgentCount of + 0 -> + 0; + _ -> + StreamCount = maps:size(StreamStates), + case StreamCount rem AgentCount of + 0 -> + StreamCount div AgentCount; + _ -> + 1 + StreamCount div AgentCount + end + end. + +stream_progresses(#{stream_states := StreamStates} = _Data, Streams) -> + lists:map( + fun(Stream) -> + StreamData = maps:get(Stream, StreamStates), + #{ + stream => Stream, + progress => maps:get(progress, StreamData) + } + end, + Streams + ). + +next_version(Version) -> + Version + 1. + +shuffle(L0) -> + L1 = lists:map( + fun(A) -> + {rand:uniform(), A} + end, + L0 + ), + L2 = lists:sort(L1), + {_, L} = lists:unzip(L2), + L. + +set_stream_ownership_to_agent(#{stream_owners := StreamOwners0} = Data, Agent, Streams) -> + StreamOwners1 = lists:foldl( + fun(Stream, Acc) -> + Acc#{Stream => Agent} + end, + StreamOwners0, + Streams + ), + Data#{ + stream_owners => StreamOwners1 + }. + +set_agent_state(#{agents := Agents} = Data, Agent, AgentState) -> + Data#{ + agents => Agents#{Agent => AgentState} + }. + +update_agent_timeout(AgentState) -> + AgentState#{ + update_deadline => now_ms_monotonic() + ?dq_config(leader_session_update_timeout_ms) + }. + +get_agent_state(#{agents := Agents} = _Data, Agent) -> + maps:get(Agent, Agents). + +this_leader(_Data) -> + self(). + +drop_agent(#{agents := Agents} = Data0, Agent) -> + AgentState = get_agent_state(Data0, Agent), + #{streams := Streams, revoked_streams := RevokedStreams} = AgentState, + AllStreams = Streams ++ RevokedStreams, + Data1 = unassign_streams(Data0, AllStreams), + ?tp(warning, shared_sub_leader_drop_agent, #{agent => Agent}), + Data1#{agents => maps:remove(Agent, Agents)}. + +invalidate_agent(#{group_id := GroupId}, Agent) -> + ok = emqx_ds_shared_sub_proto:leader_invalidate(Agent, GroupId). + +drop_invalidate_agent(Data0, Agent) -> + Data1 = drop_agent(Data0, Agent), + ok = invalidate_agent(Data1, Agent), + Data1. + +with_agent(#{agents := Agents} = Data, Agent, Fun) -> + case Agents of + #{Agent := _} -> + Fun(); + _ -> + Data + end. diff --git a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader_rank_progress.erl b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader_rank_progress.erl new file mode 100644 index 000000000..fa611463d --- /dev/null +++ b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader_rank_progress.erl @@ -0,0 +1,171 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_ds_shared_sub_leader_rank_progress). + +-include_lib("emqx/include/logger.hrl"). + +-export([ + init/0, + set_replayed/2, + add_streams/2, + replayed_up_to/2 +]). + +%% "shard" +-type rank_x() :: emqx_ds:rank_x(). + +%% "generation" +-type rank_y() :: emqx_ds:rank_y(). + +%% shard progress +-type x_progress() :: #{ + %% All streams with given rank_x and rank_y =< min_y are replayed. + min_y := rank_y(), + + ys := #{ + rank_y() => #{ + emqx_ds:stream() => _IdReplayed :: boolean() + } + } +}. + +-type t() :: #{ + rank_x() => x_progress() +}. + +-export_type([ + t/0 +]). + +%%-------------------------------------------------------------------- +%% API +%%-------------------------------------------------------------------- + +-spec init() -> t(). +init() -> #{}. + +-spec set_replayed(emqx_ds:stream_rank(), t()) -> t(). +set_replayed({{RankX, RankY}, Stream}, State) -> + case State of + #{RankX := #{ys := #{RankY := #{Stream := false} = RankYStreams} = Ys0}} -> + Ys1 = Ys0#{RankY => RankYStreams#{Stream => true}}, + {MinY, Ys2} = update_min_y(maps:to_list(Ys1)), + State#{RankX => #{min_y => MinY, ys => Ys2}}; + _ -> + ?SLOG( + warning, + #{ + msg => leader_rank_progress_double_or_invalid_update, + rank_x => RankX, + rank_y => RankY, + state => State + } + ), + State + end. + +-spec add_streams([{emqx_ds:stream_rank(), emqx_ds:stream()}], t()) -> + {[{emqx_ds:stream_rank(), emqx_ds:stream()}], t()}. +add_streams(StreamsWithRanks, State) -> + SortedStreamsWithRanks = lists:sort( + fun({{_RankX1, RankY1}, _Stream1}, {{_RankX2, RankY2}, _Stream2}) -> + RankY1 =< RankY2 + end, + StreamsWithRanks + ), + lists:foldl( + fun({Rank, Stream} = StreamWithRank, {StreamAcc, StateAcc0}) -> + case add_stream({Rank, Stream}, StateAcc0) of + {true, StateAcc1} -> + {[StreamWithRank | StreamAcc], StateAcc1}; + false -> + {StreamAcc, StateAcc0} + end + end, + {[], State}, + SortedStreamsWithRanks + ). + +-spec replayed_up_to(emqx_ds:rank_x(), t()) -> emqx_ds:rank_y(). +replayed_up_to(RankX, State) -> + case State of + #{RankX := #{min_y := MinY}} -> + MinY; + _ -> + undefined + end. + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- + +add_stream({{RankX, RankY}, Stream}, State0) -> + case State0 of + #{RankX := #{min_y := MinY}} when RankY =< MinY -> + false; + #{RankX := #{ys := #{RankY := #{Stream := true}}}} -> + false; + _ -> + XProgress = maps:get(RankX, State0, #{min_y => RankY - 1, ys => #{}}), + Ys0 = maps:get(ys, XProgress), + RankYStreams0 = maps:get(RankY, Ys0, #{}), + RankYStreams1 = RankYStreams0#{Stream => false}, + Ys1 = Ys0#{RankY => RankYStreams1}, + State1 = State0#{RankX => XProgress#{ys => Ys1}}, + {true, State1} + end. + +update_min_y([{RankY, RankYStreams} | Rest] = Ys) -> + case {has_unreplayed_streams(RankYStreams), Rest} of + {true, _} -> + {RankY - 1, maps:from_list(Ys)}; + {false, []} -> + {RankY - 1, #{}}; + {false, _} -> + update_min_y(Rest) + end. + +has_unreplayed_streams(RankYStreams) -> + lists:any( + fun(IsReplayed) -> not IsReplayed end, + maps:values(RankYStreams) + ). + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). + +add_streams_set_replayed_test() -> + State0 = init(), + {_, State1} = add_streams( + [ + {{shard1, 1}, s111}, + {{shard1, 1}, s112}, + {{shard1, 2}, s121}, + {{shard1, 2}, s122}, + {{shard1, 3}, s131}, + {{shard1, 4}, s141}, + + {{shard3, 5}, s51} + ], + State0 + ), + ?assertEqual(0, replayed_up_to(shard1, State1)), + + State2 = set_replayed({{shard1, 1}, s111}, State1), + State3 = set_replayed({{shard1, 3}, s131}, State2), + ?assertEqual(0, replayed_up_to(shard1, State3)), + State4 = set_replayed({{shard1, 1}, s112}, State3), + ?assertEqual(1, replayed_up_to(shard1, State4)), + + State5 = set_replayed({{shard1, 2}, s121}, State4), + State6 = set_replayed({{shard1, 2}, s122}, State5), + + ?assertEqual(3, replayed_up_to(shard1, State6)), + + State7 = set_replayed({{shard1, 4}, s141}, State6), + ?assertEqual(3, replayed_up_to(shard1, State7)). + +%% -ifdef(TEST) end +-endif. diff --git a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_proto.erl b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_proto.erl index d9a0b994f..383f66ff2 100644 --- a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_proto.erl +++ b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_proto.erl @@ -2,71 +2,286 @@ %% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- -%% TODO https://emqx.atlassian.net/browse/EMQX-12573 -%% This should be wrapped with a proto_v1 module. -%% For simplicity, send as simple OTP messages for now. - -module(emqx_ds_shared_sub_proto). -include("emqx_ds_shared_sub_proto.hrl"). --export([ - agent_connect_leader/3, - agent_update_stream_states/4, +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). - leader_lease_streams/4, - leader_renew_stream_lease/3 +-export([ + agent_connect_leader/4, + agent_update_stream_states/4, + agent_update_stream_states/5, + agent_disconnect/4, + + leader_lease_streams/5, + leader_renew_stream_lease/3, + leader_renew_stream_lease/4, + leader_update_streams/5, + leader_invalidate/2 ]). --type agent() :: pid(). +-export([ + format_stream_progresses/1, + format_stream_progress/1, + format_stream_key/1, + format_stream_keys/1, + format_lease_event/1, + format_lease_events/1, + agent/2 +]). + +-type agent() :: ?agent(emqx_persistent_session_ds:id(), pid()). -type leader() :: pid(). --type topic_filter() :: emqx_persistent_session_ds:share_topic_filter(). +-type share_topic_filter() :: emqx_persistent_session_ds:share_topic_filter(). -type group() :: emqx_types:group(). -type version() :: non_neg_integer(). - --type stream_progress() :: #{ - stream := emqx_ds:stream(), - iterator := emqx_ds:iterator() +-type agent_metadata() :: #{ + id := emqx_persistent_session_ds:id() }. +-type leader_stream_progress() :: #{ + stream := emqx_ds:stream(), + progress := emqx_persistent_session_ds_shared_subs:progress() +}. + +-type agent_stream_progress() :: emqx_persistent_session_ds_shared_subs:agent_stream_progress(). + -export_type([ agent/0, leader/0, group/0, version/0, - stream_progress/0 + leader_stream_progress/0, + agent_stream_progress/0, + agent_metadata/0 ]). +%%-------------------------------------------------------------------- +%% API +%%-------------------------------------------------------------------- + %% agent -> leader messages --spec agent_connect_leader(leader(), agent(), topic_filter()) -> ok. -agent_connect_leader(ToLeader, FromAgent, TopicFilter) -> - _ = erlang:send(ToLeader, ?agent_connect_leader(FromAgent, TopicFilter)), - ok. +-spec agent_connect_leader(leader(), agent(), agent_metadata(), share_topic_filter()) -> ok. +agent_connect_leader(ToLeader, FromAgent, AgentMetadata, ShareTopicFilter) when + ?is_local_leader(ToLeader) +-> + ?tp(warning, shared_sub_proto_msg, #{ + type => agent_connect_leader, + to_leader => ToLeader, + from_agent => FromAgent, + agent_metadata => AgentMetadata, + share_topic_filter => ShareTopicFilter + }), + _ = erlang:send(ToLeader, ?agent_connect_leader(FromAgent, AgentMetadata, ShareTopicFilter)), + ok; +agent_connect_leader(ToLeader, FromAgent, AgentMetadata, ShareTopicFilter) -> + emqx_ds_shared_sub_proto_v1:agent_connect_leader( + ?leader_node(ToLeader), ToLeader, FromAgent, AgentMetadata, ShareTopicFilter + ). --spec agent_update_stream_states(leader(), agent(), list(stream_progress()), version()) -> ok. -agent_update_stream_states(ToLeader, FromAgent, StreamProgresses, Version) -> +-spec agent_update_stream_states(leader(), agent(), list(agent_stream_progress()), version()) -> ok. +agent_update_stream_states(ToLeader, FromAgent, StreamProgresses, Version) when + ?is_local_leader(ToLeader) +-> + ?tp(warning, shared_sub_proto_msg, #{ + type => agent_update_stream_states, + to_leader => ToLeader, + from_agent => FromAgent, + stream_progresses => format_stream_progresses(StreamProgresses), + version => Version + }), _ = erlang:send(ToLeader, ?agent_update_stream_states(FromAgent, StreamProgresses, Version)), - ok. + ok; +agent_update_stream_states(ToLeader, FromAgent, StreamProgresses, Version) -> + emqx_ds_shared_sub_proto_v1:agent_update_stream_states( + ?leader_node(ToLeader), ToLeader, FromAgent, StreamProgresses, Version + ). -%% ... +-spec agent_update_stream_states( + leader(), agent(), list(agent_stream_progress()), version(), version() +) -> ok. +agent_update_stream_states(ToLeader, FromAgent, StreamProgresses, VersionOld, VersionNew) when + ?is_local_leader(ToLeader) +-> + ?tp(warning, shared_sub_proto_msg, #{ + type => agent_update_stream_states, + to_leader => ToLeader, + from_agent => FromAgent, + stream_progresses => format_stream_progresses(StreamProgresses), + version_old => VersionOld, + version_new => VersionNew + }), + _ = erlang:send( + ToLeader, ?agent_update_stream_states(FromAgent, StreamProgresses, VersionOld, VersionNew) + ), + ok; +agent_update_stream_states(ToLeader, FromAgent, StreamProgresses, VersionOld, VersionNew) -> + emqx_ds_shared_sub_proto_v1:agent_update_stream_states( + ?leader_node(ToLeader), ToLeader, FromAgent, StreamProgresses, VersionOld, VersionNew + ). + +agent_disconnect(ToLeader, FromAgent, StreamProgresses, Version) when + ?is_local_leader(ToLeader) +-> + ?tp(warning, shared_sub_proto_msg, #{ + type => agent_disconnect, + to_leader => ToLeader, + from_agent => FromAgent, + stream_progresses => format_stream_progresses(StreamProgresses), + version => Version + }), + _ = erlang:send(ToLeader, ?agent_disconnect(FromAgent, StreamProgresses, Version)), + ok; +agent_disconnect(ToLeader, FromAgent, StreamProgresses, Version) -> + emqx_ds_shared_sub_proto_v1:agent_disconnect( + ?leader_node(ToLeader), ToLeader, FromAgent, StreamProgresses, Version + ). %% leader -> agent messages --spec leader_lease_streams(agent(), group(), list(stream_progress()), version()) -> ok. -leader_lease_streams(ToAgent, OfGroup, Streams, Version) -> - _ = emqx_persistent_session_ds_shared_subs_agent:send( - ToAgent, - ?leader_lease_streams(OfGroup, Streams, Version) - ), +-spec leader_lease_streams(agent(), group(), leader(), list(leader_stream_progress()), version()) -> ok. +leader_lease_streams(ToAgent, OfGroup, Leader, Streams, Version) when ?is_local_agent(ToAgent) -> + ?tp(warning, shared_sub_proto_msg, #{ + type => leader_lease_streams, + to_agent => ToAgent, + of_group => OfGroup, + leader => Leader, + streams => format_stream_progresses(Streams), + version => Version + }), + _ = emqx_persistent_session_ds_shared_subs_agent:send( + ?agent_pid(ToAgent), + ?leader_lease_streams(OfGroup, Leader, Streams, Version) + ), + ok; +leader_lease_streams(ToAgent, OfGroup, Leader, Streams, Version) -> + emqx_ds_shared_sub_proto_v1:leader_lease_streams( + ?agent_node(ToAgent), ToAgent, OfGroup, Leader, Streams, Version + ). -spec leader_renew_stream_lease(agent(), group(), version()) -> ok. -leader_renew_stream_lease(ToAgent, OfGroup, Version) -> +leader_renew_stream_lease(ToAgent, OfGroup, Version) when ?is_local_agent(ToAgent) -> + ?tp(warning, shared_sub_proto_msg, #{ + type => leader_renew_stream_lease, + to_agent => ToAgent, + of_group => OfGroup, + version => Version + }), _ = emqx_persistent_session_ds_shared_subs_agent:send( - ToAgent, + ?agent_pid(ToAgent), ?leader_renew_stream_lease(OfGroup, Version) ), - ok. + ok; +leader_renew_stream_lease(ToAgent, OfGroup, Version) -> + emqx_ds_shared_sub_proto_v1:leader_renew_stream_lease( + ?agent_node(ToAgent), ToAgent, OfGroup, Version + ). -%% ... +-spec leader_renew_stream_lease(agent(), group(), version(), version()) -> ok. +leader_renew_stream_lease(ToAgent, OfGroup, VersionOld, VersionNew) when ?is_local_agent(ToAgent) -> + ?tp(warning, shared_sub_proto_msg, #{ + type => leader_renew_stream_lease, + to_agent => ToAgent, + of_group => OfGroup, + version_old => VersionOld, + version_new => VersionNew + }), + _ = emqx_persistent_session_ds_shared_subs_agent:send( + ?agent_pid(ToAgent), + ?leader_renew_stream_lease(OfGroup, VersionOld, VersionNew) + ), + ok; +leader_renew_stream_lease(ToAgent, OfGroup, VersionOld, VersionNew) -> + emqx_ds_shared_sub_proto_v1:leader_renew_stream_lease( + ?agent_node(ToAgent), ToAgent, OfGroup, VersionOld, VersionNew + ). + +-spec leader_update_streams(agent(), group(), version(), version(), list(leader_stream_progress())) -> + ok. +leader_update_streams(ToAgent, OfGroup, VersionOld, VersionNew, StreamsNew) when + ?is_local_agent(ToAgent) +-> + ?tp(warning, shared_sub_proto_msg, #{ + type => leader_update_streams, + to_agent => ToAgent, + of_group => OfGroup, + version_old => VersionOld, + version_new => VersionNew, + streams_new => format_stream_progresses(StreamsNew) + }), + _ = emqx_persistent_session_ds_shared_subs_agent:send( + ?agent_pid(ToAgent), + ?leader_update_streams(OfGroup, VersionOld, VersionNew, StreamsNew) + ), + ok; +leader_update_streams(ToAgent, OfGroup, VersionOld, VersionNew, StreamsNew) -> + emqx_ds_shared_sub_proto_v1:leader_update_streams( + ?agent_node(ToAgent), ToAgent, OfGroup, VersionOld, VersionNew, StreamsNew + ). + +-spec leader_invalidate(agent(), group()) -> ok. +leader_invalidate(ToAgent, OfGroup) when ?is_local_agent(ToAgent) -> + ?tp(warning, shared_sub_proto_msg, #{ + type => leader_invalidate, + to_agent => ToAgent, + of_group => OfGroup + }), + _ = emqx_persistent_session_ds_shared_subs_agent:send( + ?agent_pid(ToAgent), + ?leader_invalidate(OfGroup) + ), + ok; +leader_invalidate(ToAgent, OfGroup) -> + emqx_ds_shared_sub_proto_v1:leader_invalidate( + ?agent_node(ToAgent), ToAgent, OfGroup + ). + +%%-------------------------------------------------------------------- +%% Internal API +%%-------------------------------------------------------------------- + +agent(Id, Pid) -> + _ = Id, + ?agent(Id, Pid). + +format_stream_progresses(Streams) -> + lists:map( + fun format_stream_progress/1, + Streams + ). + +format_stream_progress(#{stream := Stream, progress := Progress} = Value) -> + Value#{stream => format_opaque(Stream), progress => format_progress(Progress)}. + +format_progress(#{iterator := Iterator} = Progress) -> + Progress#{iterator => format_opaque(Iterator)}. + +format_stream_key({SubId, Stream}) -> + {SubId, format_opaque(Stream)}. + +format_stream_keys(StreamKeys) -> + lists:map( + fun format_stream_key/1, + StreamKeys + ). + +format_lease_events(Events) -> + lists:map( + fun format_lease_event/1, + Events + ). + +format_lease_event(#{stream := Stream, progress := Progress} = Event) -> + Event#{stream => format_opaque(Stream), progress => format_progress(Progress)}; +format_lease_event(#{stream := Stream} = Event) -> + Event#{stream => format_opaque(Stream)}. + +%%-------------------------------------------------------------------- +%% Helpers +%%-------------------------------------------------------------------- + +format_opaque(Opaque) -> + erlang:phash2(Opaque). diff --git a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_proto.hrl b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_proto.hrl index c780ab193..bf54b2930 100644 --- a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_proto.hrl +++ b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_proto.hrl @@ -6,9 +6,6 @@ %% These messages are instantiated on the receiver's side, so they do not %% travel over the network. --ifndef(EMQX_DS_SHARED_SUB_PROTO_HRL). --define(EMQX_DS_SHARED_SUB_PROTO_HRL, true). - %% NOTE %% We do not need any kind of request/response identification, %% because the protocol is fully event-based. @@ -19,19 +16,22 @@ -define(agent_update_stream_states_msg, agent_update_stream_states). -define(agent_connect_leader_timeout_msg, agent_connect_leader_timeout). -define(agent_renew_stream_lease_timeout_msg, agent_renew_stream_lease_timeout). +-define(agent_disconnect_msg, agent_disconnect). %% Agent messages sent to the leader. %% Leader talks to many agents, `agent` field is used to identify the sender. --define(agent_connect_leader(Agent, TopicFilter), #{ +-define(agent_connect_leader(Agent, AgentMetadata, ShareTopicFilter), #{ type => ?agent_connect_leader_msg, - topic_filter => TopicFilter, + share_topic_filter => ShareTopicFilter, + agent_metadata => AgentMetadata, agent => Agent }). --define(agent_connect_leader_match(Agent, TopicFilter), #{ +-define(agent_connect_leader_match(Agent, AgentMetadata, ShareTopicFilter), #{ type := ?agent_connect_leader_msg, - topic_filter := TopicFilter, + share_topic_filter := ShareTopicFilter, + agent_metadata := AgentMetadata, agent := Agent }). @@ -49,37 +49,137 @@ agent := Agent }). +-define(agent_update_stream_states(Agent, StreamStates, VersionOld, VersionNew), #{ + type => ?agent_update_stream_states_msg, + stream_states => StreamStates, + version_old => VersionOld, + version_new => VersionNew, + agent => Agent +}). + +-define(agent_update_stream_states_match(Agent, StreamStates, VersionOld, VersionNew), #{ + type := ?agent_update_stream_states_msg, + stream_states := StreamStates, + version_old := VersionOld, + version_new := VersionNew, + agent := Agent +}). + +-define(agent_disconnect(Agent, StreamStates, Version), #{ + type => ?agent_disconnect_msg, + stream_states => StreamStates, + version => Version, + agent => Agent +}). + +-define(agent_disconnect_match(Agent, StreamStates, Version), #{ + type := ?agent_disconnect_msg, + stream_states := StreamStates, + version := Version, + agent := Agent +}). + %% leader messages, sent from the leader to the agent %% Agent may have several shared subscriptions, so may talk to several leaders -%% `group` field is used to identify the leader. +%% `group_id` field is used to identify the leader. -define(leader_lease_streams_msg, leader_lease_streams). -define(leader_renew_stream_lease_msg, leader_renew_stream_lease). --define(leader_lease_streams(Group, Streams, Version), #{ +-define(leader_lease_streams(GrouId, Leader, Streams, Version), #{ type => ?leader_lease_streams_msg, streams => Streams, version => Version, - group => Group + leader => Leader, + group_id => GrouId }). --define(leader_lease_streams_match(Group, Streams, Version), #{ +-define(leader_lease_streams_match(GroupId, Leader, Streams, Version), #{ type := ?leader_lease_streams_msg, streams := Streams, version := Version, - group := Group + leader := Leader, + group_id := GroupId }). --define(leader_renew_stream_lease(Group, Version), #{ +-define(leader_renew_stream_lease(GroupId, Version), #{ type => ?leader_renew_stream_lease_msg, version => Version, - group => Group + group_id => GroupId }). --define(leader_renew_stream_lease_match(Group, Version), #{ +-define(leader_renew_stream_lease_match(GroupId, Version), #{ type := ?leader_renew_stream_lease_msg, version := Version, - group := Group + group_id := GroupId }). +-define(leader_renew_stream_lease(GroupId, VersionOld, VersionNew), #{ + type => ?leader_renew_stream_lease_msg, + version_old => VersionOld, + version_new => VersionNew, + group_id => GroupId +}). + +-define(leader_renew_stream_lease_match(GroupId, VersionOld, VersionNew), #{ + type := ?leader_renew_stream_lease_msg, + version_old := VersionOld, + version_new := VersionNew, + group_id := GroupId +}). + +-define(leader_update_streams(GroupId, VersionOld, VersionNew, StreamsNew), #{ + type => leader_update_streams, + version_old => VersionOld, + version_new => VersionNew, + streams_new => StreamsNew, + group_id => GroupId +}). + +-define(leader_update_streams_match(GroupId, VersionOld, VersionNew, StreamsNew), #{ + type := leader_update_streams, + version_old := VersionOld, + version_new := VersionNew, + streams_new := StreamsNew, + group_id := GroupId +}). + +-define(leader_invalidate(GroupId), #{ + type => leader_invalidate, + group_id => GroupId +}). + +-define(leader_invalidate_match(GroupId), #{ + type := leader_invalidate, + group_id := GroupId +}). + +%% Helpers +%% In test mode we extend agents with (session) Id to have more +%% readable traces. + +-ifdef(TEST). + +-define(agent(Id, Pid), {Id, Pid}). + +-define(agent_pid(Agent), element(2, Agent)). + +-define(agent_node(Agent), node(element(2, Agent))). + +%% -ifdef(TEST). +-else. + +-define(agent(Id, Pid), Pid). + +-define(agent_pid(Agent), Agent). + +-define(agent_node(Agent), node(Agent)). + +%% -ifdef(TEST). -endif. + +-define(is_local_agent(Agent), (?agent_node(Agent) =:= node())). + +-define(leader_node(Leader), node(Leader)). + +-define(is_local_leader(Leader), (?leader_node(Leader) =:= node())). diff --git a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_registry.erl b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_registry.erl index 9b4a6bd11..eae212458 100644 --- a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_registry.erl +++ b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_registry.erl @@ -20,12 +20,13 @@ ]). -export([ - lookup_leader/2 + lookup_leader/3 ]). -record(lookup_leader, { agent :: emqx_ds_shared_sub_proto:agent(), - topic_filter :: emqx_persistent_session_ds:share_topic_filter() + agent_metadata :: emqx_ds_shared_sub_proto:agent_metadata(), + share_topic_filter :: emqx_persistent_session_ds:share_topic_filter() }). -define(gproc_id(ID), {n, l, ID}). @@ -35,10 +36,14 @@ %%-------------------------------------------------------------------- -spec lookup_leader( - emqx_ds_shared_sub_proto:agent(), emqx_persistent_session_ds:share_topic_filter() + emqx_ds_shared_sub_proto:agent(), + emqx_ds_shared_sub_proto:agent_metadata(), + emqx_persistent_session_ds:share_topic_filter() ) -> ok. -lookup_leader(Agent, TopicFilter) -> - gen_server:cast(?MODULE, #lookup_leader{agent = Agent, topic_filter = TopicFilter}). +lookup_leader(Agent, AgentMetadata, ShareTopicFilter) -> + gen_server:cast(?MODULE, #lookup_leader{ + agent = Agent, agent_metadata = AgentMetadata, share_topic_filter = ShareTopicFilter + }). %%-------------------------------------------------------------------- %% Internal API @@ -66,8 +71,15 @@ init([]) -> handle_call(_Request, _From, State) -> {reply, {error, unknown_request}, State}. -handle_cast(#lookup_leader{agent = Agent, topic_filter = TopicFilter}, State) -> - State1 = do_lookup_leader(Agent, TopicFilter, State), +handle_cast( + #lookup_leader{ + agent = Agent, + agent_metadata = AgentMetadata, + share_topic_filter = ShareTopicFilter + }, + State +) -> + State1 = do_lookup_leader(Agent, AgentMetadata, ShareTopicFilter, State), {noreply, State1}. handle_info(_Info, State) -> @@ -80,15 +92,15 @@ terminate(_Reason, _State) -> %% Internal functions %%-------------------------------------------------------------------- -do_lookup_leader(Agent, TopicFilter, State) -> +do_lookup_leader(Agent, AgentMetadata, ShareTopicFilter, State) -> %% TODO https://emqx.atlassian.net/browse/EMQX-12309 %% Cluster-wide unique leader election should be implemented - Id = emqx_ds_shared_sub_leader:id(TopicFilter), + Id = emqx_ds_shared_sub_leader:id(ShareTopicFilter), LeaderPid = case gproc:where(?gproc_id(Id)) of undefined -> {ok, Pid} = emqx_ds_shared_sub_leader_sup:start_leader(#{ - topic_filter => TopicFilter + share_topic_filter => ShareTopicFilter }), {ok, NewLeaderPid} = emqx_ds_shared_sub_leader:register( Pid, @@ -104,8 +116,10 @@ do_lookup_leader(Agent, TopicFilter, State) -> ?SLOG(info, #{ msg => lookup_leader, agent => Agent, - topic_filter => TopicFilter, + share_topic_filter => ShareTopicFilter, leader => LeaderPid }), - ok = emqx_ds_shared_sub_proto:agent_connect_leader(LeaderPid, Agent, TopicFilter), + ok = emqx_ds_shared_sub_proto:agent_connect_leader( + LeaderPid, Agent, AgentMetadata, ShareTopicFilter + ), State. diff --git a/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_schema.erl b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_schema.erl new file mode 100644 index 000000000..d60893678 --- /dev/null +++ b/apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_schema.erl @@ -0,0 +1,57 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_ds_shared_sub_schema). + +-include_lib("hocon/include/hoconsc.hrl"). + +-export([ + namespace/0, + roots/0, + fields/1, + desc/1 +]). + +namespace() -> emqx_shared_subs. + +roots() -> + [ + durable_queues + ]. + +fields(durable_queues) -> + [ + {enable, + ?HOCON( + boolean(), + #{ + required => false, + default => true, + desc => ?DESC(enable) + } + )}, + duration(session_find_leader_timeout_ms, 1000), + duration(session_renew_lease_timeout_ms, 5000), + duration(session_min_update_stream_state_interval_ms, 500), + + duration(leader_renew_lease_interval_ms, 1000), + duration(leader_renew_streams_interval_ms, 1000), + duration(leader_drop_timeout_interval_ms, 1000), + duration(leader_session_update_timeout_ms, 5000), + duration(leader_session_not_replaying_timeout_ms, 5000) + ]. + +duration(MsFieldName, Default) -> + {MsFieldName, + ?HOCON( + emqx_schema:timeout_duration_ms(), + #{ + required => false, + default => Default, + desc => ?DESC(MsFieldName), + importance => ?IMPORTANCE_HIDDEN + } + )}. + +desc(durable_queues) -> "Settings for durable queues". diff --git a/apps/emqx_ds_shared_sub/src/proto/emqx_ds_shared_sub_proto_v1.erl b/apps/emqx_ds_shared_sub/src/proto/emqx_ds_shared_sub_proto_v1.erl new file mode 100644 index 000000000..17ceb4876 --- /dev/null +++ b/apps/emqx_ds_shared_sub/src/proto/emqx_ds_shared_sub_proto_v1.erl @@ -0,0 +1,130 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_ds_shared_sub_proto_v1). + +-behaviour(emqx_bpapi). + +-include_lib("emqx/include/bpapi.hrl"). + +-export([ + introduced_in/0, + + agent_connect_leader/5, + agent_update_stream_states/5, + agent_update_stream_states/6, + agent_disconnect/5, + + leader_lease_streams/6, + leader_renew_stream_lease/4, + leader_renew_stream_lease/5, + leader_update_streams/6, + leader_invalidate/3 +]). + +introduced_in() -> + "5.8.0". + +-spec agent_connect_leader( + node(), + emqx_ds_shared_sub_proto:leader(), + emqx_ds_shared_sub_proto:agent(), + emqx_ds_shared_sub_proto:agent_metadata(), + emqx_persistent_session_ds:share_topic_filter() +) -> ok. +agent_connect_leader(Node, ToLeader, FromAgent, AgentMetadata, ShareTopicFilter) -> + erpc:cast(Node, emqx_ds_shared_sub_proto, agent_connect_leader, [ + ToLeader, FromAgent, AgentMetadata, ShareTopicFilter + ]). + +-spec agent_update_stream_states( + node(), + emqx_ds_shared_sub_proto:leader(), + emqx_ds_shared_sub_proto:agent(), + list(emqx_ds_shared_sub_proto:agent_stream_progress()), + emqx_ds_shared_sub_proto:version() +) -> ok. +agent_update_stream_states(Node, ToLeader, FromAgent, StreamProgresses, Version) -> + erpc:cast(Node, emqx_ds_shared_sub_proto, agent_update_stream_states, [ + ToLeader, FromAgent, StreamProgresses, Version + ]). + +-spec agent_update_stream_states( + node(), + emqx_ds_shared_sub_proto:leader(), + emqx_ds_shared_sub_proto:agent(), + list(emqx_ds_shared_sub_proto:agent_stream_progress()), + emqx_ds_shared_sub_proto:version(), + emqx_ds_shared_sub_proto:version() +) -> ok. +agent_update_stream_states(Node, ToLeader, FromAgent, StreamProgresses, VersionOld, VersionNew) -> + erpc:cast(Node, emqx_ds_shared_sub_proto, agent_update_stream_states, [ + ToLeader, FromAgent, StreamProgresses, VersionOld, VersionNew + ]). + +-spec agent_disconnect( + node(), + emqx_ds_shared_sub_proto:leader(), + emqx_ds_shared_sub_proto:agent(), + list(emqx_ds_shared_sub_proto:agent_stream_progress()), + emqx_ds_shared_sub_proto:version() +) -> ok. +agent_disconnect(Node, ToLeader, FromAgent, StreamProgresses, Version) -> + erpc:cast(Node, emqx_ds_shared_sub_proto, agent_disconnect, [ + ToLeader, FromAgent, StreamProgresses, Version + ]). + +%% leader -> agent messages + +-spec leader_lease_streams( + node(), + emqx_ds_shared_sub_proto:agent(), + emqx_ds_shared_sub_proto:group(), + emqx_ds_shared_sub_proto:leader(), + list(emqx_ds_shared_sub_proto:leader_stream_progress()), + emqx_ds_shared_sub_proto:version() +) -> ok. +leader_lease_streams(Node, ToAgent, OfGroup, Leader, Streams, Version) -> + erpc:cast(Node, emqx_ds_shared_sub_proto, leader_lease_streams, [ + ToAgent, OfGroup, Leader, Streams, Version + ]). + +-spec leader_renew_stream_lease( + node(), + emqx_ds_shared_sub_proto:agent(), + emqx_ds_shared_sub_proto:group(), + emqx_ds_shared_sub_proto:version() +) -> ok. +leader_renew_stream_lease(Node, ToAgent, OfGroup, Version) -> + erpc:cast(Node, emqx_ds_shared_sub_proto, leader_renew_stream_lease, [ToAgent, OfGroup, Version]). + +-spec leader_renew_stream_lease( + node(), + emqx_ds_shared_sub_proto:agent(), + emqx_ds_shared_sub_proto:group(), + emqx_ds_shared_sub_proto:version(), + emqx_ds_shared_sub_proto:version() +) -> ok. +leader_renew_stream_lease(Node, ToAgent, OfGroup, VersionOld, VersionNew) -> + erpc:cast(Node, emqx_ds_shared_sub_proto, leader_renew_stream_lease, [ + ToAgent, OfGroup, VersionOld, VersionNew + ]). + +-spec leader_update_streams( + node(), + emqx_ds_shared_sub_proto:agent(), + emqx_ds_shared_sub_proto:group(), + emqx_ds_shared_sub_proto:version(), + emqx_ds_shared_sub_proto:version(), + list(emqx_ds_shared_sub_proto:leader_stream_progress()) +) -> ok. +leader_update_streams(Node, ToAgent, OfGroup, VersionOld, VersionNew, StreamsNew) -> + erpc:cast(Node, emqx_ds_shared_sub_proto, leader_update_streams, [ + ToAgent, OfGroup, VersionOld, VersionNew, StreamsNew + ]). + +-spec leader_invalidate(node(), emqx_ds_shared_sub_proto:agent(), emqx_ds_shared_sub_proto:group()) -> + ok. +leader_invalidate(Node, ToAgent, OfGroup) -> + erpc:cast(Node, emqx_ds_shared_sub_proto, leader_invalidate, [ToAgent, OfGroup]). diff --git a/apps/emqx_ds_shared_sub/test/emqx_ds_shared_sub_SUITE.erl b/apps/emqx_ds_shared_sub/test/emqx_ds_shared_sub_SUITE.erl index f18114918..4f99a8455 100644 --- a/apps/emqx_ds_shared_sub/test/emqx_ds_shared_sub_SUITE.erl +++ b/apps/emqx_ds_shared_sub/test/emqx_ds_shared_sub_SUITE.erl @@ -10,10 +10,10 @@ -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). --include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("emqx/include/asserts.hrl"). -all() -> emqx_common_test_helpers:all(?MODULE). +all() -> + emqx_common_test_helpers:all(?MODULE). init_per_suite(Config) -> Apps = emqx_cth_suite:start( @@ -51,29 +51,364 @@ end_per_testcase(_TC, _Config) -> ok. t_lease_initial(_Config) -> - ConnPub = emqtt_connect_pub(<<"client_pub">>), - - %% Need to pre-create some streams in "topic/#". - %% Leader is dummy by far and won't update streams after the first lease to the agent. - %% So there should be some streams already when the agent connects. - ok = init_streams(ConnPub, <<"topic1/1">>), - ConnShared = emqtt_connect_sub(<<"client_shared">>), {ok, _, _} = emqtt:subscribe(ConnShared, <<"$share/gr1/topic1/#">>, 1), - {ok, _} = emqtt:publish(ConnPub, <<"topic1/1">>, <<"hello2">>, 1), + ConnPub = emqtt_connect_pub(<<"client_pub">>), + + {ok, _} = emqtt:publish(ConnPub, <<"topic1/1">>, <<"hello1">>, 1), + ct:sleep(2_000), + {ok, _} = emqtt:publish(ConnPub, <<"topic1/2">>, <<"hello2">>, 1), + + ?assertReceive({publish, #{payload := <<"hello1">>}}, 10_000), ?assertReceive({publish, #{payload := <<"hello2">>}}, 10_000), ok = emqtt:disconnect(ConnShared), ok = emqtt:disconnect(ConnPub). -t_lease_reconnect(_Config) -> +t_two_clients(_Config) -> + ConnShared1 = emqtt_connect_sub(<<"client_shared1">>), + {ok, _, _} = emqtt:subscribe(ConnShared1, <<"$share/gr4/topic4/#">>, 1), + + ConnShared2 = emqtt_connect_sub(<<"client_shared2">>), + {ok, _, _} = emqtt:subscribe(ConnShared2, <<"$share/gr4/topic4/#">>, 1), + ConnPub = emqtt_connect_pub(<<"client_pub">>), - %% Need to pre-create some streams in "topic/#". - %% Leader is dummy by far and won't update streams after the first lease to the agent. - %% So there should be some streams already when the agent connects. - ok = init_streams(ConnPub, <<"topic2/2">>), + {ok, _} = emqtt:publish(ConnPub, <<"topic4/1">>, <<"hello1">>, 1), + {ok, _} = emqtt:publish(ConnPub, <<"topic4/2">>, <<"hello2">>, 1), + ct:sleep(2_000), + {ok, _} = emqtt:publish(ConnPub, <<"topic4/1">>, <<"hello3">>, 1), + {ok, _} = emqtt:publish(ConnPub, <<"topic4/2">>, <<"hello4">>, 1), + + ?assertReceive({publish, #{payload := <<"hello1">>}}, 10_000), + ?assertReceive({publish, #{payload := <<"hello2">>}}, 10_000), + ?assertReceive({publish, #{payload := <<"hello3">>}}, 10_000), + ?assertReceive({publish, #{payload := <<"hello4">>}}, 10_000), + + ok = emqtt:disconnect(ConnShared1), + ok = emqtt:disconnect(ConnShared2), + ok = emqtt:disconnect(ConnPub). + +t_client_loss(_Config) -> + process_flag(trap_exit, true), + + ConnShared1 = emqtt_connect_sub(<<"client_shared1">>), + {ok, _, _} = emqtt:subscribe(ConnShared1, <<"$share/gr5/topic5/#">>, 1), + + ConnShared2 = emqtt_connect_sub(<<"client_shared2">>), + {ok, _, _} = emqtt:subscribe(ConnShared2, <<"$share/gr5/topic5/#">>, 1), + + ConnPub = emqtt_connect_pub(<<"client_pub">>), + + {ok, _} = emqtt:publish(ConnPub, <<"topic5/1">>, <<"hello1">>, 1), + {ok, _} = emqtt:publish(ConnPub, <<"topic5/2">>, <<"hello2">>, 1), + + exit(ConnShared1, kill), + + {ok, _} = emqtt:publish(ConnPub, <<"topic5/1">>, <<"hello3">>, 1), + {ok, _} = emqtt:publish(ConnPub, <<"topic5/2">>, <<"hello4">>, 1), + + ?assertReceive({publish, #{payload := <<"hello3">>}}, 10_000), + ?assertReceive({publish, #{payload := <<"hello4">>}}, 10_000), + + ok = emqtt:disconnect(ConnShared2), + ok = emqtt:disconnect(ConnPub). + +t_stream_revoke(_Config) -> + process_flag(trap_exit, true), + + ConnShared1 = emqtt_connect_sub(<<"client_shared1">>), + {ok, _, _} = emqtt:subscribe(ConnShared1, <<"$share/gr6/topic6/#">>, 1), + + ConnPub = emqtt_connect_pub(<<"client_pub">>), + + {ok, _} = emqtt:publish(ConnPub, <<"topic6/1">>, <<"hello1">>, 1), + {ok, _} = emqtt:publish(ConnPub, <<"topic6/2">>, <<"hello2">>, 1), + + ?assertReceive({publish, #{payload := <<"hello1">>}}, 10_000), + ?assertReceive({publish, #{payload := <<"hello2">>}}, 10_000), + + ConnShared2 = emqtt_connect_sub(<<"client_shared2">>), + + ?assertWaitEvent( + {ok, _, _} = emqtt:subscribe(ConnShared2, <<"$share/gr6/topic6/#">>, 1), + #{ + ?snk_kind := shared_sub_group_sm_leader_update_streams, + stream_progresses := [_ | _], + id := <<"client_shared2">> + }, + 5_000 + ), + + {ok, _} = emqtt:publish(ConnPub, <<"topic6/1">>, <<"hello3">>, 1), + {ok, _} = emqtt:publish(ConnPub, <<"topic6/2">>, <<"hello4">>, 1), + + ?assertReceive({publish, #{payload := <<"hello3">>}}, 10_000), + ?assertReceive({publish, #{payload := <<"hello4">>}}, 10_000), + + ok = emqtt:disconnect(ConnShared1), + ok = emqtt:disconnect(ConnShared2), + ok = emqtt:disconnect(ConnPub). + +t_graceful_disconnect(_Config) -> + ConnShared1 = emqtt_connect_sub(<<"client_shared1">>), + {ok, _, _} = emqtt:subscribe(ConnShared1, <<"$share/gr4/topic7/#">>, 1), + + ConnShared2 = emqtt_connect_sub(<<"client_shared2">>), + {ok, _, _} = emqtt:subscribe(ConnShared2, <<"$share/gr4/topic7/#">>, 1), + + ConnPub = emqtt_connect_pub(<<"client_pub">>), + + {ok, _} = emqtt:publish(ConnPub, <<"topic7/1">>, <<"hello1">>, 1), + {ok, _} = emqtt:publish(ConnPub, <<"topic7/2">>, <<"hello2">>, 1), + + ?assertReceive({publish, #{payload := <<"hello1">>}}, 2_000), + ?assertReceive({publish, #{payload := <<"hello2">>}}, 2_000), + + ?assertWaitEvent( + ok = emqtt:disconnect(ConnShared1), + #{?snk_kind := shared_sub_leader_disconnect_agent}, + 1_000 + ), + + {ok, _} = emqtt:publish(ConnPub, <<"topic7/1">>, <<"hello3">>, 1), + {ok, _} = emqtt:publish(ConnPub, <<"topic7/2">>, <<"hello4">>, 1), + + %% Since the disconnect is graceful, the streams should rebalance quickly, + %% before the timeout. + ?assertReceive({publish, #{payload := <<"hello3">>}}, 2_000), + ?assertReceive({publish, #{payload := <<"hello4">>}}, 2_000), + + ok = emqtt:disconnect(ConnShared2), + ok = emqtt:disconnect(ConnPub). + +t_intensive_reassign(_Config) -> + ConnPub = emqtt_connect_pub(<<"client_pub">>), + + ConnShared1 = emqtt_connect_sub(<<"client_shared1">>), + {ok, _, _} = emqtt:subscribe(ConnShared1, <<"$share/gr8/topic8/#">>, 1), + + ct:sleep(1000), + + NPubs = 10_000, + + Topics = [<<"topic8/1">>, <<"topic8/2">>, <<"topic8/3">>], + ok = publish_n(ConnPub, Topics, 1, NPubs), + + Self = self(), + _ = spawn_link(fun() -> + ok = publish_n(ConnPub, Topics, NPubs + 1, 2 * NPubs), + Self ! publish_done + end), + + ConnShared2 = emqtt_connect_sub(<<"client_shared2">>), + ConnShared3 = emqtt_connect_sub(<<"client_shared3">>), + {ok, _, _} = emqtt:subscribe(ConnShared2, <<"$share/gr8/topic8/#">>, 1), + {ok, _, _} = emqtt:subscribe(ConnShared3, <<"$share/gr8/topic8/#">>, 1), + + receive + publish_done -> ok + end, + + Pubs = drain_publishes(), + + ClientByBid = fun(Pid) -> + case Pid of + ConnShared1 -> <<"client_shared1">>; + ConnShared2 -> <<"client_shared2">>; + ConnShared3 -> <<"client_shared3">> + end + end, + + {Missing, Duplicate} = verify_received_pubs(Pubs, 2 * NPubs, ClientByBid), + + ?assertEqual([], Missing), + ?assertEqual([], Duplicate), + + ok = emqtt:disconnect(ConnShared1), + ok = emqtt:disconnect(ConnShared2), + ok = emqtt:disconnect(ConnShared3), + ok = emqtt:disconnect(ConnPub). + +t_unsubscribe(_Config) -> + ConnPub = emqtt_connect_pub(<<"client_pub">>), + + ConnShared1 = emqtt_connect_sub(<<"client_shared1">>), + {ok, _, _} = emqtt:subscribe(ConnShared1, <<"$share/gr9/topic9/#">>, 1), + + ct:sleep(1000), + + NPubs = 10_000, + + Topics = [<<"topic9/1">>, <<"topic9/2">>, <<"topic9/3">>], + ok = publish_n(ConnPub, Topics, 1, NPubs), + + Self = self(), + _ = spawn_link(fun() -> + ok = publish_n(ConnPub, Topics, NPubs + 1, 2 * NPubs), + Self ! publish_done + end), + + ConnShared2 = emqtt_connect_sub(<<"client_shared2">>), + {ok, _, _} = emqtt:subscribe(ConnShared2, <<"$share/gr9/topic9/#">>, 1), + {ok, _, _} = emqtt:unsubscribe(ConnShared1, <<"$share/gr9/topic9/#">>), + + receive + publish_done -> ok + end, + + Pubs = drain_publishes(), + + ClientByBid = fun(Pid) -> + case Pid of + ConnShared1 -> <<"client_shared1">>; + ConnShared2 -> <<"client_shared2">> + end + end, + + {Missing, Duplicate} = verify_received_pubs(Pubs, 2 * NPubs, ClientByBid), + + ?assertEqual([], Missing), + ?assertEqual([], Duplicate), + + ok = emqtt:disconnect(ConnShared1), + ok = emqtt:disconnect(ConnShared2), + ok = emqtt:disconnect(ConnPub). + +t_quick_resubscribe(_Config) -> + ConnPub = emqtt_connect_pub(<<"client_pub">>), + + ConnShared1 = emqtt_connect_sub(<<"client_shared1">>), + {ok, _, _} = emqtt:subscribe(ConnShared1, <<"$share/gr10/topic10/#">>, 1), + + ct:sleep(1000), + + NPubs = 10_000, + + Topics = [<<"topic10/1">>, <<"topic10/2">>, <<"topic10/3">>], + ok = publish_n(ConnPub, Topics, 1, NPubs), + + Self = self(), + _ = spawn_link(fun() -> + ok = publish_n(ConnPub, Topics, NPubs + 1, 2 * NPubs), + Self ! publish_done + end), + + ConnShared2 = emqtt_connect_sub(<<"client_shared2">>), + {ok, _, _} = emqtt:subscribe(ConnShared2, <<"$share/gr10/topic10/#">>, 1), + ok = lists:foreach( + fun(_) -> + {ok, _, _} = emqtt:unsubscribe(ConnShared1, <<"$share/gr10/topic10/#">>), + {ok, _, _} = emqtt:subscribe(ConnShared1, <<"$share/gr10/topic10/#">>, 1), + ct:sleep(5) + end, + lists:seq(1, 10) + ), + + receive + publish_done -> ok + end, + + Pubs = drain_publishes(), + + ClientByBid = fun(Pid) -> + case Pid of + ConnShared1 -> <<"client_shared1">>; + ConnShared2 -> <<"client_shared2">> + end + end, + + {Missing, Duplicate} = verify_received_pubs(Pubs, 2 * NPubs, ClientByBid), + + ?assertEqual([], Missing), + ?assertEqual([], Duplicate), + + ok = emqtt:disconnect(ConnShared1), + ok = emqtt:disconnect(ConnShared2), + ok = emqtt:disconnect(ConnPub). + +t_disconnect_no_double_replay1(_Config) -> + ConnPub = emqtt_connect_pub(<<"client_pub">>), + + ConnShared1 = emqtt_connect_sub(<<"client_shared1">>), + {ok, _, _} = emqtt:subscribe(ConnShared1, <<"$share/gr11/topic11/#">>, 1), + + ConnShared2 = emqtt_connect_sub(<<"client_shared2">>), + {ok, _, _} = emqtt:subscribe(ConnShared2, <<"$share/gr11/topic11/#">>, 1), + + ct:sleep(1000), + + NPubs = 10_000, + + Topics = [<<"topic11/1">>, <<"topic11/2">>, <<"topic11/3">>], + ok = publish_n(ConnPub, Topics, 1, NPubs), + + Self = self(), + _ = spawn_link(fun() -> + ok = publish_n(ConnPub, Topics, NPubs + 1, 2 * NPubs), + Self ! publish_done + end), + + ok = emqtt:disconnect(ConnShared2), + + receive + publish_done -> ok + end, + + Pubs = drain_publishes(), + + ClientByBid = fun(Pid) -> + case Pid of + ConnShared1 -> <<"client_shared1">>; + ConnShared2 -> <<"client_shared2">> + end + end, + + {Missing, _Duplicate} = verify_received_pubs(Pubs, 2 * NPubs, ClientByBid), + + ?assertEqual([], Missing), + + %% We cannnot garantee that the message are not duplicated until we are able + %% to send progress of a partially replayed stream range to the leader. + % ?assertEqual([], Duplicate), + + ok = emqtt:disconnect(ConnShared1), + ok = emqtt:disconnect(ConnPub). + +t_disconnect_no_double_replay2(_Config) -> + ConnPub = emqtt_connect_pub(<<"client_pub">>), + + ConnShared1 = emqtt_connect_sub(<<"client_shared1">>, [{auto_ack, false}]), + {ok, _, _} = emqtt:subscribe(ConnShared1, <<"$share/gr12/topic12/#">>, 1), + + ct:sleep(1000), + + ok = publish_n(ConnPub, [<<"topic12/1">>], 1, 20), + + receive + {publish, #{payload := <<"1">>, packet_id := PacketId1}} -> + ok = emqtt:puback(ConnShared1, PacketId1) + after 5000 -> + ct:fail("No publish received") + end, + + ok = emqtt:disconnect(ConnShared1), + + ConnShared12 = emqtt_connect_sub(<<"client_shared12">>), + {ok, _, _} = emqtt:subscribe(ConnShared12, <<"$share/gr12/topic12/#">>, 1), + + %% We cannnot garantee that the message is not duplicated until we are able + %% to send progress of a partially replayed stream range to the leader. + % ?assertNotReceive( + % {publish, #{payload := <<"1">>}}, + % 3000 + % ), + + ok = emqtt:disconnect(ConnShared12). + +t_lease_reconnect(_Config) -> + ConnPub = emqtt_connect_pub(<<"client_pub">>), ConnShared = emqtt_connect_sub(<<"client_shared">>), @@ -93,7 +428,6 @@ t_lease_reconnect(_Config) -> 5_000 ), - ct:sleep(1_000), {ok, _} = emqtt:publish(ConnPub, <<"topic2/2">>, <<"hello2">>, 1), ?assertReceive({publish, #{payload := <<"hello2">>}}, 10_000), @@ -114,7 +448,7 @@ t_renew_lease_timeout(_Config) -> ?wait_async_action( ok = terminate_leaders(), #{?snk_kind := leader_lease_streams}, - 5_000 + 10_000 ), fun(Trace) -> ?strict_causality( @@ -131,28 +465,24 @@ t_renew_lease_timeout(_Config) -> %% Helper functions %%-------------------------------------------------------------------- -init_streams(ConnPub, Topic) -> - ConnRegular = emqtt_connect_sub(<<"client_regular">>), - {ok, _, _} = emqtt:subscribe(ConnRegular, Topic, 1), - {ok, _} = emqtt:publish(ConnPub, Topic, <<"hello1">>, 1), - - ?assertReceive({publish, #{payload := <<"hello1">>}}, 5_000), - - ok = emqtt:disconnect(ConnRegular). - emqtt_connect_sub(ClientId) -> - {ok, C} = emqtt:start_link([ - {client_id, ClientId}, - {clean_start, true}, - {proto_ver, v5}, - {properties, #{'Session-Expiry-Interval' => 7_200}} - ]), + emqtt_connect_sub(ClientId, []). + +emqtt_connect_sub(ClientId, Options) -> + {ok, C} = emqtt:start_link( + [ + {clientid, ClientId}, + {clean_start, true}, + {proto_ver, v5}, + {properties, #{'Session-Expiry-Interval' => 7_200}} + ] ++ Options + ), {ok, _} = emqtt:connect(C), C. emqtt_connect_pub(ClientId) -> {ok, C} = emqtt:start_link([ - {client_id, ClientId}, + {clientid, ClientId}, {clean_start, true}, {proto_ver, v5} ]), @@ -163,3 +493,53 @@ terminate_leaders() -> ok = supervisor:terminate_child(emqx_ds_shared_sub_sup, emqx_ds_shared_sub_leader_sup), {ok, _} = supervisor:restart_child(emqx_ds_shared_sub_sup, emqx_ds_shared_sub_leader_sup), ok. + +publish_n(_Conn, _Topics, From, To) when From > To -> + ok; +publish_n(Conn, [Topic | RestTopics], From, To) -> + {ok, _} = emqtt:publish(Conn, Topic, integer_to_binary(From), 1), + publish_n(Conn, RestTopics ++ [Topic], From + 1, To). + +drain_publishes() -> + drain_publishes([]). + +drain_publishes(Acc) -> + receive + {publish, Msg} -> + drain_publishes([Msg | Acc]) + after 5_000 -> + lists:reverse(Acc) + end. + +verify_received_pubs(Pubs, NPubs, ClientByBid) -> + Messages = lists:foldl( + fun(#{payload := Payload, client_pid := Pid}, Acc) -> + maps:update_with( + binary_to_integer(Payload), + fun(Clients) -> + [ClientByBid(Pid) | Clients] + end, + [ClientByBid(Pid)], + Acc + ) + end, + #{}, + Pubs + ), + + Missing = lists:filter( + fun(N) -> not maps:is_key(N, Messages) end, + lists:seq(1, NPubs) + ), + Duplicate = lists:filtermap( + fun(N) -> + case Messages of + #{N := [_]} -> false; + #{N := [_ | _] = Clients} -> {true, {N, Clients}}; + _ -> false + end + end, + lists:seq(1, NPubs) + ), + + {Missing, Duplicate}. diff --git a/apps/emqx_ds_shared_sub/test/emqx_ds_shared_sub_api_SUITE.erl b/apps/emqx_ds_shared_sub/test/emqx_ds_shared_sub_api_SUITE.erl new file mode 100644 index 000000000..0969bdcd1 --- /dev/null +++ b/apps/emqx_ds_shared_sub/test/emqx_ds_shared_sub_api_SUITE.erl @@ -0,0 +1,140 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_ds_shared_sub_api_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +-import( + emqx_mgmt_api_test_util, + [ + request_api/2, + request/3, + uri/1 + ] +). + +all() -> + emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + Apps = emqx_cth_suite:start( + [ + {emqx, #{ + config => #{ + <<"durable_sessions">> => #{ + <<"enable">> => true, + <<"renew_streams_interval">> => "100ms" + }, + <<"durable_storage">> => #{ + <<"messages">> => #{ + <<"backend">> => <<"builtin_raft">> + } + } + } + }}, + emqx_ds_shared_sub, + emqx_management, + emqx_mgmt_api_test_util:emqx_dashboard() + ], + #{work_dir => ?config(priv_dir, Config)} + ), + [{apps, Apps} | Config]. + +end_per_suite(Config) -> + ok = emqx_cth_suite:stop(?config(apps, Config)), + ok. + +init_per_testcase(_TC, Config) -> + ok = snabbkaffe:start_trace(), + Config. + +end_per_testcase(_TC, _Config) -> + ok = snabbkaffe:stop(), + ok = terminate_leaders(), + ok. +%%-------------------------------------------------------------------- +%% Tests +%%-------------------------------------------------------------------- + +t_basic_crud(_Config) -> + ?assertMatch( + {ok, []}, + api_get(["durable_queues"]) + ), + + ?assertMatch( + {ok, 200, #{ + <<"id">> := <<"q1">> + }}, + api(put, ["durable_queues", "q1"], #{}) + ), + + ?assertMatch( + {error, {_, 404, _}}, + api_get(["durable_queues", "q2"]) + ), + + ?assertMatch( + {ok, 200, #{ + <<"id">> := <<"q2">> + }}, + api(put, ["durable_queues", "q2"], #{}) + ), + + ?assertMatch( + {ok, #{ + <<"id">> := <<"q2">> + }}, + api_get(["durable_queues", "q2"]) + ), + + ?assertMatch( + {ok, [#{<<"id">> := <<"q2">>}, #{<<"id">> := <<"q1">>}]}, + api_get(["durable_queues"]) + ), + + ?assertMatch( + {ok, 200, <<"Queue deleted">>}, + api(delete, ["durable_queues", "q2"], #{}) + ), + + ?assertMatch( + {ok, [#{<<"id">> := <<"q1">>}]}, + api_get(["durable_queues"]) + ). + +%%-------------------------------------------------------------------- +%% Helpers +%%-------------------------------------------------------------------- + +api_get(Path) -> + case request_api(get, uri(Path)) of + {ok, ResponseBody} -> + {ok, jiffy:decode(list_to_binary(ResponseBody), [return_maps])}; + {error, _} = Error -> + Error + end. + +api(Method, Path, Data) -> + case request(Method, uri(Path), Data) of + {ok, Code, ResponseBody} -> + Res = + case emqx_utils_json:safe_decode(ResponseBody, [return_maps]) of + {ok, Decoded} -> Decoded; + {error, _} -> ResponseBody + end, + {ok, Code, Res}; + {error, _} = Error -> + Error + end. + +terminate_leaders() -> + ok = supervisor:terminate_child(emqx_ds_shared_sub_sup, emqx_ds_shared_sub_leader_sup), + {ok, _} = supervisor:restart_child(emqx_ds_shared_sub_sup, emqx_ds_shared_sub_leader_sup), + ok. diff --git a/apps/emqx_ds_shared_sub/test/emqx_ds_shared_sub_config_SUITE.erl b/apps/emqx_ds_shared_sub/test/emqx_ds_shared_sub_config_SUITE.erl new file mode 100644 index 000000000..a3d58ebf9 --- /dev/null +++ b/apps/emqx_ds_shared_sub/test/emqx_ds_shared_sub_config_SUITE.erl @@ -0,0 +1,62 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_ds_shared_sub_config_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +-include_lib("emqx/include/asserts.hrl"). + +all() -> + emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + Apps = emqx_cth_suite:start( + [ + emqx_conf, + {emqx, #{ + config => #{ + <<"durable_sessions">> => #{ + <<"enable">> => true, + <<"renew_streams_interval">> => "100ms" + }, + <<"durable_storage">> => #{ + <<"messages">> => #{ + <<"backend">> => <<"builtin_raft">> + } + } + } + }}, + {emqx_ds_shared_sub, #{ + config => #{ + <<"durable_queues">> => #{ + <<"enable">> => true, + <<"session_find_leader_timeout_ms">> => "1200ms" + } + } + }} + ], + #{work_dir => ?config(priv_dir, Config)} + ), + [{apps, Apps} | Config]. + +end_per_suite(Config) -> + ok = emqx_cth_suite:stop(?config(apps, Config)), + ok. + +t_update_config(_Config) -> + ?assertEqual( + 1200, + emqx_ds_shared_sub_config:get(session_find_leader_timeout_ms) + ), + + {ok, _} = emqx_conf:update([durable_queues], #{session_find_leader_timeout_ms => 2000}, #{}), + ?assertEqual( + 2000, + emqx_ds_shared_sub_config:get(session_find_leader_timeout_ms) + ). diff --git a/apps/emqx_ds_shared_sub/test/emqx_ds_shared_sub_mgmt_api_subscription_SUITE.erl b/apps/emqx_ds_shared_sub/test/emqx_ds_shared_sub_mgmt_api_subscription_SUITE.erl new file mode 100644 index 000000000..fde9acbea --- /dev/null +++ b/apps/emqx_ds_shared_sub/test/emqx_ds_shared_sub_mgmt_api_subscription_SUITE.erl @@ -0,0 +1,125 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_ds_shared_sub_mgmt_api_subscription_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +-define(CLIENTID, <<"api_clientid">>). +-define(USERNAME, <<"api_username">>). + +all() -> emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + Apps = emqx_cth_suite:start( + [ + {emqx, + "durable_sessions {\n" + " enable = true\n" + " renew_streams_interval = 10ms\n" + "}"}, + {emqx_ds_shared_sub, #{ + config => #{ + <<"durable_queues">> => #{ + <<"enable">> => true, + <<"session_find_leader_timeout_ms">> => "1200ms" + } + } + }}, + emqx_management, + emqx_mgmt_api_test_util:emqx_dashboard() + ], + #{work_dir => emqx_cth_suite:work_dir(Config)} + ), + [{apps, Apps} | Config]. + +end_per_suite(Config) -> + ok = emqx_cth_suite:stop(?config(apps, Config)). + +init_per_testcase(_TC, Config) -> + ClientConfig = #{ + username => ?USERNAME, + clientid => ?CLIENTID, + proto_ver => v5, + clean_start => true, + properties => #{'Session-Expiry-Interval' => 300} + }, + + {ok, Client} = emqtt:start_link(ClientConfig), + {ok, _} = emqtt:connect(Client), + [{client_config, ClientConfig}, {client, Client} | Config]. + +end_per_testcase(_TC, Config) -> + Client = proplists:get_value(client, Config), + emqtt:disconnect(Client). + +t_list_with_shared_sub(_Config) -> + Client = proplists:get_value(client, _Config), + RealTopic = <<"t/+">>, + Topic = <<"$share/g1/", RealTopic/binary>>, + + {ok, _, _} = emqtt:subscribe(Client, Topic), + {ok, _, _} = emqtt:subscribe(Client, RealTopic), + + QS0 = [ + {"clientid", ?CLIENTID}, + {"match_topic", "t/#"} + ], + Headers = emqx_mgmt_api_test_util:auth_header_(), + + ?assertMatch( + #{<<"data">> := [#{<<"clientid">> := ?CLIENTID}, #{<<"clientid">> := ?CLIENTID}]}, + request_json(get, QS0, Headers) + ), + + QS1 = [ + {"clientid", ?CLIENTID}, + {"share_group", "g1"} + ], + + ?assertMatch( + #{<<"data">> := [#{<<"clientid">> := ?CLIENTID, <<"topic">> := <<"$share/g1/t/+">>}]}, + request_json(get, QS1, Headers) + ). + +t_list_with_invalid_match_topic(Config) -> + Client = proplists:get_value(client, Config), + RealTopic = <<"t/+">>, + Topic = <<"$share/g1/", RealTopic/binary>>, + + {ok, _, _} = emqtt:subscribe(Client, Topic), + {ok, _, _} = emqtt:subscribe(Client, RealTopic), + + QS = [ + {"clientid", ?CLIENTID}, + {"match_topic", "$share/g1/t/1"} + ], + Headers = emqx_mgmt_api_test_util:auth_header_(), + + ?assertMatch( + {error, + {{_, 400, _}, _, #{ + <<"message">> := <<"match_topic_invalid">>, + <<"code">> := <<"INVALID_PARAMETER">> + }}}, + begin + {error, {R, _H, Body}} = emqx_mgmt_api_test_util:request_api( + get, path(), uri_string:compose_query(QS), Headers, [], #{return_all => true} + ), + {error, {R, _H, emqx_utils_json:decode(Body, [return_maps])}} + end + ), + ok. + +request_json(Method, Query, Headers) when is_list(Query) -> + Qs = uri_string:compose_query(Query), + {ok, MatchRes} = emqx_mgmt_api_test_util:request_api(Method, path(), Qs, Headers), + emqx_utils_json:decode(MatchRes, [return_maps]). + +path() -> + emqx_mgmt_api_test_util:api_path(["subscriptions"]). diff --git a/apps/emqx_enterprise/src/emqx_enterprise.app.src b/apps/emqx_enterprise/src/emqx_enterprise.app.src index 93fb02287..e79a6f3a3 100644 --- a/apps/emqx_enterprise/src/emqx_enterprise.app.src +++ b/apps/emqx_enterprise/src/emqx_enterprise.app.src @@ -1,6 +1,6 @@ {application, emqx_enterprise, [ {description, "EMQX Enterprise Edition"}, - {vsn, "0.2.2"}, + {vsn, "0.2.3"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_enterprise/src/emqx_enterprise_schema.erl b/apps/emqx_enterprise/src/emqx_enterprise_schema.erl index f593dc877..eb43f67af 100644 --- a/apps/emqx_enterprise/src/emqx_enterprise_schema.erl +++ b/apps/emqx_enterprise/src/emqx_enterprise_schema.erl @@ -11,13 +11,15 @@ -export([namespace/0, roots/0, fields/1, translations/0, translation/1, desc/1, validations/0]). -export([upgrade_raw_conf/1]). +-export([injected_fields/0]). -define(EE_SCHEMA_MODULES, [ emqx_license_schema, emqx_schema_registry_schema, emqx_schema_validation_schema, emqx_message_transformation_schema, - emqx_ft_schema + emqx_ft_schema, + emqx_ds_shared_sub_schema ]). %% Callback to upgrade config after loaded from config file but before validation. @@ -53,7 +55,6 @@ fields("log_audit_handler") -> importance => ?IMPORTANCE_HIDDEN } )}, - {"path", hoconsc:mk( string(), @@ -127,6 +128,11 @@ desc(Name) -> validations() -> emqx_conf_schema:validations() ++ emqx_license_schema:validations(). +injected_fields() -> + #{ + 'node.role' => [replicant] + }. + %%------------------------------------------------------------------------------ %% helpers %%------------------------------------------------------------------------------ diff --git a/apps/emqx_exhook/src/emqx_exhook.app.src b/apps/emqx_exhook/src/emqx_exhook.app.src index a12ec7d3a..a0961f660 100644 --- a/apps/emqx_exhook/src/emqx_exhook.app.src +++ b/apps/emqx_exhook/src/emqx_exhook.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_exhook, [ {description, "EMQX Extension for Hook"}, - {vsn, "5.0.17"}, + {vsn, "5.0.18"}, {modules, []}, {registered, []}, {mod, {emqx_exhook_app, []}}, diff --git a/apps/emqx_exhook/src/emqx_exhook_schema.erl b/apps/emqx_exhook/src/emqx_exhook_schema.erl index 6f37ea385..895515cf9 100644 --- a/apps/emqx_exhook/src/emqx_exhook_schema.erl +++ b/apps/emqx_exhook/src/emqx_exhook_schema.erl @@ -54,6 +54,7 @@ fields(server) -> {enable, ?HOCON(boolean(), #{ default => true, + importance => ?IMPORTANCE_NO_DOC, desc => ?DESC(enable) })}, {url, diff --git a/apps/emqx_ft/src/emqx_ft_schema.erl b/apps/emqx_ft/src/emqx_ft_schema.erl index adf8b4241..f779121d3 100644 --- a/apps/emqx_ft/src/emqx_ft_schema.erl +++ b/apps/emqx_ft/src/emqx_ft_schema.erl @@ -66,6 +66,7 @@ fields(file_transfer) -> boolean(), #{ desc => ?DESC("enable"), + %% importance => ?IMPORTANCE_NO_DOC, required => false, default => false } @@ -242,6 +243,7 @@ common_backend_fields() -> mk( boolean(), #{ desc => ?DESC("backend_enable"), + importance => ?IMPORTANCE_NO_DOC, required => false, default => true } diff --git a/apps/emqx_gateway/src/emqx_gateway_api_authn.erl b/apps/emqx_gateway/src/emqx_gateway_api_authn.erl index 0ee16824d..c79bc8e61 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api_authn.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api_authn.erl @@ -381,6 +381,9 @@ params_fuzzy_in_qs() -> schema_authn() -> emqx_dashboard_swagger:schema_with_examples( - emqx_authn_schema:authenticator_type_without([emqx_authn_scram_mnesia_schema]), + emqx_authn_schema:authenticator_type_without([ + emqx_authn_scram_mnesia_schema, + emqx_authn_scram_restapi_schema + ]), emqx_authn_api:authenticator_examples() ). diff --git a/apps/emqx_gateway/src/emqx_gateway_schema.erl b/apps/emqx_gateway/src/emqx_gateway_schema.erl index 11488d1a3..c59736a16 100644 --- a/apps/emqx_gateway/src/emqx_gateway_schema.erl +++ b/apps/emqx_gateway/src/emqx_gateway_schema.erl @@ -240,6 +240,7 @@ gateway_common_options() -> boolean(), #{ default => true, + importance => ?IMPORTANCE_NO_DOC, desc => ?DESC(gateway_common_enable) } )}, @@ -413,6 +414,7 @@ common_listener_opts() -> boolean(), #{ default => true, + importance => ?IMPORTANCE_NO_DOC, desc => ?DESC(gateway_common_listener_enable) } )}, diff --git a/apps/emqx_ldap/src/emqx_ldap.app.src b/apps/emqx_ldap/src/emqx_ldap.app.src index b0d8ec59c..1b1b667cc 100644 --- a/apps/emqx_ldap/src/emqx_ldap.app.src +++ b/apps/emqx_ldap/src/emqx_ldap.app.src @@ -1,6 +1,6 @@ {application, emqx_ldap, [ {description, "EMQX LDAP Connector"}, - {vsn, "0.1.8"}, + {vsn, "0.1.9"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_ldap/src/emqx_ldap.erl b/apps/emqx_ldap/src/emqx_ldap.erl index d04be5d68..a2b09ccda 100644 --- a/apps/emqx_ldap/src/emqx_ldap.erl +++ b/apps/emqx_ldap/src/emqx_ldap.erl @@ -27,6 +27,7 @@ %% callbacks of behaviour emqx_resource -export([ + resource_type/0, callback_mode/0, on_start/2, on_stop/2, @@ -40,6 +41,7 @@ -export([namespace/0, roots/0, fields/1, desc/1]). -export([do_get_status/1, get_status_with_poolname/1]). +-export([search/2]). -define(LDAP_HOST_OPTIONS, #{ default_port => 389 @@ -129,6 +131,8 @@ ensure_username(Field) -> emqx_connector_schema_lib:username(Field). %% =================================================================== +resource_type() -> ldap. + callback_mode() -> always_sync. -spec on_start(binary(), hocon:config()) -> {ok, state()} | {error, _}. @@ -270,6 +274,22 @@ on_query( Error end. +search(Pid, SearchOptions) -> + case eldap:search(Pid, SearchOptions) of + {error, ldap_closed} -> + %% ldap server closing the socket does not result in + %% process restart, so we need to kill it to trigger a quick reconnect + %% instead of waiting for the next health-check + _ = exit(Pid, kill), + {error, ldap_closed}; + {error, {gen_tcp_error, _} = Reason} -> + %% kill the process to trigger reconnect + _ = exit(Pid, kill), + {error, Reason}; + Result -> + Result + end. + do_ldap_query( InstId, SearchOptions, @@ -280,7 +300,7 @@ do_ldap_query( case ecpool:pick_and_do( PoolName, - {eldap, search, [SearchOptions]}, + {?MODULE, search, [SearchOptions]}, handover ) of @@ -316,7 +336,7 @@ do_ldap_query( ?SLOG( error, LogMeta#{ - msg => "ldap_connector_do_query_failed", + msg => "ldap_connector_query_failed", reason => emqx_utils:redact(Reason) } ), diff --git a/apps/emqx_machine/priv/reboot_lists.eterm b/apps/emqx_machine/priv/reboot_lists.eterm index 4830f51c4..251b9790a 100644 --- a/apps/emqx_machine/priv/reboot_lists.eterm +++ b/apps/emqx_machine/priv/reboot_lists.eterm @@ -45,6 +45,7 @@ emqx_ds_backends, emqx_http_lib, emqx_resource, + emqx_connector_jwt, emqx_connector, emqx_auth, emqx_auth_http, diff --git a/apps/emqx_management/src/emqx_mgmt_api_ds.erl b/apps/emqx_management/src/emqx_mgmt_api_ds.erl index bc949cd8a..3eb80f50e 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_ds.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_ds.erl @@ -87,7 +87,8 @@ schema("/ds/sites") -> tags => ?TAGS, responses => #{ - 200 => mk(array(binary()), #{desc => <<"List sites">>}) + 200 => mk(array(binary()), #{desc => <<"List sites">>}), + 404 => disabled_schema() } } }; @@ -115,7 +116,8 @@ schema("/ds/storages") -> tags => ?TAGS, responses => #{ - 200 => mk(array(atom()), #{desc => <<"List durable storages">>}) + 200 => mk(array(atom()), #{desc => <<"List durable storages">>}), + 404 => disabled_schema() } } }; @@ -130,7 +132,8 @@ schema("/ds/storages/:ds") -> responses => #{ 200 => mk(ref(db), #{desc => <<"Get information about a durable storage">>}), - 400 => not_found(<<"Durable storage">>) + 400 => not_found(<<"Durable storage">>), + 404 => disabled_schema() } } }; @@ -148,7 +151,8 @@ schema("/ds/storages/:ds/replicas") -> 200 => mk(array(binary()), #{ desc => <<"List sites that contain replicas of the durable storage">> }), - 400 => not_found(<<"Durable storage">>) + 400 => not_found(<<"Durable storage">>), + 404 => disabled_schema() } }, put => @@ -159,7 +163,8 @@ schema("/ds/storages/:ds/replicas") -> responses => #{ 202 => mk(array(binary()), #{}), - 400 => bad_request() + 400 => bad_request(), + 404 => disabled_schema() }, 'requestBody' => mk(array(binary()), #{desc => <<"New list of sites">>}) } @@ -296,10 +301,15 @@ fields(db_site) -> %%================================================================================ list_sites(get, _Params) -> - {200, emqx_ds_replication_layer_meta:sites()}. + case is_enabled() of + true -> + {200, emqx_ds_replication_layer_meta:sites()}; + false -> + err_disabled() + end. get_site(get, #{bindings := #{site := Site}}) -> - case lists:member(Site, emqx_ds_replication_layer_meta:sites()) of + case is_enabled() andalso lists:member(Site, emqx_ds_replication_layer_meta:sites()) of false -> ?NOT_FOUND(<<"Site not found: ", Site/binary>>); true -> @@ -314,40 +324,70 @@ get_site(get, #{bindings := #{site := Site}}) -> end. list_dbs(get, _Params) -> - ?OK(dbs()). + case is_enabled() of + true -> + ?OK(dbs()); + false -> + err_disabled() + end. get_db(get, #{bindings := #{ds := DB}}) -> - ?OK(#{ - name => DB, - shards => list_shards(DB) - }). + case is_enabled() of + true -> + ?OK(#{ + name => DB, + shards => list_shards(DB) + }); + false -> + err_disabled() + end. db_replicas(get, #{bindings := #{ds := DB}}) -> - Replicas = emqx_ds_replication_layer_meta:db_sites(DB), - ?OK(Replicas); + case is_enabled() of + true -> + Replicas = emqx_ds_replication_layer_meta:db_sites(DB), + ?OK(Replicas); + false -> + err_disabled() + end; db_replicas(put, #{bindings := #{ds := DB}, body := Sites}) -> - case update_db_sites(DB, Sites, rest) of - {ok, _} -> - {202, <<"OK">>}; - {error, Description} -> - ?BAD_REQUEST(400, Description) + case is_enabled() of + true -> + case update_db_sites(DB, Sites, rest) of + {ok, _} -> + {202, <<"OK">>}; + {error, Description} -> + ?BAD_REQUEST(400, Description) + end; + false -> + err_disabled() end. db_replica(put, #{bindings := #{ds := DB, site := Site}}) -> - case join(DB, Site, rest) of - {ok, _} -> - {202, <<"OK">>}; - {error, Description} -> - ?BAD_REQUEST(400, Description) + case is_enabled() of + true -> + case join(DB, Site, rest) of + {ok, _} -> + {202, <<"OK">>}; + {error, Description} -> + ?BAD_REQUEST(400, Description) + end; + false -> + err_disabled() end; db_replica(delete, #{bindings := #{ds := DB, site := Site}}) -> - case leave(DB, Site, rest) of - {ok, Sites} when is_list(Sites) -> - {202, <<"OK">>}; - {ok, unchanged} -> - ?NOT_FOUND(<<"Site is not part of replica set">>); - {error, Description} -> - ?BAD_REQUEST(400, Description) + case is_enabled() of + true -> + case leave(DB, Site, rest) of + {ok, Sites} when is_list(Sites) -> + {202, <<"OK">>}; + {ok, unchanged} -> + ?NOT_FOUND(<<"Site is not part of replica set">>); + {error, Description} -> + ?BAD_REQUEST(400, Description) + end; + false -> + err_disabled() end. -spec update_db_sites(emqx_ds:db(), [emqx_ds_replication_layer_meta:site()], rest | cli) -> @@ -391,6 +431,9 @@ forget(Site, Via) -> %% site_info(Site) -> %% #{}. +disabled_schema() -> + emqx_dashboard_swagger:error_codes(['NOT_FOUND'], <<"Durable storage is disabled">>). + not_found(What) -> emqx_dashboard_swagger:error_codes(['NOT_FOUND'], <>). @@ -492,4 +535,10 @@ meta_result_to_binary({error, Err}) -> IOList = io_lib:format("Error: ~p", [Err]), {error, iolist_to_binary(IOList)}. +is_enabled() -> + emqx_persistent_message:is_persistence_enabled(). + +err_disabled() -> + ?NOT_FOUND(<<"Durable storage is disabled">>). + -endif. diff --git a/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl b/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl index b9cefeb1f..b662061a6 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_subscriptions.erl @@ -242,20 +242,25 @@ do_subscriptions_query_persistent(#{<<"page">> := Page, <<"limit">> := Limit} = %% TODO: filtering by client ID can be implemented more efficiently: FilterTopic = maps:get(<<"topic">>, QString, '_'), Stream0 = emqx_persistent_session_ds_router:stream(FilterTopic), + SubPred = fun(Sub) -> - compare_optional(<<"topic">>, QString, topic, Sub) andalso + compare_optional(<<"topic">>, QString, '_real_topic', Sub) andalso compare_optional(<<"clientid">>, QString, clientid, Sub) andalso compare_optional(<<"qos">>, QString, qos, Sub) andalso - compare_match_topic_optional(<<"match_topic">>, QString, topic, Sub) + compare_optional(<<"share_group">>, QString, '_group', Sub) andalso + compare_match_topic_optional(<<"match_topic">>, QString, '_real_topic', Sub) end, NDropped = (Page - 1) * Limit, {_, Stream} = consume_n_matching( fun persistent_route_to_subscription/1, SubPred, NDropped, Stream0 ), - {Subscriptions, Stream1} = consume_n_matching( + {Subscriptions0, Stream1} = consume_n_matching( fun persistent_route_to_subscription/1, SubPred, Limit, Stream ), HasNext = Stream1 =/= [], + Subscriptions1 = lists:map( + fun remove_temp_match_fields/1, Subscriptions0 + ), Meta = case maps:is_key(<<"match_topic">>, QString) orelse maps:is_key(<<"qos">>, QString) of true -> @@ -276,7 +281,7 @@ do_subscriptions_query_persistent(#{<<"page">> := Page, <<"limit">> := Limit} = #{ meta => Meta, - data => Subscriptions + data => Subscriptions1 }. compare_optional(QField, Query, SField, Subscription) -> @@ -328,29 +333,63 @@ consume_n_matching(Map, Pred, N, S0, Acc) -> end end. -persistent_route_to_subscription(#route{topic = Topic, dest = SessionId}) -> - case emqx_persistent_session_ds:get_client_subscription(SessionId, Topic) of - #{subopts := SubOpts} -> - #{qos := Qos, nl := Nl, rh := Rh, rap := Rap} = SubOpts, - #{ - topic => Topic, - clientid => SessionId, - node => all, +persistent_route_to_subscription(#route{dest = Dest} = Route) -> + Sub = + case get_client_subscription(Route) of + #{subopts := SubOpts} -> + #{qos := Qos, nl := Nl, rh := Rh, rap := Rap} = SubOpts, + #{ + topic => format_topic(Route), + clientid => session_id(Dest), + node => all, - qos => Qos, - nl => Nl, - rh => Rh, - rap => Rap, - durable => true - }; - undefined -> - #{ - topic => Topic, - clientid => SessionId, - node => all, - durable => true - } - end. + qos => Qos, + nl => Nl, + rh => Rh, + rap => Rap, + durable => true + }; + undefined -> + #{ + topic => format_topic(Route), + clientid => session_id(Dest), + node => all, + durable => true + } + end, + add_temp_match_fields(Route, Sub). + +get_client_subscription(#route{ + topic = Topic, dest = #share_dest{session_id = SessionId, group = Group} +}) -> + emqx_persistent_session_ds:get_client_subscription(SessionId, #share{ + topic = Topic, group = Group + }); +get_client_subscription(#route{topic = Topic, dest = SessionId}) -> + emqx_persistent_session_ds:get_client_subscription(SessionId, Topic). + +session_id(#share_dest{session_id = SessionId}) -> SessionId; +session_id(SessionId) -> SessionId. + +add_temp_match_fields(Route, Sub) -> + add_temp_match_fields(['_real_topic', '_group'], Route, Sub). + +add_temp_match_fields([], _Route, Sub) -> + Sub; +add_temp_match_fields(['_real_topic' | Rest], #route{topic = Topic} = Route, Sub) -> + add_temp_match_fields(Rest, Route, Sub#{'_real_topic' => Topic}); +add_temp_match_fields(['_group' | Rest], #route{dest = #share_dest{group = Group}} = Route, Sub) -> + add_temp_match_fields(Rest, Route, Sub#{'_group' => Group}); +add_temp_match_fields(['_group' | Rest], Route, Sub) -> + add_temp_match_fields(Rest, Route, Sub#{'_group' => undefined}). + +remove_temp_match_fields(Sub) -> + maps:without(['_real_topic', '_group'], Sub). + +format_topic(#route{topic = Topic, dest = #share_dest{group = Group}}) -> + <<"$share/", Group/binary, "/", Topic/binary>>; +format_topic(#route{topic = Topic}) -> + Topic. %% @private This function merges paginated results from two sources. %% diff --git a/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl index 398b48ab7..4845dedde 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl @@ -107,8 +107,8 @@ init_per_group(persistent_sessions, Config) -> ], Dashboard = emqx_mgmt_api_test_util:emqx_dashboard(), Cluster = [ - {emqx_mgmt_api_clients_SUITE1, #{role => core, apps => AppSpecs ++ [Dashboard]}}, - {emqx_mgmt_api_clients_SUITE2, #{role => core, apps => AppSpecs}} + {emqx_mgmt_api_clients_SUITE1, #{apps => AppSpecs ++ [Dashboard]}}, + {emqx_mgmt_api_clients_SUITE2, #{apps => AppSpecs}} ], Nodes = [N1 | _] = emqx_cth_cluster:start( @@ -128,8 +128,8 @@ init_per_group(non_persistent_cluster, Config) -> ], Dashboard = emqx_mgmt_api_test_util:emqx_dashboard(), Cluster = [ - {mgmt_api_clients_SUITE1, #{role => core, apps => AppSpecs ++ [Dashboard]}}, - {mgmt_api_clients_SUITE2, #{role => core, apps => AppSpecs}} + {mgmt_api_clients_SUITE1, #{apps => AppSpecs ++ [Dashboard]}}, + {mgmt_api_clients_SUITE2, #{apps => AppSpecs}} ], Nodes = [N1 | _] = emqx_cth_cluster:start( @@ -550,13 +550,10 @@ t_persistent_sessions5(Config) -> lists:sort(lists:map(fun(#{<<"clientid">> := CId}) -> CId end, R3 ++ R4)) ), - lists:foreach(fun emqtt:stop/1, [C3, C4]), - lists:foreach( - fun(ClientId) -> - ok = erpc:call(N1, emqx_persistent_session_ds, destroy_session, [ClientId]) - end, - ClientIds - ), + lists:foreach(fun disconnect_and_destroy_session/1, [C3, C4]), + C1B = connect_client(#{port => Port1, clientid => ClientId1}), + C2B = connect_client(#{port => Port2, clientid => ClientId2}), + lists:foreach(fun disconnect_and_destroy_session/1, [C1B, C2B]), ok end, @@ -1623,8 +1620,7 @@ t_list_clients_v2(Config) -> port => Port2, clientid => ClientId6, expiry => 0, clean_start => true }), %% offline persistent clients - ok = emqtt:stop(C3), - ok = emqtt:stop(C4), + lists:foreach(fun stop_and_commit/1, [C3, C4]), %% one by one QueryParams1 = #{limit => "1"}, @@ -2143,3 +2139,16 @@ do_traverse_in_reverse_v2(QueryParams0, Config, [Cursor | Rest], DirectOrderClie disconnect_and_destroy_session(Client) -> ok = emqtt:disconnect(Client, ?RC_SUCCESS, #{'Session-Expiry-Interval' => 0}). + +%% To avoid a race condition where we try to delete the session while it's terminating and +%% committing. This shouldn't happen realistically, because we have a safe grace period +%% before attempting to GC a session. +%% Also, we need to wait until offline metadata is committed before checking the v2 client +%% list, to avoid flaky batch results. +stop_and_commit(Client) -> + {ok, {ok, _}} = + ?wait_async_action( + emqtt:stop(Client), + #{?snk_kind := persistent_session_ds_terminate} + ), + ok. diff --git a/apps/emqx_management/test/emqx_mgmt_api_cluster_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_cluster_SUITE.erl index 113468f0b..135d2103a 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_cluster_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_cluster_SUITE.erl @@ -24,7 +24,12 @@ -define(APPS, [emqx_conf, emqx_management]). all() -> - emqx_common_test_helpers:all(?MODULE). + case emqx_cth_suite:skip_if_oss() of + false -> + emqx_common_test_helpers:all(?MODULE); + True -> + True + end. init_per_suite(Config) -> Config. diff --git a/apps/emqx_management/test/emqx_mgmt_api_data_backup_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_data_backup_SUITE.erl index 6a580fd57..994d34798 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_data_backup_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_data_backup_SUITE.erl @@ -36,7 +36,12 @@ ). all() -> - emqx_common_test_helpers:all(?MODULE). + case emqx_cth_suite:skip_if_oss() of + false -> + emqx_common_test_helpers:all(?MODULE); + True -> + True + end. init_per_suite(Config) -> Config. diff --git a/apps/emqx_management/test/emqx_mgmt_api_subscription_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_subscription_SUITE.erl index 9a55fa1a0..604239379 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_subscription_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_subscription_SUITE.erl @@ -47,9 +47,12 @@ groups() -> CommonTCs = AllTCs -- persistent_only_tcs(), [ {mem, CommonTCs}, - %% Shared subscriptions are currently not supported: + %% Persistent shared subscriptions are an EE app. + %% So they are tested outside emqx_management app which is CE. {persistent, - (CommonTCs -- [t_list_with_shared_sub, t_subscription_api]) ++ persistent_only_tcs()} + (CommonTCs -- + [t_list_with_shared_sub, t_list_with_invalid_match_topic, t_subscription_api]) ++ + persistent_only_tcs()} ]. persistent_only_tcs() -> diff --git a/apps/emqx_management/test/emqx_mgmt_api_test_util.erl b/apps/emqx_management/test/emqx_mgmt_api_test_util.erl index 106a65a9c..4b1d40651 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_test_util.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_test_util.erl @@ -293,3 +293,30 @@ format_multipart_formdata(Data, Params, Name, FileNames, MimeType, Boundary) -> FileNames ), erlang:iolist_to_binary([WithPaths, StartBoundary, <<"--">>, LineSeparator]). + +maybe_json_decode(X) -> + case emqx_utils_json:safe_decode(X, [return_maps]) of + {ok, Decoded} -> Decoded; + {error, _} -> X + end. + +simple_request(Method, Path, Params) -> + AuthHeader = auth_header_(), + Opts = #{return_all => true}, + case request_api(Method, Path, "", AuthHeader, Params, Opts) of + {ok, {{_, Status, _}, _Headers, Body0}} -> + Body = maybe_json_decode(Body0), + {Status, Body}; + {error, {{_, Status, _}, _Headers, Body0}} -> + Body = + case emqx_utils_json:safe_decode(Body0, [return_maps]) of + {ok, Decoded0 = #{<<"message">> := Msg0}} -> + Msg = maybe_json_decode(Msg0), + Decoded0#{<<"message">> := Msg}; + {ok, Decoded0} -> + Decoded0; + {error, _} -> + Body0 + end, + {Status, Body} + end. diff --git a/apps/emqx_management/test/emqx_mgmt_cli_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_cli_SUITE.erl index f3757d7ec..16871c129 100644 --- a/apps/emqx_management/test/emqx_mgmt_cli_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_cli_SUITE.erl @@ -22,7 +22,13 @@ -include_lib("common_test/include/ct.hrl"). all() -> - emqx_common_test_helpers:all(?MODULE). + All = emqx_common_test_helpers:all(?MODULE), + case emqx_cth_suite:skip_if_oss() of + false -> + All; + _ -> + All -- [t_autocluster_leave] + end. init_per_suite(Config) -> Apps = emqx_cth_suite:start( diff --git a/apps/emqx_management/test/emqx_mgmt_data_backup_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_data_backup_SUITE.erl index 9a3d3971c..80fac8c30 100644 --- a/apps/emqx_management/test/emqx_mgmt_data_backup_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_data_backup_SUITE.erl @@ -52,7 +52,12 @@ >>). all() -> - emqx_common_test_helpers:all(?MODULE). + case emqx_cth_suite:skip_if_oss() of + false -> + emqx_common_test_helpers:all(?MODULE); + True -> + True + end. init_per_suite(Config) -> Config. diff --git a/apps/emqx_modules/src/emqx_modules_schema.erl b/apps/emqx_modules/src/emqx_modules_schema.erl index 48c5a4ab5..8df7d91dc 100644 --- a/apps/emqx_modules/src/emqx_modules_schema.erl +++ b/apps/emqx_modules/src/emqx_modules_schema.erl @@ -79,7 +79,12 @@ rewrite_validator(Rules) -> fields("delayed") -> [ - {enable, ?HOCON(boolean(), #{default => true, desc => ?DESC(enable)})}, + {enable, + ?HOCON(boolean(), #{ + default => true, + importance => ?IMPORTANCE_NO_DOC, + desc => ?DESC(enable) + })}, {max_delayed_messages, ?HOCON(integer(), #{desc => ?DESC(max_delayed_messages), default => 0})} ]; diff --git a/apps/emqx_mongodb/src/emqx_mongodb.app.src b/apps/emqx_mongodb/src/emqx_mongodb.app.src index 92d7026cc..51230fd47 100644 --- a/apps/emqx_mongodb/src/emqx_mongodb.app.src +++ b/apps/emqx_mongodb/src/emqx_mongodb.app.src @@ -1,6 +1,6 @@ {application, emqx_mongodb, [ {description, "EMQX MongoDB Connector"}, - {vsn, "0.1.6"}, + {vsn, "0.1.7"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_mongodb/src/emqx_mongodb.erl b/apps/emqx_mongodb/src/emqx_mongodb.erl index e262f5ccd..8d6fad89f 100644 --- a/apps/emqx_mongodb/src/emqx_mongodb.erl +++ b/apps/emqx_mongodb/src/emqx_mongodb.erl @@ -26,6 +26,7 @@ %% callbacks of behaviour emqx_resource -export([ + resource_type/0, callback_mode/0, on_start/2, on_stop/2, @@ -172,6 +173,7 @@ desc(_) -> undefined. %% =================================================================== +resource_type() -> mongodb. callback_mode() -> always_sync. diff --git a/apps/emqx_mysql/src/emqx_mysql.app.src b/apps/emqx_mysql/src/emqx_mysql.app.src index 9637cc473..c7fcb0975 100644 --- a/apps/emqx_mysql/src/emqx_mysql.app.src +++ b/apps/emqx_mysql/src/emqx_mysql.app.src @@ -1,6 +1,6 @@ {application, emqx_mysql, [ {description, "EMQX MySQL Database Connector"}, - {vsn, "0.1.9"}, + {vsn, "0.2.0"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_mysql/src/emqx_mysql.erl b/apps/emqx_mysql/src/emqx_mysql.erl index 6311d66f2..197e33d75 100644 --- a/apps/emqx_mysql/src/emqx_mysql.erl +++ b/apps/emqx_mysql/src/emqx_mysql.erl @@ -25,6 +25,7 @@ %% callbacks of behaviour emqx_resource -export([ + resource_type/0, callback_mode/0, on_start/2, on_stop/2, @@ -91,6 +92,8 @@ server() -> emqx_schema:servers_sc(Meta, ?MYSQL_HOST_OPTIONS). %% =================================================================== +resource_type() -> mysql. + callback_mode() -> always_sync. -spec on_start(binary(), hocon:config()) -> {ok, state()} | {error, _}. diff --git a/apps/emqx_opentelemetry/src/emqx_opentelemetry.app.src b/apps/emqx_opentelemetry/src/emqx_opentelemetry.app.src index 6a84ae043..8ed649cda 100644 --- a/apps/emqx_opentelemetry/src/emqx_opentelemetry.app.src +++ b/apps/emqx_opentelemetry/src/emqx_opentelemetry.app.src @@ -1,6 +1,6 @@ {application, emqx_opentelemetry, [ {description, "OpenTelemetry for EMQX Broker"}, - {vsn, "0.2.6"}, + {vsn, "0.2.7"}, {registered, []}, {mod, {emqx_otel_app, []}}, {applications, [ diff --git a/apps/emqx_opentelemetry/src/emqx_otel_schema.erl b/apps/emqx_opentelemetry/src/emqx_otel_schema.erl index e89591fa9..a12efc7a9 100644 --- a/apps/emqx_opentelemetry/src/emqx_otel_schema.erl +++ b/apps/emqx_opentelemetry/src/emqx_otel_schema.erl @@ -72,6 +72,7 @@ fields("otel_metrics") -> boolean(), #{ default => false, + %% importance => ?IMPORTANCE_NO_DOC, required => true, desc => ?DESC(enable) } @@ -104,6 +105,7 @@ fields("otel_logs") -> #{ default => false, desc => ?DESC(enable), + %% importance => ?IMPORTANCE_NO_DOC importance => ?IMPORTANCE_HIGH } )}, @@ -143,6 +145,7 @@ fields("otel_traces") -> #{ default => false, desc => ?DESC(enable), + %% importance => ?IMPORTANCE_NO_DOC importance => ?IMPORTANCE_HIGH } )}, diff --git a/apps/emqx_opentelemetry/test/emqx_otel_trace_SUITE.erl b/apps/emqx_opentelemetry/test/emqx_otel_trace_SUITE.erl index a02cec3ef..9cf545cdc 100644 --- a/apps/emqx_opentelemetry/test/emqx_otel_trace_SUITE.erl +++ b/apps/emqx_opentelemetry/test/emqx_otel_trace_SUITE.erl @@ -414,9 +414,9 @@ mqtt_host_port(Node) -> cluster(TC, Config) -> Nodes = emqx_cth_cluster:start( [ - {otel_trace_core1, #{role => core, apps => apps_spec()}}, - {otel_trace_core2, #{role => core, apps => apps_spec()}}, - {otel_trace_replicant, #{role => replicant, apps => apps_spec()}} + {otel_trace_node1, #{apps => apps_spec()}}, + {otel_trace_node2, #{apps => apps_spec()}}, + {otel_trace_node3, #{apps => apps_spec()}} ], #{work_dir => emqx_cth_suite:work_dir(TC, Config)} ), diff --git a/apps/emqx_oracle/src/emqx_oracle.app.src b/apps/emqx_oracle/src/emqx_oracle.app.src index 3f238ae9c..80ff8da09 100644 --- a/apps/emqx_oracle/src/emqx_oracle.app.src +++ b/apps/emqx_oracle/src/emqx_oracle.app.src @@ -1,6 +1,6 @@ {application, emqx_oracle, [ {description, "EMQX Enterprise Oracle Database Connector"}, - {vsn, "0.2.2"}, + {vsn, "0.2.3"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_oracle/src/emqx_oracle.erl b/apps/emqx_oracle/src/emqx_oracle.erl index 5b25e049a..6e2a40f95 100644 --- a/apps/emqx_oracle/src/emqx_oracle.erl +++ b/apps/emqx_oracle/src/emqx_oracle.erl @@ -21,6 +21,7 @@ %% callbacks for behaviour emqx_resource -export([ + resource_type/0, callback_mode/0, on_start/2, on_stop/2, @@ -67,6 +68,8 @@ batch_params_tokens := params_tokens() }. +resource_type() -> oracle. + % As ecpool is not monitoring the worker's PID when doing a handover_async, the % request can be lost if worker crashes. Thus, it's better to force requests to % be sync for now. diff --git a/apps/emqx_plugins/src/emqx_plugins.erl b/apps/emqx_plugins/src/emqx_plugins.erl index 37df54d3f..8ae211b55 100644 --- a/apps/emqx_plugins/src/emqx_plugins.erl +++ b/apps/emqx_plugins/src/emqx_plugins.erl @@ -358,13 +358,13 @@ get_config_bin(NameVsn) -> %% RPC call from Management API or CLI. %% The plugin config Json Map was valid by avro schema %% Or: if no and plugin config ALWAYS be valid before calling this function. -put_config(NameVsn, ConfigJsonMap, AvroValue) when not is_binary(NameVsn) -> +put_config(NameVsn, ConfigJsonMap, AvroValue) when (not is_binary(NameVsn)) -> put_config(bin(NameVsn), ConfigJsonMap, AvroValue); put_config(NameVsn, ConfigJsonMap, _AvroValue) -> HoconBin = hocon_pp:do(ConfigJsonMap, #{}), ok = backup_and_write_hocon_bin(NameVsn, HoconBin), - %% TODO: callback in plugin's on_config_changed (config update by mgmt API) %% TODO: callback in plugin's on_config_upgraded (config vsn upgrade v1 -> v2) + ok = maybe_call_on_config_changed(NameVsn, ConfigJsonMap), ok = persistent_term:put(?PLUGIN_PERSIS_CONFIG_KEY(NameVsn), ConfigJsonMap), ok. @@ -375,6 +375,43 @@ restart(NameVsn) -> {error, Reason} -> {error, Reason} end. +%% @doc Call plugin's callback on_config_changed/2 +maybe_call_on_config_changed(NameVsn, NewConf) -> + FuncName = on_config_changed, + maybe + {ok, PluginAppModule} ?= app_module_name(NameVsn), + true ?= erlang:function_exported(PluginAppModule, FuncName, 2), + {ok, OldConf} = get_config(NameVsn), + try erlang:apply(PluginAppModule, FuncName, [OldConf, NewConf]) of + _ -> ok + catch + Class:CatchReason:Stacktrace -> + ?SLOG(error, #{ + msg => "failed_to_call_on_config_changed", + exception => Class, + reason => CatchReason, + stacktrace => Stacktrace + }), + ok + end + else + {error, Reason} -> + ?SLOG(info, #{msg => "failed_to_call_on_config_changed", reason => Reason}); + false -> + ?SLOG(info, #{msg => "on_config_changed_callback_not_exported"}); + _ -> + ok + end. + +app_module_name(NameVsn) -> + case read_plugin_info(NameVsn, #{}) of + {ok, #{<<"name">> := Name} = _PluginInfo} -> + emqx_utils:safe_to_existing_atom(<>); + {error, Reason} -> + ?SLOG(error, Reason#{msg => "failed_to_read_plugin_info"}), + {error, Reason} + end. + %% @doc List all installed plugins. %% Including the ones that are installed, but not enabled in config. -spec list() -> [plugin_info()]. diff --git a/apps/emqx_plugins/src/emqx_plugins_schema.erl b/apps/emqx_plugins/src/emqx_plugins_schema.erl index 8932e3ab6..d750a029c 100644 --- a/apps/emqx_plugins/src/emqx_plugins_schema.erl +++ b/apps/emqx_plugins/src/emqx_plugins_schema.erl @@ -56,6 +56,8 @@ state_fields() -> ?HOCON( boolean(), #{ + default => true, + importance => ?IMPORTANCE_NO_DOC, desc => ?DESC(enable), required => true } diff --git a/apps/emqx_postgresql/src/emqx_postgresql.app.src b/apps/emqx_postgresql/src/emqx_postgresql.app.src index 7aaf42e71..e1bd67325 100644 --- a/apps/emqx_postgresql/src/emqx_postgresql.app.src +++ b/apps/emqx_postgresql/src/emqx_postgresql.app.src @@ -1,6 +1,6 @@ {application, emqx_postgresql, [ {description, "EMQX PostgreSQL Database Connector"}, - {vsn, "0.2.2"}, + {vsn, "0.2.3"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_postgresql/src/emqx_postgresql.erl b/apps/emqx_postgresql/src/emqx_postgresql.erl index 7e64a3e83..a061fc15e 100644 --- a/apps/emqx_postgresql/src/emqx_postgresql.erl +++ b/apps/emqx_postgresql/src/emqx_postgresql.erl @@ -29,6 +29,7 @@ %% callbacks of behaviour emqx_resource -export([ + resource_type/0, callback_mode/0, on_start/2, on_stop/2, @@ -120,6 +121,8 @@ adjust_fields(Fields) -> ). %% =================================================================== +resource_type() -> pgsql. + callback_mode() -> always_sync. -spec on_start(binary(), hocon:config()) -> {ok, state()} | {error, _}. diff --git a/apps/emqx_prometheus/src/emqx_prometheus_schema.erl b/apps/emqx_prometheus/src/emqx_prometheus_schema.erl index 84925c7b6..6d3503a82 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus_schema.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus_schema.erl @@ -78,6 +78,7 @@ fields(push_gateway) -> #{ default => false, required => true, + %% importance => ?IMPORTANCE_NO_DOC, desc => ?DESC(push_gateway_enable) } )}, @@ -229,6 +230,7 @@ fields(legacy_deprecated_setting) -> #{ default => false, required => true, + %% importance => ?IMPORTANCE_NO_DOC, desc => ?DESC(legacy_enable) } )}, diff --git a/apps/emqx_psk/src/emqx_psk.app.src b/apps/emqx_psk/src/emqx_psk.app.src index 14c6ba0cc..88dd0ccef 100644 --- a/apps/emqx_psk/src/emqx_psk.app.src +++ b/apps/emqx_psk/src/emqx_psk.app.src @@ -2,7 +2,7 @@ {application, emqx_psk, [ {description, "EMQX PSK"}, % strict semver, bump manually! - {vsn, "5.0.6"}, + {vsn, "5.0.7"}, {modules, []}, {registered, [emqx_psk_sup]}, {applications, [kernel, stdlib]}, diff --git a/apps/emqx_psk/src/emqx_psk_schema.erl b/apps/emqx_psk/src/emqx_psk_schema.erl index 652f61b08..800845923 100644 --- a/apps/emqx_psk/src/emqx_psk_schema.erl +++ b/apps/emqx_psk/src/emqx_psk_schema.erl @@ -42,6 +42,7 @@ fields() -> [ {enable, ?HOCON(boolean(), #{ + %% importance => ?IMPORTANCE_NO_DOC, default => false, require => true, desc => ?DESC(enable) diff --git a/apps/emqx_redis/src/emqx_redis.erl b/apps/emqx_redis/src/emqx_redis.erl index 059e9aa23..9507913ed 100644 --- a/apps/emqx_redis/src/emqx_redis.erl +++ b/apps/emqx_redis/src/emqx_redis.erl @@ -28,6 +28,7 @@ %% callbacks of behaviour emqx_resource -export([ + resource_type/0, callback_mode/0, on_start/2, on_stop/2, @@ -119,6 +120,8 @@ redis_type(Type) -> desc => ?DESC(Type) }}. +resource_type() -> redis. + callback_mode() -> always_sync. on_start(InstId, Config0) -> diff --git a/apps/emqx_resource/include/emqx_resource.hrl b/apps/emqx_resource/include/emqx_resource.hrl index 587786cb2..8c2bb39a1 100644 --- a/apps/emqx_resource/include/emqx_resource.hrl +++ b/apps/emqx_resource/include/emqx_resource.hrl @@ -23,7 +23,8 @@ %% remind us of that. -define(rm_status_stopped, stopped). --type resource_type() :: module(). +-type resource_type() :: atom(). +-type resource_module() :: module(). -type resource_id() :: binary(). -type channel_id() :: binary(). -type raw_resource_config() :: binary() | raw_term_resource_config(). @@ -158,5 +159,12 @@ %% See `hocon_tconf` -define(TEST_ID_PREFIX, "t_probe_"). -define(RES_METRICS, resource_metrics). +-define(LOG_LEVEL(_L_), + case _L_ of + true -> info; + false -> warning + end +). +-define(TAG, "RESOURCE"). -define(RESOURCE_ALLOCATION_TAB, emqx_resource_allocations). diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl index 15c0a0293..bf47c83ab 100644 --- a/apps/emqx_resource/src/emqx_resource.erl +++ b/apps/emqx_resource/src/emqx_resource.erl @@ -39,12 +39,10 @@ -export([ %% store the config and start the instance - create_local/4, create_local/5, create_dry_run_local/2, create_dry_run_local/3, create_dry_run_local/4, - recreate_local/3, recreate_local/4, %% remove the config and stop the instance remove_local/1, @@ -98,6 +96,7 @@ -export([ %% get the callback mode of a specific module get_callback_mode/1, + get_resource_type/1, %% start the instance call_start/3, %% verify if the resource is working normally @@ -140,6 +139,8 @@ validate_name/1 ]). +-export([is_dry_run/1]). + -export_type([ query_mode/0, resource_id/0, @@ -243,6 +244,9 @@ QueryResult :: term() ) -> term(). +%% Used for tagging log entries. +-callback resource_type() -> atom(). + -define(SAFE_CALL(EXPR), (fun() -> try @@ -279,16 +283,10 @@ is_resource_mod(Module) -> %% ================================================================================= %% APIs for resource instances %% ================================================================================= - --spec create_local(resource_id(), resource_group(), resource_type(), resource_config()) -> - {ok, resource_data() | 'already_created'} | {error, Reason :: term()}. -create_local(ResId, Group, ResourceType, Config) -> - create_local(ResId, Group, ResourceType, Config, #{}). - -spec create_local( resource_id(), resource_group(), - resource_type(), + resource_module(), resource_config(), creation_opts() ) -> @@ -296,7 +294,7 @@ create_local(ResId, Group, ResourceType, Config) -> create_local(ResId, Group, ResourceType, Config, Opts) -> emqx_resource_manager:ensure_resource(ResId, Group, ResourceType, Config, Opts). --spec create_dry_run_local(resource_type(), resource_config()) -> +-spec create_dry_run_local(resource_module(), resource_config()) -> ok | {error, Reason :: term()}. create_dry_run_local(ResourceType, Config) -> emqx_resource_manager:create_dry_run(ResourceType, Config). @@ -304,19 +302,21 @@ create_dry_run_local(ResourceType, Config) -> create_dry_run_local(ResId, ResourceType, Config) -> emqx_resource_manager:create_dry_run(ResId, ResourceType, Config). --spec create_dry_run_local(resource_id(), resource_type(), resource_config(), OnReadyCallback) -> +-spec create_dry_run_local( + resource_id(), + resource_module(), + resource_config(), + OnReadyCallback +) -> ok | {error, Reason :: term()} when OnReadyCallback :: fun((resource_id()) -> ok | {error, Reason :: term()}). create_dry_run_local(ResId, ResourceType, Config, OnReadyCallback) -> emqx_resource_manager:create_dry_run(ResId, ResourceType, Config, OnReadyCallback). --spec recreate_local(resource_id(), resource_type(), resource_config()) -> - {ok, resource_data()} | {error, Reason :: term()}. -recreate_local(ResId, ResourceType, Config) -> - recreate_local(ResId, ResourceType, Config, #{}). - --spec recreate_local(resource_id(), resource_type(), resource_config(), creation_opts()) -> +-spec recreate_local( + resource_id(), resource_module(), resource_config(), creation_opts() +) -> {ok, resource_data()} | {error, Reason :: term()}. recreate_local(ResId, ResourceType, Config, Opts) -> emqx_resource_manager:recreate(ResId, ResourceType, Config, Opts). @@ -330,11 +330,15 @@ remove_local(ResId) -> ok; Error -> %% Only log, the ResId worker is always removed in manager's remove action. - ?SLOG(warning, #{ - msg => "remove_local_resource_failed", - error => Error, - resource_id => ResId - }), + ?SLOG( + warning, + #{ + msg => "remove_resource_failed", + error => Error, + resource_id => ResId + }, + #{tag => ?TAG} + ), ok end. @@ -487,6 +491,10 @@ list_group_instances(Group) -> emqx_resource_manager:list_group(Group). get_callback_mode(Mod) -> Mod:callback_mode(). +-spec get_resource_type(module()) -> resource_type(). +get_resource_type(Mod) -> + Mod:resource_type(). + -spec call_start(resource_id(), module(), resource_config()) -> {ok, resource_state()} | {error, Reason :: term()}. call_start(ResId, Mod, Config) -> @@ -599,7 +607,7 @@ query_mode(Mod, Config, Opts) -> maps:get(query_mode, Opts, sync) end. --spec check_config(resource_type(), raw_resource_config()) -> +-spec check_config(resource_module(), raw_resource_config()) -> {ok, resource_config()} | {error, term()}. check_config(ResourceType, Conf) -> emqx_hocon:check(ResourceType, Conf). @@ -607,7 +615,7 @@ check_config(ResourceType, Conf) -> -spec check_and_create_local( resource_id(), resource_group(), - resource_type(), + resource_module(), raw_resource_config() ) -> {ok, resource_data()} | {error, term()}. @@ -617,7 +625,7 @@ check_and_create_local(ResId, Group, ResourceType, RawConfig) -> -spec check_and_create_local( resource_id(), resource_group(), - resource_type(), + resource_module(), raw_resource_config(), creation_opts() ) -> {ok, resource_data()} | {error, term()}. @@ -630,7 +638,7 @@ check_and_create_local(ResId, Group, ResourceType, RawConfig, Opts) -> -spec check_and_recreate_local( resource_id(), - resource_type(), + resource_module(), raw_resource_config(), creation_opts() ) -> @@ -769,6 +777,13 @@ validate_name(Name) -> _ = validate_name(Name, #{atom_name => false}), ok. +-spec is_dry_run(resource_id()) -> boolean(). +is_dry_run(ResId) -> + case string:find(ResId, ?TEST_ID_PREFIX) of + nomatch -> false; + TestIdStart -> string:equal(TestIdStart, ResId) + end. + validate_name(<<>>, _Opts) -> invalid_data("Name cannot be empty string"); validate_name(Name, _Opts) when size(Name) >= 255 -> diff --git a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl index 8eb7b373d..e37917215 100644 --- a/apps/emqx_resource/src/emqx_resource_buffer_worker.erl +++ b/apps/emqx_resource/src/emqx_resource_buffer_worker.erl @@ -305,10 +305,10 @@ running(info, {flush_metrics, _Ref}, _Data) -> running(info, {'DOWN', _MRef, process, Pid, Reason}, Data0 = #{async_workers := AsyncWorkers0}) when is_map_key(Pid, AsyncWorkers0) -> - ?SLOG(info, #{msg => "async_worker_died", state => running, reason => Reason}), + ?SLOG(info, #{msg => "async_worker_died", state => running, reason => Reason}, #{tag => ?TAG}), handle_async_worker_down(Data0, Pid); running(info, Info, _St) -> - ?SLOG(error, #{msg => "unexpected_msg", state => running, info => Info}), + ?SLOG(error, #{msg => "unexpected_msg", state => running, info => Info}, #{tag => ?TAG}), keep_state_and_data. blocked(enter, _, #{resume_interval := ResumeT} = St0) -> @@ -338,10 +338,10 @@ blocked(info, {flush_metrics, _Ref}, _Data) -> blocked(info, {'DOWN', _MRef, process, Pid, Reason}, Data0 = #{async_workers := AsyncWorkers0}) when is_map_key(Pid, AsyncWorkers0) -> - ?SLOG(info, #{msg => "async_worker_died", state => blocked, reason => Reason}), + ?SLOG(info, #{msg => "async_worker_died", state => blocked, reason => Reason}, #{tag => ?TAG}), handle_async_worker_down(Data0, Pid); blocked(info, Info, _Data) -> - ?SLOG(error, #{msg => "unexpected_msg", state => blocked, info => Info}), + ?SLOG(error, #{msg => "unexpected_msg", state => blocked, info => Info}, #{tag => ?TAG}), keep_state_and_data. terminate(_Reason, #{id := Id, index := Index, queue := Q}) -> @@ -988,7 +988,16 @@ handle_query_result_pure(Id, {error, Reason} = Error, HasBeenSent, TraceCTX) -> true -> PostFn = fun() -> - ?SLOG(error, #{id => Id, msg => "unrecoverable_error", reason => Reason}), + ?SLOG_THROTTLE( + error, + Id, + #{ + resource_id => Id, + msg => unrecoverable_resource_error, + reason => Reason + }, + #{tag => ?TAG} + ), ok end, Counters = @@ -1028,7 +1037,16 @@ handle_query_async_result_pure(Id, {error, Reason} = Error, HasBeenSent, TraceCT true -> PostFn = fun() -> - ?SLOG(error, #{id => Id, msg => "unrecoverable_error", reason => Reason}), + ?SLOG_THROTTLE( + error, + Id, + #{ + resource_id => Id, + msg => unrecoverable_resource_error, + reason => Reason + }, + #{tag => ?TAG} + ), ok end, Counters = @@ -1148,12 +1166,16 @@ log_expired_message_count(_Data = #{id := Id, index := Index, counters := Counte false -> ok; true -> - ?SLOG(info, #{ - msg => "buffer_worker_dropped_expired_messages", - resource_id => Id, - worker_index => Index, - expired_count => ExpiredCount - }), + ?SLOG( + info, + #{ + msg => "buffer_worker_dropped_expired_messages", + resource_id => Id, + worker_index => Index, + expired_count => ExpiredCount + }, + #{tag => ?TAG} + ), ok end. @@ -1565,7 +1587,7 @@ handle_async_reply1( case is_expired(ExpireAt, Now) of true -> IsAcked = ack_inflight(InflightTID, Ref, BufferWorkerPid), - %% evalutate metrics call here since we're not inside + %% evaluate metrics call here since we're not inside %% buffer worker IsAcked andalso begin @@ -1806,12 +1828,16 @@ append_queue(Id, Index, Q, Queries) -> ok = replayq:ack(Q1, QAckRef), Dropped = length(Items2), Counters = #{dropped_queue_full => Dropped}, - ?SLOG_THROTTLE(warning, #{ - msg => data_bridge_buffer_overflow, - resource_id => Id, - worker_index => Index, - dropped => Dropped - }), + ?SLOG_THROTTLE( + warning, + #{ + msg => data_bridge_buffer_overflow, + resource_id => Id, + worker_index => Index, + dropped => Dropped + }, + #{tag => ?TAG} + ), {Items2, Q1, Counters} end, ?tp( @@ -2245,11 +2271,15 @@ adjust_batch_time(Id, RequestTTL, BatchTime0) -> BatchTime = max(0, min(BatchTime0, RequestTTL div 2)), case BatchTime =:= BatchTime0 of false -> - ?SLOG(info, #{ - id => Id, - msg => "adjusting_buffer_worker_batch_time", - new_batch_time => BatchTime - }); + ?SLOG( + info, + #{ + resource_id => Id, + msg => "adjusting_buffer_worker_batch_time", + new_batch_time => BatchTime + }, + #{tag => ?TAG} + ); true -> ok end, diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl index 1a3781a65..575116f77 100644 --- a/apps/emqx_resource/src/emqx_resource_manager.erl +++ b/apps/emqx_resource/src/emqx_resource_manager.erl @@ -75,6 +75,7 @@ -record(data, { id, group, + type, mod, callback_mode, query_mode, @@ -166,7 +167,7 @@ where(ResId) -> -spec ensure_resource( resource_id(), resource_group(), - resource_type(), + resource_module(), resource_config(), creation_opts() ) -> {ok, resource_data()}. @@ -179,7 +180,9 @@ ensure_resource(ResId, Group, ResourceType, Config, Opts) -> end. %% @doc Called from emqx_resource when recreating a resource which may or may not exist --spec recreate(resource_id(), resource_type(), resource_config(), creation_opts()) -> +-spec recreate( + resource_id(), resource_module(), resource_config(), creation_opts() +) -> {ok, resource_data()} | {error, not_found} | {error, updating_to_incorrect_resource_type}. recreate(ResId, ResourceType, NewConfig, Opts) -> case lookup(ResId) of @@ -222,8 +225,8 @@ create(ResId, Group, ResourceType, Config, Opts) -> %% @doc Called from `emqx_resource` when doing a dry run for creating a resource instance. %% %% Triggers the `emqx_resource_manager_sup` supervisor to actually create -%% and link the process itself if not already started, and then immedately stops. --spec create_dry_run(resource_type(), resource_config()) -> +%% and link the process itself if not already started, and then immediately stops. +-spec create_dry_run(resource_module(), resource_config()) -> ok | {error, Reason :: term()}. create_dry_run(ResourceType, Config) -> ResId = make_test_id(), @@ -235,7 +238,9 @@ create_dry_run(ResId, ResourceType, Config) -> do_nothing_on_ready(_ResId) -> ok. --spec create_dry_run(resource_id(), resource_type(), resource_config(), OnReadyCallback) -> +-spec create_dry_run( + resource_id(), resource_module(), resource_config(), OnReadyCallback +) -> ok | {error, Reason :: term()} when OnReadyCallback :: fun((resource_id()) -> ok | {error, Reason :: term()}). @@ -245,7 +250,9 @@ create_dry_run(ResId, ResourceType, Config, OnReadyCallback) -> true -> maps:get(resource_opts, Config, #{}); false -> #{} end, - ok = emqx_resource_manager_sup:ensure_child(ResId, <<"dry_run">>, ResourceType, Config, Opts), + ok = emqx_resource_manager_sup:ensure_child( + ResId, <<"dry_run">>, ResourceType, Config, Opts + ), HealthCheckInterval = maps:get(health_check_interval, Opts, ?HEALTHCHECK_INTERVAL), Timeout = emqx_utils:clamp(HealthCheckInterval, 5_000, 60_000), case wait_for_ready(ResId, Timeout) of @@ -526,6 +533,7 @@ start_link(ResId, Group, ResourceType, Config, Opts) -> ), Data = #data{ id = ResId, + type = emqx_resource:get_resource_type(ResourceType), group = Group, mod = ResourceType, callback_mode = emqx_resource:get_callback_mode(ResourceType), @@ -710,11 +718,13 @@ handle_event(EventType, EventData, State, Data) -> error, #{ msg => "ignore_all_other_events", + resource_id => Data#data.id, event_type => EventType, event_data => EventData, state => State, data => emqx_utils:redact(Data) - } + }, + #{tag => tag(Data#data.group, Data#data.type)} ), keep_state_and_data. @@ -779,7 +789,8 @@ handle_remove_event(From, ClearMetrics, Data) -> start_resource(Data, From) -> %% in case the emqx_resource:call_start/2 hangs, the lookup/1 can read status from the cache - case emqx_resource:call_start(Data#data.id, Data#data.mod, Data#data.config) of + #data{id = ResId, mod = Mod, config = Config, group = Group, type = Type} = Data, + case emqx_resource:call_start(ResId, Mod, Config) of {ok, ResourceState} -> UpdatedData1 = Data#data{status = ?status_connecting, state = ResourceState}, %% Perform an initial health_check immediately before transitioning into a connected state @@ -787,12 +798,17 @@ start_resource(Data, From) -> Actions = maybe_reply([{state_timeout, 0, health_check}], From, ok), {next_state, ?state_connecting, update_state(UpdatedData2, Data), Actions}; {error, Reason} = Err -> - ?SLOG(warning, #{ - msg => "start_resource_failed", - id => Data#data.id, - reason => Reason - }), - _ = maybe_alarm(?status_disconnected, Data#data.id, Err, Data#data.error), + IsDryRun = emqx_resource:is_dry_run(ResId), + ?SLOG( + log_level(IsDryRun), + #{ + msg => "start_resource_failed", + resource_id => ResId, + reason => Reason + }, + #{tag => tag(Group, Type)} + ), + _ = maybe_alarm(?status_disconnected, IsDryRun, ResId, Err, Data#data.error), %% Add channels and raise alarms NewData1 = channels_health_check(?status_disconnected, add_channels(Data)), %% Keep track of the error reason why the connection did not work @@ -823,13 +839,20 @@ add_channels(Data) -> add_channels_in_list([], Data) -> Data; add_channels_in_list([{ChannelID, ChannelConfig} | Rest], Data) -> + #data{ + id = ResId, + mod = Mod, + state = State, + added_channels = AddedChannelsMap, + group = Group, + type = Type + } = Data, case emqx_resource:call_add_channel( - Data#data.id, Data#data.mod, Data#data.state, ChannelID, ChannelConfig + ResId, Mod, State, ChannelID, ChannelConfig ) of {ok, NewState} -> - AddedChannelsMap = Data#data.added_channels, %% Set the channel status to connecting to indicate that %% we have not yet performed the initial health_check NewAddedChannelsMap = maps:put( @@ -843,12 +866,17 @@ add_channels_in_list([{ChannelID, ChannelConfig} | Rest], Data) -> }, add_channels_in_list(Rest, NewData); {error, Reason} = Error -> - ?SLOG(warning, #{ - msg => add_channel_failed, - id => Data#data.id, - channel_id => ChannelID, - reason => Reason - }), + IsDryRun = emqx_resource:is_dry_run(ResId), + ?SLOG( + log_level(IsDryRun), + #{ + msg => "add_channel_failed", + resource_id => ResId, + channel_id => ChannelID, + reason => Reason + }, + #{tag => tag(Group, Type)} + ), AddedChannelsMap = Data#data.added_channels, NewAddedChannelsMap = maps:put( ChannelID, @@ -859,7 +887,7 @@ add_channels_in_list([{ChannelID, ChannelConfig} | Rest], Data) -> added_channels = NewAddedChannelsMap }, %% Raise an alarm since the channel could not be added - _ = maybe_alarm(?status_disconnected, ChannelID, Error, no_prev_error), + _ = maybe_alarm(?status_disconnected, IsDryRun, ChannelID, Error, no_prev_error), add_channels_in_list(Rest, NewData) end. @@ -883,7 +911,8 @@ stop_resource(#data{id = ResId} = Data) -> false -> ok end, - _ = maybe_clear_alarm(ResId), + IsDryRun = emqx_resource:is_dry_run(ResId), + _ = maybe_clear_alarm(IsDryRun, ResId), ok = emqx_metrics_worker:reset_metrics(?RES_METRICS, ResId), NewData#data{status = ?rm_status_stopped}. @@ -894,16 +923,24 @@ remove_channels(Data) -> remove_channels_in_list([], Data, _KeepInChannelMap) -> Data; remove_channels_in_list([ChannelID | Rest], Data, KeepInChannelMap) -> - AddedChannelsMap = Data#data.added_channels, + #data{ + id = ResId, + added_channels = AddedChannelsMap, + mod = Mod, + state = State, + group = Group, + type = Type + } = Data, + IsDryRun = emqx_resource:is_dry_run(ResId), NewAddedChannelsMap = case KeepInChannelMap of true -> AddedChannelsMap; false -> - _ = maybe_clear_alarm(ChannelID), + _ = maybe_clear_alarm(IsDryRun, ChannelID), maps:remove(ChannelID, AddedChannelsMap) end, - case safe_call_remove_channel(Data#data.id, Data#data.mod, Data#data.state, ChannelID) of + case safe_call_remove_channel(ResId, Mod, State, ChannelID) of {ok, NewState} -> NewData = Data#data{ state = NewState, @@ -911,12 +948,18 @@ remove_channels_in_list([ChannelID | Rest], Data, KeepInChannelMap) -> }, remove_channels_in_list(Rest, NewData, KeepInChannelMap); {error, Reason} -> - ?SLOG(warning, #{ - msg => remove_channel_failed, - id => Data#data.id, - channel_id => ChannelID, - reason => Reason - }), + ?SLOG( + log_level(IsDryRun), + #{ + msg => "remove_channel_failed", + resource_id => ResId, + group => Group, + type => Type, + channel_id => ChannelID, + reason => Reason + }, + #{tag => tag(Group, Type)} + ), NewData = Data#data{ added_channels = NewAddedChannelsMap }, @@ -995,8 +1038,8 @@ handle_not_connected_add_channel(From, ChannelId, ChannelConfig, State, Data) -> handle_remove_channel(From, ChannelId, Data) -> Channels = Data#data.added_channels, - %% Deactivate alarm - _ = maybe_clear_alarm(ChannelId), + IsDryRun = emqx_resource:is_dry_run(Data#data.id), + _ = maybe_clear_alarm(IsDryRun, ChannelId), case channel_status_is_channel_added( maps:get(ChannelId, Channels, channel_status_not_added(undefined)) @@ -1017,13 +1060,18 @@ handle_remove_channel(From, ChannelId, Data) -> end. handle_remove_channel_exists(From, ChannelId, Data) -> + #data{ + id = Id, + group = Group, + type = Type, + added_channels = AddedChannelsMap + } = Data, case emqx_resource:call_remove_channel( - Data#data.id, Data#data.mod, Data#data.state, ChannelId + Id, Data#data.mod, Data#data.state, ChannelId ) of {ok, NewState} -> - AddedChannelsMap = Data#data.added_channels, NewAddedChannelsMap = maps:remove(ChannelId, AddedChannelsMap), UpdatedData = Data#data{ state = NewState, @@ -1031,13 +1079,17 @@ handle_remove_channel_exists(From, ChannelId, Data) -> }, {keep_state, update_state(UpdatedData, Data), [{reply, From, ok}]}; {error, Reason} = Error -> - %% Log the error as a warning - ?SLOG(warning, #{ - msg => remove_channel_failed, - id => Data#data.id, - channel_id => ChannelId, - reason => Reason - }), + IsDryRun = emqx_resource:is_dry_run(Id), + ?SLOG( + log_level(IsDryRun), + #{ + msg => "remove_channel_failed", + resource_id => Id, + channel_id => ChannelId, + reason => Reason + }, + #{tag => tag(Group, Type)} + ), {keep_state_and_data, [{reply, From, Error}]} end. @@ -1048,7 +1100,8 @@ handle_not_connected_and_not_connecting_remove_channel(From, ChannelId, Data) -> Channels = Data#data.added_channels, NewChannels = maps:remove(ChannelId, Channels), NewData = Data#data{added_channels = NewChannels}, - _ = maybe_clear_alarm(ChannelId), + IsDryRun = emqx_resource:is_dry_run(Data#data.id), + _ = maybe_clear_alarm(IsDryRun, ChannelId), {keep_state, update_state(NewData, Data), [{reply, From, ok}]}. handle_manual_resource_health_check(From, Data0 = #data{hc_workers = #{resource := HCWorkers}}) when @@ -1117,7 +1170,8 @@ continue_with_health_check(#data{} = Data0, CurrentState, HCRes) -> error = PrevError } = Data0, {NewStatus, NewState, Err} = parse_health_check_result(HCRes, Data0), - _ = maybe_alarm(NewStatus, ResId, Err, PrevError), + IsDryRun = emqx_resource:is_dry_run(ResId), + _ = maybe_alarm(NewStatus, IsDryRun, ResId, Err, PrevError), ok = maybe_resume_resource_workers(ResId, NewStatus), Data1 = Data0#data{ state = NewState, status = NewStatus, error = Err @@ -1141,11 +1195,17 @@ continue_resource_health_check_connected(NewStatus, Data0) -> Actions = Replies ++ resource_health_check_actions(Data), {keep_state, Data, Actions}; _ -> - ?SLOG(warning, #{ - msg => "health_check_failed", - id => Data0#data.id, - status => NewStatus - }), + #data{id = ResId, group = Group, type = Type} = Data0, + IsDryRun = emqx_resource:is_dry_run(ResId), + ?SLOG( + log_level(IsDryRun), + #{ + msg => "health_check_failed", + resource_id => ResId, + status => NewStatus + }, + #{tag => tag(Group, Type)} + ), %% Note: works because, coincidentally, channel/resource status is a %% subset of resource manager state... But there should be a conversion %% between the two here, as resource manager also has `stopped', which is @@ -1241,7 +1301,7 @@ channels_health_check(?status_connected = _ConnectorStatus, Data0) -> channels_health_check(?status_connecting = _ConnectorStatus, Data0) -> %% Whenever the resource is connecting: %% 1. Change the status of all added channels to connecting - %% 2. Raise alarms (TODO: if it is a probe we should not raise alarms) + %% 2. Raise alarms Channels = Data0#data.added_channels, ChannelsToChangeStatusFor = [ {ChannelId, Config} @@ -1267,9 +1327,10 @@ channels_health_check(?status_connecting = _ConnectorStatus, Data0) -> || {ChannelId, NewStatus} <- maps:to_list(NewChannels) ], %% Raise alarms for all channels + IsDryRun = emqx_resource:is_dry_run(Data0#data.id), lists:foreach( fun({ChannelId, Status, PrevStatus}) -> - maybe_alarm(?status_connecting, ChannelId, Status, PrevStatus) + maybe_alarm(?status_connecting, IsDryRun, ChannelId, Status, PrevStatus) end, ChannelsWithNewAndPrevErrorStatuses ), @@ -1302,9 +1363,10 @@ channels_health_check(ConnectorStatus, Data0) -> || {ChannelId, #{config := Config} = OldStatus} <- maps:to_list(Data1#data.added_channels) ], %% Raise alarms + IsDryRun = emqx_resource:is_dry_run(Data1#data.id), _ = lists:foreach( fun({ChannelId, OldStatus, NewStatus}) -> - _ = maybe_alarm(NewStatus, ChannelId, NewStatus, OldStatus) + _ = maybe_alarm(NewStatus, IsDryRun, ChannelId, NewStatus, OldStatus) end, ChannelsWithNewAndOldStatuses ), @@ -1413,13 +1475,14 @@ continue_channel_health_check_connected_no_update_during_check(ChannelId, OldSta NewStatus = maps:get(ChannelId, Data1#data.added_channels), ChannelsToRemove = [ChannelId || not channel_status_is_channel_added(NewStatus)], Data = remove_channels_in_list(ChannelsToRemove, Data1, true), + IsDryRun = emqx_resource:is_dry_run(Data1#data.id), %% Raise/clear alarms case NewStatus of #{status := ?status_connected} -> - _ = maybe_clear_alarm(ChannelId), + _ = maybe_clear_alarm(IsDryRun, ChannelId), ok; _ -> - _ = maybe_alarm(NewStatus, ChannelId, NewStatus, OldStatus), + _ = maybe_alarm(NewStatus, IsDryRun, ChannelId, NewStatus, OldStatus), ok end, Data. @@ -1583,15 +1646,21 @@ remove_runtime_data(#data{} = Data0) -> health_check_interval(Opts) -> maps:get(health_check_interval, Opts, ?HEALTHCHECK_INTERVAL). --spec maybe_alarm(resource_status(), resource_id(), _Error :: term(), _PrevError :: term()) -> ok. -maybe_alarm(?status_connected, _ResId, _Error, _PrevError) -> +-spec maybe_alarm( + resource_status(), + boolean(), + resource_id(), + _Error :: term(), + _PrevError :: term() +) -> ok. +maybe_alarm(?status_connected, _IsDryRun, _ResId, _Error, _PrevError) -> ok; -maybe_alarm(_Status, <>, _Error, _PrevError) -> +maybe_alarm(_Status, true, _ResId, _Error, _PrevError) -> ok; %% Assume that alarm is already active -maybe_alarm(_Status, _ResId, Error, Error) -> +maybe_alarm(_Status, _IsDryRun, _ResId, Error, Error) -> ok; -maybe_alarm(_Status, ResId, Error, _PrevError) -> +maybe_alarm(_Status, false, ResId, Error, _PrevError) -> HrError = case Error of {error, undefined} -> @@ -1623,10 +1692,10 @@ maybe_resume_resource_workers(ResId, ?status_connected) -> maybe_resume_resource_workers(_, _) -> ok. --spec maybe_clear_alarm(resource_id()) -> ok | {error, not_found}. -maybe_clear_alarm(<>) -> +-spec maybe_clear_alarm(boolean(), resource_id()) -> ok | {error, not_found}. +maybe_clear_alarm(true, _ResId) -> ok; -maybe_clear_alarm(ResId) -> +maybe_clear_alarm(false, ResId) -> emqx_alarm:safe_deactivate(ResId). parse_health_check_result(Status, Data) when ?IS_STATUS(Status) -> @@ -1642,7 +1711,8 @@ parse_health_check_result({error, Error}, Data) -> msg => "health_check_exception", resource_id => Data#data.id, reason => Error - } + }, + #{tag => tag(Data#data.group, Data#data.type)} ), {?status_disconnected, Data#data.state, {error, Error}}. @@ -1794,10 +1864,18 @@ add_or_update_channel_status(Data, ChannelId, ChannelConfig, State) -> ChannelStatus = channel_status({error, resource_not_operational}, ChannelConfig), NewChannels = maps:put(ChannelId, ChannelStatus, Channels), ResStatus = state_to_status(State), - maybe_alarm(ResStatus, ChannelId, ChannelStatus, no_prev), + IsDryRun = emqx_resource:is_dry_run(ChannelId), + maybe_alarm(ResStatus, IsDryRun, ChannelId, ChannelStatus, no_prev), Data#data{added_channels = NewChannels}. state_to_status(?state_stopped) -> ?rm_status_stopped; state_to_status(?state_connected) -> ?status_connected; state_to_status(?state_connecting) -> ?status_connecting; state_to_status(?state_disconnected) -> ?status_disconnected. + +log_level(true) -> info; +log_level(false) -> warning. + +tag(Group, Type) -> + Str = emqx_utils_conv:str(Group) ++ "/" ++ emqx_utils_conv:str(Type), + string:uppercase(Str). diff --git a/apps/emqx_resource/src/emqx_resource_manager_sup.erl b/apps/emqx_resource/src/emqx_resource_manager_sup.erl index 7af6eca81..8542eec1c 100644 --- a/apps/emqx_resource/src/emqx_resource_manager_sup.erl +++ b/apps/emqx_resource/src/emqx_resource_manager_sup.erl @@ -58,10 +58,11 @@ init([]) -> child_spec(ResId, Group, ResourceType, Config, Opts) -> #{ id => ResId, - start => {emqx_resource_manager, start_link, [ResId, Group, ResourceType, Config, Opts]}, + start => + {emqx_resource_manager, start_link, [ResId, Group, ResourceType, Config, Opts]}, restart => transient, %% never force kill a resource manager. - %% becasue otherwise it may lead to release leak, + %% because otherwise it may lead to release leak, %% resource_manager's terminate callback calls resource on_stop shutdown => infinity, type => worker, diff --git a/apps/emqx_resource/src/emqx_resource_metrics.erl b/apps/emqx_resource/src/emqx_resource_metrics.erl index 97a09b074..98a74dfad 100644 --- a/apps/emqx_resource/src/emqx_resource_metrics.erl +++ b/apps/emqx_resource/src/emqx_resource_metrics.erl @@ -17,6 +17,7 @@ -module(emqx_resource_metrics). -include_lib("emqx/include/logger.hrl"). +-include("emqx_resource.hrl"). -export([ events/0, @@ -74,7 +75,6 @@ success_get/1 ]). --define(RES_METRICS, resource_metrics). -define(TELEMETRY_PREFIX, emqx, resource). -spec events() -> [telemetry:event_name()]. @@ -127,15 +127,19 @@ handle_telemetry_event( %% We catch errors to avoid detaching the telemetry handler function. %% When restarting a resource while it's under load, there might be transient %% failures while the metrics are not yet created. - ?SLOG(warning, #{ - msg => "handle_resource_metrics_failed", - hint => "transient failures may occur when restarting a resource", - kind => Kind, - reason => Reason, - stacktrace => Stacktrace, - resource_id => ID, - event => Event - }), + ?SLOG( + warning, + #{ + msg => "handle_resource_metrics_failed", + hint => "transient failures may occur when restarting a resource", + kind => Kind, + reason => Reason, + stacktrace => Stacktrace, + resource_id => ID, + event => Event + }, + #{tag => ?TAG} + ), ok end; handle_telemetry_event( @@ -151,15 +155,19 @@ handle_telemetry_event( %% We catch errors to avoid detaching the telemetry handler function. %% When restarting a resource while it's under load, there might be transient %% failures while the metrics are not yet created. - ?SLOG(warning, #{ - msg => "handle_resource_metrics_failed", - hint => "transient failures may occur when restarting a resource", - kind => Kind, - reason => Reason, - stacktrace => Stacktrace, - resource_id => ID, - event => Event - }), + ?SLOG( + warning, + #{ + msg => "handle_resource_metrics_failed", + hint => "transient failures may occur when restarting a resource", + kind => Kind, + reason => Reason, + stacktrace => Stacktrace, + resource_id => ID, + event => Event + }, + #{tag => ?TAG} + ), ok end; handle_telemetry_event(_EventName, _Measurements, _Metadata, _HandlerConfig) -> diff --git a/apps/emqx_resource/src/emqx_resource_pool.erl b/apps/emqx_resource/src/emqx_resource_pool.erl index 47e7ed7ff..ba286e35c 100644 --- a/apps/emqx_resource/src/emqx_resource_pool.erl +++ b/apps/emqx_resource/src/emqx_resource_pool.erl @@ -26,6 +26,7 @@ ]). -include_lib("emqx/include/logger.hrl"). +-include("emqx_resource.hrl"). -ifndef(TEST). -define(HEALTH_CHECK_TIMEOUT, 15000). @@ -37,33 +38,43 @@ start(Name, Mod, Options) -> case ecpool:start_sup_pool(Name, Mod, Options) of {ok, _} -> - ?SLOG(info, #{msg => "start_ecpool_ok", pool_name => Name}), + ?SLOG(info, #{msg => "start_ecpool_ok", pool_name => Name}, #{tag => ?TAG}), ok; {error, {already_started, _Pid}} -> stop(Name), start(Name, Mod, Options); {error, Reason} -> NReason = parse_reason(Reason), - ?SLOG(error, #{ - msg => "start_ecpool_error", - pool_name => Name, - reason => NReason - }), + IsDryRun = emqx_resource:is_dry_run(Name), + ?SLOG( + ?LOG_LEVEL(IsDryRun), + #{ + msg => "start_ecpool_error", + resource_id => Name, + reason => NReason + }, + #{tag => ?TAG} + ), {error, {start_pool_failed, Name, NReason}} end. stop(Name) -> case ecpool:stop_sup_pool(Name) of ok -> - ?SLOG(info, #{msg => "stop_ecpool_ok", pool_name => Name}); + ?SLOG(info, #{msg => "stop_ecpool_ok", pool_name => Name}, #{tag => ?TAG}); {error, not_found} -> ok; {error, Reason} -> - ?SLOG(error, #{ - msg => "stop_ecpool_failed", - pool_name => Name, - reason => Reason - }), + IsDryRun = emqx_resource:is_dry_run(Name), + ?SLOG( + ?LOG_LEVEL(IsDryRun), + #{ + msg => "stop_ecpool_failed", + resource_id => Name, + reason => Reason + }, + #{tag => ?TAG} + ), error({stop_pool_failed, Name, Reason}) end. diff --git a/apps/emqx_resource/src/proto/emqx_resource_proto_v1.erl b/apps/emqx_resource/src/proto/emqx_resource_proto_v1.erl index 859b9fa52..47b724271 100644 --- a/apps/emqx_resource/src/proto/emqx_resource_proto_v1.erl +++ b/apps/emqx_resource/src/proto/emqx_resource_proto_v1.erl @@ -40,7 +40,7 @@ deprecated_since() -> -spec create( resource_id(), resource_group(), - resource_type(), + resource_module(), resource_config(), creation_opts() ) -> @@ -51,7 +51,7 @@ create(ResId, Group, ResourceType, Config, Opts) -> ]). -spec create_dry_run( - resource_type(), + resource_module(), resource_config() ) -> ok | {error, Reason :: term()}. @@ -60,7 +60,7 @@ create_dry_run(ResourceType, Config) -> -spec recreate( resource_id(), - resource_type(), + resource_module(), resource_config(), creation_opts() ) -> diff --git a/apps/emqx_resource/test/emqx_connector_demo.erl b/apps/emqx_resource/test/emqx_connector_demo.erl index 0fc11cc66..e068defb1 100644 --- a/apps/emqx_resource/test/emqx_connector_demo.erl +++ b/apps/emqx_resource/test/emqx_connector_demo.erl @@ -24,6 +24,7 @@ %% callbacks of behaviour emqx_resource -export([ + resource_type/0, callback_mode/0, on_start/2, on_stop/2, @@ -62,6 +63,8 @@ register(required) -> true; register(default) -> false; register(_) -> undefined. +resource_type() -> demo. + callback_mode() -> persistent_term:get(?CM_KEY). diff --git a/apps/emqx_resource/test/emqx_resource_SUITE.erl b/apps/emqx_resource/test/emqx_resource_SUITE.erl index e77a90efa..f463c741b 100644 --- a/apps/emqx_resource/test/emqx_resource_SUITE.erl +++ b/apps/emqx_resource/test/emqx_resource_SUITE.erl @@ -3492,7 +3492,7 @@ gauge_metric_set_fns() -> ]. create(Id, Group, Type, Config) -> - emqx_resource:create_local(Id, Group, Type, Config). + emqx_resource:create_local(Id, Group, Type, Config, #{}). create(Id, Group, Type, Config, Opts) -> emqx_resource:create_local(Id, Group, Type, Config, Opts). diff --git a/apps/emqx_retainer/src/emqx_retainer_schema.erl b/apps/emqx_retainer/src/emqx_retainer_schema.erl index b89ee7db6..26b65faa8 100644 --- a/apps/emqx_retainer/src/emqx_retainer_schema.erl +++ b/apps/emqx_retainer/src/emqx_retainer_schema.erl @@ -43,7 +43,7 @@ roots() -> fields("retainer") -> [ - {enable, sc(boolean(), enable, true)}, + {enable, sc(boolean(), enable, true, ?IMPORTANCE_NO_DOC)}, {msg_expiry_interval, sc( %% not used in a `receive ... after' block, just timestamp comparison @@ -126,6 +126,7 @@ fields(mnesia_config) -> {enable, ?HOCON(boolean(), #{ desc => ?DESC(mnesia_enable), + importance => ?IMPORTANCE_NO_DOC, required => false, default => true })} diff --git a/apps/emqx_rule_engine/include/rule_engine.hrl b/apps/emqx_rule_engine/include/rule_engine.hrl index 7d0000a1c..cf0aae4d8 100644 --- a/apps/emqx_rule_engine/include/rule_engine.hrl +++ b/apps/emqx_rule_engine/include/rule_engine.hrl @@ -128,3 +128,4 @@ -define(KEY_PATH, [rule_engine, rules]). -define(RULE_PATH(RULE), [rule_engine, rules, RULE]). +-define(TAG, "RULE"). diff --git a/apps/emqx_rule_engine/src/emqx_rule_actions.erl b/apps/emqx_rule_engine/src/emqx_rule_actions.erl index 7d45c47e6..57e1d02b8 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_actions.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_actions.erl @@ -134,7 +134,15 @@ republish( }, _Args ) -> - ?SLOG(error, #{msg => "recursive_republish_detected", topic => Topic}); + ?SLOG( + error, + #{ + msg => "recursive_republish_detected", + topic => Topic, + rule_id => RuleId + }, + #{tag => ?TAG} + ); republish( Selected, #{metadata := #{rule_id := RuleId}} = Env, @@ -321,6 +329,8 @@ render_pub_props(UserPropertiesTemplate, Selected, Env) -> rule_id => emqx_utils_maps:deep_get([metadata, rule_id], ENV, undefined), reason => REASON, property => K + }#{ + tag => ?TAG } ) ). diff --git a/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl b/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl index e33c1d6fb..4f68175af 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_api_schema.erl @@ -21,6 +21,7 @@ -include_lib("typerefl/include/types.hrl"). -include_lib("hocon/include/hoconsc.hrl"). -include_lib("emqx/include/logger.hrl"). +-include("rule_engine.hrl"). -export([check_params/2]). @@ -36,10 +37,14 @@ check_params(Params, Tag) -> #{Tag := Checked} -> {ok, Checked} catch throw:Reason -> - ?SLOG(error, #{ - msg => "check_rule_params_failed", - reason => Reason - }), + ?SLOG( + info, + #{ + msg => "check_rule_params_failed", + reason => Reason + }, + #{tag => ?TAG} + ), {error, Reason} end. diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.erl b/apps/emqx_rule_engine/src/emqx_rule_engine.erl index 359984228..c4b562b2b 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.erl @@ -459,15 +459,15 @@ handle_call({delete_rule, Rule}, _From, State) -> ok = do_delete_rule(Rule), {reply, ok, State}; handle_call(Req, _From, State) -> - ?SLOG(error, #{msg => "unexpected_call", request => Req}), + ?SLOG(error, #{msg => "unexpected_call", request => Req}, #{tag => ?TAG}), {reply, ignored, State}. handle_cast(Msg, State) -> - ?SLOG(error, #{msg => "unexpected_cast", request => Msg}), + ?SLOG(error, #{msg => "unexpected_cast", request => Msg}, #{tag => ?TAG}), {noreply, State}. handle_info(Info, State) -> - ?SLOG(error, #{msg => "unexpected_info", request => Info}), + ?SLOG(error, #{msg => "unexpected_info", request => Info}, #{tag => ?TAG}), {noreply, State}. terminate(_Reason, _State) -> diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl index 0ac59b36c..6c286ff2b 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine_api.erl @@ -131,6 +131,8 @@ end). {<<"like_id">>, binary}, {<<"like_from">>, binary}, {<<"match_from">>, binary}, + {<<"action">>, binary}, + {<<"source">>, binary}, {<<"like_description">>, binary} ]). @@ -194,6 +196,10 @@ schema("/rules") -> })}, {match_from, mk(binary(), #{desc => ?DESC("api1_match_from"), in => query, required => false})}, + {action, + mk(hoconsc:array(binary()), #{in => query, desc => ?DESC("api1_qs_action")})}, + {source, + mk(hoconsc:array(binary()), #{in => query, desc => ?DESC("api1_qs_source")})}, ref(emqx_dashboard_swagger, page), ref(emqx_dashboard_swagger, limit) ], @@ -390,11 +396,15 @@ param_path_id() -> {ok, #{post_config_update := #{emqx_rule_engine := Rule}}} -> {201, format_rule_info_resp(Rule)}; {error, Reason} -> - ?SLOG(error, #{ - msg => "create_rule_failed", - id => Id, - reason => Reason - }), + ?SLOG( + info, + #{ + msg => "create_rule_failed", + rule_id => Id, + reason => Reason + }, + #{tag => ?TAG} + ), {400, #{code => 'BAD_REQUEST', message => ?ERR_BADARGS(Reason)}} end end @@ -450,11 +460,15 @@ param_path_id() -> {ok, #{post_config_update := #{emqx_rule_engine := Rule}}} -> {200, format_rule_info_resp(Rule)}; {error, Reason} -> - ?SLOG(error, #{ - msg => "update_rule_failed", - id => Id, - reason => Reason - }), + ?SLOG( + info, + #{ + msg => "update_rule_failed", + rule_id => Id, + reason => Reason + }, + #{tag => ?TAG} + ), {400, #{code => 'BAD_REQUEST', message => ?ERR_BADARGS(Reason)}} end; '/rules/:id'(delete, #{bindings := #{id := Id}}) -> @@ -465,11 +479,15 @@ param_path_id() -> {ok, _} -> {204}; {error, Reason} -> - ?SLOG(error, #{ - msg => "delete_rule_failed", - id => Id, - reason => Reason - }), + ?SLOG( + error, + #{ + msg => "delete_rule_failed", + rule_id => Id, + reason => Reason + }, + #{tag => ?TAG} + ), {500, #{code => 'INTERNAL_ERROR', message => ?ERR_BADARGS(Reason)}} end; not_found -> @@ -589,10 +607,15 @@ get_rule_metrics(Id) -> NodeMetrics = [format_metrics(Node, Metrics) || {Node, {ok, Metrics}} <- NodeResults], NodeErrors = [Result || Result = {_Node, {NOk, _}} <- NodeResults, NOk =/= ok], NodeErrors == [] orelse - ?SLOG(warning, #{ - msg => "rpc_get_rule_metrics_errors", - errors => NodeErrors - }), + ?SLOG( + warning, + #{ + msg => "rpc_get_rule_metrics_errors", + rule_id => Id, + errors => NodeErrors + }, + #{tag => ?TAG} + ), NodeMetrics. format_metrics(Node, #{ @@ -731,7 +754,8 @@ filter_out_request_body(Conf) -> maps:without(ExtraConfs, Conf). -spec qs2ms(atom(), {list(), list()}) -> emqx_mgmt_api:match_spec_and_filter(). -qs2ms(_Tab, {Qs, Fuzzy}) -> +qs2ms(_Tab, {Qs0, Fuzzy0}) -> + {Qs, Fuzzy} = adapt_custom_filters(Qs0, Fuzzy0), case lists:keytake(from, 1, Qs) of false -> #{match_spec => generate_match_spec(Qs), fuzzy_fun => fuzzy_match_fun(Fuzzy)}; @@ -742,6 +766,38 @@ qs2ms(_Tab, {Qs, Fuzzy}) -> } end. +%% Some filters are run as fuzzy filters because they cannot be expressed as simple ETS +%% match specs. +-spec adapt_custom_filters(Qs, Fuzzy) -> {Qs, Fuzzy}. +adapt_custom_filters(Qs, Fuzzy) -> + lists:foldl( + fun + ({action, '=:=', X}, {QsAcc, FuzzyAcc}) -> + ActionIds = wrap(X), + Parsed = lists:map(fun emqx_rule_actions:parse_action/1, ActionIds), + {QsAcc, [{action, in, Parsed} | FuzzyAcc]}; + ({source, '=:=', X}, {QsAcc, FuzzyAcc}) -> + SourceIds = wrap(X), + Parsed = lists:flatmap( + fun(SourceId) -> + [ + emqx_bridge_resource:bridge_hookpoint(SourceId), + emqx_bridge_v2:source_hookpoint(SourceId) + ] + end, + SourceIds + ), + {QsAcc, [{source, in, Parsed} | FuzzyAcc]}; + (Clause, {QsAcc, FuzzyAcc}) -> + {[Clause | QsAcc], FuzzyAcc} + end, + {[], Fuzzy}, + Qs + ). + +wrap(Xs) when is_list(Xs) -> Xs; +wrap(X) -> [X]. + generate_match_spec(Qs) -> {MtchHead, Conds} = generate_match_spec(Qs, 2, {#{}, []}), [{{'_', MtchHead}, Conds, ['$_']}]. @@ -779,6 +835,12 @@ run_fuzzy_match(E = {_Id, #{from := Topics}}, [{from, match, Pattern} | Fuzzy]) run_fuzzy_match(E = {_Id, #{from := Topics}}, [{from, like, Pattern} | Fuzzy]) -> lists:any(fun(For) -> binary:match(For, Pattern) /= nomatch end, Topics) andalso run_fuzzy_match(E, Fuzzy); +run_fuzzy_match(E = {_Id, #{actions := Actions}}, [{action, in, ActionIds} | Fuzzy]) -> + lists:any(fun(AId) -> lists:member(AId, Actions) end, ActionIds) andalso + run_fuzzy_match(E, Fuzzy); +run_fuzzy_match(E = {_Id, #{from := Froms}}, [{source, in, SourceIds} | Fuzzy]) -> + lists:any(fun(SId) -> lists:member(SId, Froms) end, SourceIds) andalso + run_fuzzy_match(E, Fuzzy); run_fuzzy_match(E, [_ | Fuzzy]) -> run_fuzzy_match(E, Fuzzy). diff --git a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl index 3f7f24604..ee6a83ab1 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl @@ -252,6 +252,9 @@ timezone_to_offset_seconds/1 ]). +%% System functions +-export([getenv/1]). + %% See extra_functions_module/0 and set_extra_functions_module/1 in the %% emqx_rule_engine module -callback handle_rule_function(atom(), list()) -> any() | {error, no_match_for_function}. @@ -1262,3 +1265,9 @@ convert_timestamp(MillisecondsTimestamp) -> uuid_str(UUID, DisplayOpt) -> uuid:uuid_to_string(UUID, DisplayOpt). + +%%------------------------------------------------------------------------------ +%% System Funcs +%%------------------------------------------------------------------------------ +getenv(Env) -> + emqx_variform_bif:getenv(Env). diff --git a/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl b/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl index 39ed7440d..c9be82127 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_sqltester.erl @@ -15,6 +15,7 @@ -module(emqx_rule_sqltester). -include_lib("emqx/include/logger.hrl"). +-include("rule_engine.hrl"). -export([ test/1, @@ -114,11 +115,13 @@ test(#{sql := Sql, context := Context}) -> true -> %% test if the topic matches the topic filters in the rule case emqx_topic:match_any(InTopic, EventTopics) of - true -> test_rule(Sql, Select, Context, EventTopics); - false -> {error, nomatch} + true -> + test_rule(Sql, Select, Context, EventTopics); + false -> + {error, nomatch} end; false -> - case lists:member(InTopic, EventTopics) of + case emqx_topic:match_any(InTopic, EventTopics) of true -> %% the rule is for both publish and events, test it directly test_rule(Sql, Select, Context, EventTopics); @@ -127,10 +130,15 @@ test(#{sql := Sql, context := Context}) -> end end; {error, Reason} -> - ?SLOG(debug, #{ - msg => "rulesql_parse_error", - detail => Reason - }), + ?SLOG( + debug, + #{ + msg => "rulesql_parse_error", + sql => Sql, + reason => Reason + }, + #{tag => ?TAG} + ), {error, Reason} end. diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_api_2_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_api_2_SUITE.erl index 31a094055..8a866cc69 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_api_2_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_api_2_SUITE.erl @@ -33,7 +33,6 @@ init_per_suite(Config) -> app_specs(), #{work_dir => emqx_cth_suite:work_dir(Config)} ), - emqx_common_test_http:create_default_app(), [{apps, Apps} | Config]. end_per_suite(Config) -> @@ -46,7 +45,7 @@ app_specs() -> emqx_conf, emqx_rule_engine, emqx_management, - {emqx_dashboard, "dashboard.listeners.http { enable = true, bind = 18083 }"} + emqx_mgmt_api_test_util:emqx_dashboard() ]. %%------------------------------------------------------------------------------ @@ -64,8 +63,12 @@ request(Method, Path, Params) -> request(Method, Path, Params, Opts). request(Method, Path, Params, Opts) -> + request(Method, Path, Params, _QueryParams = [], Opts). + +request(Method, Path, Params, QueryParams0, Opts) when is_list(QueryParams0) -> AuthHeader = emqx_mgmt_api_test_util:auth_header_(), - case emqx_mgmt_api_test_util:request_api(Method, Path, "", AuthHeader, Params, Opts) of + QueryParams = uri_string:compose_query(QueryParams0, [{encoding, utf8}]), + case emqx_mgmt_api_test_util:request_api(Method, Path, QueryParams, AuthHeader, Params, Opts) of {ok, {Status, Headers, Body0}} -> Body = maybe_json_decode(Body0), {ok, {Status, Headers, Body}}; @@ -93,6 +96,45 @@ sql_test_api(Params) -> ct:pal("sql test (http) result:\n ~p", [Res]), Res. +list_rules(QueryParams) when is_list(QueryParams) -> + Method = get, + Path = emqx_mgmt_api_test_util:api_path(["rules"]), + Opts = #{return_all => true}, + Res = request(Method, Path, _Body = [], QueryParams, Opts), + emqx_mgmt_api_test_util:simplify_result(Res). + +list_rules_just_ids(QueryParams) when is_list(QueryParams) -> + case list_rules(QueryParams) of + {200, #{<<"data">> := Results0}} -> + Results = lists:sort([Id || #{<<"id">> := Id} <- Results0]), + {200, Results}; + Res -> + Res + end. + +create_rule() -> + create_rule(_Overrides = #{}). + +create_rule(Overrides) -> + Params0 = #{ + <<"enable">> => true, + <<"sql">> => <<"select true from t">> + }, + Params = emqx_utils_maps:deep_merge(Params0, Overrides), + Method = post, + Path = emqx_mgmt_api_test_util:api_path(["rules"]), + Res = request(Method, Path, Params), + emqx_mgmt_api_test_util:simplify_result(Res). + +sources_sql(Sources) -> + Froms = iolist_to_binary(lists:join(<<", ">>, lists:map(fun source_from/1, Sources))), + <<"select * from ", Froms/binary>>. + +source_from({v1, Id}) -> + <<"\"$bridges/", Id/binary, "\" ">>; +source_from({v2, Id}) -> + <<"\"$sources/", Id/binary, "\" ">>. + %%------------------------------------------------------------------------------ %% Test cases %%------------------------------------------------------------------------------ @@ -351,6 +393,38 @@ t_rule_test_smoke(_Config) -> } ], MultipleFrom = [ + #{ + expected => #{code => 200}, + input => + #{ + <<"context">> => + #{ + <<"clientid">> => <<"c_emqx">>, + <<"event_type">> => <<"message_publish">>, + <<"qos">> => 1, + <<"topic">> => <<"t/a">>, + <<"username">> => <<"u_emqx">> + }, + <<"sql">> => + <<"SELECT\n *\nFROM\n \"t/#\", \"$bridges/mqtt:source\" ">> + } + }, + #{ + expected => #{code => 200}, + input => + #{ + <<"context">> => + #{ + <<"clientid">> => <<"c_emqx">>, + <<"event_type">> => <<"message_publish">>, + <<"qos">> => 1, + <<"topic">> => <<"t/a">>, + <<"username">> => <<"u_emqx">> + }, + <<"sql">> => + <<"SELECT\n *\nFROM\n \"t/#\", \"$sources/mqtt:source\" ">> + } + }, #{ expected => #{code => 200}, input => @@ -446,7 +520,69 @@ do_t_rule_test_smoke(#{input := Input, expected := #{code := ExpectedCode}} = Ca {true, #{ expected => ExpectedCode, hint => maps:get(hint, Case, <<>>), + input => Input, got => Code, resp_body => Body }} end. + +%% Tests filtering the rule list by used actions and/or sources. +t_filter_by_source_and_action(_Config) -> + ?assertMatch( + {200, #{<<"data">> := []}}, + list_rules([]) + ), + + ActionId1 = <<"mqtt:a1">>, + ActionId2 = <<"mqtt:a2">>, + SourceId1 = <<"mqtt:s1">>, + SourceId2 = <<"mqtt:s2">>, + {201, #{<<"id">> := Id1}} = create_rule(#{<<"actions">> => [ActionId1]}), + {201, #{<<"id">> := Id2}} = create_rule(#{<<"actions">> => [ActionId2]}), + {201, #{<<"id">> := Id3}} = create_rule(#{<<"actions">> => [ActionId2, ActionId1]}), + {201, #{<<"id">> := Id4}} = create_rule(#{<<"sql">> => sources_sql([{v1, SourceId1}])}), + {201, #{<<"id">> := Id5}} = create_rule(#{<<"sql">> => sources_sql([{v2, SourceId2}])}), + {201, #{<<"id">> := Id6}} = create_rule(#{ + <<"sql">> => sources_sql([{v2, SourceId1}, {v2, SourceId1}]) + }), + {201, #{<<"id">> := Id7}} = create_rule(#{ + <<"sql">> => sources_sql([{v2, SourceId1}]), + <<"actions">> => [ActionId1] + }), + + ?assertMatch( + {200, [_, _, _, _, _, _, _]}, + list_rules_just_ids([]) + ), + + ?assertEqual( + {200, lists:sort([Id1, Id3, Id7])}, + list_rules_just_ids([{<<"action">>, ActionId1}]) + ), + + ?assertEqual( + {200, lists:sort([Id1, Id2, Id3, Id7])}, + list_rules_just_ids([{<<"action">>, ActionId1}, {<<"action">>, ActionId2}]) + ), + + ?assertEqual( + {200, lists:sort([Id4, Id6, Id7])}, + list_rules_just_ids([{<<"source">>, SourceId1}]) + ), + + ?assertEqual( + {200, lists:sort([Id4, Id5, Id6, Id7])}, + list_rules_just_ids([{<<"source">>, SourceId1}, {<<"source">>, SourceId2}]) + ), + + %% When mixing source and action id filters, we use AND. + ?assertEqual( + {200, lists:sort([])}, + list_rules_just_ids([{<<"source">>, SourceId2}, {<<"action">>, ActionId2}]) + ), + ?assertEqual( + {200, lists:sort([Id7])}, + list_rules_just_ids([{<<"source">>, SourceId1}, {<<"action">>, ActionId1}]) + ), + + ok. diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_test_connector.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_test_connector.erl index 6a0d6b3ec..cf5d3afbe 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_test_connector.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_test_connector.erl @@ -25,6 +25,7 @@ %% callbacks of behaviour emqx_resource -export([ + resource_type/0, callback_mode/0, on_start/2, on_stop/2, @@ -40,6 +41,8 @@ ]). %% =================================================================== +resource_type() -> test_connector. + callback_mode() -> always_sync. on_start( diff --git a/apps/emqx_slow_subs/src/emqx_slow_subs.app.src b/apps/emqx_slow_subs/src/emqx_slow_subs.app.src index 6a24bc90b..a1485337e 100644 --- a/apps/emqx_slow_subs/src/emqx_slow_subs.app.src +++ b/apps/emqx_slow_subs/src/emqx_slow_subs.app.src @@ -1,7 +1,7 @@ {application, emqx_slow_subs, [ {description, "EMQX Slow Subscribers Statistics"}, % strict semver, bump manually! - {vsn, "1.0.7"}, + {vsn, "1.0.8"}, {modules, []}, {registered, [emqx_slow_subs_sup]}, {applications, [kernel, stdlib, emqx]}, diff --git a/apps/emqx_slow_subs/src/emqx_slow_subs_schema.erl b/apps/emqx_slow_subs/src/emqx_slow_subs_schema.erl index 37ca9327f..517457930 100644 --- a/apps/emqx_slow_subs/src/emqx_slow_subs_schema.erl +++ b/apps/emqx_slow_subs/src/emqx_slow_subs_schema.erl @@ -27,6 +27,7 @@ roots() -> fields("slow_subs") -> [ + %% {enable, sc(boolean(), false, enable, ?IMPORTANCE_NO_DOC)}, {enable, sc(boolean(), false, enable)}, {threshold, sc( @@ -66,3 +67,6 @@ desc(_) -> %%-------------------------------------------------------------------- sc(Type, Default, Desc) -> ?HOCON(Type, #{default => Default, desc => ?DESC(Desc)}). + +%% sc(Type, Default, Desc, Importance) -> +%% ?HOCON(Type, #{default => Default, desc => ?DESC(Desc), importance => Importance}). diff --git a/apps/emqx_utils/include/emqx_utils_api.hrl b/apps/emqx_utils/include/emqx_utils_api.hrl index ba2941a4f..0876b9829 100644 --- a/apps/emqx_utils/include/emqx_utils_api.hrl +++ b/apps/emqx_utils/include/emqx_utils_api.hrl @@ -21,6 +21,8 @@ -define(OK(CONTENT), {200, CONTENT}). +-define(CREATED(CONTENT), {201, CONTENT}). + -define(NO_CONTENT, 204). -define(BAD_REQUEST(CODE, REASON), {400, ?ERROR_MSG(CODE, REASON)}). diff --git a/apps/emqx_utils/src/emqx_utils_scram.erl b/apps/emqx_utils/src/emqx_utils_scram.erl new file mode 100644 index 000000000..cb11082fb --- /dev/null +++ b/apps/emqx_utils/src/emqx_utils_scram.erl @@ -0,0 +1,79 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021-2024 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_utils_scram). + +-export([authenticate/7]). + +%%------------------------------------------------------------------------------ +%% Authentication +%%------------------------------------------------------------------------------ +authenticate(AuthMethod, AuthData, AuthCache, Conf, RetrieveFun, OnErrFun, ResultKeys) -> + case ensure_auth_method(AuthMethod, AuthData, Conf) of + true -> + case AuthCache of + #{next_step := client_final} -> + check_client_final_message(AuthData, AuthCache, Conf, OnErrFun, ResultKeys); + _ -> + check_client_first_message(AuthData, AuthCache, Conf, RetrieveFun, OnErrFun) + end; + false -> + ignore + end. + +ensure_auth_method(_AuthMethod, undefined, _Conf) -> + false; +ensure_auth_method(<<"SCRAM-SHA-256">>, _AuthData, #{algorithm := sha256}) -> + true; +ensure_auth_method(<<"SCRAM-SHA-512">>, _AuthData, #{algorithm := sha512}) -> + true; +ensure_auth_method(_AuthMethod, _AuthData, _Conf) -> + false. + +check_client_first_message( + Bin, _Cache, #{iteration_count := IterationCount}, RetrieveFun, OnErrFun +) -> + case + esasl_scram:check_client_first_message( + Bin, + #{ + iteration_count => IterationCount, + retrieve => RetrieveFun + } + ) + of + {continue, ServerFirstMessage, Cache} -> + {continue, ServerFirstMessage, Cache}; + ignore -> + ignore; + {error, Reason} -> + OnErrFun("check_client_first_message_error", Reason), + {error, not_authorized} + end. + +check_client_final_message(Bin, Cache, #{algorithm := Alg}, OnErrFun, ResultKeys) -> + case + esasl_scram:check_client_final_message( + Bin, + Cache#{algorithm => Alg} + ) + of + {ok, ServerFinalMessage} -> + {ok, maps:with(ResultKeys, Cache), ServerFinalMessage}; + {error, Reason} -> + OnErrFun("check_client_final_message_error", Reason), + {error, not_authorized} + end. diff --git a/apps/emqx_utils/src/emqx_variform_bif.erl b/apps/emqx_utils/src/emqx_variform_bif.erl index f30db8f7a..09048a697 100644 --- a/apps/emqx_utils/src/emqx_variform_bif.erl +++ b/apps/emqx_utils/src/emqx_variform_bif.erl @@ -79,6 +79,12 @@ %% Number compare functions -export([num_comp/2, num_eq/2, num_lt/2, num_lte/2, num_gt/2, num_gte/2]). +%% System +-export([getenv/1]). + +-define(CACHE(Key), {?MODULE, Key}). +-define(ENV_CACHE(Env), ?CACHE({env, Env})). + %%------------------------------------------------------------------------------ %% String Funcs %%------------------------------------------------------------------------------ @@ -569,3 +575,24 @@ num_lte(A, B) -> num_gte(A, B) -> R = num_comp(A, B), R =:= gt orelse R =:= eq. + +%%------------------------------------------------------------------------------ +%% System +%%------------------------------------------------------------------------------ +getenv(Bin) when is_binary(Bin) -> + EnvKey = ?ENV_CACHE(Bin), + case persistent_term:get(EnvKey, undefined) of + undefined -> + Name = "EMQXVAR_" ++ erlang:binary_to_list(Bin), + Result = + case os:getenv(Name) of + false -> + <<>>; + Value -> + erlang:list_to_binary(Value) + end, + persistent_term:put(EnvKey, Result), + Result; + Result -> + Result + end. diff --git a/apps/emqx_utils/test/emqx_variform_bif_tests.erl b/apps/emqx_utils/test/emqx_variform_bif_tests.erl index 92144ff43..aa6724de5 100644 --- a/apps/emqx_utils/test/emqx_variform_bif_tests.erl +++ b/apps/emqx_utils/test/emqx_variform_bif_tests.erl @@ -72,3 +72,10 @@ base64_encode_decode_test() -> RandBytes = crypto:strong_rand_bytes(100), Encoded = emqx_variform_bif:base64_encode(RandBytes), ?assertEqual(RandBytes, emqx_variform_bif:base64_decode(Encoded)). + +system_test() -> + EnvName = erlang:atom_to_list(?MODULE), + EnvVal = erlang:atom_to_list(?FUNCTION_NAME), + EnvNameBin = erlang:list_to_binary(EnvName), + os:putenv("EMQXVAR_" ++ EnvName, EnvVal), + ?assertEqual(erlang:list_to_binary(EnvVal), emqx_variform_bif:getenv(EnvNameBin)). diff --git a/changes/ce/breaking-13526.en.md b/changes/ce/breaking-13526.en.md new file mode 100644 index 000000000..752e58ef3 --- /dev/null +++ b/changes/ce/breaking-13526.en.md @@ -0,0 +1,5 @@ +- Core-replicant feature has been removed from the Open-Source Edition. + Starting from release 5.8, all nodes running Open-Source Edition will assume Core role. + This change doesn't affect Enterprise Edition users. + +- Obsolete and unused `cluster.core_nodes` configuration parameter has been removed. diff --git a/changes/ce/feat-13505.en.md b/changes/ce/feat-13505.en.md new file mode 100644 index 000000000..6b9ba8b94 --- /dev/null +++ b/changes/ce/feat-13505.en.md @@ -0,0 +1 @@ +Now, it's possible to filter rules in the HTTP API by the IDs of used data integration actions/sources. diff --git a/changes/ce/feat-13507.en.md b/changes/ce/feat-13507.en.md new file mode 100644 index 000000000..026cf6bf4 --- /dev/null +++ b/changes/ce/feat-13507.en.md @@ -0,0 +1,4 @@ +Added a new builtin function `getenv` in the rule engine and variform expression to access the environment variables with below limitations. + +- Prefix `EMQXVAR_` is added before reading from OS environment variables. i.e. `getenv('FOO_BAR')` is to read `EMQXVAR_FOO_BAR`. +- The values are immutable once loaded from the OS environment. diff --git a/changes/ce/feat-13521.en.md b/changes/ce/feat-13521.en.md new file mode 100644 index 000000000..6d57eee23 --- /dev/null +++ b/changes/ce/feat-13521.en.md @@ -0,0 +1,4 @@ +Fix LDAP query timeout issue. + +Previously, LDAP query timeout may cause the underlying connection to be unusable. +Fixed to always reconnect if timeout happens. diff --git a/changes/ce/feat-13528.en.md b/changes/ce/feat-13528.en.md new file mode 100644 index 000000000..f761e9565 --- /dev/null +++ b/changes/ce/feat-13528.en.md @@ -0,0 +1 @@ +Add log throttling for data integration unrecoverable errors. diff --git a/changes/ce/feat-13548.en.md b/changes/ce/feat-13548.en.md new file mode 100644 index 000000000..75b56cd43 --- /dev/null +++ b/changes/ce/feat-13548.en.md @@ -0,0 +1,6 @@ +Optionally calls the `on_config_changed/2` callback function when the plugin configuration is updated via the REST API. + +This callback function is assumed to be exported by the `_app` module. +i.e: +Plugin NameVsn: `my_plugin-1.0.0` +This callback function is assumed to be `my_plugin_app:on_config_changed/2` diff --git a/changes/ce/fix-13503.en.md b/changes/ce/fix-13503.en.md new file mode 100644 index 000000000..a4f0eb811 --- /dev/null +++ b/changes/ce/fix-13503.en.md @@ -0,0 +1 @@ +Fixed an issue where a connector wouldn't respect the configured health check interval when first starting up, and would need an update/restart for the correct value to take effect. diff --git a/changes/ce/fix-13515.en.md b/changes/ce/fix-13515.en.md new file mode 100644 index 000000000..775c21848 --- /dev/null +++ b/changes/ce/fix-13515.en.md @@ -0,0 +1 @@ +Fixed an issue where the same client could not subscribe to the same exclusive topic when the node was down for some reason. diff --git a/changes/ce/fix-13527.en.md b/changes/ce/fix-13527.en.md new file mode 100644 index 000000000..0c3324e41 --- /dev/null +++ b/changes/ce/fix-13527.en.md @@ -0,0 +1 @@ +Fixed an issue where running a SQL test in Rule Engine for the Message Publish event when a `$bridges/...` source was included in the `FROM` clause would always yield no results. diff --git a/changes/ee/feat-13452.en.md b/changes/ee/feat-13452.en.md new file mode 100644 index 000000000..7b2427329 --- /dev/null +++ b/changes/ee/feat-13452.en.md @@ -0,0 +1,5 @@ +Kafka producer action's `topic` config now supports templates. + +The topics must be already created in Kafka. If a message is rendered towards a non-existing topic in Kafka (given Kafka disabled topic auto-creation), the message will fail with an unrecoverable error. Also, if a message does not contain enough information to render to the configured template (e.g.: the template is `t-${t}` and the message context does not define `t`), this message will also fail with an unrecoverable error. + +This same feature is also available for Azure Event Hubs and Confluent Platform producer integrations. diff --git a/changes/ee/feat-13504.en.md b/changes/ee/feat-13504.en.md new file mode 100644 index 000000000..acea1241a --- /dev/null +++ b/changes/ee/feat-13504.en.md @@ -0,0 +1,5 @@ +Added a HTTP backend for the authentication mechanism `scram`. + +Note: This is not an implementation of the RFC 7804: Salted Challenge Response HTTP Authentication Mechanism. + +This backend is an implementation of scram that uses an external web resource as a source of SCRAM authentication data, including stored key of the client, server key, and the salt. It support other authentication and authorization extension fields like HTTP auth backend, namely: `is_superuser`, `client_attrs`, `expire_at` and `acl`. diff --git a/mix.exs b/mix.exs index dc2953683..96bb32632 100644 --- a/mix.exs +++ b/mix.exs @@ -184,7 +184,7 @@ defmodule EMQXUmbrella.MixProject do def common_dep(:ekka), do: {:ekka, github: "emqx/ekka", tag: "0.19.5", override: true} def common_dep(:esockd), do: {:esockd, github: "emqx/esockd", tag: "5.11.3", override: true} def common_dep(:gproc), do: {:gproc, github: "emqx/gproc", tag: "0.9.0.1", override: true} - def common_dep(:hocon), do: {:hocon, github: "emqx/hocon", tag: "0.43.1", override: true} + def common_dep(:hocon), do: {:hocon, github: "emqx/hocon", tag: "0.43.2", override: true} def common_dep(:lc), do: {:lc, github: "emqx/lc", tag: "0.3.2", override: true} # in conflict by ehttpc and emqtt def common_dep(:gun), do: {:gun, github: "emqx/gun", tag: "1.3.11", override: true} @@ -387,7 +387,7 @@ defmodule EMQXUmbrella.MixProject do {:hstreamdb_erl, github: "hstreamdb/hstreamdb_erl", tag: "0.5.18+v0.18.1+ezstd-v1.0.5-emqx1"}, {:influxdb, github: "emqx/influxdb-client-erl", tag: "1.1.13", override: true}, - {:wolff, github: "kafka4beam/wolff", tag: "2.0.0"}, + {:wolff, github: "kafka4beam/wolff", tag: "3.0.2"}, {:kafka_protocol, github: "kafka4beam/kafka_protocol", tag: "4.1.5", override: true}, {:brod_gssapi, github: "kafka4beam/brod_gssapi", tag: "v0.1.1"}, {:brod, github: "kafka4beam/brod", tag: "3.18.0"}, diff --git a/rebar.config b/rebar.config index b260561cb..68b2df1d7 100644 --- a/rebar.config +++ b/rebar.config @@ -98,7 +98,7 @@ {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.5"}}}, {getopt, "1.0.2"}, {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.10"}}}, - {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.43.1"}}}, + {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.43.2"}}}, {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.3"}}}, {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.2.1"}}}, {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.2"}}}, diff --git a/rel/config/ee-examples/file_transfer-with-local-exporter.conf.example b/rel/config/ee-examples/file_transfer-with-local-exporter.conf.example index dcb1e88d1..4eeef33c7 100644 --- a/rel/config/ee-examples/file_transfer-with-local-exporter.conf.example +++ b/rel/config/ee-examples/file_transfer-with-local-exporter.conf.example @@ -3,9 +3,6 @@ ## NOTE: This configuration is only applicable in EMQX Enterprise edition 5.1 or later. file_transfer { - ## Enable the File Transfer feature - enable = true - ## Storage backend settings storage { ## Local file system backend setting diff --git a/rel/config/ee-examples/file_transfer-with-s3-exporter.conf.example b/rel/config/ee-examples/file_transfer-with-s3-exporter.conf.example index 94061223e..7a738c097 100644 --- a/rel/config/ee-examples/file_transfer-with-s3-exporter.conf.example +++ b/rel/config/ee-examples/file_transfer-with-s3-exporter.conf.example @@ -4,9 +4,6 @@ ## Note: This configuration is only applicable for EMQX Enterprise edition 5.1 or later. file_transfer { - ## Enable the File Transfer feature - enable = true - ## Storage backend settings storage { ## Local file system backend setting @@ -51,8 +48,6 @@ file_transfer { ## Enable the HTTPS transport_options { - ssl.enable = true - ## Timeout for connection attempts connect_timeout = 15s } diff --git a/rel/config/ee-examples/ldap-authn.conf b/rel/config/ee-examples/ldap-authn.conf new file mode 100644 index 000000000..633a5cc7b --- /dev/null +++ b/rel/config/ee-examples/ldap-authn.conf @@ -0,0 +1,19 @@ +authentication = [ + { + backend = ldap + base_dn = "uid=${username},ou=testdevice,dc=emqx,dc=io" + filter = "(& (objectClass=mqttUser) (uid=${username}))" + mechanism = password_based + method { + is_superuser_attribute = isSuperuser + password_attribute = userPassword + type = hash + } + password = public + pool_size = 8 + query_timeout = "5s" + request_timeout = "10s" + server = "localhost:1389" + username = "cn=root,dc=emqx,dc=io" + } +] diff --git a/rel/config/examples/delayed.conf.example b/rel/config/examples/delayed.conf.example index 7b0d243c2..b5b2c2c08 100644 --- a/rel/config/examples/delayed.conf.example +++ b/rel/config/examples/delayed.conf.example @@ -7,8 +7,6 @@ ## you should copy and paste the below data into the emqx.conf for working delayed { - enable = true ## false for disabled - ## Maximum number of delayed messages ## Default: 0 (0 is no limit) max_delayed_messages = 0 diff --git a/rel/config/examples/exhook.conf.example b/rel/config/examples/exhook.conf.example index cf9fffc71..45a186586 100644 --- a/rel/config/examples/exhook.conf.example +++ b/rel/config/examples/exhook.conf.example @@ -7,9 +7,6 @@ exhook.servers = [ ## Name of the exhook server name = "server_1" - ## Feature switch - enable = false - ## URL of gRPC server url = "http://127.0.0.1:9090" diff --git a/rel/config/examples/flapping_detect.conf.example b/rel/config/examples/flapping_detect.conf.example index fe4522b1e..6878d1f41 100644 --- a/rel/config/examples/flapping_detect.conf.example +++ b/rel/config/examples/flapping_detect.conf.example @@ -3,9 +3,6 @@ ## Ban the client when the times of connections exceed the limit in the configured time window flapping_detect { - ## use 'true' to enable this feature - enable = false - ## Time window for flapping detection window_time = 1m diff --git a/rel/config/examples/force_gc.conf.example b/rel/config/examples/force_gc.conf.example index ee07ee4d2..08543b533 100644 --- a/rel/config/examples/force_gc.conf.example +++ b/rel/config/examples/force_gc.conf.example @@ -1,9 +1,6 @@ ## Force Elrang VM garbage collection force_gc { - ## set 'false' to disable this feature - enable = true - ## GC the process after this many received messages count = 16000 diff --git a/rel/config/examples/force_shutdown.conf.example b/rel/config/examples/force_shutdown.conf.example index d0a466aa0..1dd6747a8 100644 --- a/rel/config/examples/force_shutdown.conf.example +++ b/rel/config/examples/force_shutdown.conf.example @@ -3,9 +3,6 @@ ## Forced shutdown MQTT clients for overload protection force_shutdown { - ## set 'false' to disable force shutdown feature - enable = true - ## Maximum mailbox size for each Erlang process ## Note: Do not modify this unless you know what this is for max_mailbox_size = 1000 diff --git a/rel/config/examples/listeners.ssl.conf.example b/rel/config/examples/listeners.ssl.conf.example index 4a74f1a27..ffc79db54 100644 --- a/rel/config/examples/listeners.ssl.conf.example +++ b/rel/config/examples/listeners.ssl.conf.example @@ -3,7 +3,6 @@ listeners.ssl.my_ssl_listener_name { ## Port or Address to listen on, 0 means disable bind = 8883 ## or with an IP e.g. "127.0.0.1:8883" - enabled = true acceptors = 16 enable_authn = true max_connections = infinity diff --git a/rel/config/examples/listeners.ws.conf.example b/rel/config/examples/listeners.ws.conf.example index f6a6adae8..f8c07e84a 100644 --- a/rel/config/examples/listeners.ws.conf.example +++ b/rel/config/examples/listeners.ws.conf.example @@ -3,7 +3,6 @@ listeners.ws.my_ws_listener_name { ## Port or Address to listen on, 0 means disable bind = "0.0.0.0:8083" # or just a port number, e.g. 8083 - enabled = true enable_authn = true max_connections = infinity proxy_protocol = false diff --git a/rel/config/examples/listeners.wss.conf.example b/rel/config/examples/listeners.wss.conf.example index cbc632e30..d0a03b777 100644 --- a/rel/config/examples/listeners.wss.conf.example +++ b/rel/config/examples/listeners.wss.conf.example @@ -3,7 +3,6 @@ listeners.wss.my_wss_listener_name = { ## Port or Address to listen on, 0 means disable bind = 8084 ## or with an IP, e.g. "127.0.0.1:8084" - enabled = true enable_authn = true max_connections = infinity proxy_protocol = false diff --git a/rel/config/examples/log.console.conf.example b/rel/config/examples/log.console.conf.example index 9bb01b0d9..3b8a0436e 100644 --- a/rel/config/examples/log.console.conf.example +++ b/rel/config/examples/log.console.conf.example @@ -1,9 +1,6 @@ ## Log to console log.console { - ## set true to enable this - enable = false - ## Log level ## Type: debug | info | notice | warning | error | critical | alert | emergency level = warning diff --git a/rel/config/examples/log.file.conf.example b/rel/config/examples/log.file.conf.example index e6f408f61..f6dce6eaf 100644 --- a/rel/config/examples/log.file.conf.example +++ b/rel/config/examples/log.file.conf.example @@ -1,9 +1,6 @@ ## Log to file log.file { - ## Enable file log handler - enable = true - ## Log level ## Type: debug | info | notice | warning | error | critical | alert | emergency level = warning diff --git a/rel/config/examples/node.conf.example b/rel/config/examples/node.conf.example index 596e9884d..f4fd3288e 100644 --- a/rel/config/examples/node.conf.example +++ b/rel/config/examples/node.conf.example @@ -11,11 +11,11 @@ node { ## Secret cookie is a random string that should be the same on all nodes in the cluster, but unique per EMQX cluster cookie = "Yzc0NGExM2Rj" - ## Select a node role + ## Select a node role (Enterprise Edition feature) ## Possible values: ## - core: This is a core node which provides durability of the client states, and takes care of writes ## - replicant: This is a stateless worker node - role = core + ## role = core ## Maximum number of simultaneously existing processes for this Erlang system process_limit = 2097152 diff --git a/rel/config/examples/plugins.conf.example b/rel/config/examples/plugins.conf.example index 6fcc09cbb..b7673036e 100644 --- a/rel/config/examples/plugins.conf.example +++ b/rel/config/examples/plugins.conf.example @@ -9,8 +9,6 @@ plugins { ## Format: {name}-{version} ## Note: name and version should be what it is in the plugin application name_vsn = "my_acl-0.1.0", - - enable = true ## enable this plugin }, {name_vsn = "my_rule-0.1.1", enable = false} ] diff --git a/rel/config/examples/prometheus-pushgateway.conf.example b/rel/config/examples/prometheus-pushgateway.conf.example index 70b74794a..f463056e3 100644 --- a/rel/config/examples/prometheus-pushgateway.conf.example +++ b/rel/config/examples/prometheus-pushgateway.conf.example @@ -5,9 +5,6 @@ ## If you want to use push-gateway prometheus { - ## Set to true to make EMQX send metrics to push-gateway - enable = false - ## URL of push-gateway server push_gateway_server = "http://127.0.0.1:9091" diff --git a/rel/config/examples/prometheus.conf.example b/rel/config/examples/prometheus.conf.example index 049b11ee3..e31d0fe1d 100644 --- a/rel/config/examples/prometheus.conf.example +++ b/rel/config/examples/prometheus.conf.example @@ -7,7 +7,6 @@ prometheus { enable_basic_auth = false push_gateway { - enable = false url = "http://127.0.0.1:9091" headers {Authorization = "Basic YWRtaW46Y2JraG55eWd5QDE="} interval = 15s diff --git a/rel/config/examples/psk_authentication.conf.example b/rel/config/examples/psk_authentication.conf.example index 6c3482638..cea6f21c2 100644 --- a/rel/config/examples/psk_authentication.conf.example +++ b/rel/config/examples/psk_authentication.conf.example @@ -1,9 +1,6 @@ ## Pre-Shared Keys authentication psk_authentication { - ## Set to false to disable - enable = true - ## If init_file is specified, EMQX will import PSKs from the file into the built-in database at startup for use by the runtime init_file = "psk" diff --git a/rel/config/examples/retainer.conf.example b/rel/config/examples/retainer.conf.example index d78119ec2..b698020d4 100644 --- a/rel/config/examples/retainer.conf.example +++ b/rel/config/examples/retainer.conf.example @@ -5,9 +5,6 @@ ##-------------------------------------------------------------------- retainer { - ## set to false to disable retainer - enable = true - ## Message retention time, default is 0 means the message will never expire msg_expiry_interval = 5s diff --git a/rel/i18n/emqx_bridge_azure_event_hub.hocon b/rel/i18n/emqx_bridge_azure_event_hub.hocon index 2a3071f2c..b99ad56fa 100644 --- a/rel/i18n/emqx_bridge_azure_event_hub.hocon +++ b/rel/i18n/emqx_bridge_azure_event_hub.hocon @@ -69,7 +69,7 @@ producer_kafka_opts.label: """Azure Event Hubs Producer""" kafka_topic.desc: -"""Event Hubs name""" +"""Event Hubs name. Supports templates (e.g.: `t-${payload.t}`).""" kafka_topic.label: """Event Hubs Name""" diff --git a/rel/i18n/emqx_bridge_confluent_producer.hocon b/rel/i18n/emqx_bridge_confluent_producer.hocon index 234da3e5f..fa933a8ec 100644 --- a/rel/i18n/emqx_bridge_confluent_producer.hocon +++ b/rel/i18n/emqx_bridge_confluent_producer.hocon @@ -69,10 +69,10 @@ producer_kafka_opts.label: """Confluent Producer""" kafka_topic.desc: -"""Event Hub name""" +"""Kafka topic name. Supports templates (e.g.: `t-${payload.t}`).""" kafka_topic.label: -"""Event Hub Name""" +"""Kafka Topic Name""" kafka_message_timestamp.desc: """Which timestamp to use. The timestamp is expected to be a millisecond precision Unix epoch which can be in string format, e.g. 1661326462115 or '1661326462115'. When the desired data field for this template is not found, or if the found data is not a valid integer, the current system timestamp will be used.""" diff --git a/rel/i18n/emqx_bridge_kafka.hocon b/rel/i18n/emqx_bridge_kafka.hocon index f63e6f3eb..b807c80c2 100644 --- a/rel/i18n/emqx_bridge_kafka.hocon +++ b/rel/i18n/emqx_bridge_kafka.hocon @@ -81,7 +81,7 @@ producer_kafka_opts.label: """Kafka Producer""" kafka_topic.desc: -"""Kafka topic name""" +"""Kafka topic name. Supports templates (e.g.: `t-${payload.t}`).""" kafka_topic.label: """Kafka Topic Name""" diff --git a/rel/i18n/emqx_ds_shared_sub_api.hocon b/rel/i18n/emqx_ds_shared_sub_api.hocon new file mode 100644 index 000000000..369aeb88e --- /dev/null +++ b/rel/i18n/emqx_ds_shared_sub_api.hocon @@ -0,0 +1,34 @@ +emqx_ds_shared_sub_api { + +param_queue_id.desc: +"""The ID of the durable queue.""" + +param_queue_id.label: +"""Queue ID""" + +durable_queues_get.desc: +"""Get the list of durable queues.""" + +durable_queues_get.label: +"""Durable Queues""" + +durable_queue_get.desc: +"""Get the information of a durable queue.""" + +durable_queue_get.label: +"""Durable Queue""" + +durable_queue_delete.desc: +"""Delete a durable queue.""" + +durable_queue_delete.label: +"""Delete Durable Queue""" + +durable_queues_put.desc: +"""Create a durable queue.""" + +durable_queues_put.label: +"""Create Durable Queue""" + + +} diff --git a/rel/i18n/emqx_ds_shared_sub_schema.hocon b/rel/i18n/emqx_ds_shared_sub_schema.hocon new file mode 100644 index 000000000..2ee28cc30 --- /dev/null +++ b/rel/i18n/emqx_ds_shared_sub_schema.hocon @@ -0,0 +1,63 @@ +emqx_ds_shared_sub_schema { + +enable.desc: +"""Enable the shared subscription feature.""" + +enable.label: +"""Enable Shared Subscription""" + +session_find_leader_timeout_ms.desc: +"""The timeout in milliseconds for the session to find a leader. +If the session cannot find a leader within this time, the session will retry.""" + +session_find_leader_timeout_ms.label: +"""Session Find Leader Timeout""" + +session_renew_lease_timeout_ms.desc: +"""The timeout in milliseconds for the session to wait for the leader to renew the lease. +If the leader does not renew the lease within this time, the session will consider +the leader as lost and try to find a new leader.""" + +session_renew_lease_timeout_ms.label: +"""Session Renew Lease Timeout""" + +session_min_update_stream_state_interval_ms.desc: +"""The minimum interval in milliseconds for the session to update the stream state. +If session has no updates for the stream state within this time, the session will +send empty updates.""" + +session_min_update_stream_state_interval_ms.label: +"""Session Min Update Stream State Interval""" + +leader_renew_lease_interval_ms.desc: +"""The interval in milliseconds for the leader to renew the lease.""" + +leader_renew_lease_interval_ms.label: +"""Leader Renew Lease Interval""" + +leader_renew_streams_interval_ms.desc: +"""The interval in milliseconds for the leader to renew the streams.""" + +leader_renew_streams_interval_ms.label: +"""Leader Renew Streams Interval""" + +leader_drop_timeout_interval_ms.desc: +"""The interval in milliseconds for the leader to drop non-responsive sessions.""" + +leader_drop_timeout_interval_ms.label: +"""Leader Drop Timeout Interval""" + +leader_session_update_timeout_ms.desc: +"""The timeout in milliseconds for the leader to wait for the session to update the stream state. +If the session does not update the stream state within this time, the leader will drop the session.""" + +leader_session_update_timeout_ms.label: +"""Leader Session Update Timeout""" + +leader_session_not_replaying_timeout_ms.desc: +"""The timeout in milliseconds for the leader to wait for the session leave intermediate states.""" + +leader_session_not_replaying_timeout_ms.label: +"""Leader Session Not Replaying Timeout""" + +} diff --git a/rel/i18n/emqx_rule_engine_api.hocon b/rel/i18n/emqx_rule_engine_api.hocon index 0745a108d..fd78d7ca1 100644 --- a/rel/i18n/emqx_rule_engine_api.hocon +++ b/rel/i18n/emqx_rule_engine_api.hocon @@ -96,4 +96,10 @@ api11.desc: api11.label: """Apply Rule""" +api1_qs_action.desc: +"""Filters rules that contain any of the given action id(s). When used in conjunction with source id filtering, the rules must contain sources *and* actions that match some of the criteria.""" + +api1_qs_source.desc: +"""Filters rules that contain any of the given source id(s). When used in conjunction with action id filtering, the rules must contain sources *and* actions that match some of the criteria.""" + }