From cba3f532f894fdf3e1a4631e67e36cf60cb2151e Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Thu, 11 Jul 2024 14:16:18 +0800 Subject: [PATCH 1/9] feat: don't record dry_run log --- .../src/emqx_bridge_kafka_impl_consumer.erl | 12 +- .../src/emqx_bridge_kafka_impl_producer.erl | 9 +- .../src/emqx_bridge_pulsar.app.src | 2 +- .../src/emqx_bridge_pulsar_connector.erl | 10 +- apps/emqx_resource/src/emqx_resource.erl | 9 ++ .../src/emqx_resource_manager.erl | 103 +++++++++++------- 6 files changed, 76 insertions(+), 69 deletions(-) diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_consumer.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_consumer.erl index c4f66dfff..35ffbc90b 100644 --- a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_consumer.erl +++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_consumer.erl @@ -628,16 +628,6 @@ consumer_group_id(BridgeName0) -> BridgeName = to_bin(BridgeName0), <<"emqx-kafka-consumer-", BridgeName/binary>>. --spec is_dry_run(connector_resource_id()) -> boolean(). -is_dry_run(ConnectorResId) -> - TestIdStart = string:find(ConnectorResId, ?TEST_ID_PREFIX), - case TestIdStart of - nomatch -> - false; - _ -> - string:equal(TestIdStart, ConnectorResId) - end. - -spec check_client_connectivity(pid()) -> ?status_connected | ?status_disconnected @@ -673,7 +663,7 @@ maybe_clean_error(Reason) -> -spec make_client_id(connector_resource_id(), binary(), atom() | binary()) -> atom(). make_client_id(ConnectorResId, BridgeType, BridgeName) -> - case is_dry_run(ConnectorResId) of + case emqx_resource:is_dry_run(ConnectorResId) of false -> ClientID0 = emqx_bridge_kafka_impl:make_client_id(BridgeType, BridgeName), binary_to_atom(ClientID0); diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl index b819925ac..8b6326545 100644 --- a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl +++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl @@ -137,14 +137,7 @@ create_producers_for_bridge_v2( KafkaHeadersValEncodeMode = maps:get(kafka_header_value_encode_mode, KafkaConfig, none), MaxPartitions = maps:get(partitions_limit, KafkaConfig, all_partitions), #{name := BridgeName} = emqx_bridge_v2:parse_id(BridgeV2Id), - TestIdStart = string:find(BridgeV2Id, ?TEST_ID_PREFIX), - IsDryRun = - case TestIdStart of - nomatch -> - false; - _ -> - string:equal(TestIdStart, InstId) - end, + IsDryRun = emqx_resource:is_dry_run(BridgeV2Id), ok = check_topic_and_leader_connections(ClientId, KafkaTopic, MaxPartitions), WolffProducerConfig = producers_config( BridgeType, BridgeName, KafkaConfig, IsDryRun, BridgeV2Id diff --git a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.app.src b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.app.src index a8eeba483..dcb86a3ca 100644 --- a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.app.src +++ b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_pulsar, [ {description, "EMQX Pulsar Bridge"}, - {vsn, "0.2.2"}, + {vsn, "0.2.3"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_connector.erl b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_connector.erl index 9d269493d..4157deec2 100644 --- a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_connector.erl +++ b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_connector.erl @@ -255,7 +255,7 @@ format_servers(Servers0) -> -spec make_client_id(resource_id()) -> pulsar_client_id(). make_client_id(InstanceId) -> - case is_dry_run(InstanceId) of + case emqx_resource:is_dry_run(InstanceId) of true -> pulsar_producer_probe; false -> @@ -269,14 +269,6 @@ make_client_id(InstanceId) -> binary_to_atom(ClientIdBin) end. --spec is_dry_run(resource_id()) -> boolean(). -is_dry_run(InstanceId) -> - TestIdStart = string:find(InstanceId, ?TEST_ID_PREFIX), - case TestIdStart of - nomatch -> false; - _ -> string:equal(TestIdStart, InstanceId) - end. - conn_opts(#{authentication := none}) -> #{}; conn_opts(#{authentication := #{username := Username, password := Password}}) -> diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl index 15c0a0293..b6f01fde5 100644 --- a/apps/emqx_resource/src/emqx_resource.erl +++ b/apps/emqx_resource/src/emqx_resource.erl @@ -140,6 +140,8 @@ validate_name/1 ]). +-export([is_dry_run/1]). + -export_type([ query_mode/0, resource_id/0, @@ -769,6 +771,13 @@ validate_name(Name) -> _ = validate_name(Name, #{atom_name => false}), ok. +-spec is_dry_run(resource_id()) -> boolean(). +is_dry_run(ResId) -> + case string:find(ResId, ?TEST_ID_PREFIX) of + nomatch -> false; + TestIdStart -> string:equal(TestIdStart, ResId) + end. + validate_name(<<>>, _Opts) -> invalid_data("Name cannot be empty string"); validate_name(Name, _Opts) when size(Name) >= 255 -> diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl index 50a25620c..95b1271f4 100644 --- a/apps/emqx_resource/src/emqx_resource_manager.erl +++ b/apps/emqx_resource/src/emqx_resource_manager.erl @@ -752,7 +752,8 @@ handle_remove_event(From, ClearMetrics, Data) -> start_resource(Data, From) -> %% in case the emqx_resource:call_start/2 hangs, the lookup/1 can read status from the cache - case emqx_resource:call_start(Data#data.id, Data#data.mod, Data#data.config) of + ResId = Data#data.id, + case emqx_resource:call_start(ResId, Data#data.mod, Data#data.config) of {ok, ResourceState} -> UpdatedData1 = Data#data{status = ?status_connecting, state = ResourceState}, %% Perform an initial health_check immediately before transitioning into a connected state @@ -760,12 +761,13 @@ start_resource(Data, From) -> Actions = maybe_reply([{state_timeout, 0, health_check}], From, ok), {next_state, ?state_connecting, update_state(UpdatedData2, Data), Actions}; {error, Reason} = Err -> - ?SLOG(warning, #{ + IsDryRun = emqx_resource:is_dry_run(ResId), + ?SLOG(log_level(IsDryRun), #{ msg => "start_resource_failed", - id => Data#data.id, + id => ResId, reason => Reason }), - _ = maybe_alarm(?status_disconnected, Data#data.id, Err, Data#data.error), + _ = maybe_alarm(?status_disconnected, IsDryRun, ResId, Err, Data#data.error), %% Add channels and raise alarms NewData1 = channels_health_check(?status_disconnected, add_channels(Data)), %% Keep track of the error reason why the connection did not work @@ -796,9 +798,10 @@ add_channels(Data) -> add_channels_in_list([], Data) -> Data; add_channels_in_list([{ChannelID, ChannelConfig} | Rest], Data) -> + Id = Data#data.id, case emqx_resource:call_add_channel( - Data#data.id, Data#data.mod, Data#data.state, ChannelID, ChannelConfig + Id, Data#data.mod, Data#data.state, ChannelID, ChannelConfig ) of {ok, NewState} -> @@ -816,9 +819,10 @@ add_channels_in_list([{ChannelID, ChannelConfig} | Rest], Data) -> }, add_channels_in_list(Rest, NewData); {error, Reason} = Error -> - ?SLOG(warning, #{ - msg => add_channel_failed, - id => Data#data.id, + IsDryRun = emqx_resource:is_dry_run(Id), + ?SLOG(log_level(IsDryRun), #{ + msg => "add_channel_failed", + id => Id, channel_id => ChannelID, reason => Reason }), @@ -832,7 +836,7 @@ add_channels_in_list([{ChannelID, ChannelConfig} | Rest], Data) -> added_channels = NewAddedChannelsMap }, %% Raise an alarm since the channel could not be added - _ = maybe_alarm(?status_disconnected, ChannelID, Error, no_prev_error), + _ = maybe_alarm(?status_disconnected, IsDryRun, ChannelID, Error, no_prev_error), add_channels_in_list(Rest, NewData) end. @@ -856,7 +860,8 @@ stop_resource(#data{id = ResId} = Data) -> false -> ok end, - _ = maybe_clear_alarm(ResId), + IsDryRun = emqx_resource:is_dry_run(ResId), + _ = maybe_clear_alarm(IsDryRun, ResId), ok = emqx_metrics_worker:reset_metrics(?RES_METRICS, ResId), NewData#data{status = ?rm_status_stopped}. @@ -868,15 +873,17 @@ remove_channels_in_list([], Data, _KeepInChannelMap) -> Data; remove_channels_in_list([ChannelID | Rest], Data, KeepInChannelMap) -> AddedChannelsMap = Data#data.added_channels, + Id = Data#data.id, + IsDryRun = emqx_resource:is_dry_run(Id), NewAddedChannelsMap = case KeepInChannelMap of true -> AddedChannelsMap; false -> - _ = maybe_clear_alarm(ChannelID), + _ = maybe_clear_alarm(IsDryRun, ChannelID), maps:remove(ChannelID, AddedChannelsMap) end, - case safe_call_remove_channel(Data#data.id, Data#data.mod, Data#data.state, ChannelID) of + case safe_call_remove_channel(Id, Data#data.mod, Data#data.state, ChannelID) of {ok, NewState} -> NewData = Data#data{ state = NewState, @@ -884,9 +891,9 @@ remove_channels_in_list([ChannelID | Rest], Data, KeepInChannelMap) -> }, remove_channels_in_list(Rest, NewData, KeepInChannelMap); {error, Reason} -> - ?SLOG(warning, #{ - msg => remove_channel_failed, - id => Data#data.id, + ?SLOG(log_level(IsDryRun), #{ + msg => "remove_channel_failed", + id => Id, channel_id => ChannelID, reason => Reason }), @@ -968,8 +975,8 @@ handle_not_connected_add_channel(From, ChannelId, ChannelConfig, State, Data) -> handle_remove_channel(From, ChannelId, Data) -> Channels = Data#data.added_channels, - %% Deactivate alarm - _ = maybe_clear_alarm(ChannelId), + IsDryRun = emqx_resource:is_dry_run(Data#data.id), + _ = maybe_clear_alarm(IsDryRun, ChannelId), case channel_status_is_channel_added( maps:get(ChannelId, Channels, channel_status_not_added(undefined)) @@ -990,13 +997,13 @@ handle_remove_channel(From, ChannelId, Data) -> end. handle_remove_channel_exists(From, ChannelId, Data) -> + #data{id = Id, added_channels = AddedChannelsMap} = Data, case emqx_resource:call_remove_channel( - Data#data.id, Data#data.mod, Data#data.state, ChannelId + Id, Data#data.mod, Data#data.state, ChannelId ) of {ok, NewState} -> - AddedChannelsMap = Data#data.added_channels, NewAddedChannelsMap = maps:remove(ChannelId, AddedChannelsMap), UpdatedData = Data#data{ state = NewState, @@ -1004,10 +1011,10 @@ handle_remove_channel_exists(From, ChannelId, Data) -> }, {keep_state, update_state(UpdatedData, Data), [{reply, From, ok}]}; {error, Reason} = Error -> - %% Log the error as a warning - ?SLOG(warning, #{ - msg => remove_channel_failed, - id => Data#data.id, + IsDryRun = emqx_resource:is_dry_run(Id), + ?SLOG(log_level(IsDryRun), #{ + msg => "remove_channel_failed", + id => Id, channel_id => ChannelId, reason => Reason }), @@ -1021,7 +1028,8 @@ handle_not_connected_and_not_connecting_remove_channel(From, ChannelId, Data) -> Channels = Data#data.added_channels, NewChannels = maps:remove(ChannelId, Channels), NewData = Data#data{added_channels = NewChannels}, - _ = maybe_clear_alarm(ChannelId), + IsDryRun = emqx_resource:is_dry_run(Data#data.id), + _ = maybe_clear_alarm(IsDryRun, ChannelId), {keep_state, update_state(NewData, Data), [{reply, From, ok}]}. handle_manual_resource_health_check(From, Data0 = #data{hc_workers = #{resource := HCWorkers}}) when @@ -1090,7 +1098,8 @@ continue_with_health_check(#data{} = Data0, CurrentState, HCRes) -> error = PrevError } = Data0, {NewStatus, NewState, Err} = parse_health_check_result(HCRes, Data0), - _ = maybe_alarm(NewStatus, ResId, Err, PrevError), + IsDryRun = emqx_resource:is_dry_run(ResId), + _ = maybe_alarm(NewStatus, IsDryRun, ResId, Err, PrevError), ok = maybe_resume_resource_workers(ResId, NewStatus), Data1 = Data0#data{ state = NewState, status = NewStatus, error = Err @@ -1114,7 +1123,8 @@ continue_resource_health_check_connected(NewStatus, Data0) -> Actions = Replies ++ resource_health_check_actions(Data), {keep_state, Data, Actions}; _ -> - ?SLOG(warning, #{ + IsDryRun = emqx_resource:is_dry_run(Data0#data.id), + ?SLOG(log_level(IsDryRun), #{ msg => "health_check_failed", id => Data0#data.id, status => NewStatus @@ -1214,7 +1224,7 @@ channels_health_check(?status_connected = _ConnectorStatus, Data0) -> channels_health_check(?status_connecting = _ConnectorStatus, Data0) -> %% Whenever the resource is connecting: %% 1. Change the status of all added channels to connecting - %% 2. Raise alarms (TODO: if it is a probe we should not raise alarms) + %% 2. Raise alarms Channels = Data0#data.added_channels, ChannelsToChangeStatusFor = [ {ChannelId, Config} @@ -1240,9 +1250,10 @@ channels_health_check(?status_connecting = _ConnectorStatus, Data0) -> || {ChannelId, NewStatus} <- maps:to_list(NewChannels) ], %% Raise alarms for all channels + IsDryRun = emqx_resource:is_dry_run(Data0#data.id), lists:foreach( fun({ChannelId, Status, PrevStatus}) -> - maybe_alarm(?status_connecting, ChannelId, Status, PrevStatus) + maybe_alarm(?status_connecting, IsDryRun, ChannelId, Status, PrevStatus) end, ChannelsWithNewAndPrevErrorStatuses ), @@ -1275,9 +1286,10 @@ channels_health_check(ConnectorStatus, Data0) -> || {ChannelId, #{config := Config} = OldStatus} <- maps:to_list(Data1#data.added_channels) ], %% Raise alarms + IsDryRun = emqx_resource:is_dry_run(Data1#data.id), _ = lists:foreach( fun({ChannelId, OldStatus, NewStatus}) -> - _ = maybe_alarm(NewStatus, ChannelId, NewStatus, OldStatus) + _ = maybe_alarm(NewStatus, IsDryRun, ChannelId, NewStatus, OldStatus) end, ChannelsWithNewAndOldStatuses ), @@ -1386,13 +1398,14 @@ continue_channel_health_check_connected_no_update_during_check(ChannelId, OldSta NewStatus = maps:get(ChannelId, Data1#data.added_channels), ChannelsToRemove = [ChannelId || not channel_status_is_channel_added(NewStatus)], Data = remove_channels_in_list(ChannelsToRemove, Data1, true), + IsDryRun = emqx_resource:is_dry_run(Data1#data.id), %% Raise/clear alarms case NewStatus of #{status := ?status_connected} -> - _ = maybe_clear_alarm(ChannelId), + _ = maybe_clear_alarm(IsDryRun, ChannelId), ok; _ -> - _ = maybe_alarm(NewStatus, ChannelId, NewStatus, OldStatus), + _ = maybe_alarm(NewStatus, IsDryRun, ChannelId, NewStatus, OldStatus), ok end, Data. @@ -1556,15 +1569,21 @@ remove_runtime_data(#data{} = Data0) -> health_check_interval(Opts) -> maps:get(health_check_interval, Opts, ?HEALTHCHECK_INTERVAL). --spec maybe_alarm(resource_status(), resource_id(), _Error :: term(), _PrevError :: term()) -> ok. -maybe_alarm(?status_connected, _ResId, _Error, _PrevError) -> +-spec maybe_alarm( + resource_status(), + boolean(), + resource_id(), + _Error :: term(), + _PrevError :: term() +) -> ok. +maybe_alarm(?status_connected, _IsDryRun, _ResId, _Error, _PrevError) -> ok; -maybe_alarm(_Status, <>, _Error, _PrevError) -> +maybe_alarm(_Status, true, _ResId, _Error, _PrevError) -> ok; %% Assume that alarm is already active -maybe_alarm(_Status, _ResId, Error, Error) -> +maybe_alarm(_Status, _IsDryRun, _ResId, Error, Error) -> ok; -maybe_alarm(_Status, ResId, Error, _PrevError) -> +maybe_alarm(_Status, false, ResId, Error, _PrevError) -> HrError = case Error of {error, undefined} -> @@ -1596,10 +1615,10 @@ maybe_resume_resource_workers(ResId, ?status_connected) -> maybe_resume_resource_workers(_, _) -> ok. --spec maybe_clear_alarm(resource_id()) -> ok | {error, not_found}. -maybe_clear_alarm(<>) -> +-spec maybe_clear_alarm(boolean(), resource_id()) -> ok | {error, not_found}. +maybe_clear_alarm(true, _ResId) -> ok; -maybe_clear_alarm(ResId) -> +maybe_clear_alarm(false, ResId) -> emqx_alarm:safe_deactivate(ResId). parse_health_check_result(Status, Data) when ?IS_STATUS(Status) -> @@ -1767,10 +1786,14 @@ add_or_update_channel_status(Data, ChannelId, ChannelConfig, State) -> ChannelStatus = channel_status({error, resource_not_operational}, ChannelConfig), NewChannels = maps:put(ChannelId, ChannelStatus, Channels), ResStatus = state_to_status(State), - maybe_alarm(ResStatus, ChannelId, ChannelStatus, no_prev), + IsDryRun = emqx_resource:is_dry_run(ChannelId), + maybe_alarm(ResStatus, IsDryRun, ChannelId, ChannelStatus, no_prev), Data#data{added_channels = NewChannels}. state_to_status(?state_stopped) -> ?rm_status_stopped; state_to_status(?state_connected) -> ?status_connected; state_to_status(?state_connecting) -> ?status_connecting; state_to_status(?state_disconnected) -> ?status_disconnected. + +log_level(true) -> info; +log_level(false) -> warning. From 0a04b1ad6e73dc0ae29e1b6c15ad3ff23edd0b2e Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Thu, 11 Jul 2024 17:09:10 +0800 Subject: [PATCH 2/9] feat: add group/type to resource slog --- apps/emqx_auth/include/emqx_authn.hrl | 2 +- apps/emqx_auth/include/emqx_authz.hrl | 2 +- .../src/emqx_authn/emqx_authn_utils.erl | 11 +- .../src/emqx_authz/emqx_authz_utils.erl | 14 ++- apps/emqx_auth_http/src/emqx_authn_http.erl | 5 +- apps/emqx_auth_http/src/emqx_authz_http.erl | 6 +- apps/emqx_auth_jwt/src/emqx_authn_jwt.erl | 1 + .../emqx_auth_ldap/src/emqx_auth_ldap.app.src | 2 +- apps/emqx_auth_ldap/src/emqx_authn_ldap.erl | 4 +- apps/emqx_auth_ldap/src/emqx_authz_ldap.erl | 4 +- .../test/emqx_authn_ldap_SUITE.erl | 1 + .../test/emqx_authn_ldap_bind_SUITE.erl | 1 + .../test/emqx_authz_ldap_SUITE.erl | 1 + .../src/emqx_auth_mongodb.app.src | 2 +- .../src/emqx_authn_mongodb.erl | 3 +- .../src/emqx_authz_mongodb.erl | 4 +- apps/emqx_auth_mysql/src/emqx_authn_mysql.erl | 4 +- apps/emqx_auth_mysql/src/emqx_authz_mysql.erl | 4 +- .../test/emqx_authn_mysql_SUITE.erl | 1 + .../test/emqx_authz_mysql_SUITE.erl | 1 + .../src/emqx_authn_postgresql.erl | 3 +- .../src/emqx_authz_postgresql.erl | 2 + .../test/emqx_authn_postgresql_SUITE.erl | 5 +- .../test/emqx_authz_postgresql_SUITE.erl | 1 + apps/emqx_auth_redis/src/emqx_authn_redis.erl | 3 +- apps/emqx_auth_redis/src/emqx_authz_redis.erl | 4 +- .../test/emqx_authn_redis_SUITE.erl | 1 + .../test/emqx_authz_redis_SUITE.erl | 1 + apps/emqx_bridge/src/emqx_bridge_resource.erl | 6 +- .../emqx_bridge_cassandra_connector_SUITE.erl | 1 + ...emqx_bridge_clickhouse_connector_SUITE.erl | 2 + ...emqx_bridge_greptimedb_connector_SUITE.erl | 1 + .../emqx_bridge_influxdb_connector_SUITE.erl | 2 + .../src/emqx_bridge_pulsar_connector.erl | 2 +- .../emqx_bridge_rabbitmq_connector_SUITE.erl | 1 + .../emqx_connector/include/emqx_connector.hrl | 2 +- .../src/emqx_connector_resource.erl | 7 +- .../src/emqx_dashboard_sso_manager.erl | 5 +- .../test/emqx_dashboard_sso_ldap_SUITE.erl | 2 +- apps/emqx_ldap/test/emqx_ldap_SUITE.erl | 2 + apps/emqx_mongodb/test/emqx_mongodb_SUITE.erl | 1 + apps/emqx_mysql/test/emqx_mysql_SUITE.erl | 1 + .../test/emqx_postgresql_SUITE.erl | 1 + apps/emqx_redis/test/emqx_redis_SUITE.erl | 1 + apps/emqx_resource/include/emqx_resource.hrl | 1 + apps/emqx_resource/src/emqx_resource.erl | 70 ++++++----- .../src/emqx_resource_manager.erl | 118 ++++++++++++------ .../src/emqx_resource_manager_sup.erl | 15 ++- .../test/emqx_resource_SUITE.erl | 20 ++- 49 files changed, 232 insertions(+), 122 deletions(-) diff --git a/apps/emqx_auth/include/emqx_authn.hrl b/apps/emqx_auth/include/emqx_authn.hrl index 1f2c6b8b9..8ffab1203 100644 --- a/apps/emqx_auth/include/emqx_authn.hrl +++ b/apps/emqx_auth/include/emqx_authn.hrl @@ -30,6 +30,6 @@ -type authenticator_id() :: binary(). --define(AUTHN_RESOURCE_GROUP, <<"emqx_authn">>). +-define(AUTHN_RESOURCE_GROUP, <<"authn">>). -endif. diff --git a/apps/emqx_auth/include/emqx_authz.hrl b/apps/emqx_auth/include/emqx_authz.hrl index 6dc80cb3f..73ff44c25 100644 --- a/apps/emqx_auth/include/emqx_authz.hrl +++ b/apps/emqx_auth/include/emqx_authz.hrl @@ -158,7 +158,7 @@ count => 1 }). --define(AUTHZ_RESOURCE_GROUP, <<"emqx_authz">>). +-define(AUTHZ_RESOURCE_GROUP, <<"authz">>). -define(AUTHZ_FEATURES, [rich_actions]). diff --git a/apps/emqx_auth/src/emqx_authn/emqx_authn_utils.erl b/apps/emqx_auth/src/emqx_authn/emqx_authn_utils.erl index a08ac260c..e81145f2c 100644 --- a/apps/emqx_auth/src/emqx_authn/emqx_authn_utils.erl +++ b/apps/emqx_auth/src/emqx_authn/emqx_authn_utils.erl @@ -21,8 +21,8 @@ -include_lib("snabbkaffe/include/trace.hrl"). -export([ - create_resource/3, - update_resource/3, + create_resource/4, + update_resource/4, check_password_from_selected_map/3, parse_deep/1, parse_str/1, @@ -66,8 +66,9 @@ %% APIs %%-------------------------------------------------------------------- -create_resource(ResourceId, Module, Config) -> +create_resource(Type, ResourceId, Module, Config) -> Result = emqx_resource:create_local( + Type, ResourceId, ?AUTHN_RESOURCE_GROUP, Module, @@ -76,9 +77,9 @@ create_resource(ResourceId, Module, Config) -> ), start_resource_if_enabled(Result, ResourceId, Config). -update_resource(Module, Config, ResourceId) -> +update_resource(Type, Module, Config, ResourceId) -> Result = emqx_resource:recreate_local( - ResourceId, Module, Config, ?DEFAULT_RESOURCE_OPTS + Type, ResourceId, Module, Config, ?DEFAULT_RESOURCE_OPTS ), start_resource_if_enabled(Result, ResourceId, Config). diff --git a/apps/emqx_auth/src/emqx_authz/emqx_authz_utils.erl b/apps/emqx_auth/src/emqx_authz/emqx_authz_utils.erl index e4343b6fa..533f982c1 100644 --- a/apps/emqx_auth/src/emqx_authz/emqx_authz_utils.erl +++ b/apps/emqx_auth/src/emqx_authz/emqx_authz_utils.erl @@ -25,9 +25,9 @@ -export([ cleanup_resources/0, make_resource_id/1, - create_resource/2, create_resource/3, - update_resource/2, + create_resource/4, + update_resource/3, remove_resource/1, update_config/2, parse_deep/2, @@ -57,12 +57,13 @@ %% APIs %%-------------------------------------------------------------------- -create_resource(Module, Config) -> +create_resource(Type, Module, Config) -> ResourceId = make_resource_id(Module), - create_resource(ResourceId, Module, Config). + create_resource(Type, ResourceId, Module, Config). -create_resource(ResourceId, Module, Config) -> +create_resource(Type, ResourceId, Module, Config) -> Result = emqx_resource:create_local( + Type, ResourceId, ?AUTHZ_RESOURCE_GROUP, Module, @@ -71,10 +72,11 @@ create_resource(ResourceId, Module, Config) -> ), start_resource_if_enabled(Result, ResourceId, Config). -update_resource(Module, #{annotations := #{id := ResourceId}} = Config) -> +update_resource(Type, Module, #{annotations := #{id := ResourceId}} = Config) -> Result = case emqx_resource:recreate_local( + Type, ResourceId, Module, Config, diff --git a/apps/emqx_auth_http/src/emqx_authn_http.erl b/apps/emqx_auth_http/src/emqx_authn_http.erl index d9c5c5ed5..ed151428f 100644 --- a/apps/emqx_auth_http/src/emqx_authn_http.erl +++ b/apps/emqx_auth_http/src/emqx_authn_http.erl @@ -40,6 +40,7 @@ create(Config0) -> ResourceId = emqx_authn_utils:make_resource_id(?MODULE), % {Config, State} = parse_config(Config0), {ok, _Data} = emqx_authn_utils:create_resource( + http, ResourceId, emqx_bridge_http_connector, Config @@ -50,7 +51,9 @@ create(Config0) -> update(Config0, #{resource_id := ResourceId} = _State) -> with_validated_config(Config0, fun(Config, NState) -> % {Config, NState} = parse_config(Config0), - case emqx_authn_utils:update_resource(emqx_bridge_http_connector, Config, ResourceId) of + case + emqx_authn_utils:update_resource(http, emqx_bridge_http_connector, Config, ResourceId) + of {error, Reason} -> error({load_config_error, Reason}); {ok, _} -> diff --git a/apps/emqx_auth_http/src/emqx_authz_http.erl b/apps/emqx_auth_http/src/emqx_authz_http.erl index 6b0152b7d..2a1a2638b 100644 --- a/apps/emqx_auth_http/src/emqx_authz_http.erl +++ b/apps/emqx_auth_http/src/emqx_authz_http.erl @@ -66,12 +66,14 @@ description() -> create(Config) -> NConfig = parse_config(Config), ResourceId = emqx_authn_utils:make_resource_id(?MODULE), - {ok, _Data} = emqx_authz_utils:create_resource(ResourceId, emqx_bridge_http_connector, NConfig), + {ok, _Data} = emqx_authz_utils:create_resource( + http, ResourceId, emqx_bridge_http_connector, NConfig + ), NConfig#{annotations => #{id => ResourceId}}. update(Config) -> NConfig = parse_config(Config), - case emqx_authz_utils:update_resource(emqx_bridge_http_connector, NConfig) of + case emqx_authz_utils:update_resource(http, emqx_bridge_http_connector, NConfig) of {error, Reason} -> error({load_config_error, Reason}); {ok, Id} -> NConfig#{annotations => #{id => Id}} end. diff --git a/apps/emqx_auth_jwt/src/emqx_authn_jwt.erl b/apps/emqx_auth_jwt/src/emqx_authn_jwt.erl index 18c954ab5..e01121a01 100644 --- a/apps/emqx_auth_jwt/src/emqx_authn_jwt.erl +++ b/apps/emqx_auth_jwt/src/emqx_authn_jwt.erl @@ -183,6 +183,7 @@ do_create( ) -> ResourceId = emqx_authn_utils:make_resource_id(?MODULE), {ok, _Data} = emqx_resource:create_local( + jwt, ResourceId, ?AUTHN_RESOURCE_GROUP, emqx_authn_jwks_connector, diff --git a/apps/emqx_auth_ldap/src/emqx_auth_ldap.app.src b/apps/emqx_auth_ldap/src/emqx_auth_ldap.app.src index d84d6ff81..a58117356 100644 --- a/apps/emqx_auth_ldap/src/emqx_auth_ldap.app.src +++ b/apps/emqx_auth_ldap/src/emqx_auth_ldap.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_auth_ldap, [ {description, "EMQX LDAP Authentication and Authorization"}, - {vsn, "0.1.2"}, + {vsn, "0.1.3"}, {registered, []}, {mod, {emqx_auth_ldap_app, []}}, {applications, [ diff --git a/apps/emqx_auth_ldap/src/emqx_authn_ldap.erl b/apps/emqx_auth_ldap/src/emqx_authn_ldap.erl index a18236d15..8a9b5650a 100644 --- a/apps/emqx_auth_ldap/src/emqx_authn_ldap.erl +++ b/apps/emqx_auth_ldap/src/emqx_authn_ldap.erl @@ -40,12 +40,12 @@ create(_AuthenticatorID, Config) -> do_create(Module, Config) -> ResourceId = emqx_authn_utils:make_resource_id(Module), State = parse_config(Config), - {ok, _Data} = emqx_authn_utils:create_resource(ResourceId, emqx_ldap, Config), + {ok, _Data} = emqx_authn_utils:create_resource(ldap, ResourceId, emqx_ldap, Config), {ok, State#{resource_id => ResourceId}}. update(Config, #{resource_id := ResourceId} = _State) -> NState = parse_config(Config), - case emqx_authn_utils:update_resource(emqx_ldap, Config, ResourceId) of + case emqx_authn_utils:update_resource(ldap, emqx_ldap, Config, ResourceId) of {error, Reason} -> error({load_config_error, Reason}); {ok, _} -> diff --git a/apps/emqx_auth_ldap/src/emqx_authz_ldap.erl b/apps/emqx_auth_ldap/src/emqx_authz_ldap.erl index 24bd8c008..e70b840a3 100644 --- a/apps/emqx_auth_ldap/src/emqx_authz_ldap.erl +++ b/apps/emqx_auth_ldap/src/emqx_authz_ldap.erl @@ -56,12 +56,12 @@ description() -> create(Source) -> ResourceId = emqx_authz_utils:make_resource_id(?MODULE), - {ok, _Data} = emqx_authz_utils:create_resource(ResourceId, emqx_ldap, Source), + {ok, _Data} = emqx_authz_utils:create_resource(ldap, ResourceId, emqx_ldap, Source), Annotations = new_annotations(#{id => ResourceId}, Source), Source#{annotations => Annotations}. update(Source) -> - case emqx_authz_utils:update_resource(emqx_ldap, Source) of + case emqx_authz_utils:update_resource(ldap, emqx_ldap, Source) of {error, Reason} -> error({load_config_error, Reason}); {ok, Id} -> diff --git a/apps/emqx_auth_ldap/test/emqx_authn_ldap_SUITE.erl b/apps/emqx_auth_ldap/test/emqx_authn_ldap_SUITE.erl index ac941f268..f6ee582be 100644 --- a/apps/emqx_auth_ldap/test/emqx_authn_ldap_SUITE.erl +++ b/apps/emqx_auth_ldap/test/emqx_authn_ldap_SUITE.erl @@ -47,6 +47,7 @@ init_per_suite(Config) -> work_dir => ?config(priv_dir, Config) }), {ok, _} = emqx_resource:create_local( + ldap, ?LDAP_RESOURCE, ?AUTHN_RESOURCE_GROUP, emqx_ldap, diff --git a/apps/emqx_auth_ldap/test/emqx_authn_ldap_bind_SUITE.erl b/apps/emqx_auth_ldap/test/emqx_authn_ldap_bind_SUITE.erl index d8e9c0d8e..c39225e74 100644 --- a/apps/emqx_auth_ldap/test/emqx_authn_ldap_bind_SUITE.erl +++ b/apps/emqx_auth_ldap/test/emqx_authn_ldap_bind_SUITE.erl @@ -47,6 +47,7 @@ init_per_suite(Config) -> work_dir => ?config(priv_dir, Config) }), {ok, _} = emqx_resource:create_local( + ldap, ?LDAP_RESOURCE, ?AUTHN_RESOURCE_GROUP, emqx_ldap, diff --git a/apps/emqx_auth_ldap/test/emqx_authz_ldap_SUITE.erl b/apps/emqx_auth_ldap/test/emqx_authz_ldap_SUITE.erl index 09875a3fa..3f8c0ba63 100644 --- a/apps/emqx_auth_ldap/test/emqx_authz_ldap_SUITE.erl +++ b/apps/emqx_auth_ldap/test/emqx_authz_ldap_SUITE.erl @@ -178,6 +178,7 @@ stop_apps(Apps) -> create_ldap_resource() -> {ok, _} = emqx_resource:create_local( + ldap, ?LDAP_RESOURCE, ?AUTHZ_RESOURCE_GROUP, emqx_ldap, diff --git a/apps/emqx_auth_mongodb/src/emqx_auth_mongodb.app.src b/apps/emqx_auth_mongodb/src/emqx_auth_mongodb.app.src index df3cc1268..5ffc59787 100644 --- a/apps/emqx_auth_mongodb/src/emqx_auth_mongodb.app.src +++ b/apps/emqx_auth_mongodb/src/emqx_auth_mongodb.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_auth_mongodb, [ {description, "EMQX MongoDB Authentication and Authorization"}, - {vsn, "0.2.0"}, + {vsn, "0.2.1"}, {registered, []}, {mod, {emqx_auth_mongodb_app, []}}, {applications, [ diff --git a/apps/emqx_auth_mongodb/src/emqx_authn_mongodb.erl b/apps/emqx_auth_mongodb/src/emqx_authn_mongodb.erl index 75a474c0c..ffe78159d 100644 --- a/apps/emqx_auth_mongodb/src/emqx_authn_mongodb.erl +++ b/apps/emqx_auth_mongodb/src/emqx_authn_mongodb.erl @@ -37,6 +37,7 @@ create(Config0) -> ResourceId = emqx_authn_utils:make_resource_id(?MODULE), {Config, State} = parse_config(Config0), {ok, _Data} = emqx_authn_utils:create_resource( + mongodb, ResourceId, emqx_mongodb, Config @@ -45,7 +46,7 @@ create(Config0) -> update(Config0, #{resource_id := ResourceId} = _State) -> {Config, NState} = parse_config(Config0), - case emqx_authn_utils:update_resource(emqx_mongodb, Config, ResourceId) of + case emqx_authn_utils:update_resource(mongodb, emqx_mongodb, Config, ResourceId) of {error, Reason} -> error({load_config_error, Reason}); {ok, _} -> diff --git a/apps/emqx_auth_mongodb/src/emqx_authz_mongodb.erl b/apps/emqx_auth_mongodb/src/emqx_authz_mongodb.erl index 0bab6ef90..0d65afe0d 100644 --- a/apps/emqx_auth_mongodb/src/emqx_authz_mongodb.erl +++ b/apps/emqx_auth_mongodb/src/emqx_authz_mongodb.erl @@ -49,13 +49,13 @@ description() -> create(#{filter := Filter} = Source) -> ResourceId = emqx_authz_utils:make_resource_id(?MODULE), - {ok, _Data} = emqx_authz_utils:create_resource(ResourceId, emqx_mongodb, Source), + {ok, _Data} = emqx_authz_utils:create_resource(mongodb, ResourceId, emqx_mongodb, Source), FilterTemp = emqx_authz_utils:parse_deep(Filter, ?ALLOWED_VARS), Source#{annotations => #{id => ResourceId}, filter_template => FilterTemp}. update(#{filter := Filter} = Source) -> FilterTemp = emqx_authz_utils:parse_deep(Filter, ?ALLOWED_VARS), - case emqx_authz_utils:update_resource(emqx_mongodb, Source) of + case emqx_authz_utils:update_resource(mongodb, emqx_mongodb, Source) of {error, Reason} -> error({load_config_error, Reason}); {ok, Id} -> diff --git a/apps/emqx_auth_mysql/src/emqx_authn_mysql.erl b/apps/emqx_auth_mysql/src/emqx_authn_mysql.erl index f68c74a14..7d03ca856 100644 --- a/apps/emqx_auth_mysql/src/emqx_authn_mysql.erl +++ b/apps/emqx_auth_mysql/src/emqx_authn_mysql.erl @@ -39,12 +39,12 @@ create(_AuthenticatorID, Config) -> create(Config0) -> ResourceId = emqx_authn_utils:make_resource_id(?MODULE), {Config, State} = parse_config(Config0), - {ok, _Data} = emqx_authn_utils:create_resource(ResourceId, emqx_mysql, Config), + {ok, _Data} = emqx_authn_utils:create_resource(mysql, ResourceId, emqx_mysql, Config), {ok, State#{resource_id => ResourceId}}. update(Config0, #{resource_id := ResourceId} = _State) -> {Config, NState} = parse_config(Config0), - case emqx_authn_utils:update_resource(emqx_mysql, Config, ResourceId) of + case emqx_authn_utils:update_resource(mysql, emqx_mysql, Config, ResourceId) of {error, Reason} -> error({load_config_error, Reason}); {ok, _} -> diff --git a/apps/emqx_auth_mysql/src/emqx_authz_mysql.erl b/apps/emqx_auth_mysql/src/emqx_authz_mysql.erl index 0e2b77005..557eed14f 100644 --- a/apps/emqx_auth_mysql/src/emqx_authz_mysql.erl +++ b/apps/emqx_auth_mysql/src/emqx_authz_mysql.erl @@ -53,13 +53,13 @@ create(#{query := SQL} = Source0) -> {PrepareSQL, TmplToken} = emqx_authz_utils:parse_sql(SQL, '?', ?ALLOWED_VARS), ResourceId = emqx_authz_utils:make_resource_id(?MODULE), Source = Source0#{prepare_statement => #{?PREPARE_KEY => PrepareSQL}}, - {ok, _Data} = emqx_authz_utils:create_resource(ResourceId, emqx_mysql, Source), + {ok, _Data} = emqx_authz_utils:create_resource(mysql, ResourceId, emqx_mysql, Source), Source#{annotations => #{id => ResourceId, tmpl_token => TmplToken}}. update(#{query := SQL} = Source0) -> {PrepareSQL, TmplToken} = emqx_authz_utils:parse_sql(SQL, '?', ?ALLOWED_VARS), Source = Source0#{prepare_statement => #{?PREPARE_KEY => PrepareSQL}}, - case emqx_authz_utils:update_resource(emqx_mysql, Source) of + case emqx_authz_utils:update_resource(mysql, emqx_mysql, Source) of {error, Reason} -> error({load_config_error, Reason}); {ok, Id} -> diff --git a/apps/emqx_auth_mysql/test/emqx_authn_mysql_SUITE.erl b/apps/emqx_auth_mysql/test/emqx_authn_mysql_SUITE.erl index 8ab812fc0..78b8aa8b3 100644 --- a/apps/emqx_auth_mysql/test/emqx_authn_mysql_SUITE.erl +++ b/apps/emqx_auth_mysql/test/emqx_authn_mysql_SUITE.erl @@ -58,6 +58,7 @@ init_per_suite(Config) -> work_dir => ?config(priv_dir, Config) }), {ok, _} = emqx_resource:create_local( + mysql, ?MYSQL_RESOURCE, ?AUTHN_RESOURCE_GROUP, emqx_mysql, diff --git a/apps/emqx_auth_mysql/test/emqx_authz_mysql_SUITE.erl b/apps/emqx_auth_mysql/test/emqx_authz_mysql_SUITE.erl index ce30e203e..5fe020f1b 100644 --- a/apps/emqx_auth_mysql/test/emqx_authz_mysql_SUITE.erl +++ b/apps/emqx_auth_mysql/test/emqx_authz_mysql_SUITE.erl @@ -446,6 +446,7 @@ stop_apps(Apps) -> create_mysql_resource() -> {ok, _} = emqx_resource:create_local( + mysql, ?MYSQL_RESOURCE, ?AUTHZ_RESOURCE_GROUP, emqx_mysql, diff --git a/apps/emqx_auth_postgresql/src/emqx_authn_postgresql.erl b/apps/emqx_auth_postgresql/src/emqx_authn_postgresql.erl index 980c2ddd8..95e8ac7ce 100644 --- a/apps/emqx_auth_postgresql/src/emqx_authn_postgresql.erl +++ b/apps/emqx_auth_postgresql/src/emqx_authn_postgresql.erl @@ -45,6 +45,7 @@ create(Config0) -> ResourceId = emqx_authn_utils:make_resource_id(?MODULE), {Config, State} = parse_config(Config0, ResourceId), {ok, _Data} = emqx_authn_utils:create_resource( + postgresql, ResourceId, emqx_postgresql, Config @@ -53,7 +54,7 @@ create(Config0) -> update(Config0, #{resource_id := ResourceId} = _State) -> {Config, NState} = parse_config(Config0, ResourceId), - case emqx_authn_utils:update_resource(emqx_postgresql, Config, ResourceId) of + case emqx_authn_utils:update_resource(postgresql, emqx_postgresql, Config, ResourceId) of {error, Reason} -> error({load_config_error, Reason}); {ok, _} -> diff --git a/apps/emqx_auth_postgresql/src/emqx_authz_postgresql.erl b/apps/emqx_auth_postgresql/src/emqx_authz_postgresql.erl index d1a0b32ea..e0fc9e032 100644 --- a/apps/emqx_auth_postgresql/src/emqx_authz_postgresql.erl +++ b/apps/emqx_auth_postgresql/src/emqx_authz_postgresql.erl @@ -53,6 +53,7 @@ create(#{query := SQL0} = Source) -> {SQL, PlaceHolders} = emqx_authz_utils:parse_sql(SQL0, '$n', ?ALLOWED_VARS), ResourceID = emqx_authz_utils:make_resource_id(emqx_postgresql), {ok, _Data} = emqx_authz_utils:create_resource( + postgresql, ResourceID, emqx_postgresql, Source#{prepare_statement => #{ResourceID => SQL}} @@ -63,6 +64,7 @@ update(#{query := SQL0, annotations := #{id := ResourceID}} = Source) -> {SQL, PlaceHolders} = emqx_authz_utils:parse_sql(SQL0, '$n', ?ALLOWED_VARS), case emqx_authz_utils:update_resource( + postgresql, emqx_postgresql, Source#{prepare_statement => #{ResourceID => SQL}} ) diff --git a/apps/emqx_auth_postgresql/test/emqx_authn_postgresql_SUITE.erl b/apps/emqx_auth_postgresql/test/emqx_authn_postgresql_SUITE.erl index 50bff634d..e899edbd9 100644 --- a/apps/emqx_auth_postgresql/test/emqx_authn_postgresql_SUITE.erl +++ b/apps/emqx_auth_postgresql/test/emqx_authn_postgresql_SUITE.erl @@ -79,6 +79,7 @@ init_per_suite(Config) -> work_dir => ?config(priv_dir, Config) }), {ok, _} = emqx_resource:create_local( + postgresql, ?PGSQL_RESOURCE, ?AUTHN_RESOURCE_GROUP, emqx_postgresql, @@ -198,9 +199,9 @@ test_user_auth(#{ t_authenticate_disabled_prepared_statements(_Config) -> ResConfig = maps:merge(pgsql_config(), #{disable_prepared_statements => true}), - {ok, _} = emqx_resource:recreate_local(?PGSQL_RESOURCE, emqx_postgresql, ResConfig), + {ok, _} = emqx_resource:recreate_local(postgresql, ?PGSQL_RESOURCE, emqx_postgresql, ResConfig), on_exit(fun() -> - emqx_resource:recreate_local(?PGSQL_RESOURCE, emqx_postgresql, pgsql_config()) + emqx_resource:recreate_local(postgresql, ?PGSQL_RESOURCE, emqx_postgresql, pgsql_config()) end), ok = lists:foreach( fun(Sample0) -> diff --git a/apps/emqx_auth_postgresql/test/emqx_authz_postgresql_SUITE.erl b/apps/emqx_auth_postgresql/test/emqx_authz_postgresql_SUITE.erl index 78b1e17a8..9346bf863 100644 --- a/apps/emqx_auth_postgresql/test/emqx_authz_postgresql_SUITE.erl +++ b/apps/emqx_auth_postgresql/test/emqx_authz_postgresql_SUITE.erl @@ -437,6 +437,7 @@ pgsql_config() -> create_pgsql_resource() -> emqx_resource:create_local( + postgresql, ?PGSQL_RESOURCE, ?AUTHZ_RESOURCE_GROUP, emqx_postgresql, diff --git a/apps/emqx_auth_redis/src/emqx_authn_redis.erl b/apps/emqx_auth_redis/src/emqx_authn_redis.erl index 779c58e39..4eb5a36a3 100644 --- a/apps/emqx_auth_redis/src/emqx_authn_redis.erl +++ b/apps/emqx_auth_redis/src/emqx_authn_redis.erl @@ -42,6 +42,7 @@ create(Config0) -> Res; {Config, State} -> {ok, _Data} = emqx_authn_utils:create_resource( + redis, ResourceId, emqx_redis, Config @@ -51,7 +52,7 @@ create(Config0) -> update(Config0, #{resource_id := ResourceId} = _State) -> {Config, NState} = parse_config(Config0), - case emqx_authn_utils:update_resource(emqx_redis, Config, ResourceId) of + case emqx_authn_utils:update_resource(redis, emqx_redis, Config, ResourceId) of {error, Reason} -> error({load_config_error, Reason}); {ok, _} -> diff --git a/apps/emqx_auth_redis/src/emqx_authz_redis.erl b/apps/emqx_auth_redis/src/emqx_authz_redis.erl index 8ce975033..b83d1abad 100644 --- a/apps/emqx_auth_redis/src/emqx_authz_redis.erl +++ b/apps/emqx_auth_redis/src/emqx_authz_redis.erl @@ -50,12 +50,12 @@ description() -> create(#{cmd := CmdStr} = Source) -> CmdTemplate = parse_cmd(CmdStr), ResourceId = emqx_authz_utils:make_resource_id(?MODULE), - {ok, _Data} = emqx_authz_utils:create_resource(ResourceId, emqx_redis, Source), + {ok, _Data} = emqx_authz_utils:create_resource(redis, ResourceId, emqx_redis, Source), Source#{annotations => #{id => ResourceId}, cmd_template => CmdTemplate}. update(#{cmd := CmdStr} = Source) -> CmdTemplate = parse_cmd(CmdStr), - case emqx_authz_utils:update_resource(emqx_redis, Source) of + case emqx_authz_utils:update_resource(redis, emqx_redis, Source) of {error, Reason} -> error({load_config_error, Reason}); {ok, Id} -> diff --git a/apps/emqx_auth_redis/test/emqx_authn_redis_SUITE.erl b/apps/emqx_auth_redis/test/emqx_authn_redis_SUITE.erl index e8c8760de..1e9d825d2 100644 --- a/apps/emqx_auth_redis/test/emqx_authn_redis_SUITE.erl +++ b/apps/emqx_auth_redis/test/emqx_authn_redis_SUITE.erl @@ -63,6 +63,7 @@ init_per_suite(Config) -> work_dir => ?config(priv_dir, Config) }), {ok, _} = emqx_resource:create_local( + redis, ?REDIS_RESOURCE, ?AUTHN_RESOURCE_GROUP, emqx_redis, diff --git a/apps/emqx_auth_redis/test/emqx_authz_redis_SUITE.erl b/apps/emqx_auth_redis/test/emqx_authz_redis_SUITE.erl index 5818eea07..d0c695c73 100644 --- a/apps/emqx_auth_redis/test/emqx_authz_redis_SUITE.erl +++ b/apps/emqx_auth_redis/test/emqx_authz_redis_SUITE.erl @@ -384,6 +384,7 @@ stop_apps(Apps) -> create_redis_resource() -> {ok, _} = emqx_resource:create_local( + redis, ?REDIS_RESOURCE, ?AUTHZ_RESOURCE_GROUP, emqx_redis, diff --git a/apps/emqx_bridge/src/emqx_bridge_resource.erl b/apps/emqx_bridge/src/emqx_bridge_resource.erl index d2408ca73..7e9f5300a 100644 --- a/apps/emqx_bridge/src/emqx_bridge_resource.erl +++ b/apps/emqx_bridge/src/emqx_bridge_resource.erl @@ -195,8 +195,9 @@ create(Type, Name, Conf0, Opts) -> TypeBin = bin(Type), Conf = Conf0#{bridge_type => TypeBin, bridge_name => Name}, {ok, _Data} = emqx_resource:create_local( + Type, resource_id(Type, Name), - <<"emqx_bridge">>, + <<"bridge">>, bridge_to_resource_type(Type), parse_confs(TypeBin, Name, Conf), parse_opts(Conf, Opts) @@ -264,6 +265,7 @@ recreate(Type, Name, Conf0, Opts) -> TypeBin = bin(Type), Conf = Conf0#{bridge_type => TypeBin, bridge_name => Name}, emqx_resource:recreate_local( + Type, resource_id(Type, Name), bridge_to_resource_type(Type), parse_confs(TypeBin, Name, Conf), @@ -300,7 +302,7 @@ create_dry_run_bridge_v1(Type, Conf0) -> {error, Reason}; {ok, ConfNew} -> ParseConf = parse_confs(TypeBin, TmpName, ConfNew), - emqx_resource:create_dry_run_local(bridge_to_resource_type(Type), ParseConf) + emqx_resource:create_dry_run_local(Type, bridge_to_resource_type(Type), ParseConf) end catch %% validation errors diff --git a/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_connector_SUITE.erl b/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_connector_SUITE.erl index b784d36c0..02c102832 100644 --- a/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_connector_SUITE.erl +++ b/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_connector_SUITE.erl @@ -212,6 +212,7 @@ check_config(Config) -> create_local_resource(ResourceId, CheckedConfig) -> {ok, Bridge} = emqx_resource:create_local( + cassandra, ResourceId, ?CONNECTOR_RESOURCE_GROUP, ?CASSANDRA_RESOURCE_MOD, diff --git a/apps/emqx_bridge_clickhouse/test/emqx_bridge_clickhouse_connector_SUITE.erl b/apps/emqx_bridge_clickhouse/test/emqx_bridge_clickhouse_connector_SUITE.erl index 1c83961a5..0b0acd78a 100644 --- a/apps/emqx_bridge_clickhouse/test/emqx_bridge_clickhouse_connector_SUITE.erl +++ b/apps/emqx_bridge_clickhouse/test/emqx_bridge_clickhouse_connector_SUITE.erl @@ -109,6 +109,7 @@ t_start_passfile(Config) -> ?assertMatch( {ok, #{status := connected}}, emqx_resource:create_local( + clickhouse, ResourceID, ?CONNECTOR_RESOURCE_GROUP, ?CLICKHOUSE_RESOURCE_MOD, @@ -138,6 +139,7 @@ perform_lifecycle_check(ResourceID, InitialConfig) -> status := InitialStatus }} = emqx_resource:create_local( + clickhouse, ResourceID, ?CONNECTOR_RESOURCE_GROUP, ?CLICKHOUSE_RESOURCE_MOD, diff --git a/apps/emqx_bridge_greptimedb/test/emqx_bridge_greptimedb_connector_SUITE.erl b/apps/emqx_bridge_greptimedb/test/emqx_bridge_greptimedb_connector_SUITE.erl index be36cb167..68a32e9c2 100644 --- a/apps/emqx_bridge_greptimedb/test/emqx_bridge_greptimedb_connector_SUITE.erl +++ b/apps/emqx_bridge_greptimedb/test/emqx_bridge_greptimedb_connector_SUITE.erl @@ -83,6 +83,7 @@ perform_lifecycle_check(PoolName, InitialConfig) -> state := #{client := #{pool := ReturnedPoolName}} = State, status := InitialStatus }} = emqx_resource:create_local( + greptimedb, PoolName, ?CONNECTOR_RESOURCE_GROUP, ?GREPTIMEDB_RESOURCE_MOD, diff --git a/apps/emqx_bridge_influxdb/test/emqx_bridge_influxdb_connector_SUITE.erl b/apps/emqx_bridge_influxdb/test/emqx_bridge_influxdb_connector_SUITE.erl index 0ca693171..a7f78f253 100644 --- a/apps/emqx_bridge_influxdb/test/emqx_bridge_influxdb_connector_SUITE.erl +++ b/apps/emqx_bridge_influxdb/test/emqx_bridge_influxdb_connector_SUITE.erl @@ -86,6 +86,7 @@ perform_lifecycle_check(PoolName, InitialConfig) -> state := #{client := #{pool := ReturnedPoolName}} = State, status := InitialStatus }} = emqx_resource:create_local( + influxdb, PoolName, ?CONNECTOR_RESOURCE_GROUP, ?INFLUXDB_RESOURCE_MOD, @@ -197,6 +198,7 @@ perform_tls_opts_check(PoolName, InitialConfig, VerifyReturn) -> config := #{ssl := #{enable := SslEnabled}}, status := Status }} = emqx_resource:create_local( + influxdb, PoolName, ?CONNECTOR_RESOURCE_GROUP, ?INFLUXDB_RESOURCE_MOD, diff --git a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_connector.erl b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_connector.erl index 4157deec2..835536bda 100644 --- a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_connector.erl +++ b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_connector.erl @@ -289,7 +289,7 @@ replayq_dir(ClientId) -> filename:join([emqx:data_dir(), "pulsar", emqx_utils_conv:bin(ClientId)]). producer_name(InstanceId, ChannelId) -> - case is_dry_run(InstanceId) of + case emqx_resource:is_dry_run(InstanceId) of %% do not create more atom true -> pulsar_producer_probe_worker; diff --git a/apps/emqx_bridge_rabbitmq/test/emqx_bridge_rabbitmq_connector_SUITE.erl b/apps/emqx_bridge_rabbitmq/test/emqx_bridge_rabbitmq_connector_SUITE.erl index 77482ae0f..fe288a185 100644 --- a/apps/emqx_bridge_rabbitmq/test/emqx_bridge_rabbitmq_connector_SUITE.erl +++ b/apps/emqx_bridge_rabbitmq/test/emqx_bridge_rabbitmq_connector_SUITE.erl @@ -135,6 +135,7 @@ check_config(Config) -> create_local_resource(ResourceID, CheckedConfig) -> {ok, Bridge} = emqx_resource:create_local( + rabbitmq, ResourceID, ?CONNECTOR_RESOURCE_GROUP, emqx_bridge_rabbitmq_connector, diff --git a/apps/emqx_connector/include/emqx_connector.hrl b/apps/emqx_connector/include/emqx_connector.hrl index 4b29dd5ce..0004cd72c 100644 --- a/apps/emqx_connector/include/emqx_connector.hrl +++ b/apps/emqx_connector/include/emqx_connector.hrl @@ -37,4 +37,4 @@ "The " ++ TYPE ++ " default port " ++ DEFAULT_PORT ++ " is used if `[:Port]` is not specified." ). --define(CONNECTOR_RESOURCE_GROUP, <<"emqx_connector">>). +-define(CONNECTOR_RESOURCE_GROUP, <<"connector">>). diff --git a/apps/emqx_connector/src/emqx_connector_resource.erl b/apps/emqx_connector/src/emqx_connector_resource.erl index be8d3a32d..2a5b3bcfc 100644 --- a/apps/emqx_connector/src/emqx_connector_resource.erl +++ b/apps/emqx_connector/src/emqx_connector_resource.erl @@ -18,6 +18,7 @@ -include("../../emqx_bridge/include/emqx_bridge_resource.hrl"). -include_lib("emqx/include/logger.hrl"). -include_lib("emqx_resource/include/emqx_resource.hrl"). +-include("emqx_connector.hrl"). -export([ connector_to_resource_type/1, @@ -125,8 +126,9 @@ create(Type, Name, Conf0, Opts) -> ResourceId = resource_id(Type, Name), Conf = Conf0#{connector_type => TypeBin, connector_name => Name}, {ok, _Data} = emqx_resource:create_local( + Type, ResourceId, - <<"emqx_connector">>, + ?CONNECTOR_RESOURCE_GROUP, ?MODULE:connector_to_resource_type(Type), parse_confs(TypeBin, Name, Conf), parse_opts(Conf, Opts) @@ -198,6 +200,7 @@ recreate(Type, Name, Conf) -> recreate(Type, Name, Conf, Opts) -> TypeBin = bin(Type), emqx_resource:recreate_local( + Type, resource_id(Type, Name), ?MODULE:connector_to_resource_type(Type), parse_confs(TypeBin, Name, Conf), @@ -234,7 +237,7 @@ create_dry_run(Type, Conf0, Callback) -> {ok, ConfNew} -> ParseConf = parse_confs(bin(Type), TmpName, ConfNew), emqx_resource:create_dry_run_local( - TmpName, ?MODULE:connector_to_resource_type(Type), ParseConf, Callback + Type, TmpName, ?MODULE:connector_to_resource_type(Type), ParseConf, Callback ) end catch diff --git a/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_manager.erl b/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_manager.erl index 6834da9e9..6ac02efc6 100644 --- a/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_manager.erl +++ b/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_manager.erl @@ -45,7 +45,7 @@ -define(MOD_TAB, emqx_dashboard_sso). -define(MOD_KEY_PATH, [dashboard, sso]). -define(MOD_KEY_PATH(Sub), [dashboard, sso, Sub]). --define(RESOURCE_GROUP, <<"emqx_dashboard_sso">>). +-define(RESOURCE_GROUP, <<"dashboard_sso">>). -define(NO_ERROR, <<>>). -define(DEFAULT_RESOURCE_OPTS, #{ start_after_created => false @@ -132,6 +132,7 @@ make_resource_id(Backend) -> create_resource(ResourceId, Module, Config) -> Result = emqx_resource:create_local( + dashboard_sso, ResourceId, ?RESOURCE_GROUP, Module, @@ -142,7 +143,7 @@ create_resource(ResourceId, Module, Config) -> update_resource(ResourceId, Module, Config) -> Result = emqx_resource:recreate_local( - ResourceId, Module, Config, ?DEFAULT_RESOURCE_OPTS + dashboard_sso, ResourceId, Module, Config, ?DEFAULT_RESOURCE_OPTS ), start_resource_if_enabled(ResourceId, Result, Config). diff --git a/apps/emqx_dashboard_sso/test/emqx_dashboard_sso_ldap_SUITE.erl b/apps/emqx_dashboard_sso/test/emqx_dashboard_sso_ldap_SUITE.erl index 51524f0fd..40c5de9e5 100644 --- a/apps/emqx_dashboard_sso/test/emqx_dashboard_sso_ldap_SUITE.erl +++ b/apps/emqx_dashboard_sso/test/emqx_dashboard_sso_ldap_SUITE.erl @@ -24,7 +24,7 @@ -define(MOD_TAB, emqx_dashboard_sso). -define(MOD_KEY_PATH, [dashboard, sso, ldap]). --define(RESOURCE_GROUP, <<"emqx_dashboard_sso">>). +-define(RESOURCE_GROUP, <<"dashboard_sso">>). -import(emqx_mgmt_api_test_util, [request/2, request/3, uri/1, request_api/3]). diff --git a/apps/emqx_ldap/test/emqx_ldap_SUITE.erl b/apps/emqx_ldap/test/emqx_ldap_SUITE.erl index a15ff2775..413cbc3a5 100644 --- a/apps/emqx_ldap/test/emqx_ldap_SUITE.erl +++ b/apps/emqx_ldap/test/emqx_ldap_SUITE.erl @@ -96,6 +96,7 @@ perform_lifecycle_check(ResourceId, InitialConfig) -> state := #{pool_name := PoolName} = State, status := InitialStatus }} = emqx_resource:create_local( + ldap, ResourceId, ?CONNECTOR_RESOURCE_GROUP, ?LDAP_RESOURCE_MOD, @@ -171,6 +172,7 @@ t_get_status(Config) -> ?LDAP_RESOURCE_MOD, ldap_config(Config) ), {ok, _} = emqx_resource:create_local( + ldap, ResourceId, ?CONNECTOR_RESOURCE_GROUP, ?LDAP_RESOURCE_MOD, diff --git a/apps/emqx_mongodb/test/emqx_mongodb_SUITE.erl b/apps/emqx_mongodb/test/emqx_mongodb_SUITE.erl index 8af05e0d3..850683d99 100644 --- a/apps/emqx_mongodb/test/emqx_mongodb_SUITE.erl +++ b/apps/emqx_mongodb/test/emqx_mongodb_SUITE.erl @@ -143,6 +143,7 @@ check_config(Config) -> create_local_resource(ResourceId, CheckedConfig) -> {ok, Bridge} = emqx_resource:create_local( + mongodb, ResourceId, ?CONNECTOR_RESOURCE_GROUP, ?MONGO_RESOURCE_MOD, diff --git a/apps/emqx_mysql/test/emqx_mysql_SUITE.erl b/apps/emqx_mysql/test/emqx_mysql_SUITE.erl index 03e6c6797..be69140fc 100644 --- a/apps/emqx_mysql/test/emqx_mysql_SUITE.erl +++ b/apps/emqx_mysql/test/emqx_mysql_SUITE.erl @@ -67,6 +67,7 @@ perform_lifecycle_check(ResourceId, InitialConfig) -> state := #{pool_name := PoolName} = State, status := InitialStatus }} = emqx_resource:create_local( + mysql, ResourceId, ?CONNECTOR_RESOURCE_GROUP, ?MYSQL_RESOURCE_MOD, diff --git a/apps/emqx_postgresql/test/emqx_postgresql_SUITE.erl b/apps/emqx_postgresql/test/emqx_postgresql_SUITE.erl index d771d80d8..06210be86 100644 --- a/apps/emqx_postgresql/test/emqx_postgresql_SUITE.erl +++ b/apps/emqx_postgresql/test/emqx_postgresql_SUITE.erl @@ -75,6 +75,7 @@ perform_lifecycle_check(ResourceId, InitialConfig) -> status := InitialStatus }} = emqx_resource:create_local( + postgresql, ResourceId, ?CONNECTOR_RESOURCE_GROUP, ?PGSQL_RESOURCE_MOD, diff --git a/apps/emqx_redis/test/emqx_redis_SUITE.erl b/apps/emqx_redis/test/emqx_redis_SUITE.erl index a9064f184..06ac82143 100644 --- a/apps/emqx_redis/test/emqx_redis_SUITE.erl +++ b/apps/emqx_redis/test/emqx_redis_SUITE.erl @@ -115,6 +115,7 @@ perform_lifecycle_check(ResourceId, InitialConfig, RedisCommand) -> state := #{pool_name := PoolName} = State, status := InitialStatus }} = emqx_resource:create_local( + redis, ResourceId, ?CONNECTOR_RESOURCE_GROUP, ?REDIS_RESOURCE_MOD, diff --git a/apps/emqx_resource/include/emqx_resource.hrl b/apps/emqx_resource/include/emqx_resource.hrl index 587786cb2..aa86fe239 100644 --- a/apps/emqx_resource/include/emqx_resource.hrl +++ b/apps/emqx_resource/include/emqx_resource.hrl @@ -23,6 +23,7 @@ %% remind us of that. -define(rm_status_stopped, stopped). +-type type() :: atom() | binary(). -type resource_type() :: module(). -type resource_id() :: binary(). -type channel_id() :: binary(). diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl index b6f01fde5..71ccaa696 100644 --- a/apps/emqx_resource/src/emqx_resource.erl +++ b/apps/emqx_resource/src/emqx_resource.erl @@ -28,9 +28,9 @@ -export([ check_config/2, - check_and_create_local/4, check_and_create_local/5, - check_and_recreate_local/4 + check_and_create_local/6, + check_and_recreate_local/5 ]). %% Sync resource instances and files @@ -39,13 +39,13 @@ -export([ %% store the config and start the instance - create_local/4, create_local/5, - create_dry_run_local/2, + create_local/6, create_dry_run_local/3, create_dry_run_local/4, - recreate_local/3, + create_dry_run_local/5, recreate_local/4, + recreate_local/5, %% remove the config and stop the instance remove_local/1, reset_metrics/1, @@ -282,12 +282,13 @@ is_resource_mod(Module) -> %% APIs for resource instances %% ================================================================================= --spec create_local(resource_id(), resource_group(), resource_type(), resource_config()) -> +-spec create_local(type(), resource_id(), resource_group(), resource_type(), resource_config()) -> {ok, resource_data() | 'already_created'} | {error, Reason :: term()}. -create_local(ResId, Group, ResourceType, Config) -> - create_local(ResId, Group, ResourceType, Config, #{}). +create_local(Type, ResId, Group, ResourceType, Config) -> + create_local(Type, ResId, Group, ResourceType, Config, #{}). -spec create_local( + type(), resource_id(), resource_group(), resource_type(), @@ -295,33 +296,39 @@ create_local(ResId, Group, ResourceType, Config) -> creation_opts() ) -> {ok, resource_data()}. -create_local(ResId, Group, ResourceType, Config, Opts) -> - emqx_resource_manager:ensure_resource(ResId, Group, ResourceType, Config, Opts). +create_local(Type, ResId, Group, ResourceType, Config, Opts) -> + emqx_resource_manager:ensure_resource(Type, ResId, Group, ResourceType, Config, Opts). --spec create_dry_run_local(resource_type(), resource_config()) -> +-spec create_dry_run_local(type(), resource_type(), resource_config()) -> ok | {error, Reason :: term()}. -create_dry_run_local(ResourceType, Config) -> - emqx_resource_manager:create_dry_run(ResourceType, Config). +create_dry_run_local(Type, ResourceType, Config) -> + emqx_resource_manager:create_dry_run(Type, ResourceType, Config). -create_dry_run_local(ResId, ResourceType, Config) -> - emqx_resource_manager:create_dry_run(ResId, ResourceType, Config). +create_dry_run_local(Type, ResId, ResourceType, Config) -> + emqx_resource_manager:create_dry_run(Type, ResId, ResourceType, Config). --spec create_dry_run_local(resource_id(), resource_type(), resource_config(), OnReadyCallback) -> +-spec create_dry_run_local( + type(), + resource_id(), + resource_type(), + resource_config(), + OnReadyCallback +) -> ok | {error, Reason :: term()} when OnReadyCallback :: fun((resource_id()) -> ok | {error, Reason :: term()}). -create_dry_run_local(ResId, ResourceType, Config, OnReadyCallback) -> - emqx_resource_manager:create_dry_run(ResId, ResourceType, Config, OnReadyCallback). +create_dry_run_local(Type, ResId, ResourceType, Config, OnReadyCallback) -> + emqx_resource_manager:create_dry_run(Type, ResId, ResourceType, Config, OnReadyCallback). --spec recreate_local(resource_id(), resource_type(), resource_config()) -> +-spec recreate_local(type(), resource_id(), resource_type(), resource_config()) -> {ok, resource_data()} | {error, Reason :: term()}. -recreate_local(ResId, ResourceType, Config) -> - recreate_local(ResId, ResourceType, Config, #{}). +recreate_local(Type, ResId, ResourceType, Config) -> + recreate_local(Type, ResId, ResourceType, Config, #{}). --spec recreate_local(resource_id(), resource_type(), resource_config(), creation_opts()) -> +-spec recreate_local(type(), resource_id(), resource_type(), resource_config(), creation_opts()) -> {ok, resource_data()} | {error, Reason :: term()}. -recreate_local(ResId, ResourceType, Config, Opts) -> - emqx_resource_manager:recreate(ResId, ResourceType, Config, Opts). +recreate_local(Type, ResId, ResourceType, Config, Opts) -> + emqx_resource_manager:recreate(Type, ResId, ResourceType, Config, Opts). -spec remove_local(resource_id()) -> ok. remove_local(ResId) -> @@ -607,41 +614,44 @@ check_config(ResourceType, Conf) -> emqx_hocon:check(ResourceType, Conf). -spec check_and_create_local( + type(), resource_id(), resource_group(), resource_type(), raw_resource_config() ) -> {ok, resource_data()} | {error, term()}. -check_and_create_local(ResId, Group, ResourceType, RawConfig) -> - check_and_create_local(ResId, Group, ResourceType, RawConfig, #{}). +check_and_create_local(Type, ResId, Group, ResourceType, RawConfig) -> + check_and_create_local(Type, ResId, Group, ResourceType, RawConfig, #{}). -spec check_and_create_local( + type(), resource_id(), resource_group(), resource_type(), raw_resource_config(), creation_opts() ) -> {ok, resource_data()} | {error, term()}. -check_and_create_local(ResId, Group, ResourceType, RawConfig, Opts) -> +check_and_create_local(Type, ResId, Group, ResourceType, RawConfig, Opts) -> check_and_do( ResourceType, RawConfig, - fun(ResConf) -> create_local(ResId, Group, ResourceType, ResConf, Opts) end + fun(ResConf) -> create_local(Type, ResId, Group, ResourceType, ResConf, Opts) end ). -spec check_and_recreate_local( + type(), resource_id(), resource_type(), raw_resource_config(), creation_opts() ) -> {ok, resource_data()} | {error, term()}. -check_and_recreate_local(ResId, ResourceType, RawConfig, Opts) -> +check_and_recreate_local(Type, ResId, ResourceType, RawConfig, Opts) -> check_and_do( ResourceType, RawConfig, - fun(ResConf) -> recreate_local(ResId, ResourceType, ResConf, Opts) end + fun(ResConf) -> recreate_local(Type, ResId, ResourceType, ResConf, Opts) end ). check_and_do(ResourceType, RawConfig, Do) when is_function(Do) -> diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl index 95b1271f4..1c0b74edd 100644 --- a/apps/emqx_resource/src/emqx_resource_manager.erl +++ b/apps/emqx_resource/src/emqx_resource_manager.erl @@ -25,12 +25,12 @@ % API -export([ - ensure_resource/5, - recreate/4, + ensure_resource/6, + recreate/5, remove/1, - create_dry_run/2, create_dry_run/3, create_dry_run/4, + create_dry_run/5, restart/2, start/2, stop/1, @@ -59,7 +59,7 @@ ]). % Server --export([start_link/5]). +-export([start_link/6]). % Behaviour -export([init/1, callback_mode/0, handle_event/4, terminate/3]). @@ -75,6 +75,7 @@ -record(data, { id, group, + type, mod, callback_mode, query_mode, @@ -161,43 +162,44 @@ %% Triggers the emqx_resource_manager_sup supervisor to actually create %% and link the process itself if not already started. -spec ensure_resource( + type(), resource_id(), resource_group(), resource_type(), resource_config(), creation_opts() ) -> {ok, resource_data()}. -ensure_resource(ResId, Group, ResourceType, Config, Opts) -> +ensure_resource(Type, ResId, Group, ResourceType, Config, Opts) -> case lookup(ResId) of {ok, _Group, Data} -> {ok, Data}; {error, not_found} -> - create_and_return_data(ResId, Group, ResourceType, Config, Opts) + create_and_return_data(Type, ResId, Group, ResourceType, Config, Opts) end. %% @doc Called from emqx_resource when recreating a resource which may or may not exist --spec recreate(resource_id(), resource_type(), resource_config(), creation_opts()) -> +-spec recreate(type(), resource_id(), resource_type(), resource_config(), creation_opts()) -> {ok, resource_data()} | {error, not_found} | {error, updating_to_incorrect_resource_type}. -recreate(ResId, ResourceType, NewConfig, Opts) -> +recreate(Type, ResId, ResourceType, NewConfig, Opts) -> case lookup(ResId) of {ok, Group, #{mod := ResourceType, status := _} = _Data} -> _ = remove(ResId, false), - create_and_return_data(ResId, Group, ResourceType, NewConfig, Opts); + create_and_return_data(Type, ResId, Group, ResourceType, NewConfig, Opts); {ok, _, #{mod := Mod}} when Mod =/= ResourceType -> {error, updating_to_incorrect_resource_type}; {error, not_found} -> {error, not_found} end. -create_and_return_data(ResId, Group, ResourceType, Config, Opts) -> - _ = create(ResId, Group, ResourceType, Config, Opts), +create_and_return_data(Type, ResId, Group, ResourceType, Config, Opts) -> + _ = create(Type, ResId, Group, ResourceType, Config, Opts), {ok, _Group, Data} = lookup(ResId), {ok, Data}. %% @doc Create a resource_manager and wait until it is running -create(ResId, Group, ResourceType, Config, Opts) -> +create(Type, ResId, Group, ResourceType, Config, Opts) -> % The state machine will make the actual call to the callback/resource module after init - ok = emqx_resource_manager_sup:ensure_child(ResId, Group, ResourceType, Config, Opts), + ok = emqx_resource_manager_sup:ensure_child(Type, ResId, Group, ResourceType, Config, Opts), % Create metrics for the resource ok = emqx_resource:create_metrics(ResId), QueryMode = emqx_resource:query_mode(ResourceType, Config, Opts), @@ -219,30 +221,32 @@ create(ResId, Group, ResourceType, Config, Opts) -> %% @doc Called from `emqx_resource` when doing a dry run for creating a resource instance. %% %% Triggers the `emqx_resource_manager_sup` supervisor to actually create -%% and link the process itself if not already started, and then immedately stops. --spec create_dry_run(resource_type(), resource_config()) -> +%% and link the process itself if not already started, and then immediately stops. +-spec create_dry_run(type(), resource_type(), resource_config()) -> ok | {error, Reason :: term()}. -create_dry_run(ResourceType, Config) -> +create_dry_run(Type, ResourceType, Config) -> ResId = make_test_id(), - create_dry_run(ResId, ResourceType, Config). + create_dry_run(Type, ResId, ResourceType, Config). -create_dry_run(ResId, ResourceType, Config) -> - create_dry_run(ResId, ResourceType, Config, fun do_nothing_on_ready/1). +create_dry_run(Type, ResId, ResourceType, Config) -> + create_dry_run(Type, ResId, ResourceType, Config, fun do_nothing_on_ready/1). do_nothing_on_ready(_ResId) -> ok. --spec create_dry_run(resource_id(), resource_type(), resource_config(), OnReadyCallback) -> +-spec create_dry_run(type(), resource_id(), resource_type(), resource_config(), OnReadyCallback) -> ok | {error, Reason :: term()} when OnReadyCallback :: fun((resource_id()) -> ok | {error, Reason :: term()}). -create_dry_run(ResId, ResourceType, Config, OnReadyCallback) -> +create_dry_run(Type, ResId, ResourceType, Config, OnReadyCallback) -> Opts = case is_map(Config) of true -> maps:get(resource_opts, Config, #{}); false -> #{} end, - ok = emqx_resource_manager_sup:ensure_child(ResId, <<"dry_run">>, ResourceType, Config, Opts), + ok = emqx_resource_manager_sup:ensure_child( + Type, ResId, <<"dry_run">>, ResourceType, Config, Opts + ), HealthCheckInterval = maps:get(health_check_interval, Opts, ?HEALTHCHECK_INTERVAL), Timeout = emqx_utils:clamp(HealthCheckInterval, 5_000, 60_000), case wait_for_ready(ResId, Timeout) of @@ -491,7 +495,7 @@ try_clean_allocated_resources(ResId) -> %% Server start/stop callbacks %% @doc Function called from the supervisor to actually start the server -start_link(ResId, Group, ResourceType, Config, Opts) -> +start_link(Type, ResId, Group, ResourceType, Config, Opts) -> QueryMode = emqx_resource:query_mode( ResourceType, Config, @@ -499,6 +503,7 @@ start_link(ResId, Group, ResourceType, Config, Opts) -> ), Data = #data{ id = ResId, + type = Type, group = Group, mod = ResourceType, callback_mode = emqx_resource:get_callback_mode(ResourceType), @@ -683,6 +688,9 @@ handle_event(EventType, EventData, State, Data) -> error, #{ msg => "ignore_all_other_events", + resource_id => Data#data.id, + group => Data#data.group, + type => Data#data.type, event_type => EventType, event_data => EventData, state => State, @@ -752,8 +760,8 @@ handle_remove_event(From, ClearMetrics, Data) -> start_resource(Data, From) -> %% in case the emqx_resource:call_start/2 hangs, the lookup/1 can read status from the cache - ResId = Data#data.id, - case emqx_resource:call_start(ResId, Data#data.mod, Data#data.config) of + #data{id = ResId, mod = Mod, config = Config, group = Group, type = Type} = Data, + case emqx_resource:call_start(ResId, Mod, Config) of {ok, ResourceState} -> UpdatedData1 = Data#data{status = ?status_connecting, state = ResourceState}, %% Perform an initial health_check immediately before transitioning into a connected state @@ -764,7 +772,9 @@ start_resource(Data, From) -> IsDryRun = emqx_resource:is_dry_run(ResId), ?SLOG(log_level(IsDryRun), #{ msg => "start_resource_failed", - id => ResId, + resource_id => ResId, + group => Group, + type => Type, reason => Reason }), _ = maybe_alarm(?status_disconnected, IsDryRun, ResId, Err, Data#data.error), @@ -798,14 +808,20 @@ add_channels(Data) -> add_channels_in_list([], Data) -> Data; add_channels_in_list([{ChannelID, ChannelConfig} | Rest], Data) -> - Id = Data#data.id, + #data{ + id = ResId, + mod = Mod, + state = State, + added_channels = AddedChannelsMap, + group = Group, + type = Type + } = Data, case emqx_resource:call_add_channel( - Id, Data#data.mod, Data#data.state, ChannelID, ChannelConfig + ResId, Mod, State, ChannelID, ChannelConfig ) of {ok, NewState} -> - AddedChannelsMap = Data#data.added_channels, %% Set the channel status to connecting to indicate that %% we have not yet performed the initial health_check NewAddedChannelsMap = maps:put( @@ -819,10 +835,12 @@ add_channels_in_list([{ChannelID, ChannelConfig} | Rest], Data) -> }, add_channels_in_list(Rest, NewData); {error, Reason} = Error -> - IsDryRun = emqx_resource:is_dry_run(Id), + IsDryRun = emqx_resource:is_dry_run(ResId), ?SLOG(log_level(IsDryRun), #{ msg => "add_channel_failed", - id => Id, + resource_id => ResId, + type => Type, + group => Group, channel_id => ChannelID, reason => Reason }), @@ -872,9 +890,15 @@ remove_channels(Data) -> remove_channels_in_list([], Data, _KeepInChannelMap) -> Data; remove_channels_in_list([ChannelID | Rest], Data, KeepInChannelMap) -> - AddedChannelsMap = Data#data.added_channels, - Id = Data#data.id, - IsDryRun = emqx_resource:is_dry_run(Id), + #data{ + id = ResId, + added_channels = AddedChannelsMap, + mod = Mod, + state = State, + group = Group, + type = Type + } = Data, + IsDryRun = emqx_resource:is_dry_run(ResId), NewAddedChannelsMap = case KeepInChannelMap of true -> @@ -883,7 +907,7 @@ remove_channels_in_list([ChannelID | Rest], Data, KeepInChannelMap) -> _ = maybe_clear_alarm(IsDryRun, ChannelID), maps:remove(ChannelID, AddedChannelsMap) end, - case safe_call_remove_channel(Id, Data#data.mod, Data#data.state, ChannelID) of + case safe_call_remove_channel(ResId, Mod, State, ChannelID) of {ok, NewState} -> NewData = Data#data{ state = NewState, @@ -893,7 +917,9 @@ remove_channels_in_list([ChannelID | Rest], Data, KeepInChannelMap) -> {error, Reason} -> ?SLOG(log_level(IsDryRun), #{ msg => "remove_channel_failed", - id => Id, + resource_id => ResId, + group => Group, + type => Type, channel_id => ChannelID, reason => Reason }), @@ -997,7 +1023,12 @@ handle_remove_channel(From, ChannelId, Data) -> end. handle_remove_channel_exists(From, ChannelId, Data) -> - #data{id = Id, added_channels = AddedChannelsMap} = Data, + #data{ + id = Id, + group = Group, + type = Type, + added_channels = AddedChannelsMap + } = Data, case emqx_resource:call_remove_channel( Id, Data#data.mod, Data#data.state, ChannelId @@ -1014,7 +1045,9 @@ handle_remove_channel_exists(From, ChannelId, Data) -> IsDryRun = emqx_resource:is_dry_run(Id), ?SLOG(log_level(IsDryRun), #{ msg => "remove_channel_failed", - id => Id, + resource_id => Id, + group => Group, + type => Type, channel_id => ChannelId, reason => Reason }), @@ -1123,10 +1156,13 @@ continue_resource_health_check_connected(NewStatus, Data0) -> Actions = Replies ++ resource_health_check_actions(Data), {keep_state, Data, Actions}; _ -> - IsDryRun = emqx_resource:is_dry_run(Data0#data.id), + #data{id = ResId, group = Group, type = Type} = Data0, + IsDryRun = emqx_resource:is_dry_run(ResId), ?SLOG(log_level(IsDryRun), #{ msg => "health_check_failed", - id => Data0#data.id, + resource_id => ResId, + group => Group, + type => Type, status => NewStatus }), %% Note: works because, coincidentally, channel/resource status is a @@ -1633,6 +1669,8 @@ parse_health_check_result({error, Error}, Data) -> #{ msg => "health_check_exception", resource_id => Data#data.id, + type => Data#data.type, + group => Data#data.group, reason => Error } ), diff --git a/apps/emqx_resource/src/emqx_resource_manager_sup.erl b/apps/emqx_resource/src/emqx_resource_manager_sup.erl index 7af6eca81..c14b08f94 100644 --- a/apps/emqx_resource/src/emqx_resource_manager_sup.erl +++ b/apps/emqx_resource/src/emqx_resource_manager_sup.erl @@ -19,14 +19,16 @@ -include("emqx_resource.hrl"). --export([ensure_child/5, delete_child/1]). +-export([ensure_child/6, delete_child/1]). -export([start_link/0]). -export([init/1]). -ensure_child(ResId, Group, ResourceType, Config, Opts) -> - case supervisor:start_child(?MODULE, child_spec(ResId, Group, ResourceType, Config, Opts)) of +ensure_child(Type, ResId, Group, ResourceType, Config, Opts) -> + case + supervisor:start_child(?MODULE, child_spec(Type, ResId, Group, ResourceType, Config, Opts)) + of {error, Reason} -> %% This should not happen in production but it can be a huge time sink in %% development environments if the error is just silently ignored. @@ -55,13 +57,14 @@ init([]) -> SupFlags = #{strategy => one_for_one, intensity => 10, period => 10}, {ok, {SupFlags, ChildSpecs}}. -child_spec(ResId, Group, ResourceType, Config, Opts) -> +child_spec(Type, ResId, Group, ResourceType, Config, Opts) -> #{ id => ResId, - start => {emqx_resource_manager, start_link, [ResId, Group, ResourceType, Config, Opts]}, + start => + {emqx_resource_manager, start_link, [Type, ResId, Group, ResourceType, Config, Opts]}, restart => transient, %% never force kill a resource manager. - %% becasue otherwise it may lead to release leak, + %% because otherwise it may lead to release leak, %% resource_manager's terminate callback calls resource on_stop shutdown => infinity, type => worker, diff --git a/apps/emqx_resource/test/emqx_resource_SUITE.erl b/apps/emqx_resource/test/emqx_resource_SUITE.erl index af9abe95b..981c88edd 100644 --- a/apps/emqx_resource/test/emqx_resource_SUITE.erl +++ b/apps/emqx_resource/test/emqx_resource_SUITE.erl @@ -23,6 +23,7 @@ -include_lib("snabbkaffe/include/snabbkaffe.hrl"). -define(TEST_RESOURCE, emqx_connector_demo). +-define(TYPE, test). -define(ID, <<"id">>). -define(ID1, <<"id1">>). -define(DEFAULT_RESOURCE_GROUP, <<"default">>). @@ -90,6 +91,7 @@ t_create_remove(_) -> ?assertMatch( {error, _}, emqx_resource:check_and_create_local( + ?TYPE, ?ID, ?DEFAULT_RESOURCE_GROUP, ?TEST_RESOURCE, @@ -110,6 +112,7 @@ t_create_remove(_) -> ?assertMatch( {ok, _}, emqx_resource:recreate_local( + ?TYPE, ?ID, ?TEST_RESOURCE, #{name => test_resource}, @@ -135,6 +138,7 @@ t_create_remove_local(_) -> ?assertMatch( {error, _}, emqx_resource:check_and_create_local( + ?TYPE, ?ID, ?DEFAULT_RESOURCE_GROUP, ?TEST_RESOURCE, @@ -153,6 +157,7 @@ t_create_remove_local(_) -> ), emqx_resource:recreate_local( + ?TYPE, ?ID, ?TEST_RESOURCE, #{name => test_resource}, @@ -166,6 +171,7 @@ t_create_remove_local(_) -> emqx_resource:set_resource_status_connecting(?ID), emqx_resource:recreate_local( + ?TYPE, ?ID, ?TEST_RESOURCE, #{name => test_resource}, @@ -937,6 +943,7 @@ t_stop_start(_) -> ?assertMatch( {error, _}, emqx_resource:check_and_create_local( + ?TYPE, ?ID, ?DEFAULT_RESOURCE_GROUP, ?TEST_RESOURCE, @@ -947,6 +954,7 @@ t_stop_start(_) -> ?assertMatch( {ok, _}, emqx_resource:check_and_create_local( + ?TYPE, ?ID, ?DEFAULT_RESOURCE_GROUP, ?TEST_RESOURCE, @@ -964,6 +972,7 @@ t_stop_start(_) -> ?assertMatch( {ok, _}, emqx_resource:check_and_recreate_local( + ?TYPE, ?ID, ?TEST_RESOURCE, #{<<"name">> => <<"test_resource">>}, @@ -1013,6 +1022,7 @@ t_stop_start_local(_) -> ?assertMatch( {error, _}, emqx_resource:check_and_create_local( + ?TYPE, ?ID, ?DEFAULT_RESOURCE_GROUP, ?TEST_RESOURCE, @@ -1023,6 +1033,7 @@ t_stop_start_local(_) -> ?assertMatch( {ok, _}, emqx_resource:check_and_create_local( + ?TYPE, ?ID, ?DEFAULT_RESOURCE_GROUP, ?TEST_RESOURCE, @@ -1033,6 +1044,7 @@ t_stop_start_local(_) -> ?assertMatch( {ok, _}, emqx_resource:check_and_recreate_local( + ?TYPE, ?ID, ?TEST_RESOURCE, #{<<"name">> => <<"test_resource">>}, @@ -1108,6 +1120,7 @@ create_dry_run_local_succ() -> ?assertEqual( ok, emqx_resource:create_dry_run_local( + test, ?TEST_RESOURCE, #{name => test_resource, register => true} ) @@ -1118,6 +1131,7 @@ t_create_dry_run_local_failed(_) -> ct:timetrap({seconds, 120}), ct:pal("creating with creation error"), Res1 = emqx_resource:create_dry_run_local( + test, ?TEST_RESOURCE, #{create_error => true} ), @@ -1125,6 +1139,7 @@ t_create_dry_run_local_failed(_) -> ct:pal("creating with health check error"), Res2 = emqx_resource:create_dry_run_local( + test, ?TEST_RESOURCE, #{name => test_resource, health_check_error => true} ), @@ -1132,6 +1147,7 @@ t_create_dry_run_local_failed(_) -> ct:pal("creating with stop error"), Res3 = emqx_resource:create_dry_run_local( + test, ?TEST_RESOURCE, #{name => test_resource, stop_error => true} ), @@ -3490,10 +3506,10 @@ gauge_metric_set_fns() -> ]. create(Id, Group, Type, Config) -> - emqx_resource:create_local(Id, Group, Type, Config). + emqx_resource:create_local(test, Id, Group, Type, Config). create(Id, Group, Type, Config, Opts) -> - emqx_resource:create_local(Id, Group, Type, Config, Opts). + emqx_resource:create_local(test, Id, Group, Type, Config, Opts). log_consistency_prop() -> {"check state and cache consistency", fun ?MODULE:log_consistency_prop/1}. From e148d903e8bcdc0534a44c6f8435d70a7117cc2d Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Tue, 16 Jul 2024 15:51:10 +0800 Subject: [PATCH 3/9] feat: log resource_id --- .../src/emqx_bridge_mqtt.app.src | 2 +- .../src/emqx_bridge_mqtt_connector.erl | 17 ++++++++++------- apps/emqx_resource/include/emqx_resource.hrl | 1 + apps/emqx_resource/src/emqx_resource.erl | 2 +- apps/emqx_resource/src/emqx_resource_pool.erl | 11 +++++++---- 5 files changed, 20 insertions(+), 13 deletions(-) diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src index 26b8967f0..d43ec5591 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_bridge_mqtt, [ {description, "EMQX MQTT Broker Bridge"}, - {vsn, "0.2.2"}, + {vsn, "0.2.3"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl index d507d11b8..9c2506bab 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl @@ -207,7 +207,7 @@ start_mqtt_clients(ResourceId, Conf) -> start_mqtt_clients(ResourceId, Conf, ClientOpts). start_mqtt_clients(ResourceId, StartConf, ClientOpts) -> - PoolName = <>, + PoolName = ResourceId, #{ pool_size := PoolSize } = StartConf, @@ -227,7 +227,7 @@ start_mqtt_clients(ResourceId, StartConf, ClientOpts) -> on_stop(ResourceId, State) -> ?SLOG(info, #{ msg => "stopping_mqtt_connector", - connector => ResourceId + resource_id => ResourceId }), %% on_stop can be called with State = undefined StateMap = @@ -271,7 +271,7 @@ on_query( on_query(ResourceId, {_ChannelId, Msg}, #{}) -> ?SLOG(error, #{ msg => "forwarding_unavailable", - connector => ResourceId, + resource_id => ResourceId, message => Msg, reason => "Egress is not configured" }). @@ -298,7 +298,7 @@ on_query_async( on_query_async(ResourceId, {_ChannelId, Msg}, _Callback, #{}) -> ?SLOG(error, #{ msg => "forwarding_unavailable", - connector => ResourceId, + resource_id => ResourceId, message => Msg, reason => "Egress is not configured" }). @@ -463,8 +463,10 @@ connect(Options) -> {ok, Pid} -> connect(Pid, Name); {error, Reason} = Error -> - ?SLOG(error, #{ + IsDryRun = emqx_resource:is_dry_run(Name), + ?SLOG(?LOG_LEVEL(IsDryRun), #{ msg => "client_start_failed", + resource_id => Name, config => emqx_utils:redact(ClientOpts), reason => Reason }), @@ -508,10 +510,11 @@ connect(Pid, Name) -> {ok, _Props} -> {ok, Pid}; {error, Reason} = Error -> - ?SLOG(warning, #{ + IsDryRun = emqx_resource:is_dry_run(Name), + ?SLOG(?LOG_LEVEL(IsDryRun), #{ msg => "ingress_client_connect_failed", reason => Reason, - name => Name + resource_id => Name }), _ = catch emqtt:stop(Pid), Error diff --git a/apps/emqx_resource/include/emqx_resource.hrl b/apps/emqx_resource/include/emqx_resource.hrl index aa86fe239..3d141bca1 100644 --- a/apps/emqx_resource/include/emqx_resource.hrl +++ b/apps/emqx_resource/include/emqx_resource.hrl @@ -159,5 +159,6 @@ %% See `hocon_tconf` -define(TEST_ID_PREFIX, "t_probe_"). -define(RES_METRICS, resource_metrics). +-define(LOG_LEVEL(_L_), case _L_ of true -> info; false -> warning end). -define(RESOURCE_ALLOCATION_TAB, emqx_resource_allocations). diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl index 71ccaa696..76493b99f 100644 --- a/apps/emqx_resource/src/emqx_resource.erl +++ b/apps/emqx_resource/src/emqx_resource.erl @@ -340,7 +340,7 @@ remove_local(ResId) -> Error -> %% Only log, the ResId worker is always removed in manager's remove action. ?SLOG(warning, #{ - msg => "remove_local_resource_failed", + msg => "remove_resource_failed", error => Error, resource_id => ResId }), diff --git a/apps/emqx_resource/src/emqx_resource_pool.erl b/apps/emqx_resource/src/emqx_resource_pool.erl index 47e7ed7ff..516c483e3 100644 --- a/apps/emqx_resource/src/emqx_resource_pool.erl +++ b/apps/emqx_resource/src/emqx_resource_pool.erl @@ -26,6 +26,7 @@ ]). -include_lib("emqx/include/logger.hrl"). +-include("emqx_resource.hrl"). -ifndef(TEST). -define(HEALTH_CHECK_TIMEOUT, 15000). @@ -44,9 +45,10 @@ start(Name, Mod, Options) -> start(Name, Mod, Options); {error, Reason} -> NReason = parse_reason(Reason), - ?SLOG(error, #{ + IsDryRun = emqx_resource:is_dry_run(Name), + ?SLOG(?LOG_LEVEL(IsDryRun), #{ msg => "start_ecpool_error", - pool_name => Name, + resource_id => Name, reason => NReason }), {error, {start_pool_failed, Name, NReason}} @@ -59,9 +61,10 @@ stop(Name) -> {error, not_found} -> ok; {error, Reason} -> - ?SLOG(error, #{ + IsDryRun = emqx_resource:is_dry_run(Name), + ?SLOG(?LOG_LEVEL(IsDryRun), #{ msg => "stop_ecpool_failed", - pool_name => Name, + resource_id => Name, reason => Reason }), error({stop_pool_failed, Name, Reason}) From f29988ed8ebbda70ab15d6c24afb7778c978a961 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 17 Jul 2024 09:14:58 +0800 Subject: [PATCH 4/9] chore: add tag to resouce log --- apps/emqx_resource/include/emqx_resource.hrl | 7 +- .../src/emqx_resource_manager.erl | 102 ++++++++++-------- 2 files changed, 64 insertions(+), 45 deletions(-) diff --git a/apps/emqx_resource/include/emqx_resource.hrl b/apps/emqx_resource/include/emqx_resource.hrl index 3d141bca1..f26f6699c 100644 --- a/apps/emqx_resource/include/emqx_resource.hrl +++ b/apps/emqx_resource/include/emqx_resource.hrl @@ -159,6 +159,11 @@ %% See `hocon_tconf` -define(TEST_ID_PREFIX, "t_probe_"). -define(RES_METRICS, resource_metrics). --define(LOG_LEVEL(_L_), case _L_ of true -> info; false -> warning end). +-define(LOG_LEVEL(_L_), + case _L_ of + true -> info; + false -> warning + end +). -define(RESOURCE_ALLOCATION_TAB, emqx_resource_allocations). diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl index 1c0b74edd..a742c4486 100644 --- a/apps/emqx_resource/src/emqx_resource_manager.erl +++ b/apps/emqx_resource/src/emqx_resource_manager.erl @@ -689,13 +689,12 @@ handle_event(EventType, EventData, State, Data) -> #{ msg => "ignore_all_other_events", resource_id => Data#data.id, - group => Data#data.group, - type => Data#data.type, event_type => EventType, event_data => EventData, state => State, data => emqx_utils:redact(Data) - } + }, + #{tag => tag(Data#data.group, Data#data.type)} ), keep_state_and_data. @@ -770,13 +769,15 @@ start_resource(Data, From) -> {next_state, ?state_connecting, update_state(UpdatedData2, Data), Actions}; {error, Reason} = Err -> IsDryRun = emqx_resource:is_dry_run(ResId), - ?SLOG(log_level(IsDryRun), #{ - msg => "start_resource_failed", - resource_id => ResId, - group => Group, - type => Type, - reason => Reason - }), + ?SLOG( + log_level(IsDryRun), + #{ + msg => "start_resource_failed", + resource_id => ResId, + reason => Reason + }, + #{tag => tag(Group, Type)} + ), _ = maybe_alarm(?status_disconnected, IsDryRun, ResId, Err, Data#data.error), %% Add channels and raise alarms NewData1 = channels_health_check(?status_disconnected, add_channels(Data)), @@ -836,14 +837,16 @@ add_channels_in_list([{ChannelID, ChannelConfig} | Rest], Data) -> add_channels_in_list(Rest, NewData); {error, Reason} = Error -> IsDryRun = emqx_resource:is_dry_run(ResId), - ?SLOG(log_level(IsDryRun), #{ - msg => "add_channel_failed", - resource_id => ResId, - type => Type, - group => Group, - channel_id => ChannelID, - reason => Reason - }), + ?SLOG( + log_level(IsDryRun), + #{ + msg => "add_channel_failed", + resource_id => ResId, + channel_id => ChannelID, + reason => Reason + }, + #{tag => tag(Group, Type)} + ), AddedChannelsMap = Data#data.added_channels, NewAddedChannelsMap = maps:put( ChannelID, @@ -915,14 +918,18 @@ remove_channels_in_list([ChannelID | Rest], Data, KeepInChannelMap) -> }, remove_channels_in_list(Rest, NewData, KeepInChannelMap); {error, Reason} -> - ?SLOG(log_level(IsDryRun), #{ - msg => "remove_channel_failed", - resource_id => ResId, - group => Group, - type => Type, - channel_id => ChannelID, - reason => Reason - }), + ?SLOG( + log_level(IsDryRun), + #{ + msg => "remove_channel_failed", + resource_id => ResId, + group => Group, + type => Type, + channel_id => ChannelID, + reason => Reason + }, + #{tag => tag(Group, Type)} + ), NewData = Data#data{ added_channels = NewAddedChannelsMap }, @@ -1043,14 +1050,16 @@ handle_remove_channel_exists(From, ChannelId, Data) -> {keep_state, update_state(UpdatedData, Data), [{reply, From, ok}]}; {error, Reason} = Error -> IsDryRun = emqx_resource:is_dry_run(Id), - ?SLOG(log_level(IsDryRun), #{ - msg => "remove_channel_failed", - resource_id => Id, - group => Group, - type => Type, - channel_id => ChannelId, - reason => Reason - }), + ?SLOG( + log_level(IsDryRun), + #{ + msg => "remove_channel_failed", + resource_id => Id, + channel_id => ChannelId, + reason => Reason + }, + #{tag => tag(Group, Type)} + ), {keep_state_and_data, [{reply, From, Error}]} end. @@ -1158,13 +1167,15 @@ continue_resource_health_check_connected(NewStatus, Data0) -> _ -> #data{id = ResId, group = Group, type = Type} = Data0, IsDryRun = emqx_resource:is_dry_run(ResId), - ?SLOG(log_level(IsDryRun), #{ - msg => "health_check_failed", - resource_id => ResId, - group => Group, - type => Type, - status => NewStatus - }), + ?SLOG( + log_level(IsDryRun), + #{ + msg => "health_check_failed", + resource_id => ResId, + status => NewStatus + }, + #{tag => tag(Group, Type)} + ), %% Note: works because, coincidentally, channel/resource status is a %% subset of resource manager state... But there should be a conversion %% between the two here, as resource manager also has `stopped', which is @@ -1669,10 +1680,9 @@ parse_health_check_result({error, Error}, Data) -> #{ msg => "health_check_exception", resource_id => Data#data.id, - type => Data#data.type, - group => Data#data.group, reason => Error - } + }, + #{tag => tag(Data#data.group, Data#data.type)} ), {?status_disconnected, Data#data.state, {error, Error}}. @@ -1835,3 +1845,7 @@ state_to_status(?state_disconnected) -> ?status_disconnected. log_level(true) -> info; log_level(false) -> warning. + +tag(Group, Type) -> + Str = emqx_utils_conv:str(Group) ++ "/" ++ emqx_utils_conv:str(Type), + string:uppercase(Str). From 2bb062d3a377767e407d21ecb4f4c9ffc06ef0bf Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 17 Jul 2024 12:45:37 +0800 Subject: [PATCH 5/9] chore: create_local/5 for emqx_resource_proto_v1 --- apps/emqx_auth_jwt/src/emqx_authn_jwt.erl | 3 +- .../test/emqx_authn_postgresql_SUITE.erl | 8 +++- apps/emqx_resource/src/emqx_resource.erl | 45 +++++++++++-------- apps/emqx_resource/src/emqx_resource_pool.erl | 32 ++++++++----- .../test/emqx_resource_SUITE.erl | 2 +- 5 files changed, 56 insertions(+), 34 deletions(-) diff --git a/apps/emqx_auth_jwt/src/emqx_authn_jwt.erl b/apps/emqx_auth_jwt/src/emqx_authn_jwt.erl index e01121a01..2f8ebec36 100644 --- a/apps/emqx_auth_jwt/src/emqx_authn_jwt.erl +++ b/apps/emqx_auth_jwt/src/emqx_authn_jwt.erl @@ -187,7 +187,8 @@ do_create( ResourceId, ?AUTHN_RESOURCE_GROUP, emqx_authn_jwks_connector, - connector_opts(Config) + connector_opts(Config), + #{} ), {ok, #{ jwk_resource => ResourceId, diff --git a/apps/emqx_auth_postgresql/test/emqx_authn_postgresql_SUITE.erl b/apps/emqx_auth_postgresql/test/emqx_authn_postgresql_SUITE.erl index e899edbd9..dde5a184a 100644 --- a/apps/emqx_auth_postgresql/test/emqx_authn_postgresql_SUITE.erl +++ b/apps/emqx_auth_postgresql/test/emqx_authn_postgresql_SUITE.erl @@ -199,9 +199,13 @@ test_user_auth(#{ t_authenticate_disabled_prepared_statements(_Config) -> ResConfig = maps:merge(pgsql_config(), #{disable_prepared_statements => true}), - {ok, _} = emqx_resource:recreate_local(postgresql, ?PGSQL_RESOURCE, emqx_postgresql, ResConfig), + {ok, _} = emqx_resource:recreate_local( + postgresql, ?PGSQL_RESOURCE, emqx_postgresql, ResConfig, #{} + ), on_exit(fun() -> - emqx_resource:recreate_local(postgresql, ?PGSQL_RESOURCE, emqx_postgresql, pgsql_config()) + emqx_resource:recreate_local( + postgresql, ?PGSQL_RESOURCE, emqx_postgresql, pgsql_config(), #{} + ) end), ok = lists:foreach( fun(Sample0) -> diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl index 76493b99f..e0a2e9343 100644 --- a/apps/emqx_resource/src/emqx_resource.erl +++ b/apps/emqx_resource/src/emqx_resource.erl @@ -39,12 +39,10 @@ -export([ %% store the config and start the instance - create_local/5, create_local/6, create_dry_run_local/3, create_dry_run_local/4, create_dry_run_local/5, - recreate_local/4, recreate_local/5, %% remove the config and stop the instance remove_local/1, @@ -142,6 +140,9 @@ -export([is_dry_run/1]). +%% For emqx_resource_proto_v1 rpc only +-export([create_local/5, recreate_local/4, create_dry_run_local/2]). + -export_type([ query_mode/0, resource_id/0, @@ -281,12 +282,6 @@ is_resource_mod(Module) -> %% ================================================================================= %% APIs for resource instances %% ================================================================================= - --spec create_local(type(), resource_id(), resource_group(), resource_type(), resource_config()) -> - {ok, resource_data() | 'already_created'} | {error, Reason :: term()}. -create_local(Type, ResId, Group, ResourceType, Config) -> - create_local(Type, ResId, Group, ResourceType, Config, #{}). - -spec create_local( type(), resource_id(), @@ -320,11 +315,6 @@ when create_dry_run_local(Type, ResId, ResourceType, Config, OnReadyCallback) -> emqx_resource_manager:create_dry_run(Type, ResId, ResourceType, Config, OnReadyCallback). --spec recreate_local(type(), resource_id(), resource_type(), resource_config()) -> - {ok, resource_data()} | {error, Reason :: term()}. -recreate_local(Type, ResId, ResourceType, Config) -> - recreate_local(Type, ResId, ResourceType, Config, #{}). - -spec recreate_local(type(), resource_id(), resource_type(), resource_config(), creation_opts()) -> {ok, resource_data()} | {error, Reason :: term()}. recreate_local(Type, ResId, ResourceType, Config, Opts) -> @@ -339,11 +329,15 @@ remove_local(ResId) -> ok; Error -> %% Only log, the ResId worker is always removed in manager's remove action. - ?SLOG(warning, #{ - msg => "remove_resource_failed", - error => Error, - resource_id => ResId - }), + ?SLOG( + warning, + #{ + msg => "remove_resource_failed", + error => Error, + resource_id => ResId + }, + #{tag => ?TAG} + ), ok end. @@ -815,3 +809,18 @@ validate_name(Name, Opts) -> -spec invalid_data(binary()) -> no_return(). invalid_data(Reason) -> throw(#{kind => validation_error, reason => Reason}). + +%% Those functions is only used in the emqx_resource_proto_v1 +%% for versions that are less than version 5.6.0. +%% begin +-spec create_local( + resource_id(), resource_group(), resource_type(), resource_config(), creation_opts() +) -> + {ok, resource_data() | 'already_created'} | {error, Reason :: term()}. +create_local(ResId, Group, ResourceType, Config, Opts) -> + create_local(deprecated, ResId, Group, ResourceType, Config, Opts). +create_dry_run_local(ResourceType, Config) -> + create_dry_run_local(deprecated, ResourceType, Config). +recreate_local(ResId, ResourceType, Config, Opts) -> + recreate_local(deprecated, ResId, ResourceType, Config, Opts). +%% end diff --git a/apps/emqx_resource/src/emqx_resource_pool.erl b/apps/emqx_resource/src/emqx_resource_pool.erl index 516c483e3..ba286e35c 100644 --- a/apps/emqx_resource/src/emqx_resource_pool.erl +++ b/apps/emqx_resource/src/emqx_resource_pool.erl @@ -38,7 +38,7 @@ start(Name, Mod, Options) -> case ecpool:start_sup_pool(Name, Mod, Options) of {ok, _} -> - ?SLOG(info, #{msg => "start_ecpool_ok", pool_name => Name}), + ?SLOG(info, #{msg => "start_ecpool_ok", pool_name => Name}, #{tag => ?TAG}), ok; {error, {already_started, _Pid}} -> stop(Name), @@ -46,27 +46,35 @@ start(Name, Mod, Options) -> {error, Reason} -> NReason = parse_reason(Reason), IsDryRun = emqx_resource:is_dry_run(Name), - ?SLOG(?LOG_LEVEL(IsDryRun), #{ - msg => "start_ecpool_error", - resource_id => Name, - reason => NReason - }), + ?SLOG( + ?LOG_LEVEL(IsDryRun), + #{ + msg => "start_ecpool_error", + resource_id => Name, + reason => NReason + }, + #{tag => ?TAG} + ), {error, {start_pool_failed, Name, NReason}} end. stop(Name) -> case ecpool:stop_sup_pool(Name) of ok -> - ?SLOG(info, #{msg => "stop_ecpool_ok", pool_name => Name}); + ?SLOG(info, #{msg => "stop_ecpool_ok", pool_name => Name}, #{tag => ?TAG}); {error, not_found} -> ok; {error, Reason} -> IsDryRun = emqx_resource:is_dry_run(Name), - ?SLOG(?LOG_LEVEL(IsDryRun), #{ - msg => "stop_ecpool_failed", - resource_id => Name, - reason => Reason - }), + ?SLOG( + ?LOG_LEVEL(IsDryRun), + #{ + msg => "stop_ecpool_failed", + resource_id => Name, + reason => Reason + }, + #{tag => ?TAG} + ), error({stop_pool_failed, Name, Reason}) end. diff --git a/apps/emqx_resource/test/emqx_resource_SUITE.erl b/apps/emqx_resource/test/emqx_resource_SUITE.erl index 981c88edd..8e16ec26a 100644 --- a/apps/emqx_resource/test/emqx_resource_SUITE.erl +++ b/apps/emqx_resource/test/emqx_resource_SUITE.erl @@ -3506,7 +3506,7 @@ gauge_metric_set_fns() -> ]. create(Id, Group, Type, Config) -> - emqx_resource:create_local(test, Id, Group, Type, Config). + emqx_resource:create_local(test, Id, Group, Type, Config, #{}). create(Id, Group, Type, Config, Opts) -> emqx_resource:create_local(test, Id, Group, Type, Config, Opts). From 2a58a36e372bbedef63b01dfd7e6c51e132fd47d Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 17 Jul 2024 16:05:16 +0800 Subject: [PATCH 6/9] chore: add resource tag for log --- apps/emqx_resource/include/emqx_resource.hrl | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/emqx_resource/include/emqx_resource.hrl b/apps/emqx_resource/include/emqx_resource.hrl index f26f6699c..0c0cc8f6b 100644 --- a/apps/emqx_resource/include/emqx_resource.hrl +++ b/apps/emqx_resource/include/emqx_resource.hrl @@ -165,5 +165,6 @@ false -> warning end ). +-define(TAG, "RESOURCE"). -define(RESOURCE_ALLOCATION_TAB, emqx_resource_allocations). From e74a921d33a0dd637821eeeb889cfa3a6d298ea8 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 17 Jul 2024 16:18:53 +0800 Subject: [PATCH 7/9] chore: compile error --- .../src/emqx_resource_metrics.erl | 46 +++++++++++-------- 1 file changed, 27 insertions(+), 19 deletions(-) diff --git a/apps/emqx_resource/src/emqx_resource_metrics.erl b/apps/emqx_resource/src/emqx_resource_metrics.erl index 97a09b074..98a74dfad 100644 --- a/apps/emqx_resource/src/emqx_resource_metrics.erl +++ b/apps/emqx_resource/src/emqx_resource_metrics.erl @@ -17,6 +17,7 @@ -module(emqx_resource_metrics). -include_lib("emqx/include/logger.hrl"). +-include("emqx_resource.hrl"). -export([ events/0, @@ -74,7 +75,6 @@ success_get/1 ]). --define(RES_METRICS, resource_metrics). -define(TELEMETRY_PREFIX, emqx, resource). -spec events() -> [telemetry:event_name()]. @@ -127,15 +127,19 @@ handle_telemetry_event( %% We catch errors to avoid detaching the telemetry handler function. %% When restarting a resource while it's under load, there might be transient %% failures while the metrics are not yet created. - ?SLOG(warning, #{ - msg => "handle_resource_metrics_failed", - hint => "transient failures may occur when restarting a resource", - kind => Kind, - reason => Reason, - stacktrace => Stacktrace, - resource_id => ID, - event => Event - }), + ?SLOG( + warning, + #{ + msg => "handle_resource_metrics_failed", + hint => "transient failures may occur when restarting a resource", + kind => Kind, + reason => Reason, + stacktrace => Stacktrace, + resource_id => ID, + event => Event + }, + #{tag => ?TAG} + ), ok end; handle_telemetry_event( @@ -151,15 +155,19 @@ handle_telemetry_event( %% We catch errors to avoid detaching the telemetry handler function. %% When restarting a resource while it's under load, there might be transient %% failures while the metrics are not yet created. - ?SLOG(warning, #{ - msg => "handle_resource_metrics_failed", - hint => "transient failures may occur when restarting a resource", - kind => Kind, - reason => Reason, - stacktrace => Stacktrace, - resource_id => ID, - event => Event - }), + ?SLOG( + warning, + #{ + msg => "handle_resource_metrics_failed", + hint => "transient failures may occur when restarting a resource", + kind => Kind, + reason => Reason, + stacktrace => Stacktrace, + resource_id => ID, + event => Event + }, + #{tag => ?TAG} + ), ok end; handle_telemetry_event(_EventName, _Measurements, _Metadata, _HandlerConfig) -> From e7d07ea17c2e0d47b109af22bf0f85dbed05a307 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Tue, 23 Jul 2024 16:15:57 +0800 Subject: [PATCH 8/9] feat: add resource_type to emqx_resource behaviour --- .../src/emqx_authn/emqx_authn_utils.erl | 11 ++- .../src/emqx_authz/emqx_authz_utils.erl | 14 ++- apps/emqx_auth_http/src/emqx_authn_http.erl | 5 +- apps/emqx_auth_http/src/emqx_authz_http.erl | 6 +- .../src/emqx_authn_jwks_connector.erl | 3 + apps/emqx_auth_jwt/src/emqx_authn_jwt.erl | 1 - apps/emqx_auth_ldap/src/emqx_authn_ldap.erl | 4 +- apps/emqx_auth_ldap/src/emqx_authz_ldap.erl | 4 +- .../test/emqx_authn_ldap_SUITE.erl | 1 - .../test/emqx_authn_ldap_bind_SUITE.erl | 1 - .../test/emqx_authz_ldap_SUITE.erl | 1 - .../src/emqx_authn_mongodb.erl | 3 +- .../src/emqx_authz_mongodb.erl | 4 +- apps/emqx_auth_mysql/src/emqx_authn_mysql.erl | 4 +- apps/emqx_auth_mysql/src/emqx_authz_mysql.erl | 4 +- .../test/emqx_authn_mysql_SUITE.erl | 1 - .../test/emqx_authz_mysql_SUITE.erl | 1 - .../src/emqx_authn_postgresql.erl | 3 +- .../src/emqx_authz_postgresql.erl | 2 - .../test/emqx_authn_postgresql_SUITE.erl | 9 +- .../test/emqx_authz_postgresql_SUITE.erl | 1 - apps/emqx_auth_redis/src/emqx_authn_redis.erl | 3 +- apps/emqx_auth_redis/src/emqx_authz_redis.erl | 4 +- .../test/emqx_authn_redis_SUITE.erl | 1 - .../test/emqx_authz_redis_SUITE.erl | 1 - apps/emqx_bridge/src/emqx_bridge_resource.erl | 6 +- .../emqx_bridge/test/emqx_bridge_v2_SUITE.erl | 1 + .../test/emqx_bridge_v2_api_SUITE.erl | 1 + .../test/emqx_bridge_v2_dummy_connector.erl | 4 +- .../test/emqx_bridge_v2_test_connector.erl | 3 + .../src/emqx_bridge_cassandra.app.src | 2 +- .../src/emqx_bridge_cassandra_connector.erl | 2 + .../emqx_bridge_cassandra_connector_SUITE.erl | 1 - .../src/emqx_bridge_clickhouse.app.src | 2 +- .../src/emqx_bridge_clickhouse_connector.erl | 2 + ...emqx_bridge_clickhouse_connector_SUITE.erl | 2 - .../src/emqx_bridge_dynamo.app.src | 2 +- .../src/emqx_bridge_dynamo_connector.erl | 2 + .../emqx_bridge_es/src/emqx_bridge_es.app.src | 2 +- .../src/emqx_bridge_es_connector.erl | 3 + .../src/emqx_bridge_gcp_pubsub.app.src | 2 +- .../emqx_bridge_gcp_pubsub_impl_consumer.erl | 3 + .../emqx_bridge_gcp_pubsub_impl_producer.erl | 2 + .../src/emqx_bridge_greptimedb_connector.erl | 3 + ...emqx_bridge_greptimedb_connector_SUITE.erl | 1 - .../src/emqx_bridge_hstreamdb.app.src | 2 +- .../src/emqx_bridge_hstreamdb_connector.erl | 3 + .../src/emqx_bridge_http_connector.erl | 2 + .../src/emqx_bridge_influxdb.app.src | 2 +- .../src/emqx_bridge_influxdb_connector.erl | 3 + .../emqx_bridge_influxdb_connector_SUITE.erl | 2 - .../src/emqx_bridge_iotdb.app.src | 2 +- .../src/emqx_bridge_iotdb_connector.erl | 3 + .../src/emqx_bridge_kafka_impl_consumer.erl | 2 + .../src/emqx_bridge_kafka_impl_producer.erl | 3 + .../src/emqx_bridge_kinesis.app.src | 2 +- .../src/emqx_bridge_kinesis_impl_producer.erl | 2 + .../src/emqx_bridge_mongodb.app.src | 2 +- .../src/emqx_bridge_mongodb_connector.erl | 2 + .../src/emqx_bridge_mqtt_connector.erl | 3 + .../src/emqx_bridge_mysql.app.src | 2 +- .../src/emqx_bridge_mysql_connector.erl | 2 + .../src/emqx_bridge_opents.app.src | 2 +- .../src/emqx_bridge_opents_connector.erl | 3 + .../src/emqx_bridge_pulsar_connector.erl | 2 + .../src/emqx_bridge_rabbitmq.app.src | 2 +- .../src/emqx_bridge_rabbitmq_connector.erl | 2 + .../emqx_bridge_rabbitmq_connector_SUITE.erl | 1 - .../src/emqx_bridge_redis.app.src | 2 +- .../src/emqx_bridge_redis_connector.erl | 5 +- .../src/emqx_bridge_rocketmq.app.src | 2 +- .../src/emqx_bridge_rocketmq_connector.erl | 3 + .../src/emqx_bridge_s3_connector.erl | 3 + .../src/emqx_bridge_sqlserver.app.src | 2 +- .../src/emqx_bridge_sqlserver_connector.erl | 2 + .../src/emqx_bridge_syskeeper.app.src | 2 +- .../src/emqx_bridge_syskeeper_connector.erl | 2 + .../emqx_bridge_syskeeper_proxy_server.erl | 3 + .../src/emqx_bridge_tdengine.app.src | 2 +- .../src/emqx_bridge_tdengine_connector.erl | 2 + .../src/emqx_connector_resource.erl | 6 +- .../test/emqx_connector_SUITE.erl | 5 ++ .../test/emqx_connector_api_SUITE.erl | 1 + .../test/emqx_connector_dummy_impl.erl | 3 + .../src/emqx_dashboard_sso_manager.erl | 3 +- apps/emqx_ldap/src/emqx_ldap.app.src | 2 +- apps/emqx_ldap/src/emqx_ldap.erl | 3 + apps/emqx_ldap/test/emqx_ldap_SUITE.erl | 2 - apps/emqx_mongodb/src/emqx_mongodb.app.src | 2 +- apps/emqx_mongodb/src/emqx_mongodb.erl | 2 + apps/emqx_mongodb/test/emqx_mongodb_SUITE.erl | 1 - apps/emqx_mysql/src/emqx_mysql.app.src | 2 +- apps/emqx_mysql/src/emqx_mysql.erl | 3 + apps/emqx_mysql/test/emqx_mysql_SUITE.erl | 1 - apps/emqx_oracle/src/emqx_oracle.app.src | 2 +- apps/emqx_oracle/src/emqx_oracle.erl | 3 + .../src/emqx_postgresql.app.src | 2 +- apps/emqx_postgresql/src/emqx_postgresql.erl | 3 + .../test/emqx_postgresql_SUITE.erl | 1 - apps/emqx_redis/src/emqx_redis.erl | 3 + apps/emqx_redis/test/emqx_redis_SUITE.erl | 1 - apps/emqx_resource/include/emqx_resource.hrl | 4 +- apps/emqx_resource/src/emqx_resource.erl | 88 ++++++++----------- .../src/emqx_resource_manager.erl | 53 +++++------ .../src/emqx_resource_manager_sup.erl | 12 ++- .../src/proto/emqx_resource_proto_v1.erl | 6 +- .../test/emqx_connector_demo.erl | 3 + .../test/emqx_resource_SUITE.erl | 20 +---- .../test/emqx_rule_engine_test_connector.erl | 3 + 109 files changed, 248 insertions(+), 211 deletions(-) diff --git a/apps/emqx_auth/src/emqx_authn/emqx_authn_utils.erl b/apps/emqx_auth/src/emqx_authn/emqx_authn_utils.erl index e81145f2c..a08ac260c 100644 --- a/apps/emqx_auth/src/emqx_authn/emqx_authn_utils.erl +++ b/apps/emqx_auth/src/emqx_authn/emqx_authn_utils.erl @@ -21,8 +21,8 @@ -include_lib("snabbkaffe/include/trace.hrl"). -export([ - create_resource/4, - update_resource/4, + create_resource/3, + update_resource/3, check_password_from_selected_map/3, parse_deep/1, parse_str/1, @@ -66,9 +66,8 @@ %% APIs %%-------------------------------------------------------------------- -create_resource(Type, ResourceId, Module, Config) -> +create_resource(ResourceId, Module, Config) -> Result = emqx_resource:create_local( - Type, ResourceId, ?AUTHN_RESOURCE_GROUP, Module, @@ -77,9 +76,9 @@ create_resource(Type, ResourceId, Module, Config) -> ), start_resource_if_enabled(Result, ResourceId, Config). -update_resource(Type, Module, Config, ResourceId) -> +update_resource(Module, Config, ResourceId) -> Result = emqx_resource:recreate_local( - Type, ResourceId, Module, Config, ?DEFAULT_RESOURCE_OPTS + ResourceId, Module, Config, ?DEFAULT_RESOURCE_OPTS ), start_resource_if_enabled(Result, ResourceId, Config). diff --git a/apps/emqx_auth/src/emqx_authz/emqx_authz_utils.erl b/apps/emqx_auth/src/emqx_authz/emqx_authz_utils.erl index 533f982c1..e4343b6fa 100644 --- a/apps/emqx_auth/src/emqx_authz/emqx_authz_utils.erl +++ b/apps/emqx_auth/src/emqx_authz/emqx_authz_utils.erl @@ -25,9 +25,9 @@ -export([ cleanup_resources/0, make_resource_id/1, + create_resource/2, create_resource/3, - create_resource/4, - update_resource/3, + update_resource/2, remove_resource/1, update_config/2, parse_deep/2, @@ -57,13 +57,12 @@ %% APIs %%-------------------------------------------------------------------- -create_resource(Type, Module, Config) -> +create_resource(Module, Config) -> ResourceId = make_resource_id(Module), - create_resource(Type, ResourceId, Module, Config). + create_resource(ResourceId, Module, Config). -create_resource(Type, ResourceId, Module, Config) -> +create_resource(ResourceId, Module, Config) -> Result = emqx_resource:create_local( - Type, ResourceId, ?AUTHZ_RESOURCE_GROUP, Module, @@ -72,11 +71,10 @@ create_resource(Type, ResourceId, Module, Config) -> ), start_resource_if_enabled(Result, ResourceId, Config). -update_resource(Type, Module, #{annotations := #{id := ResourceId}} = Config) -> +update_resource(Module, #{annotations := #{id := ResourceId}} = Config) -> Result = case emqx_resource:recreate_local( - Type, ResourceId, Module, Config, diff --git a/apps/emqx_auth_http/src/emqx_authn_http.erl b/apps/emqx_auth_http/src/emqx_authn_http.erl index ed151428f..d9c5c5ed5 100644 --- a/apps/emqx_auth_http/src/emqx_authn_http.erl +++ b/apps/emqx_auth_http/src/emqx_authn_http.erl @@ -40,7 +40,6 @@ create(Config0) -> ResourceId = emqx_authn_utils:make_resource_id(?MODULE), % {Config, State} = parse_config(Config0), {ok, _Data} = emqx_authn_utils:create_resource( - http, ResourceId, emqx_bridge_http_connector, Config @@ -51,9 +50,7 @@ create(Config0) -> update(Config0, #{resource_id := ResourceId} = _State) -> with_validated_config(Config0, fun(Config, NState) -> % {Config, NState} = parse_config(Config0), - case - emqx_authn_utils:update_resource(http, emqx_bridge_http_connector, Config, ResourceId) - of + case emqx_authn_utils:update_resource(emqx_bridge_http_connector, Config, ResourceId) of {error, Reason} -> error({load_config_error, Reason}); {ok, _} -> diff --git a/apps/emqx_auth_http/src/emqx_authz_http.erl b/apps/emqx_auth_http/src/emqx_authz_http.erl index 2a1a2638b..a858ec7d7 100644 --- a/apps/emqx_auth_http/src/emqx_authz_http.erl +++ b/apps/emqx_auth_http/src/emqx_authz_http.erl @@ -67,13 +67,15 @@ create(Config) -> NConfig = parse_config(Config), ResourceId = emqx_authn_utils:make_resource_id(?MODULE), {ok, _Data} = emqx_authz_utils:create_resource( - http, ResourceId, emqx_bridge_http_connector, NConfig + ResourceId, + emqx_bridge_http_connector, + NConfig ), NConfig#{annotations => #{id => ResourceId}}. update(Config) -> NConfig = parse_config(Config), - case emqx_authz_utils:update_resource(http, emqx_bridge_http_connector, NConfig) of + case emqx_authz_utils:update_resource(emqx_bridge_http_connector, NConfig) of {error, Reason} -> error({load_config_error, Reason}); {ok, Id} -> NConfig#{annotations => #{id => Id}} end. diff --git a/apps/emqx_auth_jwt/src/emqx_authn_jwks_connector.erl b/apps/emqx_auth_jwt/src/emqx_authn_jwks_connector.erl index ffa8175b7..22aed8e57 100644 --- a/apps/emqx_auth_jwt/src/emqx_authn_jwks_connector.erl +++ b/apps/emqx_auth_jwt/src/emqx_authn_jwks_connector.erl @@ -22,6 +22,7 @@ %% callbacks of behaviour emqx_resource -export([ + resource_type/0, callback_mode/0, on_start/2, on_stop/2, @@ -32,6 +33,8 @@ -define(DEFAULT_POOL_SIZE, 8). +resource_type() -> jwks. + callback_mode() -> always_sync. on_start(InstId, Opts) -> diff --git a/apps/emqx_auth_jwt/src/emqx_authn_jwt.erl b/apps/emqx_auth_jwt/src/emqx_authn_jwt.erl index 2f8ebec36..1d0d58474 100644 --- a/apps/emqx_auth_jwt/src/emqx_authn_jwt.erl +++ b/apps/emqx_auth_jwt/src/emqx_authn_jwt.erl @@ -183,7 +183,6 @@ do_create( ) -> ResourceId = emqx_authn_utils:make_resource_id(?MODULE), {ok, _Data} = emqx_resource:create_local( - jwt, ResourceId, ?AUTHN_RESOURCE_GROUP, emqx_authn_jwks_connector, diff --git a/apps/emqx_auth_ldap/src/emqx_authn_ldap.erl b/apps/emqx_auth_ldap/src/emqx_authn_ldap.erl index 8a9b5650a..a18236d15 100644 --- a/apps/emqx_auth_ldap/src/emqx_authn_ldap.erl +++ b/apps/emqx_auth_ldap/src/emqx_authn_ldap.erl @@ -40,12 +40,12 @@ create(_AuthenticatorID, Config) -> do_create(Module, Config) -> ResourceId = emqx_authn_utils:make_resource_id(Module), State = parse_config(Config), - {ok, _Data} = emqx_authn_utils:create_resource(ldap, ResourceId, emqx_ldap, Config), + {ok, _Data} = emqx_authn_utils:create_resource(ResourceId, emqx_ldap, Config), {ok, State#{resource_id => ResourceId}}. update(Config, #{resource_id := ResourceId} = _State) -> NState = parse_config(Config), - case emqx_authn_utils:update_resource(ldap, emqx_ldap, Config, ResourceId) of + case emqx_authn_utils:update_resource(emqx_ldap, Config, ResourceId) of {error, Reason} -> error({load_config_error, Reason}); {ok, _} -> diff --git a/apps/emqx_auth_ldap/src/emqx_authz_ldap.erl b/apps/emqx_auth_ldap/src/emqx_authz_ldap.erl index e70b840a3..24bd8c008 100644 --- a/apps/emqx_auth_ldap/src/emqx_authz_ldap.erl +++ b/apps/emqx_auth_ldap/src/emqx_authz_ldap.erl @@ -56,12 +56,12 @@ description() -> create(Source) -> ResourceId = emqx_authz_utils:make_resource_id(?MODULE), - {ok, _Data} = emqx_authz_utils:create_resource(ldap, ResourceId, emqx_ldap, Source), + {ok, _Data} = emqx_authz_utils:create_resource(ResourceId, emqx_ldap, Source), Annotations = new_annotations(#{id => ResourceId}, Source), Source#{annotations => Annotations}. update(Source) -> - case emqx_authz_utils:update_resource(ldap, emqx_ldap, Source) of + case emqx_authz_utils:update_resource(emqx_ldap, Source) of {error, Reason} -> error({load_config_error, Reason}); {ok, Id} -> diff --git a/apps/emqx_auth_ldap/test/emqx_authn_ldap_SUITE.erl b/apps/emqx_auth_ldap/test/emqx_authn_ldap_SUITE.erl index f6ee582be..ac941f268 100644 --- a/apps/emqx_auth_ldap/test/emqx_authn_ldap_SUITE.erl +++ b/apps/emqx_auth_ldap/test/emqx_authn_ldap_SUITE.erl @@ -47,7 +47,6 @@ init_per_suite(Config) -> work_dir => ?config(priv_dir, Config) }), {ok, _} = emqx_resource:create_local( - ldap, ?LDAP_RESOURCE, ?AUTHN_RESOURCE_GROUP, emqx_ldap, diff --git a/apps/emqx_auth_ldap/test/emqx_authn_ldap_bind_SUITE.erl b/apps/emqx_auth_ldap/test/emqx_authn_ldap_bind_SUITE.erl index c39225e74..d8e9c0d8e 100644 --- a/apps/emqx_auth_ldap/test/emqx_authn_ldap_bind_SUITE.erl +++ b/apps/emqx_auth_ldap/test/emqx_authn_ldap_bind_SUITE.erl @@ -47,7 +47,6 @@ init_per_suite(Config) -> work_dir => ?config(priv_dir, Config) }), {ok, _} = emqx_resource:create_local( - ldap, ?LDAP_RESOURCE, ?AUTHN_RESOURCE_GROUP, emqx_ldap, diff --git a/apps/emqx_auth_ldap/test/emqx_authz_ldap_SUITE.erl b/apps/emqx_auth_ldap/test/emqx_authz_ldap_SUITE.erl index 3f8c0ba63..09875a3fa 100644 --- a/apps/emqx_auth_ldap/test/emqx_authz_ldap_SUITE.erl +++ b/apps/emqx_auth_ldap/test/emqx_authz_ldap_SUITE.erl @@ -178,7 +178,6 @@ stop_apps(Apps) -> create_ldap_resource() -> {ok, _} = emqx_resource:create_local( - ldap, ?LDAP_RESOURCE, ?AUTHZ_RESOURCE_GROUP, emqx_ldap, diff --git a/apps/emqx_auth_mongodb/src/emqx_authn_mongodb.erl b/apps/emqx_auth_mongodb/src/emqx_authn_mongodb.erl index ffe78159d..75a474c0c 100644 --- a/apps/emqx_auth_mongodb/src/emqx_authn_mongodb.erl +++ b/apps/emqx_auth_mongodb/src/emqx_authn_mongodb.erl @@ -37,7 +37,6 @@ create(Config0) -> ResourceId = emqx_authn_utils:make_resource_id(?MODULE), {Config, State} = parse_config(Config0), {ok, _Data} = emqx_authn_utils:create_resource( - mongodb, ResourceId, emqx_mongodb, Config @@ -46,7 +45,7 @@ create(Config0) -> update(Config0, #{resource_id := ResourceId} = _State) -> {Config, NState} = parse_config(Config0), - case emqx_authn_utils:update_resource(mongodb, emqx_mongodb, Config, ResourceId) of + case emqx_authn_utils:update_resource(emqx_mongodb, Config, ResourceId) of {error, Reason} -> error({load_config_error, Reason}); {ok, _} -> diff --git a/apps/emqx_auth_mongodb/src/emqx_authz_mongodb.erl b/apps/emqx_auth_mongodb/src/emqx_authz_mongodb.erl index 0d65afe0d..0bab6ef90 100644 --- a/apps/emqx_auth_mongodb/src/emqx_authz_mongodb.erl +++ b/apps/emqx_auth_mongodb/src/emqx_authz_mongodb.erl @@ -49,13 +49,13 @@ description() -> create(#{filter := Filter} = Source) -> ResourceId = emqx_authz_utils:make_resource_id(?MODULE), - {ok, _Data} = emqx_authz_utils:create_resource(mongodb, ResourceId, emqx_mongodb, Source), + {ok, _Data} = emqx_authz_utils:create_resource(ResourceId, emqx_mongodb, Source), FilterTemp = emqx_authz_utils:parse_deep(Filter, ?ALLOWED_VARS), Source#{annotations => #{id => ResourceId}, filter_template => FilterTemp}. update(#{filter := Filter} = Source) -> FilterTemp = emqx_authz_utils:parse_deep(Filter, ?ALLOWED_VARS), - case emqx_authz_utils:update_resource(mongodb, emqx_mongodb, Source) of + case emqx_authz_utils:update_resource(emqx_mongodb, Source) of {error, Reason} -> error({load_config_error, Reason}); {ok, Id} -> diff --git a/apps/emqx_auth_mysql/src/emqx_authn_mysql.erl b/apps/emqx_auth_mysql/src/emqx_authn_mysql.erl index 7d03ca856..f68c74a14 100644 --- a/apps/emqx_auth_mysql/src/emqx_authn_mysql.erl +++ b/apps/emqx_auth_mysql/src/emqx_authn_mysql.erl @@ -39,12 +39,12 @@ create(_AuthenticatorID, Config) -> create(Config0) -> ResourceId = emqx_authn_utils:make_resource_id(?MODULE), {Config, State} = parse_config(Config0), - {ok, _Data} = emqx_authn_utils:create_resource(mysql, ResourceId, emqx_mysql, Config), + {ok, _Data} = emqx_authn_utils:create_resource(ResourceId, emqx_mysql, Config), {ok, State#{resource_id => ResourceId}}. update(Config0, #{resource_id := ResourceId} = _State) -> {Config, NState} = parse_config(Config0), - case emqx_authn_utils:update_resource(mysql, emqx_mysql, Config, ResourceId) of + case emqx_authn_utils:update_resource(emqx_mysql, Config, ResourceId) of {error, Reason} -> error({load_config_error, Reason}); {ok, _} -> diff --git a/apps/emqx_auth_mysql/src/emqx_authz_mysql.erl b/apps/emqx_auth_mysql/src/emqx_authz_mysql.erl index 557eed14f..0e2b77005 100644 --- a/apps/emqx_auth_mysql/src/emqx_authz_mysql.erl +++ b/apps/emqx_auth_mysql/src/emqx_authz_mysql.erl @@ -53,13 +53,13 @@ create(#{query := SQL} = Source0) -> {PrepareSQL, TmplToken} = emqx_authz_utils:parse_sql(SQL, '?', ?ALLOWED_VARS), ResourceId = emqx_authz_utils:make_resource_id(?MODULE), Source = Source0#{prepare_statement => #{?PREPARE_KEY => PrepareSQL}}, - {ok, _Data} = emqx_authz_utils:create_resource(mysql, ResourceId, emqx_mysql, Source), + {ok, _Data} = emqx_authz_utils:create_resource(ResourceId, emqx_mysql, Source), Source#{annotations => #{id => ResourceId, tmpl_token => TmplToken}}. update(#{query := SQL} = Source0) -> {PrepareSQL, TmplToken} = emqx_authz_utils:parse_sql(SQL, '?', ?ALLOWED_VARS), Source = Source0#{prepare_statement => #{?PREPARE_KEY => PrepareSQL}}, - case emqx_authz_utils:update_resource(mysql, emqx_mysql, Source) of + case emqx_authz_utils:update_resource(emqx_mysql, Source) of {error, Reason} -> error({load_config_error, Reason}); {ok, Id} -> diff --git a/apps/emqx_auth_mysql/test/emqx_authn_mysql_SUITE.erl b/apps/emqx_auth_mysql/test/emqx_authn_mysql_SUITE.erl index 78b8aa8b3..8ab812fc0 100644 --- a/apps/emqx_auth_mysql/test/emqx_authn_mysql_SUITE.erl +++ b/apps/emqx_auth_mysql/test/emqx_authn_mysql_SUITE.erl @@ -58,7 +58,6 @@ init_per_suite(Config) -> work_dir => ?config(priv_dir, Config) }), {ok, _} = emqx_resource:create_local( - mysql, ?MYSQL_RESOURCE, ?AUTHN_RESOURCE_GROUP, emqx_mysql, diff --git a/apps/emqx_auth_mysql/test/emqx_authz_mysql_SUITE.erl b/apps/emqx_auth_mysql/test/emqx_authz_mysql_SUITE.erl index 5fe020f1b..ce30e203e 100644 --- a/apps/emqx_auth_mysql/test/emqx_authz_mysql_SUITE.erl +++ b/apps/emqx_auth_mysql/test/emqx_authz_mysql_SUITE.erl @@ -446,7 +446,6 @@ stop_apps(Apps) -> create_mysql_resource() -> {ok, _} = emqx_resource:create_local( - mysql, ?MYSQL_RESOURCE, ?AUTHZ_RESOURCE_GROUP, emqx_mysql, diff --git a/apps/emqx_auth_postgresql/src/emqx_authn_postgresql.erl b/apps/emqx_auth_postgresql/src/emqx_authn_postgresql.erl index 95e8ac7ce..980c2ddd8 100644 --- a/apps/emqx_auth_postgresql/src/emqx_authn_postgresql.erl +++ b/apps/emqx_auth_postgresql/src/emqx_authn_postgresql.erl @@ -45,7 +45,6 @@ create(Config0) -> ResourceId = emqx_authn_utils:make_resource_id(?MODULE), {Config, State} = parse_config(Config0, ResourceId), {ok, _Data} = emqx_authn_utils:create_resource( - postgresql, ResourceId, emqx_postgresql, Config @@ -54,7 +53,7 @@ create(Config0) -> update(Config0, #{resource_id := ResourceId} = _State) -> {Config, NState} = parse_config(Config0, ResourceId), - case emqx_authn_utils:update_resource(postgresql, emqx_postgresql, Config, ResourceId) of + case emqx_authn_utils:update_resource(emqx_postgresql, Config, ResourceId) of {error, Reason} -> error({load_config_error, Reason}); {ok, _} -> diff --git a/apps/emqx_auth_postgresql/src/emqx_authz_postgresql.erl b/apps/emqx_auth_postgresql/src/emqx_authz_postgresql.erl index e0fc9e032..d1a0b32ea 100644 --- a/apps/emqx_auth_postgresql/src/emqx_authz_postgresql.erl +++ b/apps/emqx_auth_postgresql/src/emqx_authz_postgresql.erl @@ -53,7 +53,6 @@ create(#{query := SQL0} = Source) -> {SQL, PlaceHolders} = emqx_authz_utils:parse_sql(SQL0, '$n', ?ALLOWED_VARS), ResourceID = emqx_authz_utils:make_resource_id(emqx_postgresql), {ok, _Data} = emqx_authz_utils:create_resource( - postgresql, ResourceID, emqx_postgresql, Source#{prepare_statement => #{ResourceID => SQL}} @@ -64,7 +63,6 @@ update(#{query := SQL0, annotations := #{id := ResourceID}} = Source) -> {SQL, PlaceHolders} = emqx_authz_utils:parse_sql(SQL0, '$n', ?ALLOWED_VARS), case emqx_authz_utils:update_resource( - postgresql, emqx_postgresql, Source#{prepare_statement => #{ResourceID => SQL}} ) diff --git a/apps/emqx_auth_postgresql/test/emqx_authn_postgresql_SUITE.erl b/apps/emqx_auth_postgresql/test/emqx_authn_postgresql_SUITE.erl index dde5a184a..1dfd30899 100644 --- a/apps/emqx_auth_postgresql/test/emqx_authn_postgresql_SUITE.erl +++ b/apps/emqx_auth_postgresql/test/emqx_authn_postgresql_SUITE.erl @@ -79,7 +79,6 @@ init_per_suite(Config) -> work_dir => ?config(priv_dir, Config) }), {ok, _} = emqx_resource:create_local( - postgresql, ?PGSQL_RESOURCE, ?AUTHN_RESOURCE_GROUP, emqx_postgresql, @@ -199,13 +198,9 @@ test_user_auth(#{ t_authenticate_disabled_prepared_statements(_Config) -> ResConfig = maps:merge(pgsql_config(), #{disable_prepared_statements => true}), - {ok, _} = emqx_resource:recreate_local( - postgresql, ?PGSQL_RESOURCE, emqx_postgresql, ResConfig, #{} - ), + {ok, _} = emqx_resource:recreate_local(?PGSQL_RESOURCE, emqx_postgresql, ResConfig, #{}), on_exit(fun() -> - emqx_resource:recreate_local( - postgresql, ?PGSQL_RESOURCE, emqx_postgresql, pgsql_config(), #{} - ) + emqx_resource:recreate_local(?PGSQL_RESOURCE, emqx_postgresql, pgsql_config(), #{}) end), ok = lists:foreach( fun(Sample0) -> diff --git a/apps/emqx_auth_postgresql/test/emqx_authz_postgresql_SUITE.erl b/apps/emqx_auth_postgresql/test/emqx_authz_postgresql_SUITE.erl index 9346bf863..78b1e17a8 100644 --- a/apps/emqx_auth_postgresql/test/emqx_authz_postgresql_SUITE.erl +++ b/apps/emqx_auth_postgresql/test/emqx_authz_postgresql_SUITE.erl @@ -437,7 +437,6 @@ pgsql_config() -> create_pgsql_resource() -> emqx_resource:create_local( - postgresql, ?PGSQL_RESOURCE, ?AUTHZ_RESOURCE_GROUP, emqx_postgresql, diff --git a/apps/emqx_auth_redis/src/emqx_authn_redis.erl b/apps/emqx_auth_redis/src/emqx_authn_redis.erl index 4eb5a36a3..779c58e39 100644 --- a/apps/emqx_auth_redis/src/emqx_authn_redis.erl +++ b/apps/emqx_auth_redis/src/emqx_authn_redis.erl @@ -42,7 +42,6 @@ create(Config0) -> Res; {Config, State} -> {ok, _Data} = emqx_authn_utils:create_resource( - redis, ResourceId, emqx_redis, Config @@ -52,7 +51,7 @@ create(Config0) -> update(Config0, #{resource_id := ResourceId} = _State) -> {Config, NState} = parse_config(Config0), - case emqx_authn_utils:update_resource(redis, emqx_redis, Config, ResourceId) of + case emqx_authn_utils:update_resource(emqx_redis, Config, ResourceId) of {error, Reason} -> error({load_config_error, Reason}); {ok, _} -> diff --git a/apps/emqx_auth_redis/src/emqx_authz_redis.erl b/apps/emqx_auth_redis/src/emqx_authz_redis.erl index b83d1abad..8ce975033 100644 --- a/apps/emqx_auth_redis/src/emqx_authz_redis.erl +++ b/apps/emqx_auth_redis/src/emqx_authz_redis.erl @@ -50,12 +50,12 @@ description() -> create(#{cmd := CmdStr} = Source) -> CmdTemplate = parse_cmd(CmdStr), ResourceId = emqx_authz_utils:make_resource_id(?MODULE), - {ok, _Data} = emqx_authz_utils:create_resource(redis, ResourceId, emqx_redis, Source), + {ok, _Data} = emqx_authz_utils:create_resource(ResourceId, emqx_redis, Source), Source#{annotations => #{id => ResourceId}, cmd_template => CmdTemplate}. update(#{cmd := CmdStr} = Source) -> CmdTemplate = parse_cmd(CmdStr), - case emqx_authz_utils:update_resource(redis, emqx_redis, Source) of + case emqx_authz_utils:update_resource(emqx_redis, Source) of {error, Reason} -> error({load_config_error, Reason}); {ok, Id} -> diff --git a/apps/emqx_auth_redis/test/emqx_authn_redis_SUITE.erl b/apps/emqx_auth_redis/test/emqx_authn_redis_SUITE.erl index 1e9d825d2..e8c8760de 100644 --- a/apps/emqx_auth_redis/test/emqx_authn_redis_SUITE.erl +++ b/apps/emqx_auth_redis/test/emqx_authn_redis_SUITE.erl @@ -63,7 +63,6 @@ init_per_suite(Config) -> work_dir => ?config(priv_dir, Config) }), {ok, _} = emqx_resource:create_local( - redis, ?REDIS_RESOURCE, ?AUTHN_RESOURCE_GROUP, emqx_redis, diff --git a/apps/emqx_auth_redis/test/emqx_authz_redis_SUITE.erl b/apps/emqx_auth_redis/test/emqx_authz_redis_SUITE.erl index d0c695c73..5818eea07 100644 --- a/apps/emqx_auth_redis/test/emqx_authz_redis_SUITE.erl +++ b/apps/emqx_auth_redis/test/emqx_authz_redis_SUITE.erl @@ -384,7 +384,6 @@ stop_apps(Apps) -> create_redis_resource() -> {ok, _} = emqx_resource:create_local( - redis, ?REDIS_RESOURCE, ?AUTHZ_RESOURCE_GROUP, emqx_redis, diff --git a/apps/emqx_bridge/src/emqx_bridge_resource.erl b/apps/emqx_bridge/src/emqx_bridge_resource.erl index 7e9f5300a..40db2aee9 100644 --- a/apps/emqx_bridge/src/emqx_bridge_resource.erl +++ b/apps/emqx_bridge/src/emqx_bridge_resource.erl @@ -195,7 +195,6 @@ create(Type, Name, Conf0, Opts) -> TypeBin = bin(Type), Conf = Conf0#{bridge_type => TypeBin, bridge_name => Name}, {ok, _Data} = emqx_resource:create_local( - Type, resource_id(Type, Name), <<"bridge">>, bridge_to_resource_type(Type), @@ -265,7 +264,6 @@ recreate(Type, Name, Conf0, Opts) -> TypeBin = bin(Type), Conf = Conf0#{bridge_type => TypeBin, bridge_name => Name}, emqx_resource:recreate_local( - Type, resource_id(Type, Name), bridge_to_resource_type(Type), parse_confs(TypeBin, Name, Conf), @@ -284,7 +282,7 @@ create_dry_run(Type0, Conf0) -> create_dry_run_bridge_v1(Type, Conf0) -> TmpName = iolist_to_binary([?TEST_ID_PREFIX, emqx_utils:gen_id(8)]), TmpPath = emqx_utils:safe_filename(TmpName), - %% Already typechecked, no need to catch errors + %% Already type checked, no need to catch errors TypeBin = bin(Type), TypeAtom = safe_atom(Type), Conf1 = maps:without([<<"name">>], Conf0), @@ -302,7 +300,7 @@ create_dry_run_bridge_v1(Type, Conf0) -> {error, Reason}; {ok, ConfNew} -> ParseConf = parse_confs(TypeBin, TmpName, ConfNew), - emqx_resource:create_dry_run_local(Type, bridge_to_resource_type(Type), ParseConf) + emqx_resource:create_dry_run_local(bridge_to_resource_type(Type), ParseConf) end catch %% validation errors diff --git a/apps/emqx_bridge/test/emqx_bridge_v2_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_v2_SUITE.erl index dc2e8f275..d81d710ff 100644 --- a/apps/emqx_bridge/test/emqx_bridge_v2_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_v2_SUITE.erl @@ -1110,6 +1110,7 @@ t_query_uses_action_query_mode(_Config) -> %% ... now we use a quite different query mode for the action meck:expect(con_mod(), query_mode, 1, simple_async_internal_buffer), + meck:expect(con_mod(), resource_type, 0, dummy), meck:expect(con_mod(), callback_mode, 0, async_if_possible), {ok, _} = emqx_bridge_v2:create(bridge_type(), ActionName, ActionConfig), diff --git a/apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl b/apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl index 039402738..e5d13f452 100644 --- a/apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl +++ b/apps/emqx_bridge/test/emqx_bridge_v2_api_SUITE.erl @@ -293,6 +293,7 @@ init_mocks() -> meck:new(emqx_connector_resource, [passthrough, no_link]), meck:expect(emqx_connector_resource, connector_to_resource_type, 1, ?CONNECTOR_IMPL), meck:new(?CONNECTOR_IMPL, [non_strict, no_link]), + meck:expect(?CONNECTOR_IMPL, resource_type, 0, dummy), meck:expect(?CONNECTOR_IMPL, callback_mode, 0, async_if_possible), meck:expect( ?CONNECTOR_IMPL, diff --git a/apps/emqx_bridge/test/emqx_bridge_v2_dummy_connector.erl b/apps/emqx_bridge/test/emqx_bridge_v2_dummy_connector.erl index 6b4001b6b..1bb7fd37f 100644 --- a/apps/emqx_bridge/test/emqx_bridge_v2_dummy_connector.erl +++ b/apps/emqx_bridge/test/emqx_bridge_v2_dummy_connector.erl @@ -15,15 +15,17 @@ %% this module is only intended to be mocked -module(emqx_bridge_v2_dummy_connector). +-behavior(emqx_resource). -export([ + resource_type/0, callback_mode/0, on_start/2, on_stop/2, on_add_channel/4, on_get_channel_status/3 ]). - +resource_type() -> dummy. callback_mode() -> error(unexpected). on_start(_, _) -> error(unexpected). on_stop(_, _) -> error(unexpected). diff --git a/apps/emqx_bridge/test/emqx_bridge_v2_test_connector.erl b/apps/emqx_bridge/test/emqx_bridge_v2_test_connector.erl index c528d097c..7daedf19a 100644 --- a/apps/emqx_bridge/test/emqx_bridge_v2_test_connector.erl +++ b/apps/emqx_bridge/test/emqx_bridge_v2_test_connector.erl @@ -19,6 +19,7 @@ -export([ query_mode/1, + resource_type/0, callback_mode/0, on_start/2, on_stop/2, @@ -34,6 +35,8 @@ query_mode(_Config) -> sync. +resource_type() -> test_connector. + callback_mode() -> always_sync. diff --git a/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.app.src b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.app.src index 946ca591a..3e7422112 100644 --- a/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.app.src +++ b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_cassandra, [ {description, "EMQX Enterprise Cassandra Bridge"}, - {vsn, "0.3.1"}, + {vsn, "0.3.2"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_connector.erl b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_connector.erl index df278b791..9f830eb69 100644 --- a/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_connector.erl +++ b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_connector.erl @@ -19,6 +19,7 @@ %% callbacks of behaviour emqx_resource -export([ + resource_type/0, callback_mode/0, on_start/2, on_stop/2, @@ -94,6 +95,7 @@ desc("connector") -> %%-------------------------------------------------------------------- %% callbacks for emqx_resource +resource_type() -> cassandra. callback_mode() -> async_if_possible. diff --git a/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_connector_SUITE.erl b/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_connector_SUITE.erl index 02c102832..b784d36c0 100644 --- a/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_connector_SUITE.erl +++ b/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_connector_SUITE.erl @@ -212,7 +212,6 @@ check_config(Config) -> create_local_resource(ResourceId, CheckedConfig) -> {ok, Bridge} = emqx_resource:create_local( - cassandra, ResourceId, ?CONNECTOR_RESOURCE_GROUP, ?CASSANDRA_RESOURCE_MOD, diff --git a/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse.app.src b/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse.app.src index f38036b83..794d067bd 100644 --- a/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse.app.src +++ b/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_clickhouse, [ {description, "EMQX Enterprise ClickHouse Bridge"}, - {vsn, "0.4.1"}, + {vsn, "0.4.2"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse_connector.erl b/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse_connector.erl index f6888cad5..c5b82122a 100644 --- a/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse_connector.erl +++ b/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse_connector.erl @@ -29,6 +29,7 @@ %% callbacks for behaviour emqx_resource -export([ + resource_type/0, callback_mode/0, on_start/2, on_stop/2, @@ -128,6 +129,7 @@ values(_) -> %% =================================================================== %% Callbacks defined in emqx_resource %% =================================================================== +resource_type() -> clickhouse. callback_mode() -> always_sync. diff --git a/apps/emqx_bridge_clickhouse/test/emqx_bridge_clickhouse_connector_SUITE.erl b/apps/emqx_bridge_clickhouse/test/emqx_bridge_clickhouse_connector_SUITE.erl index 0b0acd78a..1c83961a5 100644 --- a/apps/emqx_bridge_clickhouse/test/emqx_bridge_clickhouse_connector_SUITE.erl +++ b/apps/emqx_bridge_clickhouse/test/emqx_bridge_clickhouse_connector_SUITE.erl @@ -109,7 +109,6 @@ t_start_passfile(Config) -> ?assertMatch( {ok, #{status := connected}}, emqx_resource:create_local( - clickhouse, ResourceID, ?CONNECTOR_RESOURCE_GROUP, ?CLICKHOUSE_RESOURCE_MOD, @@ -139,7 +138,6 @@ perform_lifecycle_check(ResourceID, InitialConfig) -> status := InitialStatus }} = emqx_resource:create_local( - clickhouse, ResourceID, ?CONNECTOR_RESOURCE_GROUP, ?CLICKHOUSE_RESOURCE_MOD, diff --git a/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo.app.src b/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo.app.src index ac71e04e7..b8fee4dee 100644 --- a/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo.app.src +++ b/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_dynamo, [ {description, "EMQX Enterprise Dynamo Bridge"}, - {vsn, "0.2.2"}, + {vsn, "0.2.3"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector.erl b/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector.erl index 82f5fb18d..181de34a8 100644 --- a/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector.erl +++ b/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector.erl @@ -17,6 +17,7 @@ %% `emqx_resource' API -export([ + resource_type/0, callback_mode/0, on_start/2, on_stop/2, @@ -68,6 +69,7 @@ fields(config) -> %%======================================================================================== %% `emqx_resource' API %%======================================================================================== +resource_type() -> dynamo. callback_mode() -> always_sync. diff --git a/apps/emqx_bridge_es/src/emqx_bridge_es.app.src b/apps/emqx_bridge_es/src/emqx_bridge_es.app.src index 262ac84bd..8f3dc3a7e 100644 --- a/apps/emqx_bridge_es/src/emqx_bridge_es.app.src +++ b/apps/emqx_bridge_es/src/emqx_bridge_es.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_bridge_es, [ {description, "EMQX Enterprise Elastic Search Bridge"}, - {vsn, "0.1.3"}, + {vsn, "0.1.4"}, {modules, [ emqx_bridge_es, emqx_bridge_es_connector diff --git a/apps/emqx_bridge_es/src/emqx_bridge_es_connector.erl b/apps/emqx_bridge_es/src/emqx_bridge_es_connector.erl index 20de92e6e..feccd42f1 100644 --- a/apps/emqx_bridge_es/src/emqx_bridge_es_connector.erl +++ b/apps/emqx_bridge_es/src/emqx_bridge_es_connector.erl @@ -14,6 +14,7 @@ %% `emqx_resource' API -export([ + resource_type/0, callback_mode/0, on_start/2, on_stop/2, @@ -207,6 +208,8 @@ base_url(#{server := Server}) -> "http://" ++ Server. %%------------------------------------------------------------------------------------- %% `emqx_resource' API %%------------------------------------------------------------------------------------- +resource_type() -> elastic_search. + callback_mode() -> async_if_possible. -spec on_start(manager_id(), config()) -> {ok, state()} | no_return(). diff --git a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.app.src b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.app.src index d98355a90..eff7847f2 100644 --- a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.app.src +++ b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_gcp_pubsub, [ {description, "EMQX Enterprise GCP Pub/Sub Bridge"}, - {vsn, "0.3.1"}, + {vsn, "0.3.2"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_impl_consumer.erl b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_impl_consumer.erl index 5c51cd2d9..344fc05c6 100644 --- a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_impl_consumer.erl +++ b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_impl_consumer.erl @@ -8,6 +8,7 @@ %% `emqx_resource' API -export([ + resource_type/0, callback_mode/0, query_mode/1, on_start/2, @@ -84,6 +85,8 @@ %%------------------------------------------------------------------------------------------------- %% `emqx_resource' API %%------------------------------------------------------------------------------------------------- +-spec resource_type() -> resource_type(). +resource_type() -> gcp_pubsub_consumer. -spec callback_mode() -> callback_mode(). callback_mode() -> async_if_possible. diff --git a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_impl_producer.erl b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_impl_producer.erl index 48e50c416..aec78f74c 100644 --- a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_impl_producer.erl +++ b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_impl_producer.erl @@ -41,6 +41,7 @@ %% `emqx_resource' API -export([ + resource_type/0, callback_mode/0, query_mode/1, on_start/2, @@ -62,6 +63,7 @@ %%------------------------------------------------------------------------------------------------- %% `emqx_resource' API %%------------------------------------------------------------------------------------------------- +resource_type() -> gcp_pubsub. callback_mode() -> async_if_possible. diff --git a/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector.erl b/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector.erl index e4cc0aa31..a2ffd6219 100644 --- a/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector.erl +++ b/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector.erl @@ -16,6 +16,7 @@ %% callbacks of behaviour emqx_resource -export([ + resource_type/0, callback_mode/0, on_start/2, on_stop/2, @@ -67,6 +68,8 @@ %% ------------------------------------------------------------------------------------------------- %% resource callback +resource_type() -> greptimedb. + callback_mode() -> async_if_possible. on_add_channel( diff --git a/apps/emqx_bridge_greptimedb/test/emqx_bridge_greptimedb_connector_SUITE.erl b/apps/emqx_bridge_greptimedb/test/emqx_bridge_greptimedb_connector_SUITE.erl index 68a32e9c2..be36cb167 100644 --- a/apps/emqx_bridge_greptimedb/test/emqx_bridge_greptimedb_connector_SUITE.erl +++ b/apps/emqx_bridge_greptimedb/test/emqx_bridge_greptimedb_connector_SUITE.erl @@ -83,7 +83,6 @@ perform_lifecycle_check(PoolName, InitialConfig) -> state := #{client := #{pool := ReturnedPoolName}} = State, status := InitialStatus }} = emqx_resource:create_local( - greptimedb, PoolName, ?CONNECTOR_RESOURCE_GROUP, ?GREPTIMEDB_RESOURCE_MOD, diff --git a/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb.app.src b/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb.app.src index af232accc..7ae86bba0 100644 --- a/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb.app.src +++ b/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_hstreamdb, [ {description, "EMQX Enterprise HStreamDB Bridge"}, - {vsn, "0.2.1"}, + {vsn, "0.2.2"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb_connector.erl b/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb_connector.erl index cf53291b2..154e43b3d 100644 --- a/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb_connector.erl +++ b/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb_connector.erl @@ -16,6 +16,7 @@ %% callbacks of behaviour emqx_resource -export([ + resource_type/0, callback_mode/0, on_start/2, on_stop/2, @@ -44,6 +45,8 @@ %% ------------------------------------------------------------------------------------------------- %% resource callback +resource_type() -> hstreamsdb. + callback_mode() -> always_sync. on_start(InstId, Config) -> diff --git a/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl b/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl index 91a0878c3..616b42e75 100644 --- a/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl +++ b/apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl @@ -26,6 +26,7 @@ %% callbacks of behaviour emqx_resource -export([ + resource_type/0, callback_mode/0, on_start/2, on_stop/2, @@ -183,6 +184,7 @@ sc(Type, Meta) -> hoconsc:mk(Type, Meta). ref(Field) -> hoconsc:ref(?MODULE, Field). %% =================================================================== +resource_type() -> webhook. callback_mode() -> async_if_possible. diff --git a/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.app.src b/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.app.src index a8314541a..eae5028c6 100644 --- a/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.app.src +++ b/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_influxdb, [ {description, "EMQX Enterprise InfluxDB Bridge"}, - {vsn, "0.2.3"}, + {vsn, "0.2.4"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb_connector.erl b/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb_connector.erl index 852f78485..bf93309f8 100644 --- a/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb_connector.erl +++ b/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb_connector.erl @@ -16,6 +16,7 @@ %% callbacks of behaviour emqx_resource -export([ + resource_type/0, callback_mode/0, on_start/2, on_stop/2, @@ -70,6 +71,8 @@ %% ------------------------------------------------------------------------------------------------- %% resource callback +resource_type() -> influxdb. + callback_mode() -> async_if_possible. on_add_channel( diff --git a/apps/emqx_bridge_influxdb/test/emqx_bridge_influxdb_connector_SUITE.erl b/apps/emqx_bridge_influxdb/test/emqx_bridge_influxdb_connector_SUITE.erl index a7f78f253..0ca693171 100644 --- a/apps/emqx_bridge_influxdb/test/emqx_bridge_influxdb_connector_SUITE.erl +++ b/apps/emqx_bridge_influxdb/test/emqx_bridge_influxdb_connector_SUITE.erl @@ -86,7 +86,6 @@ perform_lifecycle_check(PoolName, InitialConfig) -> state := #{client := #{pool := ReturnedPoolName}} = State, status := InitialStatus }} = emqx_resource:create_local( - influxdb, PoolName, ?CONNECTOR_RESOURCE_GROUP, ?INFLUXDB_RESOURCE_MOD, @@ -198,7 +197,6 @@ perform_tls_opts_check(PoolName, InitialConfig, VerifyReturn) -> config := #{ssl := #{enable := SslEnabled}}, status := Status }} = emqx_resource:create_local( - influxdb, PoolName, ?CONNECTOR_RESOURCE_GROUP, ?INFLUXDB_RESOURCE_MOD, diff --git a/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.app.src b/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.app.src index 691778cfd..88ac09da9 100644 --- a/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.app.src +++ b/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_bridge_iotdb, [ {description, "EMQX Enterprise Apache IoTDB Bridge"}, - {vsn, "0.2.2"}, + {vsn, "0.2.3"}, {modules, [ emqx_bridge_iotdb, emqx_bridge_iotdb_connector diff --git a/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb_connector.erl b/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb_connector.erl index 78866ef79..ec880e785 100644 --- a/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb_connector.erl +++ b/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb_connector.erl @@ -15,6 +15,7 @@ %% `emqx_resource' API -export([ + resource_type/0, callback_mode/0, on_start/2, on_stop/2, @@ -206,6 +207,8 @@ proplists_without(Keys, List) -> %%------------------------------------------------------------------------------------- %% `emqx_resource' API %%------------------------------------------------------------------------------------- +resource_type() -> iotdb. + callback_mode() -> async_if_possible. -spec on_start(manager_id(), config()) -> {ok, state()} | no_return(). diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_consumer.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_consumer.erl index 35ffbc90b..44e007415 100644 --- a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_consumer.erl +++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_consumer.erl @@ -7,6 +7,7 @@ %% `emqx_resource' API -export([ + resource_type/0, callback_mode/0, query_mode/1, on_start/2, @@ -125,6 +126,7 @@ %%------------------------------------------------------------------------------------- %% `emqx_resource' API %%------------------------------------------------------------------------------------- +resource_type() -> kafka_consumer. callback_mode() -> async_if_possible. diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl index 8b6326545..6d88a329e 100644 --- a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl +++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl @@ -10,6 +10,7 @@ %% callbacks of behaviour emqx_resource -export([ + resource_type/0, query_mode/1, callback_mode/0, on_start/2, @@ -35,6 +36,8 @@ -define(kafka_client_id, kafka_client_id). -define(kafka_producers, kafka_producers). +resource_type() -> kafka_producer. + query_mode(#{parameters := #{query_mode := sync}}) -> simple_sync_internal_buffer; query_mode(_) -> diff --git a/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis.app.src b/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis.app.src index f411b95fb..a85121905 100644 --- a/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis.app.src +++ b/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_kinesis, [ {description, "EMQX Enterprise Amazon Kinesis Bridge"}, - {vsn, "0.2.1"}, + {vsn, "0.2.2"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis_impl_producer.erl b/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis_impl_producer.erl index 95d193d92..3143cf904 100644 --- a/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis_impl_producer.erl +++ b/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis_impl_producer.erl @@ -30,6 +30,7 @@ %% `emqx_resource' API -export([ + resource_type/0, callback_mode/0, on_start/2, on_stop/2, @@ -50,6 +51,7 @@ %%------------------------------------------------------------------------------------------------- %% `emqx_resource' API %%------------------------------------------------------------------------------------------------- +resource_type() -> kinesis_producer. callback_mode() -> always_sync. diff --git a/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.app.src b/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.app.src index df9935dbb..5bb7e396d 100644 --- a/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.app.src +++ b/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_mongodb, [ {description, "EMQX Enterprise MongoDB Bridge"}, - {vsn, "0.3.2"}, + {vsn, "0.3.3"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb_connector.erl b/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb_connector.erl index 6b6db358a..dac9bef57 100644 --- a/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb_connector.erl +++ b/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb_connector.erl @@ -11,6 +11,7 @@ %% `emqx_resource' API -export([ on_remove_channel/3, + resource_type/0, callback_mode/0, on_add_channel/4, on_get_channel_status/3, @@ -25,6 +26,7 @@ %%======================================================================================== %% `emqx_resource' API %%======================================================================================== +resource_type() -> emqx_mongodb:resource_type(). callback_mode() -> emqx_mongodb:callback_mode(). diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl index 9c2506bab..118542356 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl @@ -30,6 +30,7 @@ %% callbacks of behaviour emqx_resource -export([ + resource_type/0, callback_mode/0, on_start/2, on_stop/2, @@ -76,6 +77,8 @@ on_message_received(Msg, HookPoints, ResId) -> ok. %% =================================================================== +resource_type() -> mqtt. + callback_mode() -> async_if_possible. on_start(ResourceId, #{server := Server} = Conf) -> diff --git a/apps/emqx_bridge_mysql/src/emqx_bridge_mysql.app.src b/apps/emqx_bridge_mysql/src/emqx_bridge_mysql.app.src index 63bc61e62..fb670e072 100644 --- a/apps/emqx_bridge_mysql/src/emqx_bridge_mysql.app.src +++ b/apps/emqx_bridge_mysql/src/emqx_bridge_mysql.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_mysql, [ {description, "EMQX Enterprise MySQL Bridge"}, - {vsn, "0.1.7"}, + {vsn, "0.1.8"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_mysql/src/emqx_bridge_mysql_connector.erl b/apps/emqx_bridge_mysql/src/emqx_bridge_mysql_connector.erl index da9377814..6905c86eb 100644 --- a/apps/emqx_bridge_mysql/src/emqx_bridge_mysql_connector.erl +++ b/apps/emqx_bridge_mysql/src/emqx_bridge_mysql_connector.erl @@ -10,6 +10,7 @@ %% `emqx_resource' API -export([ on_remove_channel/3, + resource_type/0, callback_mode/0, on_add_channel/4, on_batch_query/3, @@ -24,6 +25,7 @@ %%======================================================================================== %% `emqx_resource' API %%======================================================================================== +resource_type() -> emqx_mysql:resource_type(). callback_mode() -> emqx_mysql:callback_mode(). diff --git a/apps/emqx_bridge_opents/src/emqx_bridge_opents.app.src b/apps/emqx_bridge_opents/src/emqx_bridge_opents.app.src index 65cc97e4c..a27791853 100644 --- a/apps/emqx_bridge_opents/src/emqx_bridge_opents.app.src +++ b/apps/emqx_bridge_opents/src/emqx_bridge_opents.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_opents, [ {description, "EMQX Enterprise OpenTSDB Bridge"}, - {vsn, "0.2.1"}, + {vsn, "0.2.2"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_opents/src/emqx_bridge_opents_connector.erl b/apps/emqx_bridge_opents/src/emqx_bridge_opents_connector.erl index 19e117a0d..a970bb374 100644 --- a/apps/emqx_bridge_opents/src/emqx_bridge_opents_connector.erl +++ b/apps/emqx_bridge_opents/src/emqx_bridge_opents_connector.erl @@ -18,6 +18,7 @@ %% `emqx_resource' API -export([ + resource_type/0, callback_mode/0, on_start/2, on_stop/2, @@ -114,6 +115,8 @@ connector_example_values() -> -define(HTTP_CONNECT_TIMEOUT, 1000). +resource_type() -> opents. + callback_mode() -> always_sync. on_start( diff --git a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_connector.erl b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_connector.erl index 835536bda..64dde77fb 100644 --- a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_connector.erl +++ b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_connector.erl @@ -10,6 +10,7 @@ %% `emqx_resource' API -export([ + resource_type/0, callback_mode/0, query_mode/1, on_start/2, @@ -55,6 +56,7 @@ %%------------------------------------------------------------------------------------- %% `emqx_resource' API %%------------------------------------------------------------------------------------- +resource_type() -> pulsar. callback_mode() -> async_if_possible. diff --git a/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq.app.src b/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq.app.src index 27a4fedc4..c178b1f5e 100644 --- a/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq.app.src +++ b/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_rabbitmq, [ {description, "EMQX Enterprise RabbitMQ Bridge"}, - {vsn, "0.2.1"}, + {vsn, "0.2.2"}, {registered, []}, {mod, {emqx_bridge_rabbitmq_app, []}}, {applications, [ diff --git a/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq_connector.erl b/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq_connector.erl index dacb47a57..7e3e18e5f 100644 --- a/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq_connector.erl +++ b/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq_connector.erl @@ -31,6 +31,7 @@ on_remove_channel/3, on_get_channels/1, on_stop/2, + resource_type/0, callback_mode/0, on_get_status/2, on_get_channel_status/3, @@ -60,6 +61,7 @@ fields(config) -> %% =================================================================== %% emqx_resource callback +resource_type() -> rabbitmq. callback_mode() -> always_sync. diff --git a/apps/emqx_bridge_rabbitmq/test/emqx_bridge_rabbitmq_connector_SUITE.erl b/apps/emqx_bridge_rabbitmq/test/emqx_bridge_rabbitmq_connector_SUITE.erl index fe288a185..77482ae0f 100644 --- a/apps/emqx_bridge_rabbitmq/test/emqx_bridge_rabbitmq_connector_SUITE.erl +++ b/apps/emqx_bridge_rabbitmq/test/emqx_bridge_rabbitmq_connector_SUITE.erl @@ -135,7 +135,6 @@ check_config(Config) -> create_local_resource(ResourceID, CheckedConfig) -> {ok, Bridge} = emqx_resource:create_local( - rabbitmq, ResourceID, ?CONNECTOR_RESOURCE_GROUP, emqx_bridge_rabbitmq_connector, diff --git a/apps/emqx_bridge_redis/src/emqx_bridge_redis.app.src b/apps/emqx_bridge_redis/src/emqx_bridge_redis.app.src index 2cd037ed5..61cd837bb 100644 --- a/apps/emqx_bridge_redis/src/emqx_bridge_redis.app.src +++ b/apps/emqx_bridge_redis/src/emqx_bridge_redis.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_redis, [ {description, "EMQX Enterprise Redis Bridge"}, - {vsn, "0.1.8"}, + {vsn, "0.1.9"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_redis/src/emqx_bridge_redis_connector.erl b/apps/emqx_bridge_redis/src/emqx_bridge_redis_connector.erl index f117c4e7a..162c38368 100644 --- a/apps/emqx_bridge_redis/src/emqx_bridge_redis_connector.erl +++ b/apps/emqx_bridge_redis/src/emqx_bridge_redis_connector.erl @@ -12,6 +12,7 @@ %% callbacks of behaviour emqx_resource -export([ + resource_type/0, callback_mode/0, on_add_channel/4, on_remove_channel/3, @@ -29,7 +30,9 @@ %% resource callbacks %% ------------------------------------------------------------------------------------------------- -callback_mode() -> always_sync. +resource_type() -> emqx_redis:resource_type(). + +callback_mode() -> emqx_redis:callback_mode(). on_add_channel( _InstanceId, diff --git a/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq.app.src b/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq.app.src index fc59aeeca..9657ac115 100644 --- a/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq.app.src +++ b/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_rocketmq, [ {description, "EMQX Enterprise RocketMQ Bridge"}, - {vsn, "0.2.2"}, + {vsn, "0.2.3"}, {registered, []}, {applications, [kernel, stdlib, emqx_resource, rocketmq]}, {env, [ diff --git a/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq_connector.erl b/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq_connector.erl index 4fe3ea4c4..b03602bd2 100644 --- a/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq_connector.erl +++ b/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq_connector.erl @@ -16,6 +16,7 @@ %% `emqx_resource' API -export([ + resource_type/0, callback_mode/0, on_start/2, on_stop/2, @@ -90,6 +91,8 @@ servers() -> %% `emqx_resource' API %%======================================================================================== +resource_type() -> rocketmq. + callback_mode() -> always_sync. on_start( diff --git a/apps/emqx_bridge_s3/src/emqx_bridge_s3_connector.erl b/apps/emqx_bridge_s3/src/emqx_bridge_s3_connector.erl index fdc6d255b..a7aa7eae7 100644 --- a/apps/emqx_bridge_s3/src/emqx_bridge_s3_connector.erl +++ b/apps/emqx_bridge_s3/src/emqx_bridge_s3_connector.erl @@ -13,6 +13,7 @@ -behaviour(emqx_resource). -export([ + resource_type/0, callback_mode/0, on_start/2, on_stop/2, @@ -92,6 +93,8 @@ -define(AGGREG_SUP, emqx_bridge_s3_sup). %% +-spec resource_type() -> resource_type(). +resource_type() -> s3. -spec callback_mode() -> callback_mode(). callback_mode() -> diff --git a/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver.app.src b/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver.app.src index 3bc62734c..009a8d16b 100644 --- a/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver.app.src +++ b/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_sqlserver, [ {description, "EMQX Enterprise SQL Server Bridge"}, - {vsn, "0.2.2"}, + {vsn, "0.2.3"}, {registered, []}, {applications, [kernel, stdlib, emqx_resource, odbc]}, {env, [ diff --git a/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver_connector.erl b/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver_connector.erl index 603ef18d0..e14e395c0 100644 --- a/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver_connector.erl +++ b/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver_connector.erl @@ -30,6 +30,7 @@ %% callbacks for behaviour emqx_resource -export([ + resource_type/0, callback_mode/0, on_start/2, on_stop/2, @@ -173,6 +174,7 @@ server() -> %%==================================================================== %% Callbacks defined in emqx_resource %%==================================================================== +resource_type() -> sqlserver. callback_mode() -> always_sync. diff --git a/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper.app.src b/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper.app.src index 5ae95ca67..cd1d51b01 100644 --- a/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper.app.src +++ b/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_syskeeper, [ {description, "EMQX Enterprise Data bridge for Syskeeper"}, - {vsn, "0.1.3"}, + {vsn, "0.1.4"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_connector.erl b/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_connector.erl index 898915f56..c277faa4f 100644 --- a/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_connector.erl +++ b/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_connector.erl @@ -18,6 +18,7 @@ %% `emqx_resource' API -export([ + resource_type/0, callback_mode/0, query_mode/1, on_start/2, @@ -147,6 +148,7 @@ server() -> %% ------------------------------------------------------------------------------------------------- %% `emqx_resource' API +resource_type() -> syskeeper. callback_mode() -> always_sync. diff --git a/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_proxy_server.erl b/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_proxy_server.erl index e26ae43c1..d0aa44cd3 100644 --- a/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_proxy_server.erl +++ b/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_proxy_server.erl @@ -12,6 +12,7 @@ %% `emqx_resource' API -export([ + resource_type/0, query_mode/1, on_start/2, on_stop/2, @@ -40,6 +41,8 @@ %% ------------------------------------------------------------------------------------------------- %% emqx_resource +resource_type() -> + syskeeper_proxy_server. query_mode(_) -> no_queries. diff --git a/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine.app.src b/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine.app.src index d358ba8fa..5fe325b38 100644 --- a/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine.app.src +++ b/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_tdengine, [ {description, "EMQX Enterprise TDEngine Bridge"}, - {vsn, "0.2.1"}, + {vsn, "0.2.2"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine_connector.erl b/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine_connector.erl index 324694edc..46980e768 100644 --- a/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine_connector.erl +++ b/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine_connector.erl @@ -19,6 +19,7 @@ %% `emqx_resource' API -export([ + resource_type/0, callback_mode/0, on_start/2, on_stop/2, @@ -140,6 +141,7 @@ connector_example_values() -> %%======================================================================================== %% `emqx_resource' API %%======================================================================================== +resource_type() -> tdengine. callback_mode() -> always_sync. diff --git a/apps/emqx_connector/src/emqx_connector_resource.erl b/apps/emqx_connector/src/emqx_connector_resource.erl index 2a5b3bcfc..50b2132e2 100644 --- a/apps/emqx_connector/src/emqx_connector_resource.erl +++ b/apps/emqx_connector/src/emqx_connector_resource.erl @@ -126,7 +126,6 @@ create(Type, Name, Conf0, Opts) -> ResourceId = resource_id(Type, Name), Conf = Conf0#{connector_type => TypeBin, connector_name => Name}, {ok, _Data} = emqx_resource:create_local( - Type, ResourceId, ?CONNECTOR_RESOURCE_GROUP, ?MODULE:connector_to_resource_type(Type), @@ -200,7 +199,6 @@ recreate(Type, Name, Conf) -> recreate(Type, Name, Conf, Opts) -> TypeBin = bin(Type), emqx_resource:recreate_local( - Type, resource_id(Type, Name), ?MODULE:connector_to_resource_type(Type), parse_confs(TypeBin, Name, Conf), @@ -211,7 +209,7 @@ create_dry_run(Type, Conf) -> create_dry_run(Type, Conf, fun(_) -> ok end). create_dry_run(Type, Conf0, Callback) -> - %% Already typechecked, no need to catch errors + %% Already type checked, no need to catch errors TypeBin = bin(Type), TypeAtom = safe_atom(Type), %% We use a fixed name here to avoid creating an atom @@ -237,7 +235,7 @@ create_dry_run(Type, Conf0, Callback) -> {ok, ConfNew} -> ParseConf = parse_confs(bin(Type), TmpName, ConfNew), emqx_resource:create_dry_run_local( - Type, TmpName, ?MODULE:connector_to_resource_type(Type), ParseConf, Callback + TmpName, ?MODULE:connector_to_resource_type(Type), ParseConf, Callback ) end catch diff --git a/apps/emqx_connector/test/emqx_connector_SUITE.erl b/apps/emqx_connector/test/emqx_connector_SUITE.erl index fbdece6ff..8e5a6d288 100644 --- a/apps/emqx_connector/test/emqx_connector_SUITE.erl +++ b/apps/emqx_connector/test/emqx_connector_SUITE.erl @@ -50,6 +50,7 @@ t_connector_lifecycle({init, Config}) -> meck:new(emqx_connector_resource, [passthrough]), meck:expect(emqx_connector_resource, connector_to_resource_type, 1, ?CONNECTOR), meck:new(?CONNECTOR, [non_strict]), + meck:expect(?CONNECTOR, resource_type, 0, dummy), meck:expect(?CONNECTOR, callback_mode, 0, async_if_possible), meck:expect(?CONNECTOR, on_start, 2, {ok, connector_state}), meck:expect(?CONNECTOR, on_stop, 2, ok), @@ -171,6 +172,7 @@ t_remove_fail({'init', Config}) -> meck:expect(emqx_connector_resource, connector_to_resource_type, 1, ?CONNECTOR), meck:new(?CONNECTOR, [non_strict]), meck:expect(?CONNECTOR, callback_mode, 0, async_if_possible), + meck:expect(?CONNECTOR, resource_type, 0, dummy), meck:expect(?CONNECTOR, on_start, 2, {ok, connector_state}), meck:expect(?CONNECTOR, on_get_channels, 1, [{<<"my_channel">>, #{enable => true}}]), meck:expect(?CONNECTOR, on_add_channel, 4, {ok, connector_state}), @@ -234,6 +236,7 @@ t_create_with_bad_name_direct_path({init, Config}) -> meck:new(emqx_connector_resource, [passthrough]), meck:expect(emqx_connector_resource, connector_to_resource_type, 1, ?CONNECTOR), meck:new(?CONNECTOR, [non_strict]), + meck:expect(?CONNECTOR, resource_type, 0, dummy), meck:expect(?CONNECTOR, callback_mode, 0, async_if_possible), meck:expect(?CONNECTOR, on_start, 2, {ok, connector_state}), meck:expect(?CONNECTOR, on_stop, 2, ok), @@ -265,6 +268,7 @@ t_create_with_bad_name_root_path({init, Config}) -> meck:new(emqx_connector_resource, [passthrough]), meck:expect(emqx_connector_resource, connector_to_resource_type, 1, ?CONNECTOR), meck:new(?CONNECTOR, [non_strict]), + meck:expect(?CONNECTOR, resource_type, 0, dummy), meck:expect(?CONNECTOR, callback_mode, 0, async_if_possible), meck:expect(?CONNECTOR, on_start, 2, {ok, connector_state}), meck:expect(?CONNECTOR, on_stop, 2, ok), @@ -299,6 +303,7 @@ t_no_buffer_workers({'init', Config}) -> meck:new(emqx_connector_resource, [passthrough]), meck:expect(emqx_connector_resource, connector_to_resource_type, 1, ?CONNECTOR), meck:new(?CONNECTOR, [non_strict]), + meck:expect(?CONNECTOR, resource_type, 0, dummy), meck:expect(?CONNECTOR, callback_mode, 0, async_if_possible), meck:expect(?CONNECTOR, on_start, 2, {ok, connector_state}), meck:expect(?CONNECTOR, on_get_channels, 1, []), diff --git a/apps/emqx_connector/test/emqx_connector_api_SUITE.erl b/apps/emqx_connector/test/emqx_connector_api_SUITE.erl index f3e91ef12..01f4fd188 100644 --- a/apps/emqx_connector/test/emqx_connector_api_SUITE.erl +++ b/apps/emqx_connector/test/emqx_connector_api_SUITE.erl @@ -224,6 +224,7 @@ init_mocks(_TestCase) -> meck:new(emqx_connector_resource, [passthrough, no_link]), meck:expect(emqx_connector_resource, connector_to_resource_type, 1, ?CONNECTOR_IMPL), meck:new(?CONNECTOR_IMPL, [non_strict, no_link]), + meck:expect(?CONNECTOR_IMPL, resource_type, 0, dummy), meck:expect(?CONNECTOR_IMPL, callback_mode, 0, async_if_possible), meck:expect( ?CONNECTOR_IMPL, diff --git a/apps/emqx_connector/test/emqx_connector_dummy_impl.erl b/apps/emqx_connector/test/emqx_connector_dummy_impl.erl index c5d9e4f83..d506c9633 100644 --- a/apps/emqx_connector/test/emqx_connector_dummy_impl.erl +++ b/apps/emqx_connector/test/emqx_connector_dummy_impl.erl @@ -15,8 +15,10 @@ %% this module is only intended to be mocked -module(emqx_connector_dummy_impl). +-behavior(emqx_resource). -export([ + resource_type/0, query_mode/1, callback_mode/0, on_start/2, @@ -25,6 +27,7 @@ on_get_channel_status/3 ]). +resource_type() -> dummy. query_mode(_) -> error(unexpected). callback_mode() -> error(unexpected). on_start(_, _) -> error(unexpected). diff --git a/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_manager.erl b/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_manager.erl index 6ac02efc6..60fec5171 100644 --- a/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_manager.erl +++ b/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_manager.erl @@ -132,7 +132,6 @@ make_resource_id(Backend) -> create_resource(ResourceId, Module, Config) -> Result = emqx_resource:create_local( - dashboard_sso, ResourceId, ?RESOURCE_GROUP, Module, @@ -143,7 +142,7 @@ create_resource(ResourceId, Module, Config) -> update_resource(ResourceId, Module, Config) -> Result = emqx_resource:recreate_local( - dashboard_sso, ResourceId, Module, Config, ?DEFAULT_RESOURCE_OPTS + ResourceId, Module, Config, ?DEFAULT_RESOURCE_OPTS ), start_resource_if_enabled(ResourceId, Result, Config). diff --git a/apps/emqx_ldap/src/emqx_ldap.app.src b/apps/emqx_ldap/src/emqx_ldap.app.src index b0d8ec59c..1b1b667cc 100644 --- a/apps/emqx_ldap/src/emqx_ldap.app.src +++ b/apps/emqx_ldap/src/emqx_ldap.app.src @@ -1,6 +1,6 @@ {application, emqx_ldap, [ {description, "EMQX LDAP Connector"}, - {vsn, "0.1.8"}, + {vsn, "0.1.9"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_ldap/src/emqx_ldap.erl b/apps/emqx_ldap/src/emqx_ldap.erl index d04be5d68..67b250420 100644 --- a/apps/emqx_ldap/src/emqx_ldap.erl +++ b/apps/emqx_ldap/src/emqx_ldap.erl @@ -27,6 +27,7 @@ %% callbacks of behaviour emqx_resource -export([ + resource_type/0, callback_mode/0, on_start/2, on_stop/2, @@ -129,6 +130,8 @@ ensure_username(Field) -> emqx_connector_schema_lib:username(Field). %% =================================================================== +resource_type() -> ldap. + callback_mode() -> always_sync. -spec on_start(binary(), hocon:config()) -> {ok, state()} | {error, _}. diff --git a/apps/emqx_ldap/test/emqx_ldap_SUITE.erl b/apps/emqx_ldap/test/emqx_ldap_SUITE.erl index 413cbc3a5..a15ff2775 100644 --- a/apps/emqx_ldap/test/emqx_ldap_SUITE.erl +++ b/apps/emqx_ldap/test/emqx_ldap_SUITE.erl @@ -96,7 +96,6 @@ perform_lifecycle_check(ResourceId, InitialConfig) -> state := #{pool_name := PoolName} = State, status := InitialStatus }} = emqx_resource:create_local( - ldap, ResourceId, ?CONNECTOR_RESOURCE_GROUP, ?LDAP_RESOURCE_MOD, @@ -172,7 +171,6 @@ t_get_status(Config) -> ?LDAP_RESOURCE_MOD, ldap_config(Config) ), {ok, _} = emqx_resource:create_local( - ldap, ResourceId, ?CONNECTOR_RESOURCE_GROUP, ?LDAP_RESOURCE_MOD, diff --git a/apps/emqx_mongodb/src/emqx_mongodb.app.src b/apps/emqx_mongodb/src/emqx_mongodb.app.src index 92d7026cc..51230fd47 100644 --- a/apps/emqx_mongodb/src/emqx_mongodb.app.src +++ b/apps/emqx_mongodb/src/emqx_mongodb.app.src @@ -1,6 +1,6 @@ {application, emqx_mongodb, [ {description, "EMQX MongoDB Connector"}, - {vsn, "0.1.6"}, + {vsn, "0.1.7"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_mongodb/src/emqx_mongodb.erl b/apps/emqx_mongodb/src/emqx_mongodb.erl index e262f5ccd..8d6fad89f 100644 --- a/apps/emqx_mongodb/src/emqx_mongodb.erl +++ b/apps/emqx_mongodb/src/emqx_mongodb.erl @@ -26,6 +26,7 @@ %% callbacks of behaviour emqx_resource -export([ + resource_type/0, callback_mode/0, on_start/2, on_stop/2, @@ -172,6 +173,7 @@ desc(_) -> undefined. %% =================================================================== +resource_type() -> mongodb. callback_mode() -> always_sync. diff --git a/apps/emqx_mongodb/test/emqx_mongodb_SUITE.erl b/apps/emqx_mongodb/test/emqx_mongodb_SUITE.erl index 850683d99..8af05e0d3 100644 --- a/apps/emqx_mongodb/test/emqx_mongodb_SUITE.erl +++ b/apps/emqx_mongodb/test/emqx_mongodb_SUITE.erl @@ -143,7 +143,6 @@ check_config(Config) -> create_local_resource(ResourceId, CheckedConfig) -> {ok, Bridge} = emqx_resource:create_local( - mongodb, ResourceId, ?CONNECTOR_RESOURCE_GROUP, ?MONGO_RESOURCE_MOD, diff --git a/apps/emqx_mysql/src/emqx_mysql.app.src b/apps/emqx_mysql/src/emqx_mysql.app.src index 9637cc473..c7fcb0975 100644 --- a/apps/emqx_mysql/src/emqx_mysql.app.src +++ b/apps/emqx_mysql/src/emqx_mysql.app.src @@ -1,6 +1,6 @@ {application, emqx_mysql, [ {description, "EMQX MySQL Database Connector"}, - {vsn, "0.1.9"}, + {vsn, "0.2.0"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_mysql/src/emqx_mysql.erl b/apps/emqx_mysql/src/emqx_mysql.erl index 6311d66f2..197e33d75 100644 --- a/apps/emqx_mysql/src/emqx_mysql.erl +++ b/apps/emqx_mysql/src/emqx_mysql.erl @@ -25,6 +25,7 @@ %% callbacks of behaviour emqx_resource -export([ + resource_type/0, callback_mode/0, on_start/2, on_stop/2, @@ -91,6 +92,8 @@ server() -> emqx_schema:servers_sc(Meta, ?MYSQL_HOST_OPTIONS). %% =================================================================== +resource_type() -> mysql. + callback_mode() -> always_sync. -spec on_start(binary(), hocon:config()) -> {ok, state()} | {error, _}. diff --git a/apps/emqx_mysql/test/emqx_mysql_SUITE.erl b/apps/emqx_mysql/test/emqx_mysql_SUITE.erl index be69140fc..03e6c6797 100644 --- a/apps/emqx_mysql/test/emqx_mysql_SUITE.erl +++ b/apps/emqx_mysql/test/emqx_mysql_SUITE.erl @@ -67,7 +67,6 @@ perform_lifecycle_check(ResourceId, InitialConfig) -> state := #{pool_name := PoolName} = State, status := InitialStatus }} = emqx_resource:create_local( - mysql, ResourceId, ?CONNECTOR_RESOURCE_GROUP, ?MYSQL_RESOURCE_MOD, diff --git a/apps/emqx_oracle/src/emqx_oracle.app.src b/apps/emqx_oracle/src/emqx_oracle.app.src index 3f238ae9c..80ff8da09 100644 --- a/apps/emqx_oracle/src/emqx_oracle.app.src +++ b/apps/emqx_oracle/src/emqx_oracle.app.src @@ -1,6 +1,6 @@ {application, emqx_oracle, [ {description, "EMQX Enterprise Oracle Database Connector"}, - {vsn, "0.2.2"}, + {vsn, "0.2.3"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_oracle/src/emqx_oracle.erl b/apps/emqx_oracle/src/emqx_oracle.erl index 5b25e049a..6e2a40f95 100644 --- a/apps/emqx_oracle/src/emqx_oracle.erl +++ b/apps/emqx_oracle/src/emqx_oracle.erl @@ -21,6 +21,7 @@ %% callbacks for behaviour emqx_resource -export([ + resource_type/0, callback_mode/0, on_start/2, on_stop/2, @@ -67,6 +68,8 @@ batch_params_tokens := params_tokens() }. +resource_type() -> oracle. + % As ecpool is not monitoring the worker's PID when doing a handover_async, the % request can be lost if worker crashes. Thus, it's better to force requests to % be sync for now. diff --git a/apps/emqx_postgresql/src/emqx_postgresql.app.src b/apps/emqx_postgresql/src/emqx_postgresql.app.src index 7aaf42e71..e1bd67325 100644 --- a/apps/emqx_postgresql/src/emqx_postgresql.app.src +++ b/apps/emqx_postgresql/src/emqx_postgresql.app.src @@ -1,6 +1,6 @@ {application, emqx_postgresql, [ {description, "EMQX PostgreSQL Database Connector"}, - {vsn, "0.2.2"}, + {vsn, "0.2.3"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_postgresql/src/emqx_postgresql.erl b/apps/emqx_postgresql/src/emqx_postgresql.erl index 7e64a3e83..4df8e0af1 100644 --- a/apps/emqx_postgresql/src/emqx_postgresql.erl +++ b/apps/emqx_postgresql/src/emqx_postgresql.erl @@ -29,6 +29,7 @@ %% callbacks of behaviour emqx_resource -export([ + resource_type/0, callback_mode/0, on_start/2, on_stop/2, @@ -120,6 +121,8 @@ adjust_fields(Fields) -> ). %% =================================================================== +resource_type() -> postgresql. + callback_mode() -> always_sync. -spec on_start(binary(), hocon:config()) -> {ok, state()} | {error, _}. diff --git a/apps/emqx_postgresql/test/emqx_postgresql_SUITE.erl b/apps/emqx_postgresql/test/emqx_postgresql_SUITE.erl index 06210be86..d771d80d8 100644 --- a/apps/emqx_postgresql/test/emqx_postgresql_SUITE.erl +++ b/apps/emqx_postgresql/test/emqx_postgresql_SUITE.erl @@ -75,7 +75,6 @@ perform_lifecycle_check(ResourceId, InitialConfig) -> status := InitialStatus }} = emqx_resource:create_local( - postgresql, ResourceId, ?CONNECTOR_RESOURCE_GROUP, ?PGSQL_RESOURCE_MOD, diff --git a/apps/emqx_redis/src/emqx_redis.erl b/apps/emqx_redis/src/emqx_redis.erl index 059e9aa23..9507913ed 100644 --- a/apps/emqx_redis/src/emqx_redis.erl +++ b/apps/emqx_redis/src/emqx_redis.erl @@ -28,6 +28,7 @@ %% callbacks of behaviour emqx_resource -export([ + resource_type/0, callback_mode/0, on_start/2, on_stop/2, @@ -119,6 +120,8 @@ redis_type(Type) -> desc => ?DESC(Type) }}. +resource_type() -> redis. + callback_mode() -> always_sync. on_start(InstId, Config0) -> diff --git a/apps/emqx_redis/test/emqx_redis_SUITE.erl b/apps/emqx_redis/test/emqx_redis_SUITE.erl index 06ac82143..a9064f184 100644 --- a/apps/emqx_redis/test/emqx_redis_SUITE.erl +++ b/apps/emqx_redis/test/emqx_redis_SUITE.erl @@ -115,7 +115,6 @@ perform_lifecycle_check(ResourceId, InitialConfig, RedisCommand) -> state := #{pool_name := PoolName} = State, status := InitialStatus }} = emqx_resource:create_local( - redis, ResourceId, ?CONNECTOR_RESOURCE_GROUP, ?REDIS_RESOURCE_MOD, diff --git a/apps/emqx_resource/include/emqx_resource.hrl b/apps/emqx_resource/include/emqx_resource.hrl index 0c0cc8f6b..8c2bb39a1 100644 --- a/apps/emqx_resource/include/emqx_resource.hrl +++ b/apps/emqx_resource/include/emqx_resource.hrl @@ -23,8 +23,8 @@ %% remind us of that. -define(rm_status_stopped, stopped). --type type() :: atom() | binary(). --type resource_type() :: module(). +-type resource_type() :: atom(). +-type resource_module() :: module(). -type resource_id() :: binary(). -type channel_id() :: binary(). -type raw_resource_config() :: binary() | raw_term_resource_config(). diff --git a/apps/emqx_resource/src/emqx_resource.erl b/apps/emqx_resource/src/emqx_resource.erl index e0a2e9343..721df9690 100644 --- a/apps/emqx_resource/src/emqx_resource.erl +++ b/apps/emqx_resource/src/emqx_resource.erl @@ -28,9 +28,9 @@ -export([ check_config/2, + check_and_create_local/4, check_and_create_local/5, - check_and_create_local/6, - check_and_recreate_local/5 + check_and_recreate_local/4 ]). %% Sync resource instances and files @@ -39,11 +39,11 @@ -export([ %% store the config and start the instance - create_local/6, + create_local/5, + create_dry_run_local/2, create_dry_run_local/3, create_dry_run_local/4, - create_dry_run_local/5, - recreate_local/5, + recreate_local/4, %% remove the config and stop the instance remove_local/1, reset_metrics/1, @@ -96,6 +96,7 @@ -export([ %% get the callback mode of a specific module get_callback_mode/1, + get_resource_type/1, %% start the instance call_start/3, %% verify if the resource is working normally @@ -140,9 +141,6 @@ -export([is_dry_run/1]). -%% For emqx_resource_proto_v1 rpc only --export([create_local/5, recreate_local/4, create_dry_run_local/2]). - -export_type([ query_mode/0, resource_id/0, @@ -283,42 +281,42 @@ is_resource_mod(Module) -> %% APIs for resource instances %% ================================================================================= -spec create_local( - type(), resource_id(), resource_group(), - resource_type(), + resource_module(), resource_config(), creation_opts() ) -> {ok, resource_data()}. -create_local(Type, ResId, Group, ResourceType, Config, Opts) -> - emqx_resource_manager:ensure_resource(Type, ResId, Group, ResourceType, Config, Opts). +create_local(ResId, Group, ResourceType, Config, Opts) -> + emqx_resource_manager:ensure_resource(ResId, Group, ResourceType, Config, Opts). --spec create_dry_run_local(type(), resource_type(), resource_config()) -> +-spec create_dry_run_local(resource_module(), resource_config()) -> ok | {error, Reason :: term()}. -create_dry_run_local(Type, ResourceType, Config) -> - emqx_resource_manager:create_dry_run(Type, ResourceType, Config). +create_dry_run_local(ResourceType, Config) -> + emqx_resource_manager:create_dry_run(ResourceType, Config). -create_dry_run_local(Type, ResId, ResourceType, Config) -> - emqx_resource_manager:create_dry_run(Type, ResId, ResourceType, Config). +create_dry_run_local(ResId, ResourceType, Config) -> + emqx_resource_manager:create_dry_run(ResId, ResourceType, Config). -spec create_dry_run_local( - type(), resource_id(), - resource_type(), + resource_module(), resource_config(), OnReadyCallback ) -> ok | {error, Reason :: term()} when OnReadyCallback :: fun((resource_id()) -> ok | {error, Reason :: term()}). -create_dry_run_local(Type, ResId, ResourceType, Config, OnReadyCallback) -> - emqx_resource_manager:create_dry_run(Type, ResId, ResourceType, Config, OnReadyCallback). +create_dry_run_local(ResId, ResourceType, Config, OnReadyCallback) -> + emqx_resource_manager:create_dry_run(ResId, ResourceType, Config, OnReadyCallback). --spec recreate_local(type(), resource_id(), resource_type(), resource_config(), creation_opts()) -> +-spec recreate_local( + resource_id(), resource_module(), resource_config(), creation_opts() +) -> {ok, resource_data()} | {error, Reason :: term()}. -recreate_local(Type, ResId, ResourceType, Config, Opts) -> - emqx_resource_manager:recreate(Type, ResId, ResourceType, Config, Opts). +recreate_local(ResId, ResourceType, Config, Opts) -> + emqx_resource_manager:recreate(ResId, ResourceType, Config, Opts). -spec remove_local(resource_id()) -> ok. remove_local(ResId) -> @@ -490,6 +488,10 @@ list_group_instances(Group) -> emqx_resource_manager:list_group(Group). get_callback_mode(Mod) -> Mod:callback_mode(). +-spec get_resource_type(module()) -> resource_type(). +get_resource_type(Mod) -> + Mod:resource_type(). + -spec call_start(resource_id(), module(), resource_config()) -> {ok, resource_state()} | {error, Reason :: term()}. call_start(ResId, Mod, Config) -> @@ -602,50 +604,47 @@ query_mode(Mod, Config, Opts) -> maps:get(query_mode, Opts, sync) end. --spec check_config(resource_type(), raw_resource_config()) -> +-spec check_config(resource_module(), raw_resource_config()) -> {ok, resource_config()} | {error, term()}. check_config(ResourceType, Conf) -> emqx_hocon:check(ResourceType, Conf). -spec check_and_create_local( - type(), resource_id(), resource_group(), - resource_type(), + resource_module(), raw_resource_config() ) -> {ok, resource_data()} | {error, term()}. -check_and_create_local(Type, ResId, Group, ResourceType, RawConfig) -> - check_and_create_local(Type, ResId, Group, ResourceType, RawConfig, #{}). +check_and_create_local(ResId, Group, ResourceType, RawConfig) -> + check_and_create_local(ResId, Group, ResourceType, RawConfig, #{}). -spec check_and_create_local( - type(), resource_id(), resource_group(), - resource_type(), + resource_module(), raw_resource_config(), creation_opts() ) -> {ok, resource_data()} | {error, term()}. -check_and_create_local(Type, ResId, Group, ResourceType, RawConfig, Opts) -> +check_and_create_local(ResId, Group, ResourceType, RawConfig, Opts) -> check_and_do( ResourceType, RawConfig, - fun(ResConf) -> create_local(Type, ResId, Group, ResourceType, ResConf, Opts) end + fun(ResConf) -> create_local(ResId, Group, ResourceType, ResConf, Opts) end ). -spec check_and_recreate_local( - type(), resource_id(), - resource_type(), + resource_module(), raw_resource_config(), creation_opts() ) -> {ok, resource_data()} | {error, term()}. -check_and_recreate_local(Type, ResId, ResourceType, RawConfig, Opts) -> +check_and_recreate_local(ResId, ResourceType, RawConfig, Opts) -> check_and_do( ResourceType, RawConfig, - fun(ResConf) -> recreate_local(Type, ResId, ResourceType, ResConf, Opts) end + fun(ResConf) -> recreate_local(ResId, ResourceType, ResConf, Opts) end ). check_and_do(ResourceType, RawConfig, Do) when is_function(Do) -> @@ -809,18 +808,3 @@ validate_name(Name, Opts) -> -spec invalid_data(binary()) -> no_return(). invalid_data(Reason) -> throw(#{kind => validation_error, reason => Reason}). - -%% Those functions is only used in the emqx_resource_proto_v1 -%% for versions that are less than version 5.6.0. -%% begin --spec create_local( - resource_id(), resource_group(), resource_type(), resource_config(), creation_opts() -) -> - {ok, resource_data() | 'already_created'} | {error, Reason :: term()}. -create_local(ResId, Group, ResourceType, Config, Opts) -> - create_local(deprecated, ResId, Group, ResourceType, Config, Opts). -create_dry_run_local(ResourceType, Config) -> - create_dry_run_local(deprecated, ResourceType, Config). -recreate_local(ResId, ResourceType, Config, Opts) -> - recreate_local(deprecated, ResId, ResourceType, Config, Opts). -%% end diff --git a/apps/emqx_resource/src/emqx_resource_manager.erl b/apps/emqx_resource/src/emqx_resource_manager.erl index a742c4486..fe674630c 100644 --- a/apps/emqx_resource/src/emqx_resource_manager.erl +++ b/apps/emqx_resource/src/emqx_resource_manager.erl @@ -25,12 +25,12 @@ % API -export([ - ensure_resource/6, - recreate/5, + ensure_resource/5, + recreate/4, remove/1, + create_dry_run/2, create_dry_run/3, create_dry_run/4, - create_dry_run/5, restart/2, start/2, stop/1, @@ -59,7 +59,7 @@ ]). % Server --export([start_link/6]). +-export([start_link/5]). % Behaviour -export([init/1, callback_mode/0, handle_event/4, terminate/3]). @@ -162,44 +162,45 @@ %% Triggers the emqx_resource_manager_sup supervisor to actually create %% and link the process itself if not already started. -spec ensure_resource( - type(), resource_id(), resource_group(), - resource_type(), + resource_module(), resource_config(), creation_opts() ) -> {ok, resource_data()}. -ensure_resource(Type, ResId, Group, ResourceType, Config, Opts) -> +ensure_resource(ResId, Group, ResourceType, Config, Opts) -> case lookup(ResId) of {ok, _Group, Data} -> {ok, Data}; {error, not_found} -> - create_and_return_data(Type, ResId, Group, ResourceType, Config, Opts) + create_and_return_data(ResId, Group, ResourceType, Config, Opts) end. %% @doc Called from emqx_resource when recreating a resource which may or may not exist --spec recreate(type(), resource_id(), resource_type(), resource_config(), creation_opts()) -> +-spec recreate( + resource_id(), resource_module(), resource_config(), creation_opts() +) -> {ok, resource_data()} | {error, not_found} | {error, updating_to_incorrect_resource_type}. -recreate(Type, ResId, ResourceType, NewConfig, Opts) -> +recreate(ResId, ResourceType, NewConfig, Opts) -> case lookup(ResId) of {ok, Group, #{mod := ResourceType, status := _} = _Data} -> _ = remove(ResId, false), - create_and_return_data(Type, ResId, Group, ResourceType, NewConfig, Opts); + create_and_return_data(ResId, Group, ResourceType, NewConfig, Opts); {ok, _, #{mod := Mod}} when Mod =/= ResourceType -> {error, updating_to_incorrect_resource_type}; {error, not_found} -> {error, not_found} end. -create_and_return_data(Type, ResId, Group, ResourceType, Config, Opts) -> - _ = create(Type, ResId, Group, ResourceType, Config, Opts), +create_and_return_data(ResId, Group, ResourceType, Config, Opts) -> + _ = create(ResId, Group, ResourceType, Config, Opts), {ok, _Group, Data} = lookup(ResId), {ok, Data}. %% @doc Create a resource_manager and wait until it is running -create(Type, ResId, Group, ResourceType, Config, Opts) -> +create(ResId, Group, ResourceType, Config, Opts) -> % The state machine will make the actual call to the callback/resource module after init - ok = emqx_resource_manager_sup:ensure_child(Type, ResId, Group, ResourceType, Config, Opts), + ok = emqx_resource_manager_sup:ensure_child(ResId, Group, ResourceType, Config, Opts), % Create metrics for the resource ok = emqx_resource:create_metrics(ResId), QueryMode = emqx_resource:query_mode(ResourceType, Config, Opts), @@ -222,30 +223,32 @@ create(Type, ResId, Group, ResourceType, Config, Opts) -> %% %% Triggers the `emqx_resource_manager_sup` supervisor to actually create %% and link the process itself if not already started, and then immediately stops. --spec create_dry_run(type(), resource_type(), resource_config()) -> +-spec create_dry_run(resource_module(), resource_config()) -> ok | {error, Reason :: term()}. -create_dry_run(Type, ResourceType, Config) -> +create_dry_run(ResourceType, Config) -> ResId = make_test_id(), - create_dry_run(Type, ResId, ResourceType, Config). + create_dry_run(ResId, ResourceType, Config). -create_dry_run(Type, ResId, ResourceType, Config) -> - create_dry_run(Type, ResId, ResourceType, Config, fun do_nothing_on_ready/1). +create_dry_run(ResId, ResourceType, Config) -> + create_dry_run(ResId, ResourceType, Config, fun do_nothing_on_ready/1). do_nothing_on_ready(_ResId) -> ok. --spec create_dry_run(type(), resource_id(), resource_type(), resource_config(), OnReadyCallback) -> +-spec create_dry_run( + resource_id(), resource_module(), resource_config(), OnReadyCallback +) -> ok | {error, Reason :: term()} when OnReadyCallback :: fun((resource_id()) -> ok | {error, Reason :: term()}). -create_dry_run(Type, ResId, ResourceType, Config, OnReadyCallback) -> +create_dry_run(ResId, ResourceType, Config, OnReadyCallback) -> Opts = case is_map(Config) of true -> maps:get(resource_opts, Config, #{}); false -> #{} end, ok = emqx_resource_manager_sup:ensure_child( - Type, ResId, <<"dry_run">>, ResourceType, Config, Opts + ResId, <<"dry_run">>, ResourceType, Config, Opts ), HealthCheckInterval = maps:get(health_check_interval, Opts, ?HEALTHCHECK_INTERVAL), Timeout = emqx_utils:clamp(HealthCheckInterval, 5_000, 60_000), @@ -495,7 +498,7 @@ try_clean_allocated_resources(ResId) -> %% Server start/stop callbacks %% @doc Function called from the supervisor to actually start the server -start_link(Type, ResId, Group, ResourceType, Config, Opts) -> +start_link(ResId, Group, ResourceType, Config, Opts) -> QueryMode = emqx_resource:query_mode( ResourceType, Config, @@ -503,7 +506,7 @@ start_link(Type, ResId, Group, ResourceType, Config, Opts) -> ), Data = #data{ id = ResId, - type = Type, + type = emqx_resource:get_resource_type(ResourceType), group = Group, mod = ResourceType, callback_mode = emqx_resource:get_callback_mode(ResourceType), diff --git a/apps/emqx_resource/src/emqx_resource_manager_sup.erl b/apps/emqx_resource/src/emqx_resource_manager_sup.erl index c14b08f94..8542eec1c 100644 --- a/apps/emqx_resource/src/emqx_resource_manager_sup.erl +++ b/apps/emqx_resource/src/emqx_resource_manager_sup.erl @@ -19,16 +19,14 @@ -include("emqx_resource.hrl"). --export([ensure_child/6, delete_child/1]). +-export([ensure_child/5, delete_child/1]). -export([start_link/0]). -export([init/1]). -ensure_child(Type, ResId, Group, ResourceType, Config, Opts) -> - case - supervisor:start_child(?MODULE, child_spec(Type, ResId, Group, ResourceType, Config, Opts)) - of +ensure_child(ResId, Group, ResourceType, Config, Opts) -> + case supervisor:start_child(?MODULE, child_spec(ResId, Group, ResourceType, Config, Opts)) of {error, Reason} -> %% This should not happen in production but it can be a huge time sink in %% development environments if the error is just silently ignored. @@ -57,11 +55,11 @@ init([]) -> SupFlags = #{strategy => one_for_one, intensity => 10, period => 10}, {ok, {SupFlags, ChildSpecs}}. -child_spec(Type, ResId, Group, ResourceType, Config, Opts) -> +child_spec(ResId, Group, ResourceType, Config, Opts) -> #{ id => ResId, start => - {emqx_resource_manager, start_link, [Type, ResId, Group, ResourceType, Config, Opts]}, + {emqx_resource_manager, start_link, [ResId, Group, ResourceType, Config, Opts]}, restart => transient, %% never force kill a resource manager. %% because otherwise it may lead to release leak, diff --git a/apps/emqx_resource/src/proto/emqx_resource_proto_v1.erl b/apps/emqx_resource/src/proto/emqx_resource_proto_v1.erl index 859b9fa52..47b724271 100644 --- a/apps/emqx_resource/src/proto/emqx_resource_proto_v1.erl +++ b/apps/emqx_resource/src/proto/emqx_resource_proto_v1.erl @@ -40,7 +40,7 @@ deprecated_since() -> -spec create( resource_id(), resource_group(), - resource_type(), + resource_module(), resource_config(), creation_opts() ) -> @@ -51,7 +51,7 @@ create(ResId, Group, ResourceType, Config, Opts) -> ]). -spec create_dry_run( - resource_type(), + resource_module(), resource_config() ) -> ok | {error, Reason :: term()}. @@ -60,7 +60,7 @@ create_dry_run(ResourceType, Config) -> -spec recreate( resource_id(), - resource_type(), + resource_module(), resource_config(), creation_opts() ) -> diff --git a/apps/emqx_resource/test/emqx_connector_demo.erl b/apps/emqx_resource/test/emqx_connector_demo.erl index 0fc11cc66..e068defb1 100644 --- a/apps/emqx_resource/test/emqx_connector_demo.erl +++ b/apps/emqx_resource/test/emqx_connector_demo.erl @@ -24,6 +24,7 @@ %% callbacks of behaviour emqx_resource -export([ + resource_type/0, callback_mode/0, on_start/2, on_stop/2, @@ -62,6 +63,8 @@ register(required) -> true; register(default) -> false; register(_) -> undefined. +resource_type() -> demo. + callback_mode() -> persistent_term:get(?CM_KEY). diff --git a/apps/emqx_resource/test/emqx_resource_SUITE.erl b/apps/emqx_resource/test/emqx_resource_SUITE.erl index 8e16ec26a..b8fc66c10 100644 --- a/apps/emqx_resource/test/emqx_resource_SUITE.erl +++ b/apps/emqx_resource/test/emqx_resource_SUITE.erl @@ -23,7 +23,6 @@ -include_lib("snabbkaffe/include/snabbkaffe.hrl"). -define(TEST_RESOURCE, emqx_connector_demo). --define(TYPE, test). -define(ID, <<"id">>). -define(ID1, <<"id1">>). -define(DEFAULT_RESOURCE_GROUP, <<"default">>). @@ -91,7 +90,6 @@ t_create_remove(_) -> ?assertMatch( {error, _}, emqx_resource:check_and_create_local( - ?TYPE, ?ID, ?DEFAULT_RESOURCE_GROUP, ?TEST_RESOURCE, @@ -112,7 +110,6 @@ t_create_remove(_) -> ?assertMatch( {ok, _}, emqx_resource:recreate_local( - ?TYPE, ?ID, ?TEST_RESOURCE, #{name => test_resource}, @@ -138,7 +135,6 @@ t_create_remove_local(_) -> ?assertMatch( {error, _}, emqx_resource:check_and_create_local( - ?TYPE, ?ID, ?DEFAULT_RESOURCE_GROUP, ?TEST_RESOURCE, @@ -157,7 +153,6 @@ t_create_remove_local(_) -> ), emqx_resource:recreate_local( - ?TYPE, ?ID, ?TEST_RESOURCE, #{name => test_resource}, @@ -171,7 +166,6 @@ t_create_remove_local(_) -> emqx_resource:set_resource_status_connecting(?ID), emqx_resource:recreate_local( - ?TYPE, ?ID, ?TEST_RESOURCE, #{name => test_resource}, @@ -943,7 +937,6 @@ t_stop_start(_) -> ?assertMatch( {error, _}, emqx_resource:check_and_create_local( - ?TYPE, ?ID, ?DEFAULT_RESOURCE_GROUP, ?TEST_RESOURCE, @@ -954,7 +947,6 @@ t_stop_start(_) -> ?assertMatch( {ok, _}, emqx_resource:check_and_create_local( - ?TYPE, ?ID, ?DEFAULT_RESOURCE_GROUP, ?TEST_RESOURCE, @@ -972,7 +964,6 @@ t_stop_start(_) -> ?assertMatch( {ok, _}, emqx_resource:check_and_recreate_local( - ?TYPE, ?ID, ?TEST_RESOURCE, #{<<"name">> => <<"test_resource">>}, @@ -1022,7 +1013,6 @@ t_stop_start_local(_) -> ?assertMatch( {error, _}, emqx_resource:check_and_create_local( - ?TYPE, ?ID, ?DEFAULT_RESOURCE_GROUP, ?TEST_RESOURCE, @@ -1033,7 +1023,6 @@ t_stop_start_local(_) -> ?assertMatch( {ok, _}, emqx_resource:check_and_create_local( - ?TYPE, ?ID, ?DEFAULT_RESOURCE_GROUP, ?TEST_RESOURCE, @@ -1044,7 +1033,6 @@ t_stop_start_local(_) -> ?assertMatch( {ok, _}, emqx_resource:check_and_recreate_local( - ?TYPE, ?ID, ?TEST_RESOURCE, #{<<"name">> => <<"test_resource">>}, @@ -1120,7 +1108,6 @@ create_dry_run_local_succ() -> ?assertEqual( ok, emqx_resource:create_dry_run_local( - test, ?TEST_RESOURCE, #{name => test_resource, register => true} ) @@ -1131,7 +1118,6 @@ t_create_dry_run_local_failed(_) -> ct:timetrap({seconds, 120}), ct:pal("creating with creation error"), Res1 = emqx_resource:create_dry_run_local( - test, ?TEST_RESOURCE, #{create_error => true} ), @@ -1139,7 +1125,6 @@ t_create_dry_run_local_failed(_) -> ct:pal("creating with health check error"), Res2 = emqx_resource:create_dry_run_local( - test, ?TEST_RESOURCE, #{name => test_resource, health_check_error => true} ), @@ -1147,7 +1132,6 @@ t_create_dry_run_local_failed(_) -> ct:pal("creating with stop error"), Res3 = emqx_resource:create_dry_run_local( - test, ?TEST_RESOURCE, #{name => test_resource, stop_error => true} ), @@ -3506,10 +3490,10 @@ gauge_metric_set_fns() -> ]. create(Id, Group, Type, Config) -> - emqx_resource:create_local(test, Id, Group, Type, Config, #{}). + emqx_resource:create_local(Id, Group, Type, Config, #{}). create(Id, Group, Type, Config, Opts) -> - emqx_resource:create_local(test, Id, Group, Type, Config, Opts). + emqx_resource:create_local(Id, Group, Type, Config, Opts). log_consistency_prop() -> {"check state and cache consistency", fun ?MODULE:log_consistency_prop/1}. diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_test_connector.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_test_connector.erl index 6a0d6b3ec..cf5d3afbe 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_test_connector.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_test_connector.erl @@ -25,6 +25,7 @@ %% callbacks of behaviour emqx_resource -export([ + resource_type/0, callback_mode/0, on_start/2, on_stop/2, @@ -40,6 +41,8 @@ ]). %% =================================================================== +resource_type() -> test_connector. + callback_mode() -> always_sync. on_start( From 4d7535df2df8a2c59607e5bdc97e5e0b3b273f24 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Wed, 24 Jul 2024 09:50:01 +0800 Subject: [PATCH 9/9] chore: use pgsqsl replace postgresql --- .../src/emqx_bridge_hstreamdb_connector.erl | 2 +- apps/emqx_postgresql/src/emqx_postgresql.erl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb_connector.erl b/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb_connector.erl index 154e43b3d..2d061e455 100644 --- a/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb_connector.erl +++ b/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb_connector.erl @@ -45,7 +45,7 @@ %% ------------------------------------------------------------------------------------------------- %% resource callback -resource_type() -> hstreamsdb. +resource_type() -> hstreamdb. callback_mode() -> always_sync. diff --git a/apps/emqx_postgresql/src/emqx_postgresql.erl b/apps/emqx_postgresql/src/emqx_postgresql.erl index 4df8e0af1..a061fc15e 100644 --- a/apps/emqx_postgresql/src/emqx_postgresql.erl +++ b/apps/emqx_postgresql/src/emqx_postgresql.erl @@ -121,7 +121,7 @@ adjust_fields(Fields) -> ). %% =================================================================== -resource_type() -> postgresql. +resource_type() -> pgsql. callback_mode() -> always_sync.