diff --git a/.ci/docker-compose-file/credentials.env b/.ci/docker-compose-file/credentials.env new file mode 100644 index 000000000..50cc83a3f --- /dev/null +++ b/.ci/docker-compose-file/credentials.env @@ -0,0 +1,7 @@ +MONGO_USERNAME=emqx +MONGO_PASSWORD=passw0rd +MONGO_AUTHSOURCE=admin + +# See "Environment Variables" @ https://hub.docker.com/_/mongo +MONGO_INITDB_ROOT_USERNAME=${MONGO_USERNAME} +MONGO_INITDB_ROOT_PASSWORD=${MONGO_PASSWORD} diff --git a/.ci/docker-compose-file/docker-compose-mongo-single-tcp.yaml b/.ci/docker-compose-file/docker-compose-mongo-single-tcp.yaml index 39f37e66c..0eae6c358 100644 --- a/.ci/docker-compose-file/docker-compose-mongo-single-tcp.yaml +++ b/.ci/docker-compose-file/docker-compose-mongo-single-tcp.yaml @@ -9,6 +9,9 @@ services: - emqx_bridge ports: - "27017:27017" + env_file: + - .env + - credentials.env command: --ipv6 --bind_ip_all diff --git a/.ci/docker-compose-file/docker-compose.yaml b/.ci/docker-compose-file/docker-compose.yaml index d4a44bfb0..f3943b010 100644 --- a/.ci/docker-compose-file/docker-compose.yaml +++ b/.ci/docker-compose-file/docker-compose.yaml @@ -5,6 +5,7 @@ services: container_name: erlang image: ${DOCKER_CT_RUNNER_IMAGE:-ghcr.io/emqx/emqx-builder/5.2-3:1.14.5-25.3.2-2-ubuntu22.04} env_file: + - credentials.env - conf.env environment: GITHUB_ACTIONS: ${GITHUB_ACTIONS:-} diff --git a/README-CN.md b/README-CN.md index 8c6f8d8c3..f989b9bed 100644 --- a/README-CN.md +++ b/README-CN.md @@ -77,7 +77,7 @@ EMQX Cloud 文档：[docs.emqx.com/zh/cloud/latest/](https://docs.emqx.com/zh/cl 优雅的跨平台 MQTT 5.0 客户端工具，提供了桌面端、命令行、Web 三种版本，帮助您更快的开发和调试 MQTT 服务和应用。 -- [车联网平台搭建从入门到精通 ](https://www.emqx.com/zh/blog/category/internet-of-vehicles) +- [车联网平台搭建从入门到精通](https://www.emqx.com/zh/blog/category/internet-of-vehicles) 结合 EMQ 在车联网领域的实践经验，从协议选择等理论知识，到平台架构设计等实战操作，分享如何搭建一个可靠、高效、符合行业场景需求的车联网平台。 diff --git a/apps/emqx/include/emqx.hrl b/apps/emqx/include/emqx.hrl index 86a64d8bb..654d96d8c 100644 --- a/apps/emqx/include/emqx.hrl +++ b/apps/emqx/include/emqx.hrl @@ -39,9 +39,6 @@ %% System topic -define(SYSTOP, <<"$SYS/">>). -%% Queue topic --define(QUEUE, <<"$queue/">>). - %%-------------------------------------------------------------------- %% alarms %%-------------------------------------------------------------------- diff --git a/apps/emqx/include/emqx_mqtt.hrl b/apps/emqx/include/emqx_mqtt.hrl index 4d0188f71..53fed0f9d 100644 --- a/apps/emqx/include/emqx_mqtt.hrl +++ b/apps/emqx/include/emqx_mqtt.hrl @@ -55,6 +55,17 @@ %% MQTT-3.1.1 and MQTT-5.0 [MQTT-4.7.3-3] -define(MAX_TOPIC_LEN, 65535). +%%-------------------------------------------------------------------- +%% MQTT Share-Sub Internal +%%-------------------------------------------------------------------- + +-record(share, {group :: emqx_types:group(), topic :: emqx_types:topic()}). + +%% guards +-define(IS_TOPIC(T), + (is_binary(T) orelse is_record(T, share)) +). + %%-------------------------------------------------------------------- %% MQTT QoS Levels %%-------------------------------------------------------------------- @@ -661,13 +672,10 @@ end). -define(PACKET(Type), #mqtt_packet{header = #mqtt_packet_header{type = Type}}). -define(SHARE, "$share"). +-define(QUEUE, "$queue"). -define(SHARE(Group, Topic), emqx_topic:join([<>, Group, Topic])). --define(IS_SHARE(Topic), - case Topic of - <> -> true; - _ -> false - end -). + +-define(REDISPATCH_TO(GROUP, TOPIC), {GROUP, TOPIC}). -define(SHARE_EMPTY_FILTER, share_subscription_topic_cannot_be_empty). -define(SHARE_EMPTY_GROUP, share_subscription_group_name_cannot_be_empty). diff --git a/apps/emqx/include/emqx_trace.hrl b/apps/emqx/include/emqx_trace.hrl index 62028bcc0..3f9316727 100644 --- a/apps/emqx/include/emqx_trace.hrl +++ b/apps/emqx/include/emqx_trace.hrl @@ -32,6 +32,5 @@ -define(SHARD, ?COMMON_SHARD). -define(MAX_SIZE, 30). --define(OWN_KEYS, [level, filters, filter_default, handlers]). -endif. diff --git a/apps/emqx/include/http_api.hrl b/apps/emqx/include/http_api.hrl index ba1438374..0f6372584 100644 --- a/apps/emqx/include/http_api.hrl +++ b/apps/emqx/include/http_api.hrl @@ -17,6 +17,7 @@ %% HTTP API Auth -define(BAD_USERNAME_OR_PWD, 'BAD_USERNAME_OR_PWD'). -define(BAD_API_KEY_OR_SECRET, 'BAD_API_KEY_OR_SECRET'). +-define(API_KEY_NOT_ALLOW_MSG, <<"This API Key don't have permission to access this resource">>). %% Bad Request -define(BAD_REQUEST, 'BAD_REQUEST'). diff --git a/apps/emqx/include/logger.hrl b/apps/emqx/include/logger.hrl index d803f67be..a40f9dc9c 100644 --- a/apps/emqx/include/logger.hrl +++ b/apps/emqx/include/logger.hrl @@ -40,7 +40,9 @@ end ). +-define(AUDIT_HANDLER, emqx_audit). -define(TRACE_FILTER, emqx_trace_filter). +-define(OWN_KEYS, [level, filters, filter_default, handlers]). -define(TRACE(Tag, Msg, Meta), ?TRACE(debug, Tag, Msg, Meta)). @@ -61,25 +63,35 @@ ) end). --define(AUDIT(_Level_, _From_, _Meta_), begin - case emqx_config:get([log, audit], #{enable => false}) of - #{enable := false} -> +-ifdef(EMQX_RELEASE_EDITION). + +-if(?EMQX_RELEASE_EDITION == ee). + +-define(AUDIT(_LevelFun_, _MetaFun_), begin + case logger_config:get(logger, ?AUDIT_HANDLER) of + {error, {not_found, _}} -> ok; - #{enable := true, level := _AllowLevel_} -> + {ok, Handler = #{level := _AllowLevel_}} -> + _Level_ = _LevelFun_, case logger:compare_levels(_AllowLevel_, _Level_) of _R_ when _R_ == lt; _R_ == eq -> - emqx_trace:log( - _Level_, - [{emqx_audit, fun(L, _) -> L end, undefined, undefined}], - _Msg = undefined, - _Meta_#{from => _From_} - ); - gt -> + emqx_audit:log(_Level_, _MetaFun_, Handler); + _ -> ok end end end). +-else. +%% Only for compile pass, ce edition will not call it +-define(AUDIT(_L_, _M_), _ = {_L_, _M_}). +-endif. + +-else. +%% Only for compile pass, ce edition will not call it +-define(AUDIT(_L_, _M_), _ = {_L_, _M_}). +-endif. + %% print to 'user' group leader -define(ULOG(Fmt, Args), io:format(user, Fmt, Args)). -define(ELOG(Fmt, Args), io:format(standard_error, Fmt, Args)). diff --git a/apps/emqx/src/bhvrs/emqx_db_backup.erl b/apps/emqx/src/bhvrs/emqx_db_backup.erl index fddbdb1d0..95a142c0e 100644 --- a/apps/emqx/src/bhvrs/emqx_db_backup.erl +++ b/apps/emqx/src/bhvrs/emqx_db_backup.erl @@ -16,4 +16,21 @@ -module(emqx_db_backup). +-type traverse_break_reason() :: over | migrate. + -callback backup_tables() -> [mria:table()]. + +%% validate the backup +%% return `ok` to traverse the next item +%% return `{ok, over}` to finish the traverse +%% return `{ok, migrate}` to call the migration callback +-callback validate_mnesia_backup(tuple()) -> + ok + | {ok, traverse_break_reason()} + | {error, term()}. + +-callback migrate_mnesia_backup(tuple()) -> {ok, tuple()} | {error, term()}. + +-optional_callbacks([validate_mnesia_backup/1, migrate_mnesia_backup/1]). + +-export_type([traverse_break_reason/0]). diff --git a/apps/emqx/src/config/emqx_config_logger.erl b/apps/emqx/src/config/emqx_config_logger.erl index c675edb52..ce74db8f0 100644 --- a/apps/emqx/src/config/emqx_config_logger.erl +++ b/apps/emqx/src/config/emqx_config_logger.erl @@ -23,8 +23,9 @@ -export([post_config_update/5]). -export([filter_audit/2]). +-include("logger.hrl"). + -define(LOG, [log]). --define(AUDIT_HANDLER, emqx_audit). add_handler() -> ok = emqx_config_handler:add_handler(?LOG, ?MODULE), @@ -95,6 +96,10 @@ update_log_handlers(NewHandlers) -> ok = application:set_env(kernel, logger, NewHandlers), ok. +%% Don't remove audit log handler here, we need record this removed action into audit log file. +%% we will remove audit log handler after audit log is record in emqx_audit:log/3. +update_log_handler({removed, ?AUDIT_HANDLER}) -> + ok; update_log_handler({removed, Id}) -> log_to_console("Config override: ~s is removed~n", [id_for_log(Id)]), logger:remove_handler(Id); diff --git a/apps/emqx/src/emqx_broker.erl b/apps/emqx/src/emqx_broker.erl index 403e3757f..cc9cb98a6 100644 --- a/apps/emqx/src/emqx_broker.erl +++ b/apps/emqx/src/emqx_broker.erl @@ -118,18 +118,20 @@ create_tabs() -> %% Subscribe API %%------------------------------------------------------------------------------ --spec subscribe(emqx_types:topic()) -> ok. -subscribe(Topic) when is_binary(Topic) -> +-spec subscribe(emqx_types:topic() | emqx_types:share()) -> ok. +subscribe(Topic) when ?IS_TOPIC(Topic) -> subscribe(Topic, undefined). --spec subscribe(emqx_types:topic(), emqx_types:subid() | emqx_types:subopts()) -> ok. -subscribe(Topic, SubId) when is_binary(Topic), ?IS_SUBID(SubId) -> +-spec subscribe(emqx_types:topic() | emqx_types:share(), emqx_types:subid() | emqx_types:subopts()) -> + ok. +subscribe(Topic, SubId) when ?IS_TOPIC(Topic), ?IS_SUBID(SubId) -> subscribe(Topic, SubId, ?DEFAULT_SUBOPTS); -subscribe(Topic, SubOpts) when is_binary(Topic), is_map(SubOpts) -> +subscribe(Topic, SubOpts) when ?IS_TOPIC(Topic), is_map(SubOpts) -> subscribe(Topic, undefined, SubOpts). --spec subscribe(emqx_types:topic(), emqx_types:subid(), emqx_types:subopts()) -> ok. -subscribe(Topic, SubId, SubOpts0) when is_binary(Topic), ?IS_SUBID(SubId), is_map(SubOpts0) -> +-spec subscribe(emqx_types:topic() | emqx_types:share(), emqx_types:subid(), emqx_types:subopts()) -> + ok. +subscribe(Topic, SubId, SubOpts0) when ?IS_TOPIC(Topic), ?IS_SUBID(SubId), is_map(SubOpts0) -> SubOpts = maps:merge(?DEFAULT_SUBOPTS, SubOpts0), _ = emqx_trace:subscribe(Topic, SubId, SubOpts), SubPid = self(), @@ -151,13 +153,13 @@ with_subid(undefined, SubOpts) -> with_subid(SubId, SubOpts) -> maps:put(subid, SubId, SubOpts). -%% @private do_subscribe(Topic, SubPid, SubOpts) -> true = ets:insert(?SUBSCRIPTION, {SubPid, Topic}), - Group = maps:get(share, SubOpts, undefined), - do_subscribe(Group, Topic, SubPid, SubOpts). + do_subscribe2(Topic, SubPid, SubOpts). -do_subscribe(undefined, Topic, SubPid, SubOpts) -> +do_subscribe2(Topic, SubPid, SubOpts) when is_binary(Topic) -> + %% FIXME: subscribe shard bug + %% https://emqx.atlassian.net/browse/EMQX-10214 case emqx_broker_helper:get_sub_shard(SubPid, Topic) of 0 -> true = ets:insert(?SUBSCRIBER, {Topic, SubPid}), @@ -168,34 +170,40 @@ do_subscribe(undefined, Topic, SubPid, SubOpts) -> true = ets:insert(?SUBOPTION, {{Topic, SubPid}, maps:put(shard, I, SubOpts)}), call(pick({Topic, I}), {subscribe, Topic, I}) end; -%% Shared subscription -do_subscribe(Group, Topic, SubPid, SubOpts) -> +do_subscribe2(Topic = #share{group = Group, topic = RealTopic}, SubPid, SubOpts) when + is_binary(RealTopic) +-> true = ets:insert(?SUBOPTION, {{Topic, SubPid}, SubOpts}), - emqx_shared_sub:subscribe(Group, Topic, SubPid). + emqx_shared_sub:subscribe(Group, RealTopic, SubPid). %%-------------------------------------------------------------------- %% Unsubscribe API %%-------------------------------------------------------------------- --spec unsubscribe(emqx_types:topic()) -> ok. -unsubscribe(Topic) when is_binary(Topic) -> +-spec unsubscribe(emqx_types:topic() | emqx_types:share()) -> ok. +unsubscribe(Topic) when ?IS_TOPIC(Topic) -> SubPid = self(), case ets:lookup(?SUBOPTION, {Topic, SubPid}) of [{_, SubOpts}] -> - _ = emqx_broker_helper:reclaim_seq(Topic), _ = emqx_trace:unsubscribe(Topic, SubOpts), do_unsubscribe(Topic, SubPid, SubOpts); [] -> ok end. +-spec do_unsubscribe(emqx_types:topic() | emqx_types:share(), pid(), emqx_types:subopts()) -> + ok. do_unsubscribe(Topic, SubPid, SubOpts) -> true = ets:delete(?SUBOPTION, {Topic, SubPid}), true = ets:delete_object(?SUBSCRIPTION, {SubPid, Topic}), - Group = maps:get(share, SubOpts, undefined), - do_unsubscribe(Group, Topic, SubPid, SubOpts). + do_unsubscribe2(Topic, SubPid, SubOpts). -do_unsubscribe(undefined, Topic, SubPid, SubOpts) -> +-spec do_unsubscribe2(emqx_types:topic() | emqx_types:share(), pid(), emqx_types:subopts()) -> + ok. +do_unsubscribe2(Topic, SubPid, SubOpts) when + is_binary(Topic), is_pid(SubPid), is_map(SubOpts) +-> + _ = emqx_broker_helper:reclaim_seq(Topic), case maps:get(shard, SubOpts, 0) of 0 -> true = ets:delete_object(?SUBSCRIBER, {Topic, SubPid}), @@ -205,7 +213,9 @@ do_unsubscribe(undefined, Topic, SubPid, SubOpts) -> true = ets:delete_object(?SUBSCRIBER, {{shard, Topic, I}, SubPid}), cast(pick({Topic, I}), {unsubscribed, Topic, I}) end; -do_unsubscribe(Group, Topic, SubPid, _SubOpts) -> +do_unsubscribe2(#share{group = Group, topic = Topic}, SubPid, _SubOpts) when + is_binary(Group), is_binary(Topic), is_pid(SubPid) +-> emqx_shared_sub:unsubscribe(Group, Topic, SubPid). %%-------------------------------------------------------------------- @@ -306,7 +316,9 @@ aggre([], true, Acc) -> lists:usort(Acc). %% @doc Forward message to another node. --spec forward(node(), emqx_types:topic(), emqx_types:delivery(), RpcMode :: sync | async) -> +-spec forward( + node(), emqx_types:topic() | emqx_types:share(), emqx_types:delivery(), RpcMode :: sync | async +) -> emqx_types:deliver_result(). forward(Node, To, Delivery, async) -> true = emqx_broker_proto_v1:forward_async(Node, To, Delivery), @@ -329,7 +341,8 @@ forward(Node, To, Delivery, sync) -> Result end. --spec dispatch(emqx_types:topic(), emqx_types:delivery()) -> emqx_types:deliver_result(). +-spec dispatch(emqx_types:topic() | emqx_types:share(), emqx_types:delivery()) -> + emqx_types:deliver_result(). dispatch(Topic, Delivery = #delivery{}) when is_binary(Topic) -> case emqx:is_running() of true -> @@ -353,7 +366,11 @@ inc_dropped_cnt(Msg) -> end. -compile({inline, [subscribers/1]}). --spec subscribers(emqx_types:topic() | {shard, emqx_types:topic(), non_neg_integer()}) -> +-spec subscribers( + emqx_types:topic() + | emqx_types:share() + | {shard, emqx_types:topic() | emqx_types:share(), non_neg_integer()} +) -> [pid()]. subscribers(Topic) when is_binary(Topic) -> lookup_value(?SUBSCRIBER, Topic, []); @@ -372,7 +389,7 @@ subscriber_down(SubPid) -> SubOpts when is_map(SubOpts) -> _ = emqx_broker_helper:reclaim_seq(Topic), true = ets:delete(?SUBOPTION, {Topic, SubPid}), - do_unsubscribe(undefined, Topic, SubPid, SubOpts); + do_unsubscribe2(Topic, SubPid, SubOpts); undefined -> ok end @@ -386,7 +403,7 @@ subscriber_down(SubPid) -> %%-------------------------------------------------------------------- -spec subscriptions(pid() | emqx_types:subid()) -> - [{emqx_types:topic(), emqx_types:subopts()}]. + [{emqx_types:topic() | emqx_types:share(), emqx_types:subopts()}]. subscriptions(SubPid) when is_pid(SubPid) -> [ {Topic, lookup_value(?SUBOPTION, {Topic, SubPid}, #{})} @@ -400,20 +417,22 @@ subscriptions(SubId) -> [] end. --spec subscriptions_via_topic(emqx_types:topic()) -> [emqx_types:subopts()]. +-spec subscriptions_via_topic(emqx_types:topic() | emqx_types:share()) -> [emqx_types:subopts()]. subscriptions_via_topic(Topic) -> MatchSpec = [{{{Topic, '_'}, '_'}, [], ['$_']}], ets:select(?SUBOPTION, MatchSpec). --spec subscribed(pid() | emqx_types:subid(), emqx_types:topic()) -> boolean(). +-spec subscribed( + pid() | emqx_types:subid(), emqx_types:topic() | emqx_types:share() +) -> boolean(). subscribed(SubPid, Topic) when is_pid(SubPid) -> ets:member(?SUBOPTION, {Topic, SubPid}); subscribed(SubId, Topic) when ?IS_SUBID(SubId) -> SubPid = emqx_broker_helper:lookup_subpid(SubId), ets:member(?SUBOPTION, {Topic, SubPid}). --spec get_subopts(pid(), emqx_types:topic()) -> maybe(emqx_types:subopts()). -get_subopts(SubPid, Topic) when is_pid(SubPid), is_binary(Topic) -> +-spec get_subopts(pid(), emqx_types:topic() | emqx_types:share()) -> maybe(emqx_types:subopts()). +get_subopts(SubPid, Topic) when is_pid(SubPid), ?IS_TOPIC(Topic) -> lookup_value(?SUBOPTION, {Topic, SubPid}); get_subopts(SubId, Topic) when ?IS_SUBID(SubId) -> case emqx_broker_helper:lookup_subpid(SubId) of @@ -423,7 +442,7 @@ get_subopts(SubId, Topic) when ?IS_SUBID(SubId) -> undefined end. --spec set_subopts(emqx_types:topic(), emqx_types:subopts()) -> boolean(). +-spec set_subopts(emqx_types:topic() | emqx_types:share(), emqx_types:subopts()) -> boolean(). set_subopts(Topic, NewOpts) when is_binary(Topic), is_map(NewOpts) -> set_subopts(self(), Topic, NewOpts). @@ -437,7 +456,7 @@ set_subopts(SubPid, Topic, NewOpts) -> false end. --spec topics() -> [emqx_types:topic()]. +-spec topics() -> [emqx_types:topic() | emqx_types:share()]. topics() -> emqx_router:topics(). @@ -542,7 +561,8 @@ code_change(_OldVsn, State, _Extra) -> %% Internal functions %%-------------------------------------------------------------------- --spec do_dispatch(emqx_types:topic(), emqx_types:delivery()) -> emqx_types:deliver_result(). +-spec do_dispatch(emqx_types:topic() | emqx_types:share(), emqx_types:delivery()) -> + emqx_types:deliver_result(). do_dispatch(Topic, #delivery{message = Msg}) -> DispN = lists:foldl( fun(Sub, N) -> @@ -560,6 +580,8 @@ do_dispatch(Topic, #delivery{message = Msg}) -> {ok, DispN} end. +%% Donot dispatch to share subscriber here. +%% we do it in `emqx_shared_sub.erl` with configured strategy do_dispatch(SubPid, Topic, Msg) when is_pid(SubPid) -> case erlang:is_process_alive(SubPid) of true -> diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl index 4f6d5ac6f..81e01e1bd 100644 --- a/apps/emqx/src/emqx_channel.erl +++ b/apps/emqx/src/emqx_channel.erl @@ -476,60 +476,27 @@ handle_in( ok = emqx_metrics:inc('packets.pubcomp.missed'), {ok, Channel} end; -handle_in( - SubPkt = ?SUBSCRIBE_PACKET(PacketId, Properties, TopicFilters), - Channel = #channel{clientinfo = ClientInfo} -) -> - case emqx_packet:check(SubPkt) of - ok -> - TopicFilters0 = parse_topic_filters(TopicFilters), - TopicFilters1 = enrich_subopts_subid(Properties, TopicFilters0), - TupleTopicFilters0 = check_sub_authzs(TopicFilters1, Channel), - HasAuthzDeny = lists:any( - fun({_TopicFilter, ReasonCode}) -> - ReasonCode =:= ?RC_NOT_AUTHORIZED - end, - TupleTopicFilters0 - ), - DenyAction = emqx:get_config([authorization, deny_action], ignore), - case DenyAction =:= disconnect andalso HasAuthzDeny of - true -> - handle_out(disconnect, ?RC_NOT_AUTHORIZED, Channel); - false -> - TopicFilters2 = [ - TopicFilter - || {TopicFilter, ?RC_SUCCESS} <- TupleTopicFilters0 - ], - TopicFilters3 = run_hooks( - 'client.subscribe', - [ClientInfo, Properties], - TopicFilters2 - ), - {TupleTopicFilters1, NChannel} = process_subscribe( - TopicFilters3, - Properties, - Channel - ), - TupleTopicFilters2 = - lists:foldl( - fun - ({{Topic, Opts = #{deny_subscription := true}}, _QoS}, Acc) -> - Key = {Topic, maps:without([deny_subscription], Opts)}, - lists:keyreplace(Key, 1, Acc, {Key, ?RC_UNSPECIFIED_ERROR}); - (Tuple = {Key, _Value}, Acc) -> - lists:keyreplace(Key, 1, Acc, Tuple) - end, - TupleTopicFilters0, - TupleTopicFilters1 - ), - ReasonCodes2 = [ - ReasonCode - || {_TopicFilter, ReasonCode} <- TupleTopicFilters2 - ], - handle_out(suback, {PacketId, ReasonCodes2}, NChannel) - end; - {error, ReasonCode} -> - handle_out(disconnect, ReasonCode, Channel) +handle_in(SubPkt = ?SUBSCRIBE_PACKET(PacketId, _Properties, _TopicFilters0), Channel0) -> + Pipe = pipeline( + [ + fun check_subscribe/2, + fun enrich_subscribe/2, + %% TODO && FIXME (EMQX-10786): mount topic before authz check. + fun check_sub_authzs/2, + fun check_sub_caps/2 + ], + SubPkt, + Channel0 + ), + case Pipe of + {ok, NPkt = ?SUBSCRIBE_PACKET(_PacketId, TFChecked), Channel} -> + {TFSubedWithNRC, NChannel} = process_subscribe(run_sub_hooks(NPkt, Channel), Channel), + ReasonCodes = gen_reason_codes(TFChecked, TFSubedWithNRC), + handle_out(suback, {PacketId, ReasonCodes}, NChannel); + {error, {disconnect, RC}, Channel} -> + %% funcs in pipeline always cause action: `disconnect` + %% And Only one ReasonCode in DISCONNECT packet + handle_out(disconnect, RC, Channel) end; handle_in( Packet = ?UNSUBSCRIBE_PACKET(PacketId, Properties, TopicFilters), @@ -540,7 +507,7 @@ handle_in( TopicFilters1 = run_hooks( 'client.unsubscribe', [ClientInfo, Properties], - parse_topic_filters(TopicFilters) + parse_raw_topic_filters(TopicFilters) ), {ReasonCodes, NChannel} = process_unsubscribe(TopicFilters1, Properties, Channel), handle_out(unsuback, {PacketId, ReasonCodes}, NChannel); @@ -782,32 +749,14 @@ after_message_acked(ClientInfo, Msg, PubAckProps) -> %% Process Subscribe %%-------------------------------------------------------------------- --compile({inline, [process_subscribe/3]}). -process_subscribe(TopicFilters, SubProps, Channel) -> - process_subscribe(TopicFilters, SubProps, Channel, []). +process_subscribe(TopicFilters, Channel) -> + process_subscribe(TopicFilters, Channel, []). -process_subscribe([], _SubProps, Channel, Acc) -> +process_subscribe([], Channel, Acc) -> {lists:reverse(Acc), Channel}; -process_subscribe([Topic = {TopicFilter, SubOpts} | More], SubProps, Channel, Acc) -> - case check_sub_caps(TopicFilter, SubOpts, Channel) of - ok -> - {ReasonCode, NChannel} = do_subscribe( - TopicFilter, - SubOpts#{sub_props => SubProps}, - Channel - ), - process_subscribe(More, SubProps, NChannel, [{Topic, ReasonCode} | Acc]); - {error, ReasonCode} -> - ?SLOG( - warning, - #{ - msg => "cannot_subscribe_topic_filter", - reason => emqx_reason_codes:name(ReasonCode) - }, - #{topic => TopicFilter} - ), - process_subscribe(More, SubProps, Channel, [{Topic, ReasonCode} | Acc]) - end. +process_subscribe([Filter = {TopicFilter, SubOpts} | More], Channel, Acc) -> + {NReasonCode, NChannel} = do_subscribe(TopicFilter, SubOpts, Channel), + process_subscribe(More, NChannel, [{Filter, NReasonCode} | Acc]). do_subscribe( TopicFilter, @@ -818,11 +767,13 @@ do_subscribe( session = Session } ) -> + %% TODO && FIXME (EMQX-10786): mount topic before authz check. NTopicFilter = emqx_mountpoint:mount(MountPoint, TopicFilter), - NSubOpts = enrich_subopts(maps:merge(?DEFAULT_SUBOPTS, SubOpts), Channel), - case emqx_session:subscribe(ClientInfo, NTopicFilter, NSubOpts, Session) of + case emqx_session:subscribe(ClientInfo, NTopicFilter, SubOpts, Session) of {ok, NSession} -> - {QoS, Channel#channel{session = NSession}}; + %% TODO && FIXME (EMQX-11216): QoS as ReasonCode(max granted QoS) for now + RC = QoS, + {RC, Channel#channel{session = NSession}}; {error, RC} -> ?SLOG( warning, @@ -835,6 +786,30 @@ do_subscribe( {RC, Channel} end. +gen_reason_codes(TFChecked, TFSubedWitNhRC) -> + do_gen_reason_codes([], TFChecked, TFSubedWitNhRC). + +%% Initial RC is `RC_SUCCESS | RC_NOT_AUTHORIZED`, generated by check_sub_authzs/2 +%% And then TF with `RC_SUCCESS` will passing through `process_subscribe/2` and +%% NRC should override the initial RC. +do_gen_reason_codes(Acc, [], []) -> + lists:reverse(Acc); +do_gen_reason_codes( + Acc, + [{_, ?RC_SUCCESS} | RestTF], + [{_, NRC} | RestWithNRC] +) -> + %% will passing through `process_subscribe/2` + %% use NRC to override IintialRC + do_gen_reason_codes([NRC | Acc], RestTF, RestWithNRC); +do_gen_reason_codes( + Acc, + [{_, InitialRC} | Rest], + RestWithNRC +) -> + %% InitialRC is not `RC_SUCCESS`, use it. + do_gen_reason_codes([InitialRC | Acc], Rest, RestWithNRC). + %%-------------------------------------------------------------------- %% Process Unsubscribe %%-------------------------------------------------------------------- @@ -1213,13 +1188,8 @@ handle_call(Req, Channel) -> ok | {ok, channel()} | {shutdown, Reason :: term(), channel()}. handle_info({subscribe, TopicFilters}, Channel) -> - {_, NChannel} = lists:foldl( - fun({TopicFilter, SubOpts}, {_, ChannelAcc}) -> - do_subscribe(TopicFilter, SubOpts, ChannelAcc) - end, - {[], Channel}, - parse_topic_filters(TopicFilters) - ), + NTopicFilters = enrich_subscribe(TopicFilters, Channel), + {_TopicFiltersWithRC, NChannel} = process_subscribe(NTopicFilters, Channel), {ok, NChannel}; handle_info({unsubscribe, TopicFilters}, Channel) -> {_RC, NChannel} = process_unsubscribe(TopicFilters, #{}, Channel), @@ -1857,49 +1827,156 @@ check_pub_caps( ) -> emqx_mqtt_caps:check_pub(Zone, #{qos => QoS, retain => Retain, topic => Topic}). +%%-------------------------------------------------------------------- +%% Check Subscribe Packet + +check_subscribe(SubPkt, _Channel) -> + case emqx_packet:check(SubPkt) of + ok -> ok; + {error, RC} -> {error, {disconnect, RC}} + end. + %%-------------------------------------------------------------------- %% Check Sub Authorization -check_sub_authzs(TopicFilters, Channel) -> - check_sub_authzs(TopicFilters, Channel, []). - check_sub_authzs( - [TopicFilter = {Topic, _} | More], - Channel = #channel{clientinfo = ClientInfo}, - Acc + ?SUBSCRIBE_PACKET(PacketId, SubProps, TopicFilters0), + Channel = #channel{clientinfo = ClientInfo} ) -> + CheckResult = do_check_sub_authzs(TopicFilters0, ClientInfo), + HasAuthzDeny = lists:any( + fun({{_TopicFilter, _SubOpts}, ReasonCode}) -> + ReasonCode =:= ?RC_NOT_AUTHORIZED + end, + CheckResult + ), + DenyAction = emqx:get_config([authorization, deny_action], ignore), + case DenyAction =:= disconnect andalso HasAuthzDeny of + true -> + {error, {disconnect, ?RC_NOT_AUTHORIZED}, Channel}; + false -> + {ok, ?SUBSCRIBE_PACKET(PacketId, SubProps, CheckResult), Channel} + end. + +do_check_sub_authzs(TopicFilters, ClientInfo) -> + do_check_sub_authzs(ClientInfo, TopicFilters, []). + +do_check_sub_authzs(_ClientInfo, [], Acc) -> + lists:reverse(Acc); +do_check_sub_authzs(ClientInfo, [TopicFilter = {Topic, _SubOpts} | More], Acc) -> + %% subsclibe authz check only cares the real topic filter when shared-sub + %% e.g. only check <<"t/#">> for <<"$share/g/t/#">> Action = authz_action(TopicFilter), - case emqx_access_control:authorize(ClientInfo, Action, Topic) of + case + emqx_access_control:authorize( + ClientInfo, + Action, + emqx_topic:get_shared_real_topic(Topic) + ) + of + %% TODO: support maximum QoS granted + %% MQTT-3.1.1 [MQTT-3.8.4-6] and MQTT-5.0 [MQTT-3.8.4-7] + %% Not implemented yet: + %% {allow, RC} -> do_check_sub_authzs(ClientInfo, More, [{TopicFilter, RC} | Acc]); allow -> - check_sub_authzs(More, Channel, [{TopicFilter, ?RC_SUCCESS} | Acc]); + do_check_sub_authzs(ClientInfo, More, [{TopicFilter, ?RC_SUCCESS} | Acc]); deny -> - check_sub_authzs(More, Channel, [{TopicFilter, ?RC_NOT_AUTHORIZED} | Acc]) - end; -check_sub_authzs([], _Channel, Acc) -> - lists:reverse(Acc). + do_check_sub_authzs(ClientInfo, More, [{TopicFilter, ?RC_NOT_AUTHORIZED} | Acc]) + end. %%-------------------------------------------------------------------- %% Check Sub Caps -check_sub_caps(TopicFilter, SubOpts, #channel{clientinfo = ClientInfo}) -> - emqx_mqtt_caps:check_sub(ClientInfo, TopicFilter, SubOpts). +check_sub_caps( + ?SUBSCRIBE_PACKET(PacketId, SubProps, TopicFilters), + Channel = #channel{clientinfo = ClientInfo} +) -> + CheckResult = do_check_sub_caps(ClientInfo, TopicFilters), + {ok, ?SUBSCRIBE_PACKET(PacketId, SubProps, CheckResult), Channel}. + +do_check_sub_caps(ClientInfo, TopicFilters) -> + do_check_sub_caps(ClientInfo, TopicFilters, []). + +do_check_sub_caps(_ClientInfo, [], Acc) -> + lists:reverse(Acc); +do_check_sub_caps(ClientInfo, [TopicFilter = {{Topic, SubOpts}, ?RC_SUCCESS} | More], Acc) -> + case emqx_mqtt_caps:check_sub(ClientInfo, Topic, SubOpts) of + ok -> + do_check_sub_caps(ClientInfo, More, [TopicFilter | Acc]); + {error, NRC} -> + ?SLOG( + warning, + #{ + msg => "cannot_subscribe_topic_filter", + reason => emqx_reason_codes:name(NRC) + }, + #{topic => Topic} + ), + do_check_sub_caps(ClientInfo, More, [{{Topic, SubOpts}, NRC} | Acc]) + end; +do_check_sub_caps(ClientInfo, [TopicFilter = {{_Topic, _SubOpts}, _OtherRC} | More], Acc) -> + do_check_sub_caps(ClientInfo, More, [TopicFilter | Acc]). %%-------------------------------------------------------------------- -%% Enrich SubId +%% Run Subscribe Hooks -enrich_subopts_subid(#{'Subscription-Identifier' := SubId}, TopicFilters) -> - [{Topic, SubOpts#{subid => SubId}} || {Topic, SubOpts} <- TopicFilters]; -enrich_subopts_subid(_Properties, TopicFilters) -> - TopicFilters. +run_sub_hooks( + ?SUBSCRIBE_PACKET(_PacketId, Properties, TopicFilters0), + _Channel = #channel{clientinfo = ClientInfo} +) -> + TopicFilters = [ + TopicFilter + || {TopicFilter, ?RC_SUCCESS} <- TopicFilters0 + ], + _NTopicFilters = run_hooks('client.subscribe', [ClientInfo, Properties], TopicFilters). %%-------------------------------------------------------------------- %% Enrich SubOpts -enrich_subopts(SubOpts, _Channel = ?IS_MQTT_V5) -> - SubOpts; -enrich_subopts(SubOpts, #channel{clientinfo = #{zone := Zone, is_bridge := IsBridge}}) -> +%% for api subscribe without sub-authz check and sub-caps check. +enrich_subscribe(TopicFilters, Channel) when is_list(TopicFilters) -> + do_enrich_subscribe(#{}, TopicFilters, Channel); +%% for mqtt clients sent subscribe packet. +enrich_subscribe(?SUBSCRIBE_PACKET(PacketId, Properties, TopicFilters), Channel) -> + NTopicFilters = do_enrich_subscribe(Properties, TopicFilters, Channel), + {ok, ?SUBSCRIBE_PACKET(PacketId, Properties, NTopicFilters), Channel}. + +do_enrich_subscribe(Properties, TopicFilters, Channel) -> + _NTopicFilters = run_fold( + [ + %% TODO: do try catch with reason code here + fun(TFs, _) -> parse_raw_topic_filters(TFs) end, + fun enrich_subopts_subid/2, + fun enrich_subopts_porps/2, + fun enrich_subopts_flags/2 + ], + TopicFilters, + #{sub_props => Properties, channel => Channel} + ). + +enrich_subopts_subid(TopicFilters, #{sub_props := #{'Subscription-Identifier' := SubId}}) -> + [{Topic, SubOpts#{subid => SubId}} || {Topic, SubOpts} <- TopicFilters]; +enrich_subopts_subid(TopicFilters, _State) -> + TopicFilters. + +enrich_subopts_porps(TopicFilters, #{sub_props := SubProps}) -> + [{Topic, SubOpts#{sub_props => SubProps}} || {Topic, SubOpts} <- TopicFilters]. + +enrich_subopts_flags(TopicFilters, #{channel := Channel}) -> + do_enrich_subopts_flags(TopicFilters, Channel). + +do_enrich_subopts_flags(TopicFilters, ?IS_MQTT_V5) -> + [{Topic, merge_default_subopts(SubOpts)} || {Topic, SubOpts} <- TopicFilters]; +do_enrich_subopts_flags(TopicFilters, #channel{clientinfo = #{zone := Zone, is_bridge := IsBridge}}) -> + Rap = flag(IsBridge), NL = flag(get_mqtt_conf(Zone, ignore_loop_deliver)), - SubOpts#{rap => flag(IsBridge), nl => NL}. + [ + {Topic, (merge_default_subopts(SubOpts))#{rap => Rap, nl => NL}} + || {Topic, SubOpts} <- TopicFilters + ]. + +merge_default_subopts(SubOpts) -> + maps:merge(?DEFAULT_SUBOPTS, SubOpts). %%-------------------------------------------------------------------- %% Enrich ConnAck Caps @@ -2089,8 +2166,8 @@ maybe_shutdown(Reason, _Intent = shutdown, Channel) -> %%-------------------------------------------------------------------- %% Parse Topic Filters --compile({inline, [parse_topic_filters/1]}). -parse_topic_filters(TopicFilters) -> +%% [{<<"$share/group/topic">>, _SubOpts = #{}} | _] +parse_raw_topic_filters(TopicFilters) -> lists:map(fun emqx_topic:parse/1, TopicFilters). %%-------------------------------------------------------------------- diff --git a/apps/emqx/src/emqx_mountpoint.erl b/apps/emqx/src/emqx_mountpoint.erl index 5b5dac954..c19736690 100644 --- a/apps/emqx/src/emqx_mountpoint.erl +++ b/apps/emqx/src/emqx_mountpoint.erl @@ -17,6 +17,7 @@ -module(emqx_mountpoint). -include("emqx.hrl"). +-include("emqx_mqtt.hrl"). -include("emqx_placeholder.hrl"). -include("types.hrl"). @@ -34,38 +35,54 @@ -spec mount(maybe(mountpoint()), Any) -> Any when Any :: emqx_types:topic() + | emqx_types:share() | emqx_types:message() | emqx_types:topic_filters(). mount(undefined, Any) -> Any; -mount(MountPoint, Topic) when is_binary(Topic) -> - prefix(MountPoint, Topic); -mount(MountPoint, Msg = #message{topic = Topic}) -> - Msg#message{topic = prefix(MountPoint, Topic)}; +mount(MountPoint, Topic) when ?IS_TOPIC(Topic) -> + prefix_maybe_share(MountPoint, Topic); +mount(MountPoint, Msg = #message{topic = Topic}) when is_binary(Topic) -> + Msg#message{topic = prefix_maybe_share(MountPoint, Topic)}; mount(MountPoint, TopicFilters) when is_list(TopicFilters) -> - [{prefix(MountPoint, Topic), SubOpts} || {Topic, SubOpts} <- TopicFilters]. + [{prefix_maybe_share(MountPoint, Topic), SubOpts} || {Topic, SubOpts} <- TopicFilters]. -%% @private --compile({inline, [prefix/2]}). -prefix(MountPoint, Topic) -> - <>. +-spec prefix_maybe_share(maybe(mountpoint()), Any) -> Any when + Any :: + emqx_types:topic() + | emqx_types:share(). +prefix_maybe_share(MountPoint, Topic) when + is_binary(MountPoint) andalso is_binary(Topic) +-> + <>; +prefix_maybe_share(MountPoint, #share{group = Group, topic = Topic}) when + is_binary(MountPoint) andalso is_binary(Topic) +-> + #share{group = Group, topic = prefix_maybe_share(MountPoint, Topic)}. -spec unmount(maybe(mountpoint()), Any) -> Any when Any :: emqx_types:topic() + | emqx_types:share() | emqx_types:message(). unmount(undefined, Any) -> Any; -unmount(MountPoint, Topic) when is_binary(Topic) -> +unmount(MountPoint, Topic) when ?IS_TOPIC(Topic) -> + unmount_maybe_share(MountPoint, Topic); +unmount(MountPoint, Msg = #message{topic = Topic}) when is_binary(Topic) -> + Msg#message{topic = unmount_maybe_share(MountPoint, Topic)}. + +unmount_maybe_share(MountPoint, Topic) when + is_binary(MountPoint) andalso is_binary(Topic) +-> case string:prefix(Topic, MountPoint) of nomatch -> Topic; Topic1 -> Topic1 end; -unmount(MountPoint, Msg = #message{topic = Topic}) -> - case string:prefix(Topic, MountPoint) of - nomatch -> Msg; - Topic1 -> Msg#message{topic = Topic1} - end. +unmount_maybe_share(MountPoint, TopicFilter = #share{topic = Topic}) when + is_binary(MountPoint) andalso is_binary(Topic) +-> + TopicFilter#share{topic = unmount_maybe_share(MountPoint, Topic)}. -spec replvar(maybe(mountpoint()), map()) -> maybe(mountpoint()). replvar(undefined, _Vars) -> diff --git a/apps/emqx/src/emqx_mqtt_caps.erl b/apps/emqx/src/emqx_mqtt_caps.erl index 11f495dbd..5cf10691d 100644 --- a/apps/emqx/src/emqx_mqtt_caps.erl +++ b/apps/emqx/src/emqx_mqtt_caps.erl @@ -102,16 +102,19 @@ do_check_pub(_Flags, _Caps) -> -spec check_sub( emqx_types:clientinfo(), - emqx_types:topic(), + emqx_types:topic() | emqx_types:share(), emqx_types:subopts() ) -> ok_or_error(emqx_types:reason_code()). check_sub(ClientInfo = #{zone := Zone}, Topic, SubOpts) -> Caps = emqx_config:get_zone_conf(Zone, [mqtt]), Flags = #{ + %% TODO: qos check + %% (max_qos_allowed, Map) -> + %% max_qos_allowed => maps:get(max_qos_allowed, Caps, 2), topic_levels => emqx_topic:levels(Topic), is_wildcard => emqx_topic:wildcard(Topic), - is_shared => maps:is_key(share, SubOpts), + is_shared => erlang:is_record(Topic, share), is_exclusive => maps:get(is_exclusive, SubOpts, false) }, do_check_sub(Flags, Caps, ClientInfo, Topic). @@ -126,13 +129,19 @@ do_check_sub(#{is_shared := true}, #{shared_subscription := false}, _, _) -> {error, ?RC_SHARED_SUBSCRIPTIONS_NOT_SUPPORTED}; do_check_sub(#{is_exclusive := true}, #{exclusive_subscription := false}, _, _) -> {error, ?RC_TOPIC_FILTER_INVALID}; -do_check_sub(#{is_exclusive := true}, #{exclusive_subscription := true}, ClientInfo, Topic) -> +do_check_sub(#{is_exclusive := true}, #{exclusive_subscription := true}, ClientInfo, Topic) when + is_binary(Topic) +-> case emqx_exclusive_subscription:check_subscribe(ClientInfo, Topic) of deny -> {error, ?RC_QUOTA_EXCEEDED}; _ -> ok end; +%% for max_qos_allowed +%% see: RC_GRANTED_QOS_0, RC_GRANTED_QOS_1, RC_GRANTED_QOS_2 +%% do_check_sub(_, _) -> +%% {ok, RC}; do_check_sub(_Flags, _Caps, _, _) -> ok. diff --git a/apps/emqx/src/emqx_reason_codes.erl b/apps/emqx/src/emqx_reason_codes.erl index 77a8c1be2..543a62216 100644 --- a/apps/emqx/src/emqx_reason_codes.erl +++ b/apps/emqx/src/emqx_reason_codes.erl @@ -177,6 +177,7 @@ compat(connack, 16#9D) -> ?CONNACK_SERVER; compat(connack, 16#9F) -> ?CONNACK_SERVER; compat(suback, Code) when Code =< ?QOS_2 -> Code; compat(suback, Code) when Code >= 16#80 -> 16#80; +%% TODO: 16#80(qos0) 16#81(qos1) 16#82(qos2) for mqtt-v3.1.1 compat(unsuback, _Code) -> undefined; compat(_Other, _Code) -> undefined. diff --git a/apps/emqx/src/emqx_schema_secret.erl b/apps/emqx/src/emqx_schema_secret.erl new file mode 100644 index 000000000..635285ce7 --- /dev/null +++ b/apps/emqx/src/emqx_schema_secret.erl @@ -0,0 +1,85 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% @doc HOCON schema that defines _secret_ concept. +-module(emqx_schema_secret). + +-include_lib("emqx/include/logger.hrl"). +-include_lib("typerefl/include/types.hrl"). + +-export([mk/1]). + +%% HOCON Schema API +-export([convert_secret/2]). + +%% @doc Secret value. +-type t() :: binary(). + +%% @doc Source of the secret value. +%% * "file://...": file path to a file containing secret value. +%% * other binaries: secret value itself. +-type source() :: iodata(). + +-type secret() :: binary() | function(). +-reflect_type([secret/0]). + +-define(SCHEMA, #{ + required => false, + format => <<"password">>, + sensitive => true, + converter => fun ?MODULE:convert_secret/2 +}). + +-dialyzer({nowarn_function, source/1}). + +%% + +-spec mk(#{atom() => _}) -> hocon_schema:field_schema(). +mk(Overrides = #{}) -> + hoconsc:mk(secret(), maps:merge(?SCHEMA, Overrides)). + +convert_secret(undefined, #{}) -> + undefined; +convert_secret(Secret, #{make_serializable := true}) -> + unicode:characters_to_binary(source(Secret)); +convert_secret(Secret, #{}) when is_function(Secret, 0) -> + Secret; +convert_secret(Secret, #{}) when is_integer(Secret) -> + wrap(integer_to_binary(Secret)); +convert_secret(Secret, #{}) -> + try unicode:characters_to_binary(Secret) of + String when is_binary(String) -> + wrap(String); + {error, _, _} -> + throw(invalid_string) + catch + error:_ -> + throw(invalid_type) + end. + +-spec wrap(source()) -> emqx_secret:t(t()). +wrap(<<"file://", Filename/binary>>) -> + emqx_secret:wrap_load({file, Filename}); +wrap(Secret) -> + emqx_secret:wrap(Secret). + +-spec source(emqx_secret:t(t())) -> source(). +source(Secret) when is_function(Secret) -> + source(emqx_secret:term(Secret)); +source({file, Filename}) -> + <<"file://", Filename/binary>>; +source(Secret) -> + Secret. diff --git a/apps/emqx/src/emqx_secret.erl b/apps/emqx/src/emqx_secret.erl index 72c4f3c08..dfbfa488e 100644 --- a/apps/emqx/src/emqx_secret.erl +++ b/apps/emqx/src/emqx_secret.erl @@ -19,23 +19,52 @@ -module(emqx_secret). %% API: --export([wrap/1, unwrap/1]). +-export([wrap/1, wrap_load/1, unwrap/1, term/1]). -export_type([t/1]). -opaque t(T) :: T | fun(() -> t(T)). +%% Secret loader module. +%% Any changes related to processing of secrets should be made there. +-define(LOADER, emqx_secret_loader). + %%================================================================================ %% API funcions %%================================================================================ +%% @doc Wrap a term in a secret closure. +%% This effectively hides the term from any term formatting / printing code. +-spec wrap(T) -> t(T). wrap(Term) -> fun() -> Term end. +%% @doc Wrap a loader function call over a term in a secret closure. +%% This is slightly more flexible form of `wrap/1` with the same basic purpose. +-spec wrap_load(emqx_secret_loader:source()) -> t(_). +wrap_load(Source) -> + fun() -> + apply(?LOADER, load, [Source]) + end. + +%% @doc Unwrap a secret closure, revealing the secret. +%% This is either `Term` or `Module:Function(Term)` depending on how it was wrapped. +-spec unwrap(t(T)) -> T. unwrap(Term) when is_function(Term, 0) -> %% Handle potentially nested funs unwrap(Term()); unwrap(Term) -> Term. + +%% @doc Inspect the term wrapped in a secret closure. +-spec term(t(_)) -> _Term. +term(Wrap) when is_function(Wrap, 0) -> + case erlang:fun_info(Wrap, module) of + {module, ?MODULE} -> + {env, Env} = erlang:fun_info(Wrap, env), + lists:last(Env); + _ -> + error(badarg, [Wrap]) + end. diff --git a/apps/emqx/src/emqx_secret_loader.erl b/apps/emqx/src/emqx_secret_loader.erl new file mode 100644 index 000000000..2e99587bf --- /dev/null +++ b/apps/emqx/src/emqx_secret_loader.erl @@ -0,0 +1,42 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_secret_loader). + +%% API +-export([load/1]). +-export([file/1]). + +-export_type([source/0]). + +-type source() :: {file, file:filename_all()}. + +-spec load(source()) -> binary() | no_return(). +load({file, Filename}) -> + file(Filename). + +-spec file(file:filename_all()) -> binary() | no_return(). +file(Filename) -> + case file:read_file(Filename) of + {ok, Secret} -> + string:trim(Secret, trailing); + {error, Reason} -> + throw(#{ + msg => failed_to_read_secret_file, + path => Filename, + reason => emqx_utils:explain_posix(Reason) + }) + end. diff --git a/apps/emqx/src/emqx_session.erl b/apps/emqx/src/emqx_session.erl index 52342d7ee..4bae4ce03 100644 --- a/apps/emqx/src/emqx_session.erl +++ b/apps/emqx/src/emqx_session.erl @@ -267,7 +267,7 @@ destroy(Session) -> -spec subscribe( clientinfo(), - emqx_types:topic(), + emqx_types:topic() | emqx_types:share(), emqx_types:subopts(), t() ) -> @@ -287,7 +287,7 @@ subscribe(ClientInfo, TopicFilter, SubOpts, Session) -> -spec unsubscribe( clientinfo(), - emqx_types:topic(), + emqx_types:topic() | emqx_types:share(), emqx_types:subopts(), t() ) -> @@ -418,7 +418,13 @@ enrich_delivers(ClientInfo, [D | Rest], UpgradeQoS, Session) -> end. enrich_deliver(ClientInfo, {deliver, Topic, Msg}, UpgradeQoS, Session) -> - SubOpts = ?IMPL(Session):get_subscription(Topic, Session), + SubOpts = + case Msg of + #message{headers = #{redispatch_to := ?REDISPATCH_TO(Group, T)}} -> + ?IMPL(Session):get_subscription(emqx_topic:make_shared_record(Group, T), Session); + _ -> + ?IMPL(Session):get_subscription(Topic, Session) + end, enrich_message(ClientInfo, Msg, SubOpts, UpgradeQoS). enrich_message( diff --git a/apps/emqx/src/emqx_session_mem.erl b/apps/emqx/src/emqx_session_mem.erl index 3ea4f9f3b..d609435c0 100644 --- a/apps/emqx/src/emqx_session_mem.erl +++ b/apps/emqx/src/emqx_session_mem.erl @@ -316,7 +316,7 @@ unsubscribe( {error, ?RC_NO_SUBSCRIPTION_EXISTED} end. --spec get_subscription(emqx_types:topic(), session()) -> +-spec get_subscription(emqx_types:topic() | emqx_types:share(), session()) -> emqx_types:subopts() | undefined. get_subscription(Topic, #session{subscriptions = Subs}) -> maps:get(Topic, Subs, undefined). diff --git a/apps/emqx/src/emqx_shared_sub.erl b/apps/emqx/src/emqx_shared_sub.erl index 84921be6b..89a785590 100644 --- a/apps/emqx/src/emqx_shared_sub.erl +++ b/apps/emqx/src/emqx_shared_sub.erl @@ -95,7 +95,6 @@ -define(ACK, shared_sub_ack). -define(NACK(Reason), {shared_sub_nack, Reason}). -define(NO_ACK, no_ack). --define(REDISPATCH_TO(GROUP, TOPIC), {GROUP, TOPIC}). -define(SUBSCRIBER_DOWN, noproc). -type redispatch_to() :: ?REDISPATCH_TO(emqx_types:group(), emqx_types:topic()). @@ -234,19 +233,16 @@ without_group_ack(Msg) -> get_group_ack(Msg) -> emqx_message:get_header(shared_dispatch_ack, Msg, ?NO_ACK). -with_redispatch_to(#message{qos = ?QOS_0} = Msg, _Group, _Topic) -> - Msg; +%% always add `redispatch_to` header to the message +%% for QOS_0 msgs, redispatch_to is not needed and filtered out in is_redispatch_needed/1 with_redispatch_to(Msg, Group, Topic) -> emqx_message:set_headers(#{redispatch_to => ?REDISPATCH_TO(Group, Topic)}, Msg). -%% @hidden Redispatch is needed only for the messages with redispatch_to header added. -is_redispatch_needed(#message{} = Msg) -> - case get_redispatch_to(Msg) of - ?REDISPATCH_TO(_, _) -> - true; - _ -> - false - end. +%% @hidden Redispatch is needed only for the messages which not QOS_0 +is_redispatch_needed(#message{qos = ?QOS_0}) -> + false; +is_redispatch_needed(#message{headers = #{redispatch_to := ?REDISPATCH_TO(_, _)}}) -> + true. %% @doc Redispatch shared deliveries to other members in the group. redispatch(Messages0) -> diff --git a/apps/emqx/src/emqx_topic.erl b/apps/emqx/src/emqx_topic.erl index 6d232c68d..76c6ef34e 100644 --- a/apps/emqx/src/emqx_topic.erl +++ b/apps/emqx/src/emqx_topic.erl @@ -36,9 +36,16 @@ parse/2 ]). +-export([ + maybe_format_share/1, + get_shared_real_topic/1, + make_shared_record/2 +]). + -type topic() :: emqx_types:topic(). -type word() :: emqx_types:word(). -type words() :: emqx_types:words(). +-type share() :: emqx_types:share(). %% Guards -define(MULTI_LEVEL_WILDCARD_NOT_LAST(C, REST), @@ -50,7 +57,9 @@ %%-------------------------------------------------------------------- %% @doc Is wildcard topic? --spec wildcard(topic() | words()) -> true | false. +-spec wildcard(topic() | share() | words()) -> true | false. +wildcard(#share{topic = Topic}) when is_binary(Topic) -> + wildcard(Topic); wildcard(Topic) when is_binary(Topic) -> wildcard(words(Topic)); wildcard([]) -> @@ -64,7 +73,7 @@ wildcard([_H | T]) -> %% @doc Match Topic name with filter. -spec match(Name, Filter) -> boolean() when - Name :: topic() | words(), + Name :: topic() | share() | words(), Filter :: topic() | words(). match(<<$$, _/binary>>, <<$+, _/binary>>) -> false; @@ -72,6 +81,10 @@ match(<<$$, _/binary>>, <<$#, _/binary>>) -> false; match(Name, Filter) when is_binary(Name), is_binary(Filter) -> match(words(Name), words(Filter)); +match(#share{} = Name, Filter) -> + match_share(Name, Filter); +match(Name, #share{} = Filter) -> + match_share(Name, Filter); match([], []) -> true; match([H | T1], [H | T2]) -> @@ -87,12 +100,29 @@ match([_H1 | _], []) -> match([], [_H | _T2]) -> false. +-spec match_share(Name, Filter) -> boolean() when + Name :: share(), + Filter :: topic() | share(). +match_share(#share{topic = Name}, Filter) when is_binary(Filter) -> + %% only match real topic filter for normal topic filter. + match(words(Name), words(Filter)); +match_share(#share{group = Group, topic = Name}, #share{group = Group, topic = Filter}) -> + %% Matching real topic filter When subed same share group. + match(words(Name), words(Filter)); +match_share(#share{}, _) -> + %% Otherwise, non-matched. + false; +match_share(Name, #share{topic = Filter}) when is_binary(Name) -> + %% Only match real topic filter for normal topic_filter/topic_name. + match(Name, Filter). + -spec match_any(Name, [Filter]) -> boolean() when Name :: topic() | words(), Filter :: topic() | words(). match_any(Topic, Filters) -> lists:any(fun(Filter) -> match(Topic, Filter) end, Filters). +%% TODO: validate share topic #share{} for emqx_trace.erl %% @doc Validate topic name or filter -spec validate(topic() | {name | filter, topic()}) -> true. validate(Topic) when is_binary(Topic) -> @@ -107,7 +137,7 @@ validate(_, <<>>) -> validate(_, Topic) when is_binary(Topic) andalso (size(Topic) > ?MAX_TOPIC_LEN) -> %% MQTT-5.0 [MQTT-4.7.3-3] error(topic_too_long); -validate(filter, SharedFilter = <<"$share/", _Rest/binary>>) -> +validate(filter, SharedFilter = <>) -> validate_share(SharedFilter); validate(filter, Filter) when is_binary(Filter) -> validate2(words(Filter)); @@ -139,12 +169,12 @@ validate3(<>) when C == $#; C == $+; C == 0 -> validate3(<<_/utf8, Rest/binary>>) -> validate3(Rest). -validate_share(<<"$share/", Rest/binary>>) when +validate_share(<>) when Rest =:= <<>> orelse Rest =:= <<"/">> -> %% MQTT-5.0 [MQTT-4.8.2-1] error(?SHARE_EMPTY_FILTER); -validate_share(<<"$share/", Rest/binary>>) -> +validate_share(<>) -> case binary:split(Rest, <<"/">>) of %% MQTT-5.0 [MQTT-4.8.2-1] [<<>>, _] -> @@ -156,7 +186,7 @@ validate_share(<<"$share/", Rest/binary>>) -> validate_share(ShareName, Filter) end. -validate_share(_, <<"$share/", _Rest/binary>>) -> +validate_share(_, <>) -> error(?SHARE_RECURSIVELY); validate_share(ShareName, Filter) -> case binary:match(ShareName, [<<"+">>, <<"#">>]) of @@ -185,7 +215,9 @@ bin('#') -> <<"#">>; bin(B) when is_binary(B) -> B; bin(L) when is_list(L) -> list_to_binary(L). --spec levels(topic()) -> pos_integer(). +-spec levels(topic() | share()) -> pos_integer(). +levels(#share{topic = Topic}) when is_binary(Topic) -> + levels(Topic); levels(Topic) when is_binary(Topic) -> length(tokens(Topic)). @@ -197,6 +229,8 @@ tokens(Topic) -> %% @doc Split Topic Path to Words -spec words(topic()) -> words(). +words(#share{topic = Topic}) when is_binary(Topic) -> + words(Topic); words(Topic) when is_binary(Topic) -> [word(W) || W <- tokens(Topic)]. @@ -237,26 +271,29 @@ do_join(_TopicAcc, [C | Words]) when ?MULTI_LEVEL_WILDCARD_NOT_LAST(C, Words) -> do_join(TopicAcc, [Word | Words]) -> do_join(<>, Words). --spec parse(topic() | {topic(), map()}) -> {topic(), #{share => binary()}}. +-spec parse(topic() | {topic(), map()}) -> {topic() | share(), map()}. parse(TopicFilter) when is_binary(TopicFilter) -> parse(TopicFilter, #{}); parse({TopicFilter, Options}) when is_binary(TopicFilter) -> parse(TopicFilter, Options). --spec parse(topic(), map()) -> {topic(), map()}. -parse(TopicFilter = <<"$queue/", _/binary>>, #{share := _Group}) -> - error({invalid_topic_filter, TopicFilter}); -parse(TopicFilter = <<"$share/", _/binary>>, #{share := _Group}) -> - error({invalid_topic_filter, TopicFilter}); -parse(<<"$queue/", TopicFilter/binary>>, Options) -> - parse(TopicFilter, Options#{share => <<"$queue">>}); -parse(TopicFilter = <<"$share/", Rest/binary>>, Options) -> +-spec parse(topic() | share(), map()) -> {topic() | share(), map()}. +%% <<"$queue/[real_topic_filter]>">> equivalent to <<"$share/$queue/[real_topic_filter]">> +%% So the head of `real_topic_filter` MUST NOT be `<<$queue>>` or `<<$share>>` +parse(#share{topic = Topic = <>}, _Options) -> + error({invalid_topic_filter, Topic}); +parse(#share{topic = Topic = <>}, _Options) -> + error({invalid_topic_filter, Topic}); +parse(<>, Options) -> + parse(#share{group = <>, topic = Topic}, Options); +parse(TopicFilter = <>, Options) -> case binary:split(Rest, <<"/">>) of [_Any] -> error({invalid_topic_filter, TopicFilter}); - [ShareName, Filter] -> - case binary:match(ShareName, [<<"+">>, <<"#">>]) of - nomatch -> parse(Filter, Options#{share => ShareName}); + %% `Group` could be `$share` or `$queue` + [Group, Topic] -> + case binary:match(Group, [<<"+">>, <<"#">>]) of + nomatch -> parse(#share{group = Group, topic = Topic}, Options); _ -> error({invalid_topic_filter, TopicFilter}) end end; @@ -267,5 +304,22 @@ parse(TopicFilter = <<"$exclusive/", Topic/binary>>, Options) -> _ -> {Topic, Options#{is_exclusive => true}} end; -parse(TopicFilter, Options) -> +parse(TopicFilter, Options) when + ?IS_TOPIC(TopicFilter) +-> {TopicFilter, Options}. + +get_shared_real_topic(#share{topic = TopicFilter}) -> + TopicFilter; +get_shared_real_topic(TopicFilter) when is_binary(TopicFilter) -> + TopicFilter. + +make_shared_record(Group, Topic) -> + #share{group = Group, topic = Topic}. + +maybe_format_share(#share{group = <>, topic = Topic}) -> + join([<>, Topic]); +maybe_format_share(#share{group = Group, topic = Topic}) -> + join([<>, Group, Topic]); +maybe_format_share(Topic) -> + join([Topic]). diff --git a/apps/emqx/src/emqx_trace/emqx_trace.erl b/apps/emqx/src/emqx_trace/emqx_trace.erl index 99bbcc5f9..6588c99dc 100644 --- a/apps/emqx/src/emqx_trace/emqx_trace.erl +++ b/apps/emqx/src/emqx_trace/emqx_trace.erl @@ -105,7 +105,7 @@ log_filter([{Id, FilterFun, Filter, Name} | Rest], Log0) -> ignore -> ignore; Log -> - case logger_config:get(ets:whereis(logger), Id) of + case logger_config:get(logger, Id) of {ok, #{module := Module} = HandlerConfig0} -> HandlerConfig = maps:without(?OWN_KEYS, HandlerConfig0), try diff --git a/apps/emqx/src/emqx_types.erl b/apps/emqx/src/emqx_types.erl index 504540cf6..1a4825736 100644 --- a/apps/emqx/src/emqx_types.erl +++ b/apps/emqx/src/emqx_types.erl @@ -40,6 +40,10 @@ words/0 ]). +-export_type([ + share/0 +]). + -export_type([ socktype/0, sockstate/0, @@ -136,11 +140,14 @@ -type subid() :: binary() | atom(). --type group() :: binary() | undefined. +%% '_' for match spec +-type group() :: binary() | '_'. -type topic() :: binary(). -type word() :: '' | '+' | '#' | binary(). -type words() :: list(word()). +-type share() :: #share{}. + -type socktype() :: tcp | udp | ssl | proxy | atom(). -type sockstate() :: idle | running | blocked | closed. -type conninfo() :: #{ @@ -207,7 +214,6 @@ rap := 0 | 1, nl := 0 | 1, qos := qos(), - share => binary(), atom() => term() }. -type reason_code() :: 0..16#FF. diff --git a/apps/emqx/test/emqx_broker_SUITE.erl b/apps/emqx/test/emqx_broker_SUITE.erl index a205f6fcd..b416f1730 100644 --- a/apps/emqx/test/emqx_broker_SUITE.erl +++ b/apps/emqx/test/emqx_broker_SUITE.erl @@ -299,14 +299,19 @@ t_nosub_pub(Config) when is_list(Config) -> ?assertEqual(1, emqx_metrics:val('messages.dropped')). t_shared_subscribe({init, Config}) -> - emqx_broker:subscribe(<<"topic">>, <<"clientid">>, #{share => <<"group">>}), + emqx_broker:subscribe( + emqx_topic:make_shared_record(<<"group">>, <<"topic">>), <<"clientid">>, #{} + ), ct:sleep(100), Config; t_shared_subscribe(Config) when is_list(Config) -> emqx_broker:safe_publish(emqx_message:make(ct, <<"topic">>, <<"hello">>)), ?assert( receive - {deliver, <<"topic">>, #message{payload = <<"hello">>}} -> + {deliver, <<"topic">>, #message{ + headers = #{redispatch_to := ?REDISPATCH_TO(<<"group">>, <<"topic">>)}, + payload = <<"hello">> + }} -> true; Msg -> ct:pal("Msg: ~p", [Msg]), @@ -316,7 +321,7 @@ t_shared_subscribe(Config) when is_list(Config) -> end ); t_shared_subscribe({'end', _Config}) -> - emqx_broker:unsubscribe(<<"$share/group/topic">>). + emqx_broker:unsubscribe(emqx_topic:make_shared_record(<<"group">>, <<"topic">>)). t_shared_subscribe_2({init, Config}) -> Config; @@ -723,24 +728,6 @@ t_connack_auth_error(Config) when is_list(Config) -> ?assertEqual(2, emqx_metrics:val('packets.connack.auth_error')), ok. -t_handle_in_empty_client_subscribe_hook({init, Config}) -> - Hook = {?MODULE, client_subscribe_delete_all_hook, []}, - ok = emqx_hooks:put('client.subscribe', Hook, _Priority = 100), - Config; -t_handle_in_empty_client_subscribe_hook({'end', _Config}) -> - emqx_hooks:del('client.subscribe', {?MODULE, client_subscribe_delete_all_hook}), - ok; -t_handle_in_empty_client_subscribe_hook(Config) when is_list(Config) -> - {ok, C} = emqtt:start_link(), - {ok, _} = emqtt:connect(C), - try - {ok, _, RCs} = emqtt:subscribe(C, <<"t">>), - ?assertEqual([?RC_UNSPECIFIED_ERROR], RCs), - ok - after - emqtt:disconnect(C) - end. - authenticate_deny(_Credentials, _Default) -> {stop, {error, bad_username_or_password}}. @@ -800,7 +787,3 @@ recv_msgs(Count, Msgs) -> after 100 -> Msgs end. - -client_subscribe_delete_all_hook(_ClientInfo, _Username, TopicFilter) -> - EmptyFilters = [{T, Opts#{deny_subscription => true}} || {T, Opts} <- TopicFilter], - {stop, EmptyFilters}. diff --git a/apps/emqx/test/emqx_channel_SUITE.erl b/apps/emqx/test/emqx_channel_SUITE.erl index 8f6a2baaa..c6b4c0518 100644 --- a/apps/emqx/test/emqx_channel_SUITE.erl +++ b/apps/emqx/test/emqx_channel_SUITE.erl @@ -456,7 +456,7 @@ t_process_subscribe(_) -> ok = meck:expect(emqx_session, subscribe, fun(_, _, _, Session) -> {ok, Session} end), TopicFilters = [TopicFilter = {<<"+">>, ?DEFAULT_SUBOPTS}], {[{TopicFilter, ?RC_SUCCESS}], _Channel} = - emqx_channel:process_subscribe(TopicFilters, #{}, channel()). + emqx_channel:process_subscribe(TopicFilters, channel()). t_process_unsubscribe(_) -> ok = meck:expect(emqx_session, unsubscribe, fun(_, _, _, Session) -> {ok, Session} end), @@ -914,7 +914,13 @@ t_check_pub_alias(_) -> t_check_sub_authzs(_) -> emqx_config:put_zone_conf(default, [authorization, enable], true), TopicFilter = {<<"t">>, ?DEFAULT_SUBOPTS}, - [{TopicFilter, 0}] = emqx_channel:check_sub_authzs([TopicFilter], channel()). + SubPkt = ?SUBSCRIBE_PACKET(1, #{}, [TopicFilter]), + CheckedSubPkt = ?SUBSCRIBE_PACKET(1, #{}, [{TopicFilter, ?RC_SUCCESS}]), + Channel = channel(), + ?assertEqual( + {ok, CheckedSubPkt, Channel}, + emqx_channel:check_sub_authzs(SubPkt, Channel) + ). t_enrich_connack_caps(_) -> ok = meck:new(emqx_mqtt_caps, [passthrough, no_history]), @@ -1061,6 +1067,7 @@ clientinfo(InitProps) -> clientid => <<"clientid">>, username => <<"username">>, is_superuser => false, + is_bridge => false, mountpoint => undefined }, InitProps diff --git a/apps/emqx/test/emqx_common_test_http.erl b/apps/emqx/test/emqx_common_test_http.erl index 2d1128f05..30ebe409f 100644 --- a/apps/emqx/test/emqx_common_test_http.erl +++ b/apps/emqx/test/emqx_common_test_http.erl @@ -34,6 +34,9 @@ -define(DEFAULT_APP_KEY, <<"default_app_key">>). -define(DEFAULT_APP_SECRET, <<"default_app_secret">>). +%% from emqx_dashboard/include/emqx_dashboard_rbac.hrl +-define(ROLE_API_SUPERUSER, <<"administrator">>). + request_api(Method, Url, Auth) -> request_api(Method, Url, [], Auth, []). @@ -96,7 +99,8 @@ create_default_app() -> ?DEFAULT_APP_SECRET, true, ExpiredAt, - <<"default app key for test">> + <<"default app key for test">>, + ?ROLE_API_SUPERUSER ). delete_default_app() -> diff --git a/apps/emqx/test/emqx_mountpoint_SUITE.erl b/apps/emqx/test/emqx_mountpoint_SUITE.erl index 6d065d521..0bfde981c 100644 --- a/apps/emqx/test/emqx_mountpoint_SUITE.erl +++ b/apps/emqx/test/emqx_mountpoint_SUITE.erl @@ -29,6 +29,7 @@ ). -include_lib("emqx/include/emqx.hrl"). +-include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("eunit/include/eunit.hrl"). all() -> emqx_common_test_helpers:all(?MODULE). @@ -52,6 +53,27 @@ t_mount(_) -> mount(<<"device/1/">>, TopicFilters) ). +t_mount_share(_) -> + T = {TopicFilter, Opts} = emqx_topic:parse(<<"$share/group/topic">>), + TopicFilters = [T], + ?assertEqual(TopicFilter, #share{group = <<"group">>, topic = <<"topic">>}), + + %% should not mount share topic when make message. + Msg = emqx_message:make(<<"clientid">>, TopicFilter, <<"payload">>), + + ?assertEqual( + TopicFilter, + mount(undefined, TopicFilter) + ), + ?assertEqual( + #share{group = <<"group">>, topic = <<"device/1/topic">>}, + mount(<<"device/1/">>, TopicFilter) + ), + ?assertEqual( + [{#share{group = <<"group">>, topic = <<"device/1/topic">>}, Opts}], + mount(<<"device/1/">>, TopicFilters) + ). + t_unmount(_) -> Msg = emqx_message:make(<<"clientid">>, <<"device/1/topic">>, <<"payload">>), ?assertEqual(<<"topic">>, unmount(undefined, <<"topic">>)), @@ -61,6 +83,23 @@ t_unmount(_) -> ?assertEqual(<<"device/1/topic">>, unmount(<<"device/2/">>, <<"device/1/topic">>)), ?assertEqual(Msg#message{topic = <<"device/1/topic">>}, unmount(<<"device/2/">>, Msg)). +t_unmount_share(_) -> + {TopicFilter, _Opts} = emqx_topic:parse(<<"$share/group/topic">>), + MountedTopicFilter = #share{group = <<"group">>, topic = <<"device/1/topic">>}, + + ?assertEqual(TopicFilter, #share{group = <<"group">>, topic = <<"topic">>}), + + %% should not unmount share topic when make message. + Msg = emqx_message:make(<<"clientid">>, TopicFilter, <<"payload">>), + ?assertEqual( + TopicFilter, + unmount(undefined, TopicFilter) + ), + ?assertEqual( + #share{group = <<"group">>, topic = <<"topic">>}, + unmount(<<"device/1/">>, MountedTopicFilter) + ). + t_replvar(_) -> ?assertEqual(undefined, replvar(undefined, #{})), ?assertEqual( diff --git a/apps/emqx/test/emqx_mqtt_caps_SUITE.erl b/apps/emqx/test/emqx_mqtt_caps_SUITE.erl index 297ee7f7d..e97684b74 100644 --- a/apps/emqx/test/emqx_mqtt_caps_SUITE.erl +++ b/apps/emqx/test/emqx_mqtt_caps_SUITE.erl @@ -76,6 +76,8 @@ t_check_sub(_) -> ), ?assertEqual( {error, ?RC_SHARED_SUBSCRIPTIONS_NOT_SUPPORTED}, - emqx_mqtt_caps:check_sub(ClientInfo, <<"topic">>, SubOpts#{share => true}) + emqx_mqtt_caps:check_sub( + ClientInfo, #share{group = <<"group">>, topic = <<"topic">>}, SubOpts + ) ), emqx_config:put([zones], OldConf). diff --git a/apps/emqx/test/emqx_proper_types.erl b/apps/emqx/test/emqx_proper_types.erl index 0a66b3628..6c2ad56f9 100644 --- a/apps/emqx/test/emqx_proper_types.erl +++ b/apps/emqx/test/emqx_proper_types.erl @@ -511,13 +511,7 @@ peercert() -> conn_mod() -> oneof([ emqx_connection, - emqx_ws_connection, - emqx_coap_mqtt_adapter, - emqx_sn_gateway, - emqx_lwm2m_protocol, - emqx_gbt32960_conn, - emqx_jt808_connection, - emqx_tcp_connection + emqx_ws_connection ]). proto_name() -> diff --git a/apps/emqx/test/emqx_secret_tests.erl b/apps/emqx/test/emqx_secret_tests.erl new file mode 100644 index 000000000..cd6588c83 --- /dev/null +++ b/apps/emqx/test/emqx_secret_tests.erl @@ -0,0 +1,76 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_secret_tests). + +-include_lib("eunit/include/eunit.hrl"). + +wrap_unwrap_test() -> + ?assertEqual( + 42, + emqx_secret:unwrap(emqx_secret:wrap(42)) + ). + +unwrap_immediate_test() -> + ?assertEqual( + 42, + emqx_secret:unwrap(42) + ). + +wrap_unwrap_load_test_() -> + Secret = <<"foobaz">>, + { + setup, + fun() -> write_temp_file(Secret) end, + fun(Filename) -> file:delete(Filename) end, + fun(Filename) -> + ?_assertEqual( + Secret, + emqx_secret:unwrap(emqx_secret:wrap_load({file, Filename})) + ) + end + }. + +wrap_load_term_test() -> + ?assertEqual( + {file, "no/such/file/i/swear"}, + emqx_secret:term(emqx_secret:wrap_load({file, "no/such/file/i/swear"})) + ). + +wrap_unwrap_missing_file_test() -> + ?assertThrow( + #{msg := failed_to_read_secret_file, reason := "No such file or directory"}, + emqx_secret:unwrap(emqx_secret:wrap_load({file, "no/such/file/i/swear"})) + ). + +wrap_term_test() -> + ?assertEqual( + 42, + emqx_secret:term(emqx_secret:wrap(42)) + ). + +external_fun_term_error_test() -> + Term = {foo, bar}, + ?assertError( + badarg, + emqx_secret:term(fun() -> Term end) + ). + +write_temp_file(Bytes) -> + Ts = erlang:system_time(millisecond), + Filename = filename:join("/tmp", ?MODULE_STRING ++ integer_to_list(-Ts)), + ok = file:write_file(Filename, Bytes), + Filename. diff --git a/apps/emqx/test/emqx_shared_sub_SUITE.erl b/apps/emqx/test/emqx_shared_sub_SUITE.erl index 4b4535cea..86887eff0 100644 --- a/apps/emqx/test/emqx_shared_sub_SUITE.erl +++ b/apps/emqx/test/emqx_shared_sub_SUITE.erl @@ -137,7 +137,8 @@ t_random_basic(Config) when is_list(Config) -> ClientId = <<"ClientId">>, Topic = <<"foo">>, Payload = <<"hello">>, - emqx:subscribe(Topic, #{qos => 2, share => <<"group1">>}), + Group = <<"group1">>, + emqx_broker:subscribe(emqx_topic:make_shared_record(Group, Topic), #{qos => 2}), MsgQoS2 = emqx_message:make(ClientId, 2, Topic, Payload), %% wait for the subscription to show up ct:sleep(200), @@ -402,7 +403,7 @@ t_hash(Config) when is_list(Config) -> ok = ensure_config(hash_clientid, false), test_two_messages(hash_clientid). -t_hash_clinetid(Config) when is_list(Config) -> +t_hash_clientid(Config) when is_list(Config) -> ok = ensure_config(hash_clientid, false), test_two_messages(hash_clientid). @@ -528,14 +529,15 @@ last_message(ExpectedPayload, Pids, Timeout) -> t_dispatch(Config) when is_list(Config) -> ok = ensure_config(random), Topic = <<"foo">>, + Group = <<"group1">>, ?assertEqual( {error, no_subscribers}, - emqx_shared_sub:dispatch(<<"group1">>, Topic, #delivery{message = #message{}}) + emqx_shared_sub:dispatch(Group, Topic, #delivery{message = #message{}}) ), - emqx:subscribe(Topic, #{qos => 2, share => <<"group1">>}), + emqx_broker:subscribe(emqx_topic:make_shared_record(Group, Topic), #{qos => 2}), ?assertEqual( {ok, 1}, - emqx_shared_sub:dispatch(<<"group1">>, Topic, #delivery{message = #message{}}) + emqx_shared_sub:dispatch(Group, Topic, #delivery{message = #message{}}) ). t_uncovered_func(Config) when is_list(Config) -> @@ -991,37 +993,110 @@ t_session_kicked(Config) when is_list(Config) -> ?assertEqual([], collect_msgs(0)), ok. -%% FIXME: currently doesn't work -%% t_different_groups_same_topic({init, Config}) -> -%% TestName = atom_to_binary(?FUNCTION_NAME), -%% ClientId = <>, -%% {ok, C} = emqtt:start_link([{clientid, ClientId}, {proto_ver, v5}]), -%% {ok, _} = emqtt:connect(C), -%% [{client, C}, {clientid, ClientId} | Config]; -%% t_different_groups_same_topic({'end', Config}) -> -%% C = ?config(client, Config), -%% emqtt:stop(C), -%% ok; -%% t_different_groups_same_topic(Config) when is_list(Config) -> -%% C = ?config(client, Config), -%% ClientId = ?config(clientid, Config), -%% %% Subscribe and unsubscribe to both $queue and $shared topics -%% Topic = <<"t/1">>, -%% SharedTopic0 = <<"$share/aa/", Topic/binary>>, -%% SharedTopic1 = <<"$share/bb/", Topic/binary>>, -%% {ok, _, [2]} = emqtt:subscribe(C, {SharedTopic0, 2}), -%% {ok, _, [2]} = emqtt:subscribe(C, {SharedTopic1, 2}), +-define(UPDATE_SUB_QOS(ConnPid, Topic, QoS), + ?assertMatch({ok, _, [QoS]}, emqtt:subscribe(ConnPid, {Topic, QoS})) +). -%% Message0 = emqx_message:make(ClientId, _QoS = 2, Topic, <<"hi">>), -%% emqx:publish(Message0), -%% ?assertMatch([ {publish, #{payload := <<"hi">>}} -%% , {publish, #{payload := <<"hi">>}} -%% ], collect_msgs(5_000), #{routes => ets:tab2list(emqx_route)}), +t_different_groups_same_topic({init, Config}) -> + TestName = atom_to_binary(?FUNCTION_NAME), + ClientId = <>, + {ok, C} = emqtt:start_link([{clientid, ClientId}, {proto_ver, v5}]), + {ok, _} = emqtt:connect(C), + [{client, C}, {clientid, ClientId} | Config]; +t_different_groups_same_topic({'end', Config}) -> + C = ?config(client, Config), + emqtt:stop(C), + ok; +t_different_groups_same_topic(Config) when is_list(Config) -> + C = ?config(client, Config), + ClientId = ?config(clientid, Config), + %% Subscribe and unsubscribe to different group `aa` and `bb` with same topic + GroupA = <<"aa">>, + GroupB = <<"bb">>, + Topic = <<"t/1">>, -%% {ok, _, [0]} = emqtt:unsubscribe(C, SharedTopic0), -%% {ok, _, [0]} = emqtt:unsubscribe(C, SharedTopic1), + SharedTopicGroupA = ?SHARE(GroupA, Topic), + ?UPDATE_SUB_QOS(C, SharedTopicGroupA, ?QOS_2), + SharedTopicGroupB = ?SHARE(GroupB, Topic), + ?UPDATE_SUB_QOS(C, SharedTopicGroupB, ?QOS_2), -%% ok. + ?retry( + _Sleep0 = 100, + _Attempts0 = 50, + begin + ?assertEqual(2, length(emqx_router:match_routes(Topic))) + end + ), + + Message0 = emqx_message:make(ClientId, ?QOS_2, Topic, <<"hi">>), + emqx:publish(Message0), + ?assertMatch( + [ + {publish, #{payload := <<"hi">>}}, + {publish, #{payload := <<"hi">>}} + ], + collect_msgs(5_000), + #{routes => ets:tab2list(emqx_route)} + ), + + {ok, _, [?RC_SUCCESS]} = emqtt:unsubscribe(C, SharedTopicGroupA), + {ok, _, [?RC_SUCCESS]} = emqtt:unsubscribe(C, SharedTopicGroupB), + + ok. + +t_different_groups_update_subopts({init, Config}) -> + TestName = atom_to_binary(?FUNCTION_NAME), + ClientId = <>, + {ok, C} = emqtt:start_link([{clientid, ClientId}, {proto_ver, v5}]), + {ok, _} = emqtt:connect(C), + [{client, C}, {clientid, ClientId} | Config]; +t_different_groups_update_subopts({'end', Config}) -> + C = ?config(client, Config), + emqtt:stop(C), + ok; +t_different_groups_update_subopts(Config) when is_list(Config) -> + C = ?config(client, Config), + ClientId = ?config(clientid, Config), + %% Subscribe and unsubscribe to different group `aa` and `bb` with same topic + Topic = <<"t/1">>, + GroupA = <<"aa">>, + GroupB = <<"bb">>, + SharedTopicGroupA = ?SHARE(GroupA, Topic), + SharedTopicGroupB = ?SHARE(GroupB, Topic), + + Fun = fun(Group, QoS) -> + ?UPDATE_SUB_QOS(C, ?SHARE(Group, Topic), QoS), + ?assertMatch( + #{qos := QoS}, + emqx_broker:get_subopts(ClientId, emqx_topic:make_shared_record(Group, Topic)) + ) + end, + + [Fun(Group, QoS) || QoS <- [?QOS_0, ?QOS_1, ?QOS_2], Group <- [GroupA, GroupB]], + + ?retry( + _Sleep0 = 100, + _Attempts0 = 50, + begin + ?assertEqual(2, length(emqx_router:match_routes(Topic))) + end + ), + + Message0 = emqx_message:make(ClientId, _QoS = 2, Topic, <<"hi">>), + emqx:publish(Message0), + ?assertMatch( + [ + {publish, #{payload := <<"hi">>}}, + {publish, #{payload := <<"hi">>}} + ], + collect_msgs(5_000), + #{routes => ets:tab2list(emqx_route)} + ), + + {ok, _, [?RC_SUCCESS]} = emqtt:unsubscribe(C, SharedTopicGroupA), + {ok, _, [?RC_SUCCESS]} = emqtt:unsubscribe(C, SharedTopicGroupB), + + ok. t_queue_subscription({init, Config}) -> TestName = atom_to_binary(?FUNCTION_NAME), @@ -1038,23 +1113,19 @@ t_queue_subscription({'end', Config}) -> t_queue_subscription(Config) when is_list(Config) -> C = ?config(client, Config), ClientId = ?config(clientid, Config), - %% Subscribe and unsubscribe to both $queue and $shared topics + %% Subscribe and unsubscribe to both $queue share and $share/ with same topic Topic = <<"t/1">>, QueueTopic = <<"$queue/", Topic/binary>>, SharedTopic = <<"$share/aa/", Topic/binary>>, - {ok, _, [?RC_GRANTED_QOS_2]} = emqtt:subscribe(C, {QueueTopic, 2}), - {ok, _, [?RC_GRANTED_QOS_2]} = emqtt:subscribe(C, {SharedTopic, 2}), - %% FIXME: we should actually see 2 routes, one for each group - %% ($queue and aa), but currently the latest subscription - %% overwrites the existing one. + ?UPDATE_SUB_QOS(C, QueueTopic, ?QOS_2), + ?UPDATE_SUB_QOS(C, SharedTopic, ?QOS_2), + ?retry( _Sleep0 = 100, _Attempts0 = 50, begin - ct:pal("routes: ~p", [ets:tab2list(emqx_route)]), - %% FIXME: should ensure we have 2 subscriptions - [_] = emqx_router:lookup_routes(Topic) + ?assertEqual(2, length(emqx_router:match_routes(Topic))) end ), @@ -1063,37 +1134,29 @@ t_queue_subscription(Config) when is_list(Config) -> emqx:publish(Message0), ?assertMatch( [ + {publish, #{payload := <<"hi">>}}, {publish, #{payload := <<"hi">>}} - %% FIXME: should receive one message from each group - %% , {publish, #{payload := <<"hi">>}} ], - collect_msgs(5_000) + collect_msgs(5_000), + #{routes => ets:tab2list(emqx_route)} ), {ok, _, [?RC_SUCCESS]} = emqtt:unsubscribe(C, QueueTopic), - %% FIXME: return code should be success instead of 17 ("no_subscription_existed") - {ok, _, [?RC_NO_SUBSCRIPTION_EXISTED]} = emqtt:unsubscribe(C, SharedTopic), + {ok, _, [?RC_SUCCESS]} = emqtt:unsubscribe(C, SharedTopic), - %% FIXME: this should eventually be true, but currently we leak - %% the previous group subscription... - %% ?retry( - %% _Sleep0 = 100, - %% _Attempts0 = 50, - %% begin - %% ct:pal("routes: ~p", [ets:tab2list(emqx_route)]), - %% [] = emqx_router:lookup_routes(Topic) - %% end - %% ), + ?retry( + _Sleep0 = 100, + _Attempts0 = 50, + begin + ?assertEqual(0, length(emqx_router:match_routes(Topic))) + end + ), ct:sleep(500), Message1 = emqx_message:make(ClientId, _QoS = 2, Topic, <<"hello">>), emqx:publish(Message1), - %% FIXME: we should *not* receive any messages... - %% ?assertEqual([], collect_msgs(1_000), #{routes => ets:tab2list(emqx_route)}), - %% This is from the leaked group... - ?assertMatch([{publish, #{topic := Topic}}], collect_msgs(1_000), #{ - routes => ets:tab2list(emqx_route) - }), + %% we should *not* receive any messages. + ?assertEqual([], collect_msgs(1_000), #{routes => ets:tab2list(emqx_route)}), ok. diff --git a/apps/emqx/test/emqx_topic_SUITE.erl b/apps/emqx/test/emqx_topic_SUITE.erl index c49c93fb2..4761ea17d 100644 --- a/apps/emqx/test/emqx_topic_SUITE.erl +++ b/apps/emqx/test/emqx_topic_SUITE.erl @@ -238,11 +238,11 @@ long_topic() -> t_parse(_) -> ?assertError( {invalid_topic_filter, <<"$queue/t">>}, - parse(<<"$queue/t">>, #{share => <<"g">>}) + parse(#share{group = <<"$queue">>, topic = <<"$queue/t">>}, #{}) ), ?assertError( {invalid_topic_filter, <<"$share/g/t">>}, - parse(<<"$share/g/t">>, #{share => <<"g">>}) + parse(#share{group = <<"g">>, topic = <<"$share/g/t">>}, #{}) ), ?assertError( {invalid_topic_filter, <<"$share/t">>}, @@ -254,8 +254,12 @@ t_parse(_) -> ), ?assertEqual({<<"a/b/+/#">>, #{}}, parse(<<"a/b/+/#">>)), ?assertEqual({<<"a/b/+/#">>, #{qos => 1}}, parse({<<"a/b/+/#">>, #{qos => 1}})), - ?assertEqual({<<"topic">>, #{share => <<"$queue">>}}, parse(<<"$queue/topic">>)), - ?assertEqual({<<"topic">>, #{share => <<"group">>}}, parse(<<"$share/group/topic">>)), + ?assertEqual( + {#share{group = <<"$queue">>, topic = <<"topic">>}, #{}}, parse(<<"$queue/topic">>) + ), + ?assertEqual( + {#share{group = <<"group">>, topic = <<"topic">>}, #{}}, parse(<<"$share/group/topic">>) + ), %% The '$local' and '$fastlane' topics have been deprecated. ?assertEqual({<<"$local/topic">>, #{}}, parse(<<"$local/topic">>)), ?assertEqual({<<"$local/$queue/topic">>, #{}}, parse(<<"$local/$queue/topic">>)), diff --git a/apps/emqx/test/emqx_topic_index_SUITE.erl b/apps/emqx/test/emqx_topic_index_SUITE.erl index 9df9743f1..71e508306 100644 --- a/apps/emqx/test/emqx_topic_index_SUITE.erl +++ b/apps/emqx/test/emqx_topic_index_SUITE.erl @@ -209,9 +209,6 @@ t_match_fast_forward(Config) -> M:insert(<<"a/b/1/2/3/4/5/6/7/8/9/#">>, id1, <<>>, Tab), M:insert(<<"z/y/x/+/+">>, id2, <<>>, Tab), M:insert(<<"a/b/c/+">>, id3, <<>>, Tab), - % dbg:tracer(), - % dbg:p(all, c), - % dbg:tpl({ets, next, '_'}, x), ?assertEqual(id1, id(match(M, <<"a/b/1/2/3/4/5/6/7/8/9/0">>, Tab))), ?assertEqual([id1], [id(X) || X <- matches(M, <<"a/b/1/2/3/4/5/6/7/8/9/0">>, Tab)]). diff --git a/apps/emqx_audit/BSL.txt b/apps/emqx_audit/BSL.txt new file mode 100644 index 000000000..0acc0e696 --- /dev/null +++ b/apps/emqx_audit/BSL.txt @@ -0,0 +1,94 @@ +Business Source License 1.1 + +Licensor: Hangzhou EMQ Technologies Co., Ltd. +Licensed Work: EMQX Enterprise Edition + The Licensed Work is (c) 2023 + Hangzhou EMQ Technologies Co., Ltd. +Additional Use Grant: Students and educators are granted right to copy, + modify, and create derivative work for research + or education. +Change Date: 2027-02-01 +Change License: Apache License, Version 2.0 + +For information about alternative licensing arrangements for the Software, +please contact Licensor: https://www.emqx.com/en/contact + +Notice + +The Business Source License (this document, or the “License”) is not an Open +Source license. However, the Licensed Work will eventually be made available +under an Open Source License, as stated in this License. + +License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved. +“Business Source License” is a trademark of MariaDB Corporation Ab. + +----------------------------------------------------------------------------- + +Business Source License 1.1 + +Terms + +The Licensor hereby grants you the right to copy, modify, create derivative +works, redistribute, and make non-production use of the Licensed Work. The +Licensor may make an Additional Use Grant, above, permitting limited +production use. + +Effective on the Change Date, or the fourth anniversary of the first publicly +available distribution of a specific version of the Licensed Work under this +License, whichever comes first, the Licensor hereby grants you rights under +the terms of the Change License, and the rights granted in the paragraph +above terminate. + +If your use of the Licensed Work does not comply with the requirements +currently in effect as described in this License, you must purchase a +commercial license from the Licensor, its affiliated entities, or authorized +resellers, or you must refrain from using the Licensed Work. + +All copies of the original and modified Licensed Work, and derivative works +of the Licensed Work, are subject to this License. This License applies +separately for each version of the Licensed Work and the Change Date may vary +for each version of the Licensed Work released by Licensor. + +You must conspicuously display this License on each original or modified copy +of the Licensed Work. If you receive the Licensed Work in original or +modified form from a third party, the terms and conditions set forth in this +License apply to your use of that work. + +Any use of the Licensed Work in violation of this License will automatically +terminate your rights under this License for the current and all other +versions of the Licensed Work. + +This License does not grant you any right in any trademark or logo of +Licensor or its affiliates (provided that you may use a trademark or logo of +Licensor as expressly required by this License). + +TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON +AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS, +EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND +TITLE. + +MariaDB hereby grants you permission to use this License’s text to license +your works, and to refer to it using the trademark “Business Source License”, +as long as you comply with the Covenants of Licensor below. + +Covenants of Licensor + +In consideration of the right to use this License’s text and the “Business +Source License” name and trademark, Licensor covenants to MariaDB, and to all +other recipients of the licensed work to be provided by Licensor: + +1. To specify as the Change License the GPL Version 2.0 or any later version, + or a license that is compatible with GPL Version 2.0 or a later version, + where “compatible” means that software provided under the Change License can + be included in a program with software provided under GPL Version 2.0 or a + later version. Licensor may specify additional Change Licenses without + limitation. + +2. To either: (a) specify an additional grant of rights to use that does not + impose any additional restriction on the right granted in this License, as + the Additional Use Grant; or (b) insert the text “None”. + +3. To specify a Change Date. + +4. Not to modify this License in any other way. diff --git a/apps/emqx_audit/README.md b/apps/emqx_audit/README.md new file mode 100644 index 000000000..48c625ed5 --- /dev/null +++ b/apps/emqx_audit/README.md @@ -0,0 +1,5 @@ +emqx_audit +===== + +Audit log for EMQX, empowers users to efficiently access the desired audit trail data +and facilitates auditing, compliance, troubleshooting, and security analysis. diff --git a/apps/emqx_audit/include/emqx_audit.hrl b/apps/emqx_audit/include/emqx_audit.hrl new file mode 100644 index 000000000..8304a9060 --- /dev/null +++ b/apps/emqx_audit/include/emqx_audit.hrl @@ -0,0 +1,26 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-define(AUDIT, emqx_audit). + +-record(?AUDIT, { + %% basic info + created_at, + node, + from, + source, + source_ip, + %% operation info + operation_id, + operation_type, + args, + operation_result, + failure, + %% request detail + http_method, + http_request, + http_status_code, + duration_ms, + extra +}). diff --git a/apps/emqx_audit/rebar.config b/apps/emqx_audit/rebar.config new file mode 100644 index 000000000..fac0f9b07 --- /dev/null +++ b/apps/emqx_audit/rebar.config @@ -0,0 +1,5 @@ +{erl_opts, [debug_info]}. +{deps, [ + {emqx, {path, "../emqx"}}, + {emqx_utils, {path, "../emqx_utils"}} +]}. diff --git a/apps/emqx_audit/src/emqx_audit.app.src b/apps/emqx_audit/src/emqx_audit.app.src new file mode 100644 index 000000000..96cdd11ce --- /dev/null +++ b/apps/emqx_audit/src/emqx_audit.app.src @@ -0,0 +1,10 @@ +{application, emqx_audit, [ + {description, "Audit log for EMQX"}, + {vsn, "0.1.0"}, + {registered, []}, + {mod, {emqx_audit_app, []}}, + {applications, [kernel, stdlib, emqx]}, + {env, []}, + {modules, []}, + {links, []} +]}. diff --git a/apps/emqx_audit/src/emqx_audit.erl b/apps/emqx_audit/src/emqx_audit.erl new file mode 100644 index 000000000..98f4a70e8 --- /dev/null +++ b/apps/emqx_audit/src/emqx_audit.erl @@ -0,0 +1,245 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_audit). + +-behaviour(gen_server). + +-include_lib("emqx/include/emqx.hrl"). +-include_lib("emqx/include/logger.hrl"). +-include("emqx_audit.hrl"). + +%% API +-export([start_link/0]). +-export([log/3]). + +-export([trans_clean_expired/2]). + +%% gen_server callbacks +-export([ + init/1, + handle_continue/2, + handle_call/3, + handle_cast/2, + handle_info/2, + terminate/2, + code_change/3 +]). + +-define(FILTER_REQ, [cert, host_info, has_sent_resp, pid, path_info, peer, ref, sock, streamid]). + +-ifdef(TEST). +-define(INTERVAL, 100). +-else. +-define(INTERVAL, 10000). +-endif. + +to_audit(#{from := cli, cmd := Cmd, args := Args, duration_ms := DurationMs}) -> + #?AUDIT{ + operation_id = <<"">>, + operation_type = atom_to_binary(Cmd), + args = Args, + operation_result = <<"">>, + failure = <<"">>, + duration_ms = DurationMs, + from = cli, + source = <<"">>, + source_ip = <<"">>, + http_status_code = <<"">>, + http_method = <<"">>, + http_request = <<"">> + }; +to_audit(#{from := From} = Log) when From =:= dashboard orelse From =:= rest_api -> + #{ + source := Source, + source_ip := SourceIp, + %% operation info + operation_id := OperationId, + operation_type := OperationType, + operation_result := OperationResult, + %% request detail + http_status_code := StatusCode, + http_method := Method, + http_request := Request, + duration_ms := DurationMs + } = Log, + #?AUDIT{ + from = From, + source = Source, + source_ip = SourceIp, + %% operation info + operation_id = OperationId, + operation_type = OperationType, + operation_result = OperationResult, + failure = maps:get(failure, Log, <<"">>), + %% request detail + http_status_code = StatusCode, + http_method = Method, + http_request = Request, + duration_ms = DurationMs, + args = <<"">> + }; +to_audit(#{from := erlang_console, function := F, args := Args}) -> + #?AUDIT{ + from = erlang_console, + source = <<"">>, + source_ip = <<"">>, + %% operation info + operation_id = <<"">>, + operation_type = <<"">>, + operation_result = <<"">>, + failure = <<"">>, + %% request detail + http_status_code = <<"">>, + http_method = <<"">>, + http_request = <<"">>, + duration_ms = 0, + args = iolist_to_binary(io_lib:format("~p: ~p~n", [F, Args])) + }. + +log(_Level, undefined, _Handler) -> + ok; +log(Level, Meta1, Handler) -> + Meta2 = Meta1#{time => logger:timestamp(), level => Level}, + log_to_file(Level, Meta2, Handler), + log_to_db(Meta2), + remove_handler_when_disabled(). + +remove_handler_when_disabled() -> + case emqx_config:get([log, audit, enable], false) of + true -> + ok; + false -> + _ = logger:remove_handler(?AUDIT_HANDLER), + ok + end. + +log_to_db(Log) -> + Audit0 = to_audit(Log), + Audit = Audit0#?AUDIT{ + node = node(), + created_at = erlang:system_time(microsecond) + }, + mria:dirty_write(?AUDIT, Audit). + +start_link() -> + gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). + +init([]) -> + ok = mria:create_table(?AUDIT, [ + {type, ordered_set}, + {rlog_shard, ?COMMON_SHARD}, + {storage, disc_copies}, + {record_name, ?AUDIT}, + {attributes, record_info(fields, ?AUDIT)} + ]), + {ok, #{}, {continue, setup}}. + +handle_continue(setup, State) -> + ok = mria:wait_for_tables([?AUDIT]), + NewState = State#{role => mria_rlog:role()}, + ?AUDIT(alert, #{ + cmd => emqx, + args => ["start"], + version => emqx_release:version(), + from => cli, + duration_ms => 0 + }), + {noreply, NewState, interval(NewState)}. + +handle_call(_Request, _From, State) -> + {reply, ignore, State, interval(State)}. + +handle_cast(_Request, State) -> + {noreply, State, interval(State)}. + +handle_info(timeout, State) -> + ExtraWait = clean_expired_logs(), + {noreply, State, interval(State) + ExtraWait}; +handle_info(_Info, State) -> + {noreply, State, interval(State)}. + +terminate(_Reason, _State) -> + ok. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%%=================================================================== +%%% Internal functions +%%%=================================================================== + +%% if clean_expired transaction aborted, it will be scheduled with extra 60 seconds. +clean_expired_logs() -> + MaxSize = max_size(), + Oldest = mnesia:dirty_first(?AUDIT), + CurSize = mnesia:table_info(?AUDIT, size), + case CurSize - MaxSize of + DelSize when DelSize > 0 -> + case + mria:transaction( + ?COMMON_SHARD, + fun ?MODULE:trans_clean_expired/2, + [Oldest, DelSize] + ) + of + {atomic, ok} -> + 0; + {aborted, Reason} -> + ?SLOG(error, #{ + msg => "clean_expired_audit_aborted", + reason => Reason, + delete_size => DelSize, + current_size => CurSize, + max_count => MaxSize + }), + 60000 + end; + _ -> + 0 + end. + +trans_clean_expired(Oldest, DelCount) -> + First = mnesia:first(?AUDIT), + %% Other node already clean from the oldest record. + %% ensure not delete twice, otherwise records that should not be deleted will be deleted. + case First =:= Oldest of + true -> do_clean_expired(First, DelCount); + false -> ok + end. + +do_clean_expired(_, DelSize) when DelSize =< 0 -> ok; +do_clean_expired('$end_of_table', _DelSize) -> + ok; +do_clean_expired(CurKey, DeleteSize) -> + mnesia:delete(?AUDIT, CurKey, sticky_write), + do_clean_expired(mnesia:next(?AUDIT, CurKey), DeleteSize - 1). + +max_size() -> + emqx_conf:get([log, audit, max_filter_size], 5000). + +interval(#{role := replicant}) -> hibernate; +interval(#{role := core}) -> ?INTERVAL + rand:uniform(?INTERVAL). + +log_to_file(Level, Meta, #{module := Module} = Handler) -> + Log = #{level => Level, meta => Meta, msg => undefined}, + Handler1 = maps:without(?OWN_KEYS, Handler), + try + erlang:apply(Module, log, [Log, Handler1]) + catch + C:R:S -> + case logger:remove_handler(?AUDIT_HANDLER) of + ok -> + logger:internal_log( + error, {removed_failing_handler, ?AUDIT_HANDLER, C, R, S} + ); + {error, {not_found, _}} -> + ok; + {error, Reason} -> + logger:internal_log( + error, + {removed_handler_failed, ?AUDIT_HANDLER, Reason, C, R, S} + ) + end + end. diff --git a/apps/emqx_audit/src/emqx_audit_api.erl b/apps/emqx_audit/src/emqx_audit_api.erl new file mode 100644 index 000000000..a7fd8f4ad --- /dev/null +++ b/apps/emqx_audit/src/emqx_audit_api.erl @@ -0,0 +1,398 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_audit_api). + +-behaviour(minirest_api). + +%% API +-export([api_spec/0, paths/0, schema/1, namespace/0, fields/1]). +-export([audit/2]). +-export([qs2ms/2, format/1]). + +-include_lib("emqx/include/logger.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). +-include_lib("typerefl/include/types.hrl"). +-include("emqx_audit.hrl"). + +-import(hoconsc, [mk/2, ref/2, array/1]). + +-define(TAGS, ["Audit"]). + +-define(AUDIT_QS_SCHEMA, [ + {<<"node">>, atom}, + {<<"from">>, atom}, + {<<"source">>, binary}, + {<<"source_ip">>, binary}, + {<<"operation_id">>, binary}, + {<<"operation_type">>, binary}, + {<<"operation_result">>, atom}, + {<<"http_status_code">>, integer}, + {<<"http_method">>, atom}, + {<<"gte_created_at">>, timestamp}, + {<<"lte_created_at">>, timestamp}, + {<<"gte_duration_ms">>, timestamp}, + {<<"lte_duration_ms">>, timestamp} +]). +-define(DISABLE_MSG, <<"Audit is disabled">>). + +namespace() -> "audit". + +api_spec() -> + emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}). + +paths() -> + ["/audit"]. + +schema("/audit") -> + #{ + 'operationId' => audit, + get => #{ + tags => ?TAGS, + description => ?DESC(audit_get), + parameters => [ + {node, + ?HOCON(binary(), #{ + in => query, + required => false, + example => <<"emqx@127.0.0.1">>, + desc => ?DESC(filter_node) + })}, + {from, + ?HOCON(?ENUM([dashboard, rest_api, cli, erlang_console]), #{ + in => query, + required => false, + example => <<"dashboard">>, + desc => ?DESC(filter_from) + })}, + {source, + ?HOCON(binary(), #{ + in => query, + required => false, + example => <<"admin">>, + desc => ?DESC(filter_source) + })}, + {source_ip, + ?HOCON(binary(), #{ + in => query, + required => false, + example => <<"127.0.0.1">>, + desc => ?DESC(filter_source_ip) + })}, + {operation_id, + ?HOCON(binary(), #{ + in => query, + required => false, + example => <<"/rules/{id}">>, + desc => ?DESC(filter_operation_id) + })}, + {operation_type, + ?HOCON(binary(), #{ + in => query, + example => <<"rules">>, + required => false, + desc => ?DESC(filter_operation_type) + })}, + {operation_result, + ?HOCON(?ENUM([success, failure]), #{ + in => query, + example => failure, + required => false, + desc => ?DESC(filter_operation_result) + })}, + {http_status_code, + ?HOCON(integer(), #{ + in => query, + example => 200, + required => false, + desc => ?DESC(filter_http_status_code) + })}, + {http_method, + ?HOCON(?ENUM([post, put, delete]), #{ + in => query, + example => post, + required => false, + desc => ?DESC(filter_http_method) + })}, + {gte_duration_ms, + ?HOCON(integer(), #{ + in => query, + example => 0, + required => false, + desc => ?DESC(filter_gte_duration_ms) + })}, + {lte_duration_ms, + ?HOCON(integer(), #{ + in => query, + example => 1000, + required => false, + desc => ?DESC(filter_lte_duration_ms) + })}, + {gte_created_at, + ?HOCON(emqx_utils_calendar:epoch_millisecond(), #{ + in => query, + required => false, + example => <<"2023-10-15T00:00:00.820384+08:00">>, + desc => ?DESC(filter_gte_created_at) + })}, + {lte_created_at, + ?HOCON(emqx_utils_calendar:epoch_millisecond(), #{ + in => query, + example => <<"2023-10-16T00:00:00.820384+08:00">>, + required => false, + desc => ?DESC(filter_lte_created_at) + })}, + ref(emqx_dashboard_swagger, page), + ref(emqx_dashboard_swagger, limit) + ], + summary => <<"List audit logs">>, + responses => #{ + 200 => + emqx_dashboard_swagger:schema_with_example( + array(?REF(audit_list)), + audit_log_list_example() + ), + 400 => emqx_dashboard_swagger:error_codes( + ['BAD_REQUEST'], + ?DISABLE_MSG + ) + } + } + }. + +fields(audit_list) -> + [ + {data, mk(array(?REF(audit)), #{desc => ?DESC("audit_resp")})}, + {meta, mk(ref(emqx_dashboard_swagger, meta), #{})} + ]; +fields(audit) -> + [ + {created_at, + ?HOCON( + emqx_utils_calendar:epoch_millisecond(), + #{ + desc => "The time when the log is created" + } + )}, + {node, + ?HOCON(binary(), #{ + desc => "The node name to which the log is created" + })}, + {from, + ?HOCON(?ENUM([dashboard, rest_api, cli, erlang_console]), #{ + desc => "The source type of the log" + })}, + {source, + ?HOCON(binary(), #{ + desc => "The source of the log" + })}, + {source_ip, + ?HOCON(binary(), #{ + desc => "The source ip of the log" + })}, + {operation_id, + ?HOCON(binary(), #{ + desc => "The operation id of the log" + })}, + {operation_type, + ?HOCON(binary(), #{ + desc => "The operation type of the log" + })}, + {operation_result, + ?HOCON(?ENUM([success, failure]), #{ + desc => "The operation result of the log" + })}, + {http_status_code, + ?HOCON(integer(), #{ + desc => "The http status code of the log" + })}, + {http_method, + ?HOCON(?ENUM([post, put, delete]), #{ + desc => "The http method of the log" + })}, + {duration_ms, + ?HOCON(integer(), #{ + desc => "The duration of the log" + })}, + {args, + ?HOCON(?ARRAY(binary()), #{ + desc => "The args of the log" + })}, + {failure, + ?HOCON(?ARRAY(binary()), #{ + desc => "The failure of the log" + })}, + {http_request, + ?HOCON(?REF(http_request), #{ + desc => "The http request of the log" + })} + ]; +fields(http_request) -> + [ + {bindings, ?HOCON(map(), #{})}, + {body, ?HOCON(map(), #{})}, + {headers, ?HOCON(map(), #{})}, + {method, ?HOCON(?ENUM([post, put, delete]), #{})} + ]. + +audit(get, #{query_string := QueryString}) -> + case emqx_config:get([log, audit, enable], false) of + false -> + {400, #{code => 'BAD_REQUEST', message => ?DISABLE_MSG}}; + true -> + case + emqx_mgmt_api:node_query( + node(), + ?AUDIT, + QueryString, + ?AUDIT_QS_SCHEMA, + fun ?MODULE:qs2ms/2, + fun ?MODULE:format/1 + ) + of + {error, page_limit_invalid} -> + {400, #{code => 'BAD_REQUEST', message => <<"page_limit_invalid">>}}; + {error, Node, Error} -> + Message = list_to_binary( + io_lib:format("bad rpc call ~p, Reason ~p", [Node, Error]) + ), + {500, #{code => <<"NODE_DOWN">>, message => Message}}; + Result -> + {200, Result} + end + end. + +qs2ms(_Tab, {Qs, _}) -> + #{ + match_spec => gen_match_spec(Qs, #?AUDIT{_ = '_'}, []), + fuzzy_fun => undefined + }. + +gen_match_spec([], Audit, Conn) -> + [{Audit, Conn, ['$_']}]; +gen_match_spec([{node, '=:=', T} | Qs], Audit, Conn) -> + gen_match_spec(Qs, Audit#?AUDIT{node = T}, Conn); +gen_match_spec([{from, '=:=', T} | Qs], Audit, Conn) -> + gen_match_spec(Qs, Audit#?AUDIT{from = T}, Conn); +gen_match_spec([{source, '=:=', T} | Qs], Audit, Conn) -> + gen_match_spec(Qs, Audit#?AUDIT{source = T}, Conn); +gen_match_spec([{source_ip, '=:=', T} | Qs], Audit, Conn) -> + gen_match_spec(Qs, Audit#?AUDIT{source_ip = T}, Conn); +gen_match_spec([{operation_id, '=:=', T} | Qs], Audit, Conn) -> + gen_match_spec(Qs, Audit#?AUDIT{operation_id = T}, Conn); +gen_match_spec([{operation_type, '=:=', T} | Qs], Audit, Conn) -> + gen_match_spec(Qs, Audit#?AUDIT{operation_type = T}, Conn); +gen_match_spec([{operation_result, '=:=', T} | Qs], Audit, Conn) -> + gen_match_spec(Qs, Audit#?AUDIT{operation_result = T}, Conn); +gen_match_spec([{http_status_code, '=:=', T} | Qs], Audit, Conn) -> + gen_match_spec(Qs, Audit#?AUDIT{http_status_code = T}, Conn); +gen_match_spec([{http_method, '=:=', T} | Qs], Audit, Conn) -> + gen_match_spec(Qs, Audit#?AUDIT{http_method = T}, Conn); +gen_match_spec([{created_at, Hold, T} | Qs], Audit, Conn) -> + gen_match_spec(Qs, Audit#?AUDIT{created_at = '$1'}, [{'$1', Hold, T} | Conn]); +gen_match_spec([{created_at, Hold1, T1, Hold2, T2} | Qs], Audit, Conn) -> + gen_match_spec(Qs, Audit#?AUDIT{created_at = '$1'}, [ + {'$1', Hold1, T1}, {'$1', Hold2, T2} | Conn + ]); +gen_match_spec([{duration_ms, Hold, T} | Qs], Audit, Conn) -> + gen_match_spec(Qs, Audit#?AUDIT{duration_ms = '$2'}, [{'$2', Hold, T} | Conn]); +gen_match_spec([{duration_ms, Hold1, T1, Hold2, T2} | Qs], Audit, Conn) -> + gen_match_spec(Qs, Audit#?AUDIT{duration_ms = '$2'}, [ + {'$2', Hold1, T1}, {'$2', Hold2, T2} | Conn + ]). + +format(Audit) -> + #?AUDIT{ + created_at = CreatedAt, + node = Node, + from = From, + source = Source, + source_ip = SourceIp, + operation_id = OperationId, + operation_type = OperationType, + operation_result = OperationResult, + http_status_code = HttpStatusCode, + http_method = HttpMethod, + duration_ms = DurationMs, + args = Args, + failure = Failure, + http_request = HttpRequest + } = Audit, + #{ + created_at => emqx_utils_calendar:epoch_to_rfc3339(CreatedAt, microsecond), + node => Node, + from => From, + source => Source, + source_ip => SourceIp, + operation_id => OperationId, + operation_type => OperationType, + operation_result => OperationResult, + http_status_code => HttpStatusCode, + http_method => HttpMethod, + duration_ms => DurationMs, + args => Args, + failure => Failure, + http_request => HttpRequest + }. + +audit_log_list_example() -> + #{ + data => [api_example(), cli_example()], + meta => #{ + <<"count">> => 2, + <<"hasnext">> => false, + <<"limit">> => 50, + <<"page">> => 1 + } + }. + +api_example() -> + #{ + <<"args">> => "", + <<"created_at">> => "2023-10-17T10:41:20.383993+08:00", + <<"duration_ms">> => 0, + <<"failure">> => "", + <<"from">> => "dashboard", + <<"http_method">> => "post", + <<"http_request">> => #{ + <<"bindings">> => #{}, + <<"body">> => #{ + <<"password">> => "******", + <<"username">> => "admin" + }, + <<"headers">> => #{ + <<"accept">> => "*/*", + <<"authorization">> => "******", + <<"connection">> => "keep-alive", + <<"content-length">> => "45", + <<"content-type">> => "application/json" + }, + <<"method">> => "post" + }, + <<"http_status_code">> => 200, + <<"node">> => "emqx@127.0.0.1", + <<"operation_id">> => "/login", + <<"operation_result">> => "success", + <<"operation_type">> => "login", + <<"source">> => "admin", + <<"source_ip">> => "127.0.0.1" + }. + +cli_example() -> + #{ + <<"args">> => [<<"show">>, <<"log">>], + <<"created_at">> => "2023-10-17T10:45:13.100426+08:00", + <<"duration_ms">> => 7, + <<"failure">> => "", + <<"from">> => "cli", + <<"http_method">> => "", + <<"http_request">> => "", + <<"http_status_code">> => "", + <<"node">> => "emqx@127.0.0.1", + <<"operation_id">> => "", + <<"operation_result">> => "", + <<"operation_type">> => "conf", + <<"source">> => "", + <<"source_ip">> => "" + }. diff --git a/apps/emqx_audit/src/emqx_audit_app.erl b/apps/emqx_audit/src/emqx_audit_app.erl new file mode 100644 index 000000000..aa8fa1a39 --- /dev/null +++ b/apps/emqx_audit/src/emqx_audit_app.erl @@ -0,0 +1,15 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_audit_app). + +-behaviour(application). + +-export([start/2, stop/1]). + +start(_StartType, _StartArgs) -> + emqx_audit_sup:start_link(). + +stop(_State) -> + ok. diff --git a/apps/emqx_audit/src/emqx_audit_sup.erl b/apps/emqx_audit/src/emqx_audit_sup.erl new file mode 100644 index 000000000..b3a5ca985 --- /dev/null +++ b/apps/emqx_audit/src/emqx_audit_sup.erl @@ -0,0 +1,33 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_audit_sup). + +-behaviour(supervisor). + +-export([start_link/0]). + +-export([init/1]). + +-define(SERVER, ?MODULE). + +start_link() -> + supervisor:start_link({local, ?SERVER}, ?MODULE, []). + +init([]) -> + SupFlags = #{ + strategy => one_for_all, + intensity => 10, + period => 10 + }, + ChildSpecs = [ + #{ + id => emqx_audit, + start => {emqx_audit, start_link, []}, + type => worker, + restart => transient, + shutdown => 1000 + } + ], + {ok, {SupFlags, ChildSpecs}}. diff --git a/apps/emqx_audit/test/emqx_audit_api_SUITE.erl b/apps/emqx_audit/test/emqx_audit_api_SUITE.erl new file mode 100644 index 000000000..50b39d240 --- /dev/null +++ b/apps/emqx_audit/test/emqx_audit_api_SUITE.erl @@ -0,0 +1,248 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_audit_api_SUITE). +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("eunit/include/eunit.hrl"). + +all() -> + [ + {group, audit, [sequence]} + ]. + +groups() -> + [ + {audit, [sequence], common_tests()} + ]. + +common_tests() -> + emqx_common_test_helpers:all(?MODULE). + +-define(CONF_DEFAULT, #{ + node => + #{ + name => "emqx1@127.0.0.1", + cookie => "emqxsecretcookie", + data_dir => "data" + }, + log => #{ + audit => + #{ + enable => true, + ignore_high_frequency_request => true, + level => info, + max_filter_size => 15, + rotation_count => 2, + rotation_size => "10MB", + time_offset => "system" + } + } +}). + +init_per_suite(Config) -> + _ = application:load(emqx_conf), + emqx_config:erase_all(), + emqx_mgmt_api_test_util:init_suite([emqx_ctl, emqx_conf, emqx_audit]), + ok = emqx_common_test_helpers:load_config(emqx_enterprise_schema, ?CONF_DEFAULT), + emqx_config:save_schema_mod_and_names(emqx_enterprise_schema), + ok = emqx_config_logger:refresh_config(), + application:set_env(emqx, boot_modules, []), + emqx_conf_cli:load(), + Config. + +end_per_suite(_) -> + emqx_mgmt_api_test_util:end_suite([emqx_audit, emqx_conf, emqx_ctl]). + +t_http_api(_) -> + process_flag(trap_exit, true), + AuditPath = emqx_mgmt_api_test_util:api_path(["audit"]), + AuthHeader = emqx_mgmt_api_test_util:auth_header_(), + {ok, Zones} = emqx_mgmt_api_configs_SUITE:get_global_zone(), + NewZones = emqx_utils_maps:deep_put([<<"mqtt">>, <<"max_qos_allowed">>], Zones, 1), + {ok, #{<<"mqtt">> := Res}} = emqx_mgmt_api_configs_SUITE:update_global_zone(NewZones), + ?assertMatch(#{<<"max_qos_allowed">> := 1}, Res), + {ok, Res1} = emqx_mgmt_api_test_util:request_api(get, AuditPath, "limit=1", AuthHeader), + ?assertMatch( + #{ + <<"data">> := [ + #{ + <<"from">> := <<"rest_api">>, + <<"operation_id">> := <<"/configs/global_zone">>, + <<"source_ip">> := <<"127.0.0.1">>, + <<"source">> := _, + <<"http_request">> := #{ + <<"method">> := <<"put">>, + <<"body">> := #{<<"mqtt">> := #{<<"max_qos_allowed">> := 1}}, + <<"bindings">> := _, + <<"headers">> := #{<<"authorization">> := <<"******">>} + }, + <<"http_status_code">> := 200, + <<"operation_result">> := <<"success">>, + <<"operation_type">> := <<"configs">> + } + ] + }, + emqx_utils_json:decode(Res1, [return_maps]) + ), + ok. + +t_disabled(_) -> + Enable = [log, audit, enable], + ?assertEqual(true, emqx:get_config(Enable)), + AuditPath = emqx_mgmt_api_test_util:api_path(["audit"]), + AuthHeader = emqx_mgmt_api_test_util:auth_header_(), + {ok, _} = emqx_mgmt_api_test_util:request_api(get, AuditPath, "limit=1", AuthHeader), + Size1 = mnesia:table_info(emqx_audit, size), + + {ok, Logs} = emqx_mgmt_api_configs_SUITE:get_config("log"), + Logs1 = emqx_utils_maps:deep_put([<<"audit">>, <<"max_filter_size">>], Logs, 100), + NewLogs = emqx_utils_maps:deep_put([<<"audit">>, <<"enable">>], Logs1, false), + {ok, _} = emqx_mgmt_api_configs_SUITE:update_config("log", NewLogs), + {ok, GetLog1} = emqx_mgmt_api_configs_SUITE:get_config("log"), + ?assertEqual(NewLogs, GetLog1), + ?assertMatch( + {error, _}, + emqx_mgmt_api_test_util:request_api(get, AuditPath, "limit=1", AuthHeader) + ), + + Size2 = mnesia:table_info(emqx_audit, size), + %% Record the audit disable action, so the size + 1 + ?assertEqual(Size1 + 1, Size2), + + {ok, Zones} = emqx_mgmt_api_configs_SUITE:get_global_zone(), + NewZones = emqx_utils_maps:deep_put([<<"mqtt">>, <<"max_topic_levels">>], Zones, 111), + {ok, #{<<"mqtt">> := Res}} = emqx_mgmt_api_configs_SUITE:update_global_zone(NewZones), + ?assertMatch(#{<<"max_topic_levels">> := 111}, Res), + Size3 = mnesia:table_info(emqx_audit, size), + %% Don't record mqtt update request. + ?assertEqual(Size2, Size3), + %% enabled again + {ok, _} = emqx_mgmt_api_configs_SUITE:update_config("log", Logs1), + {ok, GetLog2} = emqx_mgmt_api_configs_SUITE:get_config("log"), + ?assertEqual(Logs1, GetLog2), + Size4 = mnesia:table_info(emqx_audit, size), + ?assertEqual(Size3 + 1, Size4), + ok. + +t_cli(_Config) -> + ok = emqx_ctl:run_command(["conf", "show", "log"]), + AuditPath = emqx_mgmt_api_test_util:api_path(["audit"]), + AuthHeader = emqx_mgmt_api_test_util:auth_header_(), + {ok, Res} = emqx_mgmt_api_test_util:request_api(get, AuditPath, "limit=1", AuthHeader), + #{<<"data">> := Data} = emqx_utils_json:decode(Res, [return_maps]), + ?assertMatch( + [ + #{ + <<"from">> := <<"cli">>, + <<"operation_id">> := <<"">>, + <<"source_ip">> := <<"">>, + <<"operation_type">> := <<"conf">>, + <<"args">> := [<<"show">>, <<"log">>], + <<"node">> := _, + <<"source">> := <<"">>, + <<"http_request">> := <<"">> + } + ], + Data + ), + + %% check filter + {ok, Res1} = emqx_mgmt_api_test_util:request_api(get, AuditPath, "from=cli", AuthHeader), + #{<<"data">> := Data1} = emqx_utils_json:decode(Res1, [return_maps]), + ?assertEqual(Data, Data1), + {ok, Res2} = emqx_mgmt_api_test_util:request_api( + get, AuditPath, "from=erlang_console", AuthHeader + ), + ?assertMatch(#{<<"data">> := []}, emqx_utils_json:decode(Res2, [return_maps])), + ok. + +t_max_size(_Config) -> + {ok, _} = emqx:update_config([log, audit, max_filter_size], 1000), + SizeFun = + fun() -> + AuditPath = emqx_mgmt_api_test_util:api_path(["audit"]), + AuthHeader = emqx_mgmt_api_test_util:auth_header_(), + Limit = "limit=1000", + {ok, Res} = emqx_mgmt_api_test_util:request_api(get, AuditPath, Limit, AuthHeader), + #{<<"data">> := Data} = emqx_utils_json:decode(Res, [return_maps]), + erlang:length(Data) + end, + InitSize = SizeFun(), + lists:foreach( + fun(_) -> + ok = emqx_ctl:run_command(["conf", "show", "log"]) + end, + lists:duplicate(100, 1) + ), + timer:sleep(110), + Size1 = SizeFun(), + ?assert(Size1 - InitSize >= 100, {Size1, InitSize}), + {ok, _} = emqx:update_config([log, audit, max_filter_size], 10), + %% wait for clean_expired + timer:sleep(250), + ExpectSize = emqx:get_config([log, audit, max_filter_size]), + Size2 = SizeFun(), + ?assertEqual(ExpectSize, Size2, {sys:get_state(emqx_audit)}), + ok. + +t_kickout_clients_without_log(_) -> + process_flag(trap_exit, true), + AuditPath = emqx_mgmt_api_test_util:api_path(["audit"]), + {ok, AuditLogs1} = emqx_mgmt_api_test_util:request_api(get, AuditPath), + kickout_clients(), + {ok, AuditLogs2} = emqx_mgmt_api_test_util:request_api(get, AuditPath), + ?assertEqual(AuditLogs1, AuditLogs2), + ok. + +kickout_clients() -> + ClientId1 = <<"client1">>, + ClientId2 = <<"client2">>, + ClientId3 = <<"client3">>, + + {ok, C1} = emqtt:start_link(#{ + clientid => ClientId1, + proto_ver => v5, + properties => #{'Session-Expiry-Interval' => 120} + }), + {ok, _} = emqtt:connect(C1), + {ok, C2} = emqtt:start_link(#{clientid => ClientId2}), + {ok, _} = emqtt:connect(C2), + {ok, C3} = emqtt:start_link(#{clientid => ClientId3}), + {ok, _} = emqtt:connect(C3), + + timer:sleep(300), + + %% get /clients + ClientsPath = emqx_mgmt_api_test_util:api_path(["clients"]), + {ok, Clients} = emqx_mgmt_api_test_util:request_api(get, ClientsPath), + ClientsResponse = emqx_utils_json:decode(Clients, [return_maps]), + ClientsMeta = maps:get(<<"meta">>, ClientsResponse), + ClientsPage = maps:get(<<"page">>, ClientsMeta), + ClientsLimit = maps:get(<<"limit">>, ClientsMeta), + ClientsCount = maps:get(<<"count">>, ClientsMeta), + ?assertEqual(ClientsPage, 1), + ?assertEqual(ClientsLimit, emqx_mgmt:default_row_limit()), + ?assertEqual(ClientsCount, 3), + + %% kickout clients + KickoutPath = emqx_mgmt_api_test_util:api_path(["clients", "kickout", "bulk"]), + KickoutBody = [ClientId1, ClientId2, ClientId3], + {ok, 204, _} = emqx_mgmt_api_test_util:request_api_with_body(post, KickoutPath, KickoutBody), + + {ok, Clients2} = emqx_mgmt_api_test_util:request_api(get, ClientsPath), + ClientsResponse2 = emqx_utils_json:decode(Clients2, [return_maps]), + ?assertMatch(#{<<"data">> := []}, ClientsResponse2). diff --git a/apps/emqx_auth_ldap/include/emqx_auth_ldap.hrl b/apps/emqx_auth_ldap/include/emqx_auth_ldap.hrl index 9cf6ac3c0..dcf0c07af 100644 --- a/apps/emqx_auth_ldap/include/emqx_auth_ldap.hrl +++ b/apps/emqx_auth_ldap/include/emqx_auth_ldap.hrl @@ -26,10 +26,6 @@ -define(AUTHN_BACKEND, ldap). -define(AUTHN_BACKEND_BIN, <<"ldap">>). --define(AUTHN_BACKEND_BIND, ldap_bind). --define(AUTHN_BACKEND_BIND_BIN, <<"ldap_bind">>). - -define(AUTHN_TYPE, {?AUTHN_MECHANISM, ?AUTHN_BACKEND}). --define(AUTHN_TYPE_BIND, {?AUTHN_MECHANISM, ?AUTHN_BACKEND_BIND}). -endif. diff --git a/apps/emqx_auth_ldap/src/emqx_auth_ldap_app.erl b/apps/emqx_auth_ldap/src/emqx_auth_ldap_app.erl index 7d05faab9..5e7bd2bc6 100644 --- a/apps/emqx_auth_ldap/src/emqx_auth_ldap_app.erl +++ b/apps/emqx_auth_ldap/src/emqx_auth_ldap_app.erl @@ -25,12 +25,10 @@ start(_StartType, _StartArgs) -> ok = emqx_authz:register_source(?AUTHZ_TYPE, emqx_authz_ldap), ok = emqx_authn:register_provider(?AUTHN_TYPE, emqx_authn_ldap), - ok = emqx_authn:register_provider(?AUTHN_TYPE_BIND, emqx_authn_ldap_bind), {ok, Sup} = emqx_auth_ldap_sup:start_link(), {ok, Sup}. stop(_State) -> ok = emqx_authn:deregister_provider(?AUTHN_TYPE), - ok = emqx_authn:deregister_provider(?AUTHN_TYPE_BIND), ok = emqx_authz:unregister_source(?AUTHZ_TYPE), ok. diff --git a/apps/emqx_auth_ldap/src/emqx_authn_ldap.erl b/apps/emqx_auth_ldap/src/emqx_authn_ldap.erl index 975a7f828..acdd08f50 100644 --- a/apps/emqx_auth_ldap/src/emqx_authn_ldap.erl +++ b/apps/emqx_auth_ldap/src/emqx_authn_ldap.erl @@ -16,19 +16,10 @@ -module(emqx_authn_ldap). --include_lib("emqx_auth/include/emqx_authn.hrl"). -include_lib("emqx/include/logger.hrl"). --include_lib("eldap/include/eldap.hrl"). -behaviour(emqx_authn_provider). -%% a compatible attribute for version 4.x --define(ISENABLED_ATTR, "isEnabled"). --define(VALID_ALGORITHMS, [md5, ssha, sha, sha256, sha384, sha512]). -%% TODO -%% 1. Supports more salt algorithms, SMD5 SSHA 256/384/512 -%% 2. Supports https://datatracker.ietf.org/doc/html/rfc3112 - -export([ create/2, update/2, @@ -69,163 +60,25 @@ authenticate(#{auth_method := _}, _) -> ignore; authenticate(#{password := undefined}, _) -> {error, bad_username_or_password}; -authenticate( - #{password := Password} = Credential, - #{ - password_attribute := PasswordAttr, - is_superuser_attribute := IsSuperuserAttr, - query_timeout := Timeout, - resource_id := ResourceId - } = State -) -> - case - emqx_resource:simple_sync_query( - ResourceId, - {query, Credential, [PasswordAttr, IsSuperuserAttr, ?ISENABLED_ATTR], Timeout} - ) - of - {ok, []} -> - ignore; - {ok, [Entry]} -> - is_enabled(Password, Entry, State); - {error, Reason} -> - ?TRACE_AUTHN_PROVIDER(error, "ldap_query_failed", #{ - resource => ResourceId, - timeout => Timeout, - reason => Reason - }), - ignore +authenticate(Credential, #{method := #{type := Type}} = State) -> + case Type of + hash -> + emqx_authn_ldap_hash:authenticate(Credential, State); + bind -> + emqx_authn_ldap_bind:authenticate(Credential, State) end. +%% it used the deprecated config form +parse_config( + #{password_attribute := PasswordAttr, is_superuser_attribute := IsSuperuserAttr} = Config0 +) -> + Config = maps:without([password_attribute, is_superuser_attribute], Config0), + parse_config(Config#{ + method => #{ + type => hash, + password_attribute => PasswordAttr, + is_superuser_attribute => IsSuperuserAttr + } + }); parse_config(Config) -> - maps:with([query_timeout, password_attribute, is_superuser_attribute], Config). - -%% To compatible v4.x -is_enabled(Password, #eldap_entry{attributes = Attributes} = Entry, State) -> - IsEnabled = get_lower_bin_value(?ISENABLED_ATTR, Attributes, "true"), - case emqx_authn_utils:to_bool(IsEnabled) of - true -> - ensure_password(Password, Entry, State); - _ -> - {error, user_disabled} - end. - -ensure_password( - Password, - #eldap_entry{attributes = Attributes} = Entry, - #{password_attribute := PasswordAttr} = State -) -> - case get_value(PasswordAttr, Attributes) of - undefined -> - {error, no_password}; - [LDAPPassword | _] -> - extract_hash_algorithm(LDAPPassword, Password, fun try_decode_password/4, Entry, State) - end. - -%% RFC 2307 format password -%% https://datatracker.ietf.org/doc/html/rfc2307 -extract_hash_algorithm(LDAPPassword, Password, OnFail, Entry, State) -> - case - re:run( - LDAPPassword, - "{([^{}]+)}(.+)", - [{capture, all_but_first, list}, global] - ) - of - {match, [[HashTypeStr, PasswordHashStr]]} -> - case emqx_utils:safe_to_existing_atom(string:to_lower(HashTypeStr)) of - {ok, HashType} -> - PasswordHash = to_binary(PasswordHashStr), - is_valid_algorithm(HashType, PasswordHash, Password, Entry, State); - _Error -> - {error, invalid_hash_type} - end; - _ -> - OnFail(LDAPPassword, Password, Entry, State) - end. - -is_valid_algorithm(HashType, PasswordHash, Password, Entry, State) -> - case lists:member(HashType, ?VALID_ALGORITHMS) of - true -> - verify_password(HashType, PasswordHash, Password, Entry, State); - _ -> - {error, {invalid_hash_type, HashType}} - end. - -%% this password is in LDIF format which is base64 encoding -try_decode_password(LDAPPassword, Password, Entry, State) -> - case safe_base64_decode(LDAPPassword) of - {ok, Decode} -> - extract_hash_algorithm( - Decode, - Password, - fun(_, _, _, _) -> - {error, invalid_password} - end, - Entry, - State - ); - {error, Reason} -> - {error, {invalid_password, Reason}} - end. - -%% sha with salt -%% https://www.openldap.org/faq/data/cache/347.html -verify_password(ssha, PasswordData, Password, Entry, State) -> - case safe_base64_decode(PasswordData) of - {ok, <>} -> - verify_password(sha, hash, PasswordHash, Salt, suffix, Password, Entry, State); - {ok, _} -> - {error, invalid_ssha_password}; - {error, Reason} -> - {error, {invalid_password, Reason}} - end; -verify_password( - Algorithm, - Base64HashData, - Password, - Entry, - State -) -> - verify_password(Algorithm, base64, Base64HashData, <<>>, disable, Password, Entry, State). - -verify_password(Algorithm, LDAPPasswordType, LDAPPassword, Salt, Position, Password, Entry, State) -> - PasswordHash = hash_password(Algorithm, Salt, Position, Password), - case compare_password(LDAPPasswordType, LDAPPassword, PasswordHash) of - true -> - {ok, is_superuser(Entry, State)}; - _ -> - {error, bad_username_or_password} - end. - -is_superuser(Entry, #{is_superuser_attribute := Attr} = _State) -> - Value = get_lower_bin_value(Attr, Entry#eldap_entry.attributes, "false"), - #{is_superuser => emqx_authn_utils:to_bool(Value)}. - -safe_base64_decode(Data) -> - try - {ok, base64:decode(Data)} - catch - _:Reason -> - {error, {invalid_base64_data, Reason}} - end. - -get_lower_bin_value(Key, Proplists, Default) -> - [Value | _] = get_value(Key, Proplists, [Default]), - to_binary(string:to_lower(Value)). - -to_binary(Value) -> - erlang:list_to_binary(Value). - -hash_password(Algorithm, _Salt, disable, Password) -> - hash_password(Algorithm, Password); -hash_password(Algorithm, Salt, suffix, Password) -> - hash_password(Algorithm, <>). - -hash_password(Algorithm, Data) -> - crypto:hash(Algorithm, Data). - -compare_password(hash, LDAPPasswordHash, PasswordHash) -> - emqx_passwd:compare_secure(LDAPPasswordHash, PasswordHash); -compare_password(base64, Base64HashData, PasswordHash) -> - emqx_passwd:compare_secure(Base64HashData, base64:encode(PasswordHash)). + maps:with([query_timeout, method], Config). diff --git a/apps/emqx_auth_ldap/src/emqx_authn_ldap_bind.erl b/apps/emqx_auth_ldap/src/emqx_authn_ldap_bind.erl index 000d545b9..1f2af261e 100644 --- a/apps/emqx_auth_ldap/src/emqx_authn_ldap_bind.erl +++ b/apps/emqx_auth_ldap/src/emqx_authn_ldap_bind.erl @@ -20,32 +20,13 @@ -include_lib("emqx/include/logger.hrl"). -include_lib("eldap/include/eldap.hrl"). --behaviour(emqx_authn_provider). - -export([ - create/2, - update/2, - authenticate/2, - destroy/1 + authenticate/2 ]). %%------------------------------------------------------------------------------ %% APIs %%------------------------------------------------------------------------------ - -create(_AuthenticatorID, Config) -> - emqx_authn_ldap:do_create(?MODULE, Config). - -update(Config, State) -> - emqx_authn_ldap:update(Config, State). - -destroy(State) -> - emqx_authn_ldap:destroy(State). - -authenticate(#{auth_method := _}, _) -> - ignore; -authenticate(#{password := undefined}, _) -> - {error, bad_username_or_password}; authenticate( #{password := _Password} = Credential, #{ diff --git a/apps/emqx_auth_ldap/src/emqx_authn_ldap_bind_schema.erl b/apps/emqx_auth_ldap/src/emqx_authn_ldap_bind_schema.erl deleted file mode 100644 index e5e83daa1..000000000 --- a/apps/emqx_auth_ldap/src/emqx_authn_ldap_bind_schema.erl +++ /dev/null @@ -1,66 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_authn_ldap_bind_schema). - --behaviour(emqx_authn_schema). - --export([ - fields/1, - desc/1, - refs/0, - select_union_member/1, - namespace/0 -]). - --include("emqx_auth_ldap.hrl"). --include_lib("hocon/include/hoconsc.hrl"). - -namespace() -> "authn". - -refs() -> - [?R_REF(ldap_bind)]. - -select_union_member(#{ - <<"mechanism">> := ?AUTHN_MECHANISM_BIN, <<"backend">> := ?AUTHN_BACKEND_BIND_BIN -}) -> - refs(); -select_union_member(#{<<"backend">> := ?AUTHN_BACKEND_BIND_BIN}) -> - throw(#{ - reason => "unknown_mechanism", - expected => ?AUTHN_MECHANISM - }); -select_union_member(_) -> - undefined. - -fields(ldap_bind) -> - [ - {mechanism, emqx_authn_schema:mechanism(?AUTHN_MECHANISM)}, - {backend, emqx_authn_schema:backend(?AUTHN_BACKEND_BIND)}, - {query_timeout, fun query_timeout/1} - ] ++ - emqx_authn_schema:common_fields() ++ - emqx_ldap:fields(config) ++ emqx_ldap:fields(bind_opts). - -desc(ldap_bind) -> - ?DESC(ldap_bind); -desc(_) -> - undefined. - -query_timeout(type) -> emqx_schema:timeout_duration_ms(); -query_timeout(desc) -> ?DESC(?FUNCTION_NAME); -query_timeout(default) -> <<"5s">>; -query_timeout(_) -> undefined. diff --git a/apps/emqx_auth_ldap/src/emqx_authn_ldap_hash.erl b/apps/emqx_auth_ldap/src/emqx_authn_ldap_hash.erl new file mode 100644 index 000000000..e051e57e9 --- /dev/null +++ b/apps/emqx_auth_ldap/src/emqx_authn_ldap_hash.erl @@ -0,0 +1,197 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_authn_ldap_hash). + +-include_lib("emqx_auth/include/emqx_authn.hrl"). +-include_lib("emqx/include/logger.hrl"). +-include_lib("eldap/include/eldap.hrl"). + +%% a compatible attribute for version 4.x +-define(ISENABLED_ATTR, "isEnabled"). +-define(VALID_ALGORITHMS, [md5, ssha, sha, sha256, sha384, sha512]). +%% TODO +%% 1. Supports more salt algorithms, SMD5 SSHA 256/384/512 +%% 2. Supports https://datatracker.ietf.org/doc/html/rfc3112 + +-export([ + authenticate/2 +]). + +-import(proplists, [get_value/2, get_value/3]). + +%%------------------------------------------------------------------------------ +%% APIs +%%------------------------------------------------------------------------------ +authenticate( + #{password := Password} = Credential, + #{ + method := #{ + password_attribute := PasswordAttr, + is_superuser_attribute := IsSuperuserAttr + }, + query_timeout := Timeout, + resource_id := ResourceId + } = State +) -> + case + emqx_resource:simple_sync_query( + ResourceId, + {query, Credential, [PasswordAttr, IsSuperuserAttr, ?ISENABLED_ATTR], Timeout} + ) + of + {ok, []} -> + ignore; + {ok, [Entry]} -> + is_enabled(Password, Entry, State); + {error, Reason} -> + ?TRACE_AUTHN_PROVIDER(error, "ldap_query_failed", #{ + resource => ResourceId, + timeout => Timeout, + reason => Reason + }), + ignore + end. + +%% To compatible v4.x +is_enabled(Password, #eldap_entry{attributes = Attributes} = Entry, State) -> + IsEnabled = get_lower_bin_value(?ISENABLED_ATTR, Attributes, "true"), + case emqx_authn_utils:to_bool(IsEnabled) of + true -> + ensure_password(Password, Entry, State); + _ -> + {error, user_disabled} + end. + +ensure_password( + Password, + #eldap_entry{attributes = Attributes} = Entry, + #{method := #{password_attribute := PasswordAttr}} = State +) -> + case get_value(PasswordAttr, Attributes) of + undefined -> + {error, no_password}; + [LDAPPassword | _] -> + extract_hash_algorithm(LDAPPassword, Password, fun try_decode_password/4, Entry, State) + end. + +%% RFC 2307 format password +%% https://datatracker.ietf.org/doc/html/rfc2307 +extract_hash_algorithm(LDAPPassword, Password, OnFail, Entry, State) -> + case + re:run( + LDAPPassword, + "{([^{}]+)}(.+)", + [{capture, all_but_first, list}, global] + ) + of + {match, [[HashTypeStr, PasswordHashStr]]} -> + case emqx_utils:safe_to_existing_atom(string:to_lower(HashTypeStr)) of + {ok, HashType} -> + PasswordHash = to_binary(PasswordHashStr), + is_valid_algorithm(HashType, PasswordHash, Password, Entry, State); + _Error -> + {error, invalid_hash_type} + end; + _ -> + OnFail(LDAPPassword, Password, Entry, State) + end. + +is_valid_algorithm(HashType, PasswordHash, Password, Entry, State) -> + case lists:member(HashType, ?VALID_ALGORITHMS) of + true -> + verify_password(HashType, PasswordHash, Password, Entry, State); + _ -> + {error, {invalid_hash_type, HashType}} + end. + +%% this password is in LDIF format which is base64 encoding +try_decode_password(LDAPPassword, Password, Entry, State) -> + case safe_base64_decode(LDAPPassword) of + {ok, Decode} -> + extract_hash_algorithm( + Decode, + Password, + fun(_, _, _, _) -> + {error, invalid_password} + end, + Entry, + State + ); + {error, Reason} -> + {error, {invalid_password, Reason}} + end. + +%% sha with salt +%% https://www.openldap.org/faq/data/cache/347.html +verify_password(ssha, PasswordData, Password, Entry, State) -> + case safe_base64_decode(PasswordData) of + {ok, <>} -> + verify_password(sha, hash, PasswordHash, Salt, suffix, Password, Entry, State); + {ok, _} -> + {error, invalid_ssha_password}; + {error, Reason} -> + {error, {invalid_password, Reason}} + end; +verify_password( + Algorithm, + Base64HashData, + Password, + Entry, + State +) -> + verify_password(Algorithm, base64, Base64HashData, <<>>, disable, Password, Entry, State). + +verify_password(Algorithm, LDAPPasswordType, LDAPPassword, Salt, Position, Password, Entry, State) -> + PasswordHash = hash_password(Algorithm, Salt, Position, Password), + case compare_password(LDAPPasswordType, LDAPPassword, PasswordHash) of + true -> + {ok, is_superuser(Entry, State)}; + _ -> + {error, bad_username_or_password} + end. + +is_superuser(Entry, #{method := #{is_superuser_attribute := Attr}} = _State) -> + Value = get_lower_bin_value(Attr, Entry#eldap_entry.attributes, "false"), + #{is_superuser => emqx_authn_utils:to_bool(Value)}. + +safe_base64_decode(Data) -> + try + {ok, base64:decode(Data)} + catch + _:Reason -> + {error, {invalid_base64_data, Reason}} + end. + +get_lower_bin_value(Key, Proplists, Default) -> + [Value | _] = get_value(Key, Proplists, [Default]), + to_binary(string:to_lower(Value)). + +to_binary(Value) -> + erlang:list_to_binary(Value). + +hash_password(Algorithm, _Salt, disable, Password) -> + hash_password(Algorithm, Password); +hash_password(Algorithm, Salt, suffix, Password) -> + hash_password(Algorithm, <>). + +hash_password(Algorithm, Data) -> + crypto:hash(Algorithm, Data). + +compare_password(hash, LDAPPasswordHash, PasswordHash) -> + emqx_passwd:compare_secure(LDAPPasswordHash, PasswordHash); +compare_password(base64, Base64HashData, PasswordHash) -> + emqx_passwd:compare_secure(Base64HashData, base64:encode(PasswordHash)). diff --git a/apps/emqx_auth_ldap/src/emqx_authn_ldap_schema.erl b/apps/emqx_auth_ldap/src/emqx_authn_ldap_schema.erl index fe9917fa1..99acf25bb 100644 --- a/apps/emqx_auth_ldap/src/emqx_authn_ldap_schema.erl +++ b/apps/emqx_auth_ldap/src/emqx_authn_ldap_schema.erl @@ -32,7 +32,7 @@ namespace() -> "authn". refs() -> - [?R_REF(ldap)]. + [?R_REF(ldap), ?R_REF(ldap_deprecated)]. select_union_member(#{<<"mechanism">> := ?AUTHN_MECHANISM_BIN, <<"backend">> := ?AUTHN_BACKEND_BIN}) -> refs(); @@ -44,12 +44,34 @@ select_union_member(#{<<"backend">> := ?AUTHN_BACKEND_BIN}) -> select_union_member(_) -> undefined. +fields(ldap_deprecated) -> + common_fields() ++ + [ + {password_attribute, password_attribute()}, + {is_superuser_attribute, is_superuser_attribute()} + ]; fields(ldap) -> + common_fields() ++ + [ + {method, + ?HOCON( + ?UNION([?R_REF(hash_method), ?R_REF(bind_method)]), + #{desc => ?DESC(method)} + )} + ]; +fields(hash_method) -> + [ + {type, method_type(hash)}, + {password_attribute, password_attribute()}, + {is_superuser_attribute, is_superuser_attribute()} + ]; +fields(bind_method) -> + [{type, method_type(bind)}] ++ emqx_ldap:fields(bind_opts). + +common_fields() -> [ {mechanism, emqx_authn_schema:mechanism(?AUTHN_MECHANISM)}, {backend, emqx_authn_schema:backend(?AUTHN_BACKEND)}, - {password_attribute, fun password_attribute/1}, - {is_superuser_attribute, fun is_superuser_attribute/1}, {query_timeout, fun query_timeout/1} ] ++ emqx_authn_schema:common_fields() ++ @@ -57,18 +79,35 @@ fields(ldap) -> desc(ldap) -> ?DESC(ldap); +desc(ldap_deprecated) -> + ?DESC(ldap_deprecated); +desc(hash_method) -> + ?DESC(hash_method); +desc(bind_method) -> + ?DESC(bind_method); desc(_) -> undefined. -password_attribute(type) -> string(); -password_attribute(desc) -> ?DESC(?FUNCTION_NAME); -password_attribute(default) -> <<"userPassword">>; -password_attribute(_) -> undefined. +method_type(Type) -> + ?HOCON(?ENUM([Type]), #{desc => ?DESC(?FUNCTION_NAME), default => Type}). -is_superuser_attribute(type) -> string(); -is_superuser_attribute(desc) -> ?DESC(?FUNCTION_NAME); -is_superuser_attribute(default) -> <<"isSuperuser">>; -is_superuser_attribute(_) -> undefined. +password_attribute() -> + ?HOCON( + string(), + #{ + desc => ?DESC(?FUNCTION_NAME), + default => <<"userPassword">> + } + ). + +is_superuser_attribute() -> + ?HOCON( + string(), + #{ + desc => ?DESC(?FUNCTION_NAME), + default => <<"isSuperuser">> + } + ). query_timeout(type) -> emqx_schema:timeout_duration_ms(); query_timeout(desc) -> ?DESC(?FUNCTION_NAME); diff --git a/apps/emqx_auth_ldap/test/emqx_authn_ldap_SUITE.erl b/apps/emqx_auth_ldap/test/emqx_authn_ldap_SUITE.erl index 63bceee85..2aa1c5c96 100644 --- a/apps/emqx_auth_ldap/test/emqx_authn_ldap_SUITE.erl +++ b/apps/emqx_auth_ldap/test/emqx_authn_ldap_SUITE.erl @@ -70,6 +70,29 @@ end_per_suite(Config) -> %% Tests %%------------------------------------------------------------------------------ +t_create_with_deprecated_cfg(_Config) -> + AuthConfig = deprecated_raw_ldap_auth_config(), + + {ok, _} = emqx:update_config( + ?PATH, + {create_authenticator, ?GLOBAL, AuthConfig} + ), + + {ok, [#{provider := emqx_authn_ldap, state := State}]} = emqx_authn_chains:list_authenticators( + ?GLOBAL + ), + ?assertMatch( + #{ + method := #{ + type := hash, + is_superuser_attribute := _, + password_attribute := "not_the_default_value" + } + }, + State + ), + emqx_authn_test_lib:delete_config(?ResourceID). + t_create(_Config) -> AuthConfig = raw_ldap_auth_config(), @@ -225,6 +248,19 @@ raw_ldap_auth_config() -> <<"pool_size">> => 8 }. +deprecated_raw_ldap_auth_config() -> + #{ + <<"mechanism">> => <<"password_based">>, + <<"backend">> => <<"ldap">>, + <<"server">> => ldap_server(), + <<"is_superuser_attribute">> => <<"isSuperuser">>, + <<"password_attribute">> => <<"not_the_default_value">>, + <<"base_dn">> => <<"uid=${username},ou=testdevice,dc=emqx,dc=io">>, + <<"username">> => <<"cn=root,dc=emqx,dc=io">>, + <<"password">> => <<"public">>, + <<"pool_size">> => 8 + }. + user_seeds() -> New = fun(Username, Password, Result) -> #{ diff --git a/apps/emqx_auth_ldap/test/emqx_authn_ldap_bind_SUITE.erl b/apps/emqx_auth_ldap/test/emqx_authn_ldap_bind_SUITE.erl index 1f390264b..d2b3c371c 100644 --- a/apps/emqx_auth_ldap/test/emqx_authn_ldap_bind_SUITE.erl +++ b/apps/emqx_auth_ldap/test/emqx_authn_ldap_bind_SUITE.erl @@ -27,7 +27,7 @@ -define(LDAP_RESOURCE, <<"emqx_authn_ldap_bind_SUITE">>). -define(PATH, [authentication]). --define(ResourceID, <<"password_based:ldap_bind">>). +-define(ResourceID, <<"password_based:ldap">>). all() -> emqx_common_test_helpers:all(?MODULE). @@ -78,7 +78,7 @@ t_create(_Config) -> {create_authenticator, ?GLOBAL, AuthConfig} ), - {ok, [#{provider := emqx_authn_ldap_bind}]} = emqx_authn_chains:list_authenticators(?GLOBAL), + {ok, [#{provider := emqx_authn_ldap}]} = emqx_authn_chains:list_authenticators(?GLOBAL), emqx_authn_test_lib:delete_config(?ResourceID). t_create_invalid(_Config) -> @@ -146,10 +146,10 @@ t_destroy(_Config) -> {create_authenticator, ?GLOBAL, AuthConfig} ), - {ok, [#{provider := emqx_authn_ldap_bind, state := State}]} = + {ok, [#{provider := emqx_authn_ldap, state := State}]} = emqx_authn_chains:list_authenticators(?GLOBAL), - {ok, _} = emqx_authn_ldap_bind:authenticate( + {ok, _} = emqx_authn_ldap:authenticate( #{ username => <<"mqttuser0001">>, password => <<"mqttuser0001">> @@ -165,7 +165,7 @@ t_destroy(_Config) -> % Authenticator should not be usable anymore ?assertMatch( ignore, - emqx_authn_ldap_bind:authenticate( + emqx_authn_ldap:authenticate( #{ username => <<"mqttuser0001">>, password => <<"mqttuser0001">> @@ -199,7 +199,7 @@ t_update(_Config) -> % We update with config with correct query, provider should update and work properly {ok, _} = emqx:update_config( ?PATH, - {update_authenticator, ?GLOBAL, <<"password_based:ldap_bind">>, CorrectConfig} + {update_authenticator, ?GLOBAL, <<"password_based:ldap">>, CorrectConfig} ), {ok, _} = emqx_access_control:authenticate( @@ -218,14 +218,17 @@ t_update(_Config) -> raw_ldap_auth_config() -> #{ <<"mechanism">> => <<"password_based">>, - <<"backend">> => <<"ldap_bind">>, + <<"backend">> => <<"ldap">>, <<"server">> => ldap_server(), <<"base_dn">> => <<"ou=testdevice,dc=emqx,dc=io">>, <<"filter">> => <<"(uid=${username})">>, <<"username">> => <<"cn=root,dc=emqx,dc=io">>, <<"password">> => <<"public">>, <<"pool_size">> => 8, - <<"bind_password">> => <<"${password}">> + <<"method">> => #{ + <<"type">> => <<"bind">>, + <<"bind_password">> => <<"${password}">> + } }. user_seeds() -> diff --git a/apps/emqx_auth_mongodb/test/emqx_authn_mongodb_SUITE.erl b/apps/emqx_auth_mongodb/test/emqx_authn_mongodb_SUITE.erl index c6623c11f..9ccad551d 100644 --- a/apps/emqx_auth_mongodb/test/emqx_authn_mongodb_SUITE.erl +++ b/apps/emqx_auth_mongodb/test/emqx_authn_mongodb_SUITE.erl @@ -278,6 +278,10 @@ raw_mongo_auth_config() -> <<"server">> => mongo_server(), <<"w_mode">> => <<"unsafe">>, + <<"auth_source">> => mongo_authsource(), + <<"username">> => mongo_username(), + <<"password">> => mongo_password(), + <<"filter">> => #{<<"username">> => <<"${username}">>}, <<"password_hash_field">> => <<"password_hash">>, <<"salt_field">> => <<"salt">>, @@ -464,9 +468,21 @@ mongo_config() -> {database, <<"mqtt">>}, {host, ?MONGO_HOST}, {port, ?MONGO_DEFAULT_PORT}, + {auth_source, mongo_authsource()}, + {login, mongo_username()}, + {password, mongo_password()}, {register, ?MONGO_CLIENT} ]. +mongo_authsource() -> + iolist_to_binary(os:getenv("MONGO_AUTHSOURCE", "admin")). + +mongo_username() -> + iolist_to_binary(os:getenv("MONGO_USERNAME", "")). + +mongo_password() -> + iolist_to_binary(os:getenv("MONGO_PASSWORD", "")). + start_apps(Apps) -> lists:foreach(fun application:ensure_all_started/1, Apps). diff --git a/apps/emqx_auth_mongodb/test/emqx_authz_mongodb_SUITE.erl b/apps/emqx_auth_mongodb/test/emqx_authz_mongodb_SUITE.erl index c57dce860..b19d7fba2 100644 --- a/apps/emqx_auth_mongodb/test/emqx_authz_mongodb_SUITE.erl +++ b/apps/emqx_auth_mongodb/test/emqx_authz_mongodb_SUITE.erl @@ -397,6 +397,10 @@ raw_mongo_authz_config() -> <<"collection">> => <<"acl">>, <<"server">> => mongo_server(), + <<"auth_source">> => mongo_authsource(), + <<"username">> => mongo_username(), + <<"password">> => mongo_password(), + <<"filter">> => #{<<"username">> => <<"${username}">>} }. @@ -408,9 +412,21 @@ mongo_config() -> {database, <<"mqtt">>}, {host, ?MONGO_HOST}, {port, ?MONGO_DEFAULT_PORT}, + {auth_source, mongo_authsource()}, + {login, mongo_username()}, + {password, mongo_password()}, {register, ?MONGO_CLIENT} ]. +mongo_authsource() -> + iolist_to_binary(os:getenv("MONGO_AUTHSOURCE", "admin")). + +mongo_username() -> + iolist_to_binary(os:getenv("MONGO_USERNAME", "")). + +mongo_password() -> + iolist_to_binary(os:getenv("MONGO_PASSWORD", "")). + start_apps(Apps) -> lists:foreach(fun application:ensure_all_started/1, Apps). diff --git a/apps/emqx_bridge/src/emqx_bridge.erl b/apps/emqx_bridge/src/emqx_bridge.erl index 51bdfb084..a3d54586a 100644 --- a/apps/emqx_bridge/src/emqx_bridge.erl +++ b/apps/emqx_bridge/src/emqx_bridge.erl @@ -93,7 +93,8 @@ T == iotdb; T == kinesis_producer; T == greptimedb; - T == azure_event_hub_producer + T == azure_event_hub_producer; + T == syskeeper_forwarder ). -define(ROOT_KEY, bridges). diff --git a/apps/emqx_bridge/src/emqx_bridge_resource.erl b/apps/emqx_bridge/src/emqx_bridge_resource.erl index 674eceb81..231548f30 100644 --- a/apps/emqx_bridge/src/emqx_bridge_resource.erl +++ b/apps/emqx_bridge/src/emqx_bridge_resource.erl @@ -356,9 +356,10 @@ parse_confs(<<"iotdb">>, Name, Conf) -> authentication := #{ username := Username, - password := Password + password := Secret } } = Conf, + Password = emqx_secret:unwrap(Secret), BasicToken = base64:encode(<>), %% This version atom correspond to the macro ?VSN_1_1_X in %% emqx_bridge_iotdb.hrl. It would be better to use the macro directly, but diff --git a/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.app.src b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.app.src index 59661d7c0..97be100d2 100644 --- a/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.app.src +++ b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_cassandra, [ {description, "EMQX Enterprise Cassandra Bridge"}, - {vsn, "0.1.5"}, + {vsn, "0.1.6"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_connector.erl b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_connector.erl index afea652ef..e29dc7931 100644 --- a/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_connector.erl +++ b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_connector.erl @@ -70,7 +70,7 @@ cassandra_db_fields() -> {keyspace, fun keyspace/1}, {pool_size, fun emqx_connector_schema_lib:pool_size/1}, {username, fun emqx_connector_schema_lib:username/1}, - {password, fun emqx_connector_schema_lib:password/1}, + {password, emqx_connector_schema_lib:password_field()}, {auto_reconnect, fun emqx_connector_schema_lib:auto_reconnect/1} ]. @@ -111,14 +111,14 @@ on_start( emqx_schema:parse_servers(Servers0, ?DEFAULT_SERVER_OPTION) ), - Options = [ - {nodes, Servers}, - {keyspace, Keyspace}, - {auto_reconnect, ?AUTO_RECONNECT_INTERVAL}, - {pool_size, PoolSize} - ], - Options1 = maybe_add_opt(username, Config, Options), - Options2 = maybe_add_opt(password, Config, Options1, _IsSensitive = true), + Options = + maps:to_list(maps:with([username, password], Config)) ++ + [ + {nodes, Servers}, + {keyspace, Keyspace}, + {auto_reconnect, ?AUTO_RECONNECT_INTERVAL}, + {pool_size, PoolSize} + ], SslOpts = case maps:get(enable, SSL) of @@ -131,7 +131,7 @@ on_start( [] end, State = parse_prepare_cql(Config), - case emqx_resource_pool:start(InstId, ?MODULE, Options2 ++ SslOpts) of + case emqx_resource_pool:start(InstId, ?MODULE, Options ++ SslOpts) of ok -> {ok, init_prepare(State#{pool_name => InstId, prepare_statement => #{}})}; {error, Reason} -> @@ -387,6 +387,7 @@ conn_opts(Opts) -> conn_opts([], Acc) -> Acc; conn_opts([{password, Password} | Opts], Acc) -> + %% TODO: teach `ecql` to accept 0-arity closures as passwords. conn_opts(Opts, [{password, emqx_secret:unwrap(Password)} | Acc]); conn_opts([Opt | Opts], Acc) -> conn_opts(Opts, [Opt | Acc]). @@ -512,19 +513,3 @@ maybe_assign_type(V) when is_integer(V) -> maybe_assign_type(V) when is_float(V) -> {double, V}; maybe_assign_type(V) -> V. - -maybe_add_opt(Key, Conf, Opts) -> - maybe_add_opt(Key, Conf, Opts, _IsSensitive = false). - -maybe_add_opt(Key, Conf, Opts, IsSensitive) -> - case Conf of - #{Key := Val} -> - [{Key, maybe_wrap(IsSensitive, Val)} | Opts]; - _ -> - Opts - end. - -maybe_wrap(false = _IsSensitive, Val) -> - Val; -maybe_wrap(true, Val) -> - emqx_secret:wrap(Val). diff --git a/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_connector_SUITE.erl b/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_connector_SUITE.erl index fcd482b47..de306e3f0 100644 --- a/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_connector_SUITE.erl +++ b/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_connector_SUITE.erl @@ -40,10 +40,9 @@ all() -> ]. groups() -> - TCs = emqx_common_test_helpers:all(?MODULE), [ - {auth, TCs}, - {noauth, TCs} + {auth, [t_lifecycle, t_start_passfile]}, + {noauth, [t_lifecycle]} ]. cassandra_servers(CassandraHost) -> @@ -115,32 +114,37 @@ end_per_testcase(_, _Config) -> t_lifecycle(Config) -> perform_lifecycle_check( - <<"emqx_connector_cassandra_SUITE">>, + <>, cassandra_config(Config) ). -show(X) -> - erlang:display(X), - X. - -show(Label, What) -> - erlang:display({Label, What}), - What. +t_start_passfile(Config) -> + ResourceID = atom_to_binary(?FUNCTION_NAME), + PasswordFilename = filename:join(?config(priv_dir, Config), "passfile"), + ok = file:write_file(PasswordFilename, ?CASSA_PASSWORD), + InitialConfig = emqx_utils_maps:deep_merge( + cassandra_config(Config), + #{ + <<"config">> => #{ + password => iolist_to_binary(["file://", PasswordFilename]) + } + } + ), + ?assertMatch( + #{status := connected}, + create_local_resource(ResourceID, check_config(InitialConfig)) + ), + ?assertEqual( + ok, + emqx_resource:remove_local(ResourceID) + ). perform_lifecycle_check(ResourceId, InitialConfig) -> - {ok, #{config := CheckedConfig}} = - emqx_resource:check_config(?CASSANDRA_RESOURCE_MOD, InitialConfig), - {ok, #{ + CheckedConfig = check_config(InitialConfig), + #{ state := #{pool_name := PoolName} = State, status := InitialStatus - }} = - emqx_resource:create_local( - ResourceId, - ?CONNECTOR_RESOURCE_GROUP, - ?CASSANDRA_RESOURCE_MOD, - CheckedConfig, - #{} - ), + } = create_local_resource(ResourceId, CheckedConfig), ?assertEqual(InitialStatus, connected), % Instance should match the state and status of the just started resource {ok, ?CONNECTOR_RESOURCE_GROUP, #{ @@ -191,6 +195,21 @@ perform_lifecycle_check(ResourceId, InitialConfig) -> %% utils %%-------------------------------------------------------------------- +check_config(Config) -> + {ok, #{config := CheckedConfig}} = emqx_resource:check_config(?CASSANDRA_RESOURCE_MOD, Config), + CheckedConfig. + +create_local_resource(ResourceId, CheckedConfig) -> + {ok, Bridge} = + emqx_resource:create_local( + ResourceId, + ?CONNECTOR_RESOURCE_GROUP, + ?CASSANDRA_RESOURCE_MOD, + CheckedConfig, + #{} + ), + Bridge. + cassandra_config(Config) -> Host = ?config(cassa_host, Config), AuthOpts = maps:from_list(?config(cassa_auth_opts, Config)), diff --git a/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse.app.src b/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse.app.src index 4f7519440..85c035be1 100644 --- a/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse.app.src +++ b/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_clickhouse, [ {description, "EMQX Enterprise ClickHouse Bridge"}, - {vsn, "0.2.3"}, + {vsn, "0.2.4"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse_connector.erl b/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse_connector.erl index 97b855ad2..8f575dd8d 100644 --- a/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse_connector.erl +++ b/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse_connector.erl @@ -145,7 +145,7 @@ on_start( Options = [ {url, URL}, {user, maps:get(username, Config, "default")}, - {key, emqx_secret:wrap(maps:get(password, Config, "public"))}, + {key, maps:get(password, Config, emqx_secret:wrap("public"))}, {database, DB}, {auto_reconnect, ?AUTO_RECONNECT_INTERVAL}, {pool_size, PoolSize}, @@ -243,6 +243,7 @@ connect(Options) -> URL = iolist_to_binary(emqx_http_lib:normalize(proplists:get_value(url, Options))), User = proplists:get_value(user, Options), Database = proplists:get_value(database, Options), + %% TODO: teach `clickhouse` to accept 0-arity closures as passwords. Key = emqx_secret:unwrap(proplists:get_value(key, Options)), Pool = proplists:get_value(pool, Options), PoolSize = proplists:get_value(pool_size, Options), diff --git a/apps/emqx_bridge_clickhouse/test/emqx_bridge_clickhouse_connector_SUITE.erl b/apps/emqx_bridge_clickhouse/test/emqx_bridge_clickhouse_connector_SUITE.erl index 12d678e85..e1d3149db 100644 --- a/apps/emqx_bridge_clickhouse/test/emqx_bridge_clickhouse_connector_SUITE.erl +++ b/apps/emqx_bridge_clickhouse/test/emqx_bridge_clickhouse_connector_SUITE.erl @@ -10,10 +10,12 @@ -include("emqx_connector.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("stdlib/include/assert.hrl"). +-include_lib("common_test/include/ct.hrl"). -define(APP, emqx_bridge_clickhouse). -define(CLICKHOUSE_HOST, "clickhouse"). -define(CLICKHOUSE_RESOURCE_MOD, emqx_bridge_clickhouse_connector). +-define(CLICKHOUSE_PASSWORD, "public"). %% This test SUITE requires a running clickhouse instance. If you don't want to %% bring up the whole CI infrastuctucture with the `scripts/ct/run.sh` script @@ -57,7 +59,7 @@ init_per_suite(Config) -> clickhouse:start_link([ {url, clickhouse_url()}, {user, <<"default">>}, - {key, "public"}, + {key, ?CLICKHOUSE_PASSWORD}, {pool, tmp_pool} ]), {ok, _, _} = clickhouse:query(Conn, <<"CREATE DATABASE IF NOT EXISTS mqtt">>, #{}), @@ -92,6 +94,31 @@ t_lifecycle(_Config) -> clickhouse_config() ). +t_start_passfile(Config) -> + ResourceID = atom_to_binary(?FUNCTION_NAME), + PasswordFilename = filename:join(?config(priv_dir, Config), "passfile"), + ok = file:write_file(PasswordFilename, <>), + InitialConfig = clickhouse_config(#{ + password => iolist_to_binary(["file://", PasswordFilename]) + }), + {ok, #{config := ResourceConfig}} = + emqx_resource:check_config(?CLICKHOUSE_RESOURCE_MOD, InitialConfig), + ?assertMatch( + {ok, #{status := connected}}, + emqx_resource:create_local( + ResourceID, + ?CONNECTOR_RESOURCE_GROUP, + ?CLICKHOUSE_RESOURCE_MOD, + ResourceConfig, + #{} + ) + ), + ?assertEqual( + ok, + emqx_resource:remove_local(ResourceID) + ), + ok. + show(X) -> erlang:display(X), X. @@ -168,12 +195,15 @@ perform_lifecycle_check(ResourceID, InitialConfig) -> % %%------------------------------------------------------------------------------ clickhouse_config() -> + clickhouse_config(#{}). + +clickhouse_config(Overrides) -> Config = #{ auto_reconnect => true, database => <<"mqtt">>, username => <<"default">>, - password => <<"public">>, + password => <>, pool_size => 8, url => iolist_to_binary( io_lib:format( @@ -186,7 +216,7 @@ clickhouse_config() -> ), connect_timeout => <<"10s">> }, - #{<<"config">> => Config}. + #{<<"config">> => maps:merge(Config, Overrides)}. test_query_no_params() -> {query, <<"SELECT 1">>}. diff --git a/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo.app.src b/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo.app.src index ed5078432..a4b372056 100644 --- a/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo.app.src +++ b/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_dynamo, [ {description, "EMQX Enterprise Dynamo Bridge"}, - {vsn, "0.1.3"}, + {vsn, "0.1.4"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector.erl b/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector.erl index 0d62845fd..9cdb8886c 100644 --- a/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector.erl +++ b/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector.erl @@ -45,12 +45,10 @@ fields(config) -> #{required => true, desc => ?DESC("aws_access_key_id")} )}, {aws_secret_access_key, - mk( - binary(), + emqx_schema_secret:mk( #{ required => true, - desc => ?DESC("aws_secret_access_key"), - sensitive => true + desc => ?DESC("aws_secret_access_key") } )}, {pool_size, fun emqx_connector_schema_lib:pool_size/1}, @@ -89,7 +87,7 @@ on_start( host => Host, port => Port, aws_access_key_id => to_str(AccessKeyID), - aws_secret_access_key => to_str(SecretAccessKey), + aws_secret_access_key => SecretAccessKey, schema => Schema }}, {pool_size, PoolSize} @@ -182,9 +180,8 @@ do_query( end. connect(Opts) -> - Options = proplists:get_value(config, Opts), - {ok, _Pid} = Result = emqx_bridge_dynamo_connector_client:start_link(Options), - Result. + Config = proplists:get_value(config, Opts), + {ok, _Pid} = emqx_bridge_dynamo_connector_client:start_link(Config). parse_template(Config) -> Templates = diff --git a/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector_client.erl b/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector_client.erl index 1b379298f..1cb326cf7 100644 --- a/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector_client.erl +++ b/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector_client.erl @@ -20,8 +20,7 @@ handle_cast/2, handle_info/2, terminate/2, - code_change/3, - format_status/2 + code_change/3 ]). -ifdef(TEST). @@ -62,11 +61,13 @@ start_link(Options) -> %% Initialize dynamodb data bridge init(#{ aws_access_key_id := AccessKeyID, - aws_secret_access_key := SecretAccessKey, + aws_secret_access_key := Secret, host := Host, port := Port, schema := Schema }) -> + %% TODO: teach `erlcloud` to to accept 0-arity closures as passwords. + SecretAccessKey = to_str(emqx_secret:unwrap(Secret)), erlcloud_ddb2:configure(AccessKeyID, SecretAccessKey, Host, Port, Schema), {ok, #{}}. @@ -101,13 +102,6 @@ terminate(_Reason, _State) -> code_change(_OldVsn, State, _Extra) -> {ok, State}. --spec format_status( - Opt :: normal | terminate, - Status :: list() -) -> Status :: term(). -format_status(_Opt, Status) -> - Status. - %%%=================================================================== %%% Internal functions %%%=================================================================== @@ -184,3 +178,8 @@ convert2binary(Value) when is_list(Value) -> unicode:characters_to_binary(Value); convert2binary(Value) when is_map(Value) -> emqx_utils_json:encode(Value). + +to_str(List) when is_list(List) -> + List; +to_str(Bin) when is_binary(Bin) -> + erlang:binary_to_list(Bin). diff --git a/apps/emqx_bridge_dynamo/test/emqx_bridge_dynamo_SUITE.erl b/apps/emqx_bridge_dynamo/test/emqx_bridge_dynamo_SUITE.erl index 9490e6455..936d2d506 100644 --- a/apps/emqx_bridge_dynamo/test/emqx_bridge_dynamo_SUITE.erl +++ b/apps/emqx_bridge_dynamo/test/emqx_bridge_dynamo_SUITE.erl @@ -22,8 +22,6 @@ -define(BATCH_SIZE, 10). -define(PAYLOAD, <<"HELLO">>). --define(GET_CONFIG(KEY__, CFG__), proplists:get_value(KEY__, CFG__)). - %% How to run it locally (all commands are run in $PROJ_ROOT dir): %% run ct in docker container %% run script: @@ -84,7 +82,9 @@ end_per_group(_Group, _Config) -> ok. init_per_suite(Config) -> - Config. + SecretFile = filename:join(?config(priv_dir, Config), "secret"), + ok = file:write_file(SecretFile, <>), + [{dynamo_secretfile, SecretFile} | Config]. end_per_suite(_Config) -> emqx_mgmt_api_test_util:end_suite(), @@ -158,32 +158,35 @@ common_init(ConfigT) -> end. dynamo_config(BridgeType, Config) -> - Port = integer_to_list(?GET_CONFIG(port, Config)), - Url = "http://" ++ ?GET_CONFIG(host, Config) ++ ":" ++ Port, + Host = ?config(host, Config), + Port = ?config(port, Config), Name = atom_to_binary(?MODULE), - BatchSize = ?GET_CONFIG(batch_size, Config), - QueryMode = ?GET_CONFIG(query_mode, Config), + BatchSize = ?config(batch_size, Config), + QueryMode = ?config(query_mode, Config), + SecretFile = ?config(dynamo_secretfile, Config), ConfigString = io_lib:format( - "bridges.~s.~s {\n" - " enable = true\n" - " url = ~p\n" - " table = ~p\n" - " aws_access_key_id = ~p\n" - " aws_secret_access_key = ~p\n" - " resource_opts = {\n" - " request_ttl = 500ms\n" - " batch_size = ~b\n" - " query_mode = ~s\n" - " }\n" - "}", + "bridges.~s.~s {" + "\n enable = true" + "\n url = \"http://~s:~p\"" + "\n table = ~p" + "\n aws_access_key_id = ~p" + "\n aws_secret_access_key = ~p" + "\n resource_opts = {" + "\n request_ttl = 500ms" + "\n batch_size = ~b" + "\n query_mode = ~s" + "\n }" + "\n }", [ BridgeType, Name, - Url, + Host, + Port, ?TABLE, ?ACCESS_KEY_ID, - ?SECRET_ACCESS_KEY, + %% NOTE: using file-based secrets with HOCON configs + "file://" ++ SecretFile, BatchSize, QueryMode ] @@ -252,8 +255,8 @@ delete_table(_Config) -> erlcloud_ddb2:delete_table(?TABLE_BIN). setup_dynamo(Config) -> - Host = ?GET_CONFIG(host, Config), - Port = ?GET_CONFIG(port, Config), + Host = ?config(host, Config), + Port = ?config(port, Config), erlcloud_ddb2:configure(?ACCESS_KEY_ID, ?SECRET_ACCESS_KEY, Host, Port, ?SCHEMA). directly_setup_dynamo() -> @@ -313,7 +316,9 @@ t_setup_via_http_api_and_publish(Config) -> PgsqlConfig0 = ?config(dynamo_config, Config), PgsqlConfig = PgsqlConfig0#{ <<"name">> => Name, - <<"type">> => BridgeType + <<"type">> => BridgeType, + %% NOTE: using literal secret with HTTP API requests. + <<"aws_secret_access_key">> => <> }, ?assertMatch( {ok, _}, @@ -400,7 +405,7 @@ t_simple_query(Config) -> ), Request = {get_item, {<<"id">>, <<"not_exists">>}}, Result = query_resource(Config, Request), - case ?GET_CONFIG(batch_size, Config) of + case ?config(batch_size, Config) of ?BATCH_SIZE -> ?assertMatch({error, {unrecoverable_error, {invalid_request, _}}}, Result); 1 -> diff --git a/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector.erl b/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector.erl index ff4ba313e..d588f7f8c 100644 --- a/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector.erl +++ b/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector.erl @@ -147,13 +147,7 @@ fields(greptimedb) -> [ {dbname, mk(binary(), #{required => true, desc => ?DESC("dbname")})}, {username, mk(binary(), #{desc => ?DESC("username")})}, - {password, - mk(binary(), #{ - desc => ?DESC("password"), - format => <<"password">>, - sensitive => true, - converter => fun emqx_schema:password_converter/2 - })} + {password, emqx_schema_secret:mk(#{desc => ?DESC("password")})} ] ++ emqx_connector_schema_lib:ssl_fields(). server() -> @@ -302,7 +296,8 @@ ssl_config(SSL = #{enable := true}) -> auth(#{username := Username, password := Password}) -> [ - {auth, {basic, #{username => str(Username), password => str(Password)}}} + %% TODO: teach `greptimedb` to accept 0-arity closures as passwords. + {auth, {basic, #{username => str(Username), password => emqx_secret:unwrap(Password)}}} ]; auth(_) -> []. diff --git a/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb_connector.erl b/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb_connector.erl index c8053a53d..2b4fb8d74 100644 --- a/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb_connector.erl +++ b/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb_connector.erl @@ -192,20 +192,14 @@ fields(influxdb_api_v1) -> [ {database, mk(binary(), #{required => true, desc => ?DESC("database")})}, {username, mk(binary(), #{desc => ?DESC("username")})}, - {password, - mk(binary(), #{ - desc => ?DESC("password"), - format => <<"password">>, - sensitive => true, - converter => fun emqx_schema:password_converter/2 - })} + {password, emqx_schema_secret:mk(#{desc => ?DESC("password")})} ] ++ emqx_connector_schema_lib:ssl_fields(); fields(influxdb_api_v2) -> fields(common) ++ [ {bucket, mk(binary(), #{required => true, desc => ?DESC("bucket")})}, {org, mk(binary(), #{required => true, desc => ?DESC("org")})}, - {token, mk(binary(), #{required => true, desc => ?DESC("token")})} + {token, emqx_schema_secret:mk(#{required => true, desc => ?DESC("token")})} ] ++ emqx_connector_schema_lib:ssl_fields(). server() -> @@ -363,7 +357,8 @@ protocol_config(#{ {version, v2}, {bucket, str(Bucket)}, {org, str(Org)}, - {token, Token} + %% TODO: teach `influxdb` to accept 0-arity closures as passwords. + {token, emqx_secret:unwrap(Token)} ] ++ ssl_config(SSL). ssl_config(#{enable := false}) -> @@ -383,7 +378,8 @@ username(_) -> []. password(#{password := Password}) -> - [{password, str(Password)}]; + %% TODO: teach `influxdb` to accept 0-arity closures as passwords. + [{password, str(emqx_secret:unwrap(Password))}]; password(_) -> []. diff --git a/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.app.src b/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.app.src index b79c4c2ce..42b3c165f 100644 --- a/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.app.src +++ b/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_bridge_iotdb, [ {description, "EMQX Enterprise Apache IoTDB Bridge"}, - {vsn, "0.1.3"}, + {vsn, "0.1.4"}, {modules, [ emqx_bridge_iotdb, emqx_bridge_iotdb_impl diff --git a/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.erl b/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.erl index 38dfebe97..25bafbd00 100644 --- a/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.erl +++ b/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.erl @@ -51,12 +51,9 @@ fields(auth_basic) -> [ {username, mk(binary(), #{required => true, desc => ?DESC("config_auth_basic_username")})}, {password, - mk(binary(), #{ + emqx_schema_secret:mk(#{ required => true, - desc => ?DESC("config_auth_basic_password"), - format => <<"password">>, - sensitive => true, - converter => fun emqx_schema:password_converter/2 + desc => ?DESC("config_auth_basic_password") })} ]. diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl index 0eb015cd3..d193738bb 100644 --- a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl +++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl @@ -283,11 +283,9 @@ fields(auth_username_password) -> })}, {username, mk(binary(), #{required => true, desc => ?DESC(auth_sasl_username)})}, {password, - mk(binary(), #{ + emqx_connector_schema_lib:password_field(#{ required => true, - sensitive => true, - desc => ?DESC(auth_sasl_password), - converter => fun emqx_schema:password_converter/2 + desc => ?DESC(auth_sasl_password) })} ]; fields(auth_gssapi_kerberos) -> diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl.erl index b3ad2ca36..eb8f36fb5 100644 --- a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl.erl +++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl.erl @@ -31,8 +31,8 @@ make_client_id(BridgeType0, BridgeName0) -> sasl(none) -> undefined; -sasl(#{mechanism := Mechanism, username := Username, password := Password}) -> - {Mechanism, Username, emqx_secret:wrap(Password)}; +sasl(#{mechanism := Mechanism, username := Username, password := Secret}) -> + {Mechanism, Username, Secret}; sasl(#{ kerberos_principal := Principal, kerberos_keytab_file := KeyTabFile diff --git a/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl index 943f30629..4fd08c154 100644 --- a/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl +++ b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl @@ -30,29 +30,41 @@ all() -> ]. groups() -> - AllTCs = emqx_common_test_helpers:all(?MODULE), - SASLAuths = [ - sasl_auth_plain, - sasl_auth_scram256, - sasl_auth_scram512, - sasl_auth_kerberos + SASLGroups = [ + {sasl_auth_plain, testcases(sasl)}, + {sasl_auth_scram256, testcases(sasl)}, + {sasl_auth_scram512, testcases(sasl)}, + {sasl_auth_kerberos, testcases(sasl_auth_kerberos)} ], - SASLAuthGroups = [{group, Type} || Type <- SASLAuths], - OnlyOnceTCs = only_once_tests(), - MatrixTCs = AllTCs -- OnlyOnceTCs, - SASLTests = [{Group, MatrixTCs} || Group <- SASLAuths], + SASLAuthGroups = [{group, Group} || {Group, _} <- SASLGroups], [ - {plain, MatrixTCs ++ OnlyOnceTCs}, - {ssl, MatrixTCs}, + {plain, testcases(plain)}, + {ssl, testcases(common)}, {sasl_plain, SASLAuthGroups}, {sasl_ssl, SASLAuthGroups} - ] ++ SASLTests. + | SASLGroups + ]. -sasl_only_tests() -> - [t_failed_creation_then_fixed]. - -%% tests that do not need to be run on all groups -only_once_tests() -> +testcases(all) -> + emqx_common_test_helpers:all(?MODULE); +testcases(plain) -> + %% NOTE: relevant only for a subset of SASL testcases + Exclude = [t_failed_creation_then_fixed], + testcases(all) -- Exclude; +testcases(common) -> + testcases(plain) -- testcases(once); +testcases(sasl) -> + testcases(all) -- testcases(once); +testcases(sasl_auth_kerberos) -> + %% NOTE: need a proxy to run these tests + Exclude = [ + t_failed_creation_then_fixed, + t_on_get_status, + t_receive_after_recovery + ], + testcases(sasl) -- Exclude; +testcases(once) -> + %% tests that do not need to be run on all groups [ t_begin_offset_earliest, t_bridge_rule_action_source, @@ -220,7 +232,7 @@ init_per_group(sasl_auth_kerberos, Config0) -> (KV) -> KV end, - [{has_proxy, false}, {sasl_auth_mechanism, kerberos} | Config0] + [{sasl_auth_mechanism, kerberos} | Config0] ), Config; init_per_group(_Group, Config) -> @@ -264,43 +276,6 @@ end_per_group(Group, Config) when end_per_group(_Group, _Config) -> ok. -init_per_testcase(TestCase, Config) when - TestCase =:= t_failed_creation_then_fixed --> - KafkaType = ?config(kafka_type, Config), - AuthMechanism = ?config(sasl_auth_mechanism, Config), - IsSASL = lists:member(KafkaType, [sasl_plain, sasl_ssl]), - case {IsSASL, AuthMechanism} of - {true, kerberos} -> - [{skip_does_not_apply, true}]; - {true, _} -> - common_init_per_testcase(TestCase, Config); - {false, _} -> - [{skip_does_not_apply, true}] - end; -init_per_testcase(TestCase, Config) when - TestCase =:= t_failed_creation_then_fixed --> - %% test with one partiton only for this case because - %% the wait probe may not be always sent to the same partition - HasProxy = proplists:get_value(has_proxy, Config, true), - case HasProxy of - false -> - [{skip_does_not_apply, true}]; - true -> - common_init_per_testcase(TestCase, [{num_partitions, 1} | Config]) - end; -init_per_testcase(TestCase, Config) when - TestCase =:= t_on_get_status; - TestCase =:= t_receive_after_recovery --> - HasProxy = proplists:get_value(has_proxy, Config, true), - case HasProxy of - false -> - [{skip_does_not_apply, true}]; - true -> - common_init_per_testcase(TestCase, Config) - end; init_per_testcase(t_cluster_group = TestCase, Config0) -> Config = emqx_utils:merge_opts(Config0, [{num_partitions, 6}]), common_init_per_testcase(TestCase, Config); @@ -393,30 +368,24 @@ common_init_per_testcase(TestCase, Config0) -> ]. end_per_testcase(_Testcase, Config) -> - case proplists:get_bool(skip_does_not_apply, Config) of - true -> - ok; - false -> - ProxyHost = ?config(proxy_host, Config), - ProxyPort = ?config(proxy_port, Config), - ProducersConfigs = ?config(kafka_producers, Config), - emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), - delete_all_bridges(), - #{clientid := KafkaProducerClientId, producers := ProducersMapping} = - ProducersConfigs, - lists:foreach( - fun(Producers) -> - ok = wolff:stop_and_delete_supervised_producers(Producers) - end, - maps:values(ProducersMapping) - ), - ok = wolff:stop_and_delete_supervised_client(KafkaProducerClientId), - %% in CI, apparently this needs more time since the - %% machines struggle with all the containers running... - emqx_common_test_helpers:call_janitor(60_000), - ok = snabbkaffe:stop(), - ok - end. + ProxyHost = ?config(proxy_host, Config), + ProxyPort = ?config(proxy_port, Config), + ProducersConfigs = ?config(kafka_producers, Config), + emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), + delete_all_bridges(), + #{clientid := KafkaProducerClientId, producers := ProducersMapping} = + ProducersConfigs, + lists:foreach( + fun(Producers) -> + ok = wolff:stop_and_delete_supervised_producers(Producers) + end, + maps:values(ProducersMapping) + ), + ok = wolff:stop_and_delete_supervised_client(KafkaProducerClientId), + %% in CI, apparently this needs more time since the + %% machines struggle with all the containers running... + emqx_common_test_helpers:call_janitor(60_000), + ok = snabbkaffe:stop(). %%------------------------------------------------------------------------------ %% Helper fns @@ -1391,14 +1360,6 @@ t_multiple_topic_mappings(Config) -> ok. t_on_get_status(Config) -> - case proplists:get_bool(skip_does_not_apply, Config) of - true -> - ok; - false -> - do_t_on_get_status(Config) - end. - -do_t_on_get_status(Config) -> ProxyPort = ?config(proxy_port, Config), ProxyHost = ?config(proxy_host, Config), ProxyName = ?config(proxy_name, Config), @@ -1421,14 +1382,6 @@ do_t_on_get_status(Config) -> %% ensure that we can create and use the bridge successfully after %% creating it with bad config. t_failed_creation_then_fixed(Config) -> - case proplists:get_bool(skip_does_not_apply, Config) of - true -> - ok; - false -> - ?check_trace(do_t_failed_creation_then_fixed(Config), []) - end. - -do_t_failed_creation_then_fixed(Config) -> ct:timetrap({seconds, 180}), MQTTTopic = ?config(mqtt_topic, Config), MQTTQoS = ?config(mqtt_qos, Config), @@ -1516,14 +1469,6 @@ do_t_failed_creation_then_fixed(Config) -> %% recovering from a network partition will make the subscribers %% consume the messages produced during the down time. t_receive_after_recovery(Config) -> - case proplists:get_bool(skip_does_not_apply, Config) of - true -> - ok; - false -> - do_t_receive_after_recovery(Config) - end. - -do_t_receive_after_recovery(Config) -> ct:timetrap(120_000), ProxyPort = ?config(proxy_port, Config), ProxyHost = ?config(proxy_host, Config), diff --git a/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_producer_SUITE.erl b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_producer_SUITE.erl index b37ef00e9..2a8a42a09 100644 --- a/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_producer_SUITE.erl +++ b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_producer_SUITE.erl @@ -28,13 +28,8 @@ ). -include_lib("eunit/include/eunit.hrl"). --include_lib("emqx/include/emqx.hrl"). --include_lib("emqx_dashboard/include/emqx_dashboard.hrl"). -define(HOST, "http://127.0.0.1:18083"). - -%% -define(API_VERSION, "v5"). - -define(BASE_PATH, "/api/v5"). %% NOTE: it's "kafka", but not "kafka_producer" @@ -48,13 +43,6 @@ %%------------------------------------------------------------------------------ all() -> - case code:get_object_code(cthr) of - {Module, Code, Filename} -> - {module, Module} = code:load_binary(Module, Filename, Code), - ok; - error -> - error - end, All0 = emqx_common_test_helpers:all(?MODULE), All = All0 -- matrix_cases(), Groups = lists:map(fun({G, _, _}) -> {group, G} end, groups()), @@ -105,23 +93,12 @@ init_per_suite(Config0) -> emqx_connector, emqx_bridge_kafka, emqx_bridge, - emqx_rule_engine + emqx_rule_engine, + {emqx_dashboard, "dashboard.listeners.http { enable = true, bind = 18083 }"} ], #{work_dir => emqx_cth_suite:work_dir(Config)} ), - emqx_mgmt_api_test_util:init_suite(), wait_until_kafka_is_up(), - %% Wait until bridges API is up - (fun WaitUntilRestApiUp() -> - case http_get(["bridges"]) of - {ok, 200, _Res} -> - ok; - Val -> - ct:pal("REST API for bridges not up. Wait and try again. Response: ~p", [Val]), - timer:sleep(1000), - WaitUntilRestApiUp() - end - end)(), [{apps, Apps} | Config]. end_per_suite(Config) -> @@ -183,6 +160,7 @@ t_query_mode_async(CtConfig) -> t_publish(matrix) -> {publish, [ [tcp, none, key_dispatch, sync], + [ssl, plain_passfile, random, sync], [ssl, scram_sha512, random, async], [ssl, kerberos, random, sync] ]}; @@ -200,9 +178,15 @@ t_publish(Config) -> end, Auth1 = case Auth of - none -> "none"; - scram_sha512 -> valid_sasl_scram512_settings(); - kerberos -> valid_sasl_kerberos_settings() + none -> + "none"; + plain_passfile -> + Passfile = filename:join(?config(priv_dir, Config), "passfile"), + valid_sasl_plain_passfile_settings(Passfile); + scram_sha512 -> + valid_sasl_scram512_settings(); + kerberos -> + valid_sasl_kerberos_settings() end, ConnCfg = #{ "bootstrap_hosts" => Hosts, @@ -1018,112 +1002,89 @@ hocon_config(Args, ConfigTemplateFun) -> ), Hocon. -%% erlfmt-ignore hocon_config_template() -> -""" -bridges.kafka.{{ bridge_name }} { - bootstrap_hosts = \"{{ kafka_hosts_string }}\" - enable = true - authentication = {{{ authentication }}} - ssl = {{{ ssl }}} - local_topic = \"{{ local_topic }}\" - kafka = { - message = { - key = \"${clientid}\" - value = \"${.payload}\" - timestamp = \"${timestamp}\" - } - buffer = { - memory_overload_protection = false - } - partition_strategy = {{ partition_strategy }} - topic = \"{{ kafka_topic }}\" - query_mode = {{ query_mode }} - } - metadata_request_timeout = 5s - min_metadata_refresh_interval = 3s - socket_opts { - nodelay = true - } - connect_timeout = 5s -} -""". + "bridges.kafka.{{ bridge_name }} {" + "\n bootstrap_hosts = \"{{ kafka_hosts_string }}\"" + "\n enable = true" + "\n authentication = {{{ authentication }}}" + "\n ssl = {{{ ssl }}}" + "\n local_topic = \"{{ local_topic }}\"" + "\n kafka = {" + "\n message = {" + "\n key = \"${clientid}\"" + "\n value = \"${.payload}\"" + "\n timestamp = \"${timestamp}\"" + "\n }" + "\n buffer = {" + "\n memory_overload_protection = false" + "\n }" + "\n partition_strategy = {{ partition_strategy }}" + "\n topic = \"{{ kafka_topic }}\"" + "\n query_mode = {{ query_mode }}" + "\n }" + "\n metadata_request_timeout = 5s" + "\n min_metadata_refresh_interval = 3s" + "\n socket_opts {" + "\n nodelay = true" + "\n }" + "\n connect_timeout = 5s" + "\n }". -%% erlfmt-ignore hocon_config_template_with_headers() -> -""" -bridges.kafka.{{ bridge_name }} { - bootstrap_hosts = \"{{ kafka_hosts_string }}\" - enable = true - authentication = {{{ authentication }}} - ssl = {{{ ssl }}} - local_topic = \"{{ local_topic }}\" - kafka = { - message = { - key = \"${clientid}\" - value = \"${.payload}\" - timestamp = \"${timestamp}\" - } - buffer = { - memory_overload_protection = false - } - kafka_headers = \"{{ kafka_headers }}\" - kafka_header_value_encode_mode: json - kafka_ext_headers: {{{ kafka_ext_headers }}} - partition_strategy = {{ partition_strategy }} - topic = \"{{ kafka_topic }}\" - query_mode = {{ query_mode }} - } - metadata_request_timeout = 5s - min_metadata_refresh_interval = 3s - socket_opts { - nodelay = true - } - connect_timeout = 5s -} -""". + "bridges.kafka.{{ bridge_name }} {" + "\n bootstrap_hosts = \"{{ kafka_hosts_string }}\"" + "\n enable = true" + "\n authentication = {{{ authentication }}}" + "\n ssl = {{{ ssl }}}" + "\n local_topic = \"{{ local_topic }}\"" + "\n kafka = {" + "\n message = {" + "\n key = \"${clientid}\"" + "\n value = \"${.payload}\"" + "\n timestamp = \"${timestamp}\"" + "\n }" + "\n buffer = {" + "\n memory_overload_protection = false" + "\n }" + "\n kafka_headers = \"{{ kafka_headers }}\"" + "\n kafka_header_value_encode_mode: json" + "\n kafka_ext_headers: {{{ kafka_ext_headers }}}" + "\n partition_strategy = {{ partition_strategy }}" + "\n topic = \"{{ kafka_topic }}\"" + "\n query_mode = {{ query_mode }}" + "\n }" + "\n metadata_request_timeout = 5s" + "\n min_metadata_refresh_interval = 3s" + "\n socket_opts {" + "\n nodelay = true" + "\n }" + "\n connect_timeout = 5s" + "\n }". -%% erlfmt-ignore hocon_config_template_authentication("none") -> "none"; hocon_config_template_authentication(#{"mechanism" := _}) -> -""" -{ - mechanism = {{ mechanism }} - password = {{ password }} - username = {{ username }} -} -"""; + "{" + "\n mechanism = {{ mechanism }}" + "\n password = \"{{ password }}\"" + "\n username = \"{{ username }}\"" + "\n }"; hocon_config_template_authentication(#{"kerberos_principal" := _}) -> -""" -{ - kerberos_principal = \"{{ kerberos_principal }}\" - kerberos_keytab_file = \"{{ kerberos_keytab_file }}\" -} -""". + "{" + "\n kerberos_principal = \"{{ kerberos_principal }}\"" + "\n kerberos_keytab_file = \"{{ kerberos_keytab_file }}\"" + "\n }". -%% erlfmt-ignore hocon_config_template_ssl(Map) when map_size(Map) =:= 0 -> -""" -{ - enable = false -} -"""; + "{ enable = false }"; hocon_config_template_ssl(#{"enable" := "false"}) -> -""" -{ - enable = false -} -"""; + "{ enable = false }"; hocon_config_template_ssl(#{"enable" := "true"}) -> -""" -{ - enable = true - cacertfile = \"{{{cacertfile}}}\" - certfile = \"{{{certfile}}}\" - keyfile = \"{{{keyfile}}}\" -} -""". + "{ enable = true" + "\n cacertfile = \"{{{cacertfile}}}\"" + "\n certfile = \"{{{certfile}}}\"" + "\n keyfile = \"{{{keyfile}}}\"" + "\n }". kafka_hosts_string(tcp, none) -> kafka_hosts_string(); @@ -1197,6 +1158,13 @@ valid_sasl_kerberos_settings() -> "kerberos_keytab_file" => shared_secret(rig_keytab) }. +valid_sasl_plain_passfile_settings(Passfile) -> + Auth = valid_sasl_plain_settings(), + ok = file:write_file(Passfile, maps:get("password", Auth)), + Auth#{ + "password" := "file://" ++ Passfile + }. + kafka_hosts() -> kpro:parse_endpoints(kafka_hosts_string()). diff --git a/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_tests.erl b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_tests.erl index 1d9682b9b..ff4334a85 100644 --- a/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_tests.erl +++ b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_tests.erl @@ -223,144 +223,136 @@ check_atom_key(Conf) when is_map(Conf) -> %% Data section %%=========================================================================== -%% erlfmt-ignore kafka_producer_old_hocon(_WithLocalTopic = true) -> kafka_producer_old_hocon("mqtt {topic = \"mqtt/local\"}\n"); kafka_producer_old_hocon(_WithLocalTopic = false) -> kafka_producer_old_hocon("mqtt {}\n"); kafka_producer_old_hocon(MQTTConfig) when is_list(MQTTConfig) -> -""" -bridges.kafka { - myproducer { - authentication = \"none\" - bootstrap_hosts = \"toxiproxy:9292\" - connect_timeout = \"5s\" - metadata_request_timeout = \"5s\" - min_metadata_refresh_interval = \"3s\" - producer { - kafka { - buffer { - memory_overload_protection = false - mode = \"memory\" - per_partition_limit = \"2GB\" - segment_bytes = \"100MB\" - } - compression = \"no_compression\" - max_batch_bytes = \"896KB\" - max_inflight = 10 - message { - key = \"${.clientid}\" - timestamp = \"${.timestamp}\" - value = \"${.}\" - } - partition_count_refresh_interval = \"60s\" - partition_strategy = \"random\" - required_acks = \"all_isr\" - topic = \"test-topic-two-partitions\" - } -""" ++ MQTTConfig ++ -""" - } - socket_opts { - nodelay = true - recbuf = \"1024KB\" - sndbuf = \"1024KB\" - } - ssl {enable = false, verify = \"verify_peer\"} - } -} -""". + [ + "bridges.kafka {" + "\n myproducer {" + "\n authentication = \"none\"" + "\n bootstrap_hosts = \"toxiproxy:9292\"" + "\n connect_timeout = \"5s\"" + "\n metadata_request_timeout = \"5s\"" + "\n min_metadata_refresh_interval = \"3s\"" + "\n producer {" + "\n kafka {" + "\n buffer {" + "\n memory_overload_protection = false" + "\n mode = \"memory\"" + "\n per_partition_limit = \"2GB\"" + "\n segment_bytes = \"100MB\"" + "\n }" + "\n compression = \"no_compression\"" + "\n max_batch_bytes = \"896KB\"" + "\n max_inflight = 10" + "\n message {" + "\n key = \"${.clientid}\"" + "\n timestamp = \"${.timestamp}\"" + "\n value = \"${.}\"" + "\n }" + "\n partition_count_refresh_interval = \"60s\"" + "\n partition_strategy = \"random\"" + "\n required_acks = \"all_isr\"" + "\n topic = \"test-topic-two-partitions\"" + "\n }", + MQTTConfig, + "\n }" + "\n socket_opts {" + "\n nodelay = true" + "\n recbuf = \"1024KB\"" + "\n sndbuf = \"1024KB\"" + "\n }" + "\n ssl {enable = false, verify = \"verify_peer\"}" + "\n }" + "\n}" + ]. kafka_producer_new_hocon() -> - "" - "\n" - "bridges.kafka {\n" - " myproducer {\n" - " authentication = \"none\"\n" - " bootstrap_hosts = \"toxiproxy:9292\"\n" - " connect_timeout = \"5s\"\n" - " metadata_request_timeout = \"5s\"\n" - " min_metadata_refresh_interval = \"3s\"\n" - " kafka {\n" - " buffer {\n" - " memory_overload_protection = false\n" - " mode = \"memory\"\n" - " per_partition_limit = \"2GB\"\n" - " segment_bytes = \"100MB\"\n" - " }\n" - " compression = \"no_compression\"\n" - " max_batch_bytes = \"896KB\"\n" - " max_inflight = 10\n" - " message {\n" - " key = \"${.clientid}\"\n" - " timestamp = \"${.timestamp}\"\n" - " value = \"${.}\"\n" - " }\n" - " partition_count_refresh_interval = \"60s\"\n" - " partition_strategy = \"random\"\n" - " required_acks = \"all_isr\"\n" - " topic = \"test-topic-two-partitions\"\n" - " }\n" - " local_topic = \"mqtt/local\"\n" - " socket_opts {\n" - " nodelay = true\n" - " recbuf = \"1024KB\"\n" - " sndbuf = \"1024KB\"\n" - " }\n" - " ssl {enable = false, verify = \"verify_peer\"}\n" - " resource_opts {\n" - " health_check_interval = 10s\n" - " }\n" - " }\n" - "}\n" - "". + "bridges.kafka {" + "\n myproducer {" + "\n authentication = \"none\"" + "\n bootstrap_hosts = \"toxiproxy:9292\"" + "\n connect_timeout = \"5s\"" + "\n metadata_request_timeout = \"5s\"" + "\n min_metadata_refresh_interval = \"3s\"" + "\n kafka {" + "\n buffer {" + "\n memory_overload_protection = false" + "\n mode = \"memory\"" + "\n per_partition_limit = \"2GB\"" + "\n segment_bytes = \"100MB\"" + "\n }" + "\n compression = \"no_compression\"" + "\n max_batch_bytes = \"896KB\"" + "\n max_inflight = 10" + "\n message {" + "\n key = \"${.clientid}\"" + "\n timestamp = \"${.timestamp}\"" + "\n value = \"${.}\"" + "\n }" + "\n partition_count_refresh_interval = \"60s\"" + "\n partition_strategy = \"random\"" + "\n required_acks = \"all_isr\"" + "\n topic = \"test-topic-two-partitions\"" + "\n }" + "\n local_topic = \"mqtt/local\"" + "\n socket_opts {" + "\n nodelay = true" + "\n recbuf = \"1024KB\"" + "\n sndbuf = \"1024KB\"" + "\n }" + "\n ssl {enable = false, verify = \"verify_peer\"}" + "\n resource_opts {" + "\n health_check_interval = 10s" + "\n }" + "\n }" + "\n}". -%% erlfmt-ignore kafka_consumer_hocon() -> -""" -bridges.kafka_consumer.my_consumer { - enable = true - bootstrap_hosts = \"kafka-1.emqx.net:9292\" - connect_timeout = 5s - min_metadata_refresh_interval = 3s - metadata_request_timeout = 5s - authentication = { - mechanism = plain - username = emqxuser - password = password - } - kafka { - max_batch_bytes = 896KB - max_rejoin_attempts = 5 - offset_commit_interval_seconds = 3s - offset_reset_policy = latest - } - topic_mapping = [ - { - kafka_topic = \"kafka-topic-1\" - mqtt_topic = \"mqtt/topic/1\" - qos = 1 - payload_template = \"${.}\" - }, - { - kafka_topic = \"kafka-topic-2\" - mqtt_topic = \"mqtt/topic/2\" - qos = 2 - payload_template = \"v = ${.value}\" - } - ] - key_encoding_mode = none - value_encoding_mode = none - ssl { - enable = false - verify = verify_none - server_name_indication = \"auto\" - } - resource_opts { - health_check_interval = 10s - } -} -""". + "bridges.kafka_consumer.my_consumer {" + "\n enable = true" + "\n bootstrap_hosts = \"kafka-1.emqx.net:9292\"" + "\n connect_timeout = 5s" + "\n min_metadata_refresh_interval = 3s" + "\n metadata_request_timeout = 5s" + "\n authentication = {" + "\n mechanism = plain" + "\n username = emqxuser" + "\n password = password" + "\n }" + "\n kafka {" + "\n max_batch_bytes = 896KB" + "\n max_rejoin_attempts = 5" + "\n offset_commit_interval_seconds = 3s" + "\n offset_reset_policy = latest" + "\n }" + "\n topic_mapping = [" + "\n {" + "\n kafka_topic = \"kafka-topic-1\"" + "\n mqtt_topic = \"mqtt/topic/1\"" + "\n qos = 1" + "\n payload_template = \"${.}\"" + "\n }," + "\n {" + "\n kafka_topic = \"kafka-topic-2\"" + "\n mqtt_topic = \"mqtt/topic/2\"" + "\n qos = 2" + "\n payload_template = \"v = ${.value}\"" + "\n }" + "\n ]" + "\n key_encoding_mode = none" + "\n value_encoding_mode = none" + "\n ssl {" + "\n enable = false" + "\n verify = verify_none" + "\n server_name_indication = \"auto\"" + "\n }" + "\n resource_opts {" + "\n health_check_interval = 10s" + "\n }" + "\n }". %% assert compatibility bridge_schema_json_test() -> diff --git a/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis.app.src b/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis.app.src index 6066e2495..74d7dc94f 100644 --- a/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis.app.src +++ b/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_kinesis, [ {description, "EMQX Enterprise Amazon Kinesis Bridge"}, - {vsn, "0.1.2"}, + {vsn, "0.1.3"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis.erl b/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis.erl index d98e7ab11..14e197113 100644 --- a/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis.erl +++ b/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis.erl @@ -62,12 +62,10 @@ fields(connector_config) -> } )}, {aws_secret_access_key, - mk( - binary(), + emqx_schema_secret:mk( #{ required => true, - desc => ?DESC("aws_secret_access_key"), - sensitive => true + desc => ?DESC("aws_secret_access_key") } )}, {endpoint, diff --git a/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis_connector_client.erl b/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis_connector_client.erl index d9dc0220f..959b539a0 100644 --- a/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis_connector_client.erl +++ b/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis_connector_client.erl @@ -97,7 +97,13 @@ init(#{ partition_key => PartitionKey, stream_name => StreamName }, - New = + %% TODO: teach `erlcloud` to to accept 0-arity closures as passwords. + ok = erlcloud_config:configure( + to_str(AwsAccessKey), + to_str(emqx_secret:unwrap(AwsSecretAccessKey)), + Host, + Port, + Scheme, fun(AccessKeyID, SecretAccessKey, HostAddr, HostPort, ConnectionScheme) -> Config0 = erlcloud_kinesis:new( AccessKeyID, @@ -107,9 +113,7 @@ init(#{ ConnectionScheme ++ "://" ), Config0#aws_config{retry_num = MaxRetries} - end, - erlcloud_config:configure( - to_str(AwsAccessKey), to_str(AwsSecretAccessKey), Host, Port, Scheme, New + end ), % check the connection case erlcloud_kinesis:list_streams() of diff --git a/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis_impl_producer.erl b/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis_impl_producer.erl index 1e07ae96e..decf3e83b 100644 --- a/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis_impl_producer.erl +++ b/apps/emqx_bridge_kinesis/src/emqx_bridge_kinesis_impl_producer.erl @@ -15,7 +15,7 @@ -type config() :: #{ aws_access_key_id := binary(), - aws_secret_access_key := binary(), + aws_secret_access_key := emqx_secret:t(binary()), endpoint := binary(), stream_name := binary(), partition_key := binary(), diff --git a/apps/emqx_bridge_kinesis/test/emqx_bridge_kinesis_impl_producer_SUITE.erl b/apps/emqx_bridge_kinesis/test/emqx_bridge_kinesis_impl_producer_SUITE.erl index ea926fc33..61b354ea3 100644 --- a/apps/emqx_bridge_kinesis/test/emqx_bridge_kinesis_impl_producer_SUITE.erl +++ b/apps/emqx_bridge_kinesis/test/emqx_bridge_kinesis_impl_producer_SUITE.erl @@ -11,10 +11,11 @@ -include_lib("common_test/include/ct.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). --define(PRODUCER, emqx_bridge_kinesis_impl_producer). -define(BRIDGE_TYPE, kinesis_producer). -define(BRIDGE_TYPE_BIN, <<"kinesis_producer">>). -define(KINESIS_PORT, 4566). +-define(KINESIS_ACCESS_KEY, "aws_access_key_id"). +-define(KINESIS_SECRET_KEY, "aws_secret_access_key"). -define(TOPIC, <<"t/topic">>). %%------------------------------------------------------------------------------ @@ -38,6 +39,8 @@ init_per_suite(Config) -> ProxyHost = os:getenv("PROXY_HOST", "toxiproxy.emqx.net"), ProxyPort = list_to_integer(os:getenv("PROXY_PORT", "8474")), ProxyName = "kinesis", + SecretFile = filename:join(?config(priv_dir, Config), "secret"), + ok = file:write_file(SecretFile, <>), ok = emqx_common_test_helpers:start_apps([emqx_conf]), ok = emqx_connector_test_helpers:start_apps([emqx_resource, emqx_bridge, emqx_rule_engine]), {ok, _} = application:ensure_all_started(emqx_connector), @@ -46,6 +49,7 @@ init_per_suite(Config) -> {proxy_host, ProxyHost}, {proxy_port, ProxyPort}, {kinesis_port, ?KINESIS_PORT}, + {kinesis_secretfile, SecretFile}, {proxy_name, ProxyName} | Config ]. @@ -130,6 +134,7 @@ kinesis_config(Config) -> Scheme = proplists:get_value(connection_scheme, Config, "http"), ProxyHost = proplists:get_value(proxy_host, Config), KinesisPort = proplists:get_value(kinesis_port, Config), + SecretFile = proplists:get_value(kinesis_secretfile, Config), BatchSize = proplists:get_value(batch_size, Config, 100), BatchTime = proplists:get_value(batch_time, Config, <<"500ms">>), PayloadTemplate = proplists:get_value(payload_template, Config, "${payload}"), @@ -140,29 +145,32 @@ kinesis_config(Config) -> Name = <<(atom_to_binary(?MODULE))/binary, (GUID)/binary>>, ConfigString = io_lib:format( - "bridges.kinesis_producer.~s {\n" - " enable = true\n" - " aws_access_key_id = \"aws_access_key_id\"\n" - " aws_secret_access_key = \"aws_secret_access_key\"\n" - " endpoint = \"~s://~s:~b\"\n" - " stream_name = \"~s\"\n" - " partition_key = \"~s\"\n" - " payload_template = \"~s\"\n" - " max_retries = ~b\n" - " pool_size = 1\n" - " resource_opts = {\n" - " health_check_interval = \"3s\"\n" - " request_ttl = 30s\n" - " resume_interval = 1s\n" - " metrics_flush_interval = \"700ms\"\n" - " worker_pool_size = 1\n" - " query_mode = ~s\n" - " batch_size = ~b\n" - " batch_time = \"~s\"\n" - " }\n" - "}\n", + "bridges.kinesis_producer.~s {" + "\n enable = true" + "\n aws_access_key_id = ~p" + "\n aws_secret_access_key = ~p" + "\n endpoint = \"~s://~s:~b\"" + "\n stream_name = \"~s\"" + "\n partition_key = \"~s\"" + "\n payload_template = \"~s\"" + "\n max_retries = ~b" + "\n pool_size = 1" + "\n resource_opts = {" + "\n health_check_interval = \"3s\"" + "\n request_ttl = 30s" + "\n resume_interval = 1s" + "\n metrics_flush_interval = \"700ms\"" + "\n worker_pool_size = 1" + "\n query_mode = ~s" + "\n batch_size = ~b" + "\n batch_time = \"~s\"" + "\n }" + "\n }", [ Name, + ?KINESIS_ACCESS_KEY, + %% NOTE: using file-based secrets with HOCON configs. + "file://" ++ SecretFile, Scheme, ProxyHost, KinesisPort, @@ -203,9 +211,6 @@ delete_bridge(Config) -> ct:pal("deleting bridge ~p", [{Type, Name}]), emqx_bridge:remove(Type, Name). -create_bridge_http(Config) -> - create_bridge_http(Config, _KinesisConfigOverrides = #{}). - create_bridge_http(Config, KinesisConfigOverrides) -> TypeBin = ?BRIDGE_TYPE_BIN, Name = ?config(kinesis_name, Config), @@ -489,7 +494,11 @@ to_bin(Str) when is_list(Str) -> %%------------------------------------------------------------------------------ t_create_via_http(Config) -> - ?assertMatch({ok, _}, create_bridge_http(Config)), + Overrides = #{ + %% NOTE: using literal secret with HTTP API requests. + <<"aws_secret_access_key">> => <> + }, + ?assertMatch({ok, _}, create_bridge_http(Config, Overrides)), ok. t_start_failed_then_fix(Config) -> diff --git a/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.app.src b/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.app.src index 35bcc3fc4..5545ac967 100644 --- a/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.app.src +++ b/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_mongodb, [ {description, "EMQX Enterprise MongoDB Bridge"}, - {vsn, "0.2.1"}, + {vsn, "0.2.2"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb_connector.erl b/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb_connector.erl index 8c004d829..741db9550 100644 --- a/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb_connector.erl +++ b/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb_connector.erl @@ -6,9 +6,6 @@ -behaviour(emqx_resource). --include_lib("emqx_connector/include/emqx_connector_tables.hrl"). --include_lib("emqx_resource/include/emqx_resource.hrl"). --include_lib("typerefl/include/types.hrl"). -include_lib("emqx/include/logger.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). diff --git a/apps/emqx_bridge_mongodb/test/emqx_bridge_mongodb_SUITE.erl b/apps/emqx_bridge_mongodb/test/emqx_bridge_mongodb_SUITE.erl index f2d0bc1c5..cedb19b88 100644 --- a/apps/emqx_bridge_mongodb/test/emqx_bridge_mongodb_SUITE.erl +++ b/apps/emqx_bridge_mongodb/test/emqx_bridge_mongodb_SUITE.erl @@ -11,6 +11,8 @@ -include_lib("common_test/include/ct.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). +-import(emqx_utils_conv, [bin/1]). + %%------------------------------------------------------------------------------ %% CT boilerplate %%------------------------------------------------------------------------------ @@ -96,14 +98,27 @@ init_per_group(Type = single, Config) -> true -> ok = start_apps(), emqx_mgmt_api_test_util:init_suite(), - {Name, MongoConfig} = mongo_config(MongoHost, MongoPort, Type, Config), + %% NOTE: `mongo-single` has auth enabled, see `credentials.env`. + AuthSource = bin(os:getenv("MONGO_AUTHSOURCE", "admin")), + Username = bin(os:getenv("MONGO_USERNAME", "")), + Password = bin(os:getenv("MONGO_PASSWORD", "")), + Passfile = filename:join(?config(priv_dir, Config), "passfile"), + ok = file:write_file(Passfile, Password), + NConfig = [ + {mongo_authsource, AuthSource}, + {mongo_username, Username}, + {mongo_password, Password}, + {mongo_passfile, Passfile} + | Config + ], + {Name, MongoConfig} = mongo_config(MongoHost, MongoPort, Type, NConfig), [ {mongo_host, MongoHost}, {mongo_port, MongoPort}, {mongo_config, MongoConfig}, {mongo_type, Type}, {mongo_name, Name} - | Config + | NConfig ]; false -> {skip, no_mongo} @@ -121,13 +136,13 @@ end_per_suite(_Config) -> ok. init_per_testcase(_Testcase, Config) -> - catch clear_db(Config), + clear_db(Config), delete_bridge(Config), snabbkaffe:start_trace(), Config. end_per_testcase(_Testcase, Config) -> - catch clear_db(Config), + clear_db(Config), delete_bridge(Config), snabbkaffe:stop(), ok. @@ -175,19 +190,19 @@ mongo_config(MongoHost, MongoPort0, rs = Type, Config) -> Name = atom_to_binary(?MODULE), ConfigString = io_lib:format( - "bridges.mongodb_rs.~s {\n" - " enable = true\n" - " collection = mycol\n" - " replica_set_name = rs0\n" - " servers = [~p]\n" - " w_mode = safe\n" - " use_legacy_protocol = auto\n" - " database = mqtt\n" - " resource_opts = {\n" - " query_mode = ~s\n" - " worker_pool_size = 1\n" - " }\n" - "}", + "bridges.mongodb_rs.~s {" + "\n enable = true" + "\n collection = mycol" + "\n replica_set_name = rs0" + "\n servers = [~p]" + "\n w_mode = safe" + "\n use_legacy_protocol = auto" + "\n database = mqtt" + "\n resource_opts = {" + "\n query_mode = ~s" + "\n worker_pool_size = 1" + "\n }" + "\n }", [ Name, Servers, @@ -202,18 +217,18 @@ mongo_config(MongoHost, MongoPort0, sharded = Type, Config) -> Name = atom_to_binary(?MODULE), ConfigString = io_lib:format( - "bridges.mongodb_sharded.~s {\n" - " enable = true\n" - " collection = mycol\n" - " servers = [~p]\n" - " w_mode = safe\n" - " use_legacy_protocol = auto\n" - " database = mqtt\n" - " resource_opts = {\n" - " query_mode = ~s\n" - " worker_pool_size = 1\n" - " }\n" - "}", + "bridges.mongodb_sharded.~s {" + "\n enable = true" + "\n collection = mycol" + "\n servers = [~p]" + "\n w_mode = safe" + "\n use_legacy_protocol = auto" + "\n database = mqtt" + "\n resource_opts = {" + "\n query_mode = ~s" + "\n worker_pool_size = 1" + "\n }" + "\n }", [ Name, Servers, @@ -228,21 +243,27 @@ mongo_config(MongoHost, MongoPort0, single = Type, Config) -> Name = atom_to_binary(?MODULE), ConfigString = io_lib:format( - "bridges.mongodb_single.~s {\n" - " enable = true\n" - " collection = mycol\n" - " server = ~p\n" - " w_mode = safe\n" - " use_legacy_protocol = auto\n" - " database = mqtt\n" - " resource_opts = {\n" - " query_mode = ~s\n" - " worker_pool_size = 1\n" - " }\n" - "}", + "bridges.mongodb_single.~s {" + "\n enable = true" + "\n collection = mycol" + "\n server = ~p" + "\n w_mode = safe" + "\n use_legacy_protocol = auto" + "\n database = mqtt" + "\n auth_source = ~s" + "\n username = ~s" + "\n password = \"file://~s\"" + "\n resource_opts = {" + "\n query_mode = ~s" + "\n worker_pool_size = 1" + "\n }" + "\n }", [ Name, Server, + ?config(mongo_authsource, Config), + ?config(mongo_username, Config), + ?config(mongo_passfile, Config), QueryMode ] ), @@ -284,8 +305,24 @@ clear_db(Config) -> Host = ?config(mongo_host, Config), Port = ?config(mongo_port, Config), Server = Host ++ ":" ++ integer_to_list(Port), - #{<<"database">> := Db, <<"collection">> := Collection} = ?config(mongo_config, Config), - {ok, Client} = mongo_api:connect(Type, [Server], [], [{database, Db}, {w_mode, unsafe}]), + #{ + <<"database">> := Db, + <<"collection">> := Collection + } = ?config(mongo_config, Config), + WorkerOpts = [ + {database, Db}, + {w_mode, unsafe} + | lists:flatmap( + fun + ({mongo_authsource, AS}) -> [{auth_source, AS}]; + ({mongo_username, User}) -> [{login, User}]; + ({mongo_password, Pass}) -> [{password, Pass}]; + (_) -> [] + end, + Config + ) + ], + {ok, Client} = mongo_api:connect(Type, [Server], [], WorkerOpts), {true, _} = mongo_api:delete(Client, Collection, _Selector = #{}), mongo_api:disconnect(Client). @@ -386,13 +423,21 @@ t_setup_via_config_and_publish(Config) -> ok. t_setup_via_http_api_and_publish(Config) -> - Type = mongo_type_bin(?config(mongo_type, Config)), + Type = ?config(mongo_type, Config), Name = ?config(mongo_name, Config), MongoConfig0 = ?config(mongo_config, Config), - MongoConfig = MongoConfig0#{ + MongoConfig1 = MongoConfig0#{ <<"name">> => Name, - <<"type">> => Type + <<"type">> => mongo_type_bin(Type) }, + MongoConfig = + case Type of + single -> + %% NOTE: using literal password with HTTP API requests. + MongoConfig1#{<<"password">> => ?config(mongo_password, Config)}; + _ -> + MongoConfig1 + end, ?assertMatch( {ok, _}, create_bridge_http(MongoConfig) diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src index e39c4df69..cbef0dda8 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_bridge_mqtt, [ {description, "EMQX MQTT Broker Bridge"}, - {vsn, "0.1.4"}, + {vsn, "0.1.5"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl index eb81c4b6e..61e9353ce 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl @@ -96,7 +96,7 @@ choose_ingress_pool_size( #{remote := #{topic := RemoteTopic}, pool_size := PoolSize} ) -> case emqx_topic:parse(RemoteTopic) of - {_Filter, #{share := _Name}} -> + {#share{} = _Filter, _SubOpts} -> % NOTE: this is shared subscription, many workers may subscribe PoolSize; {_Filter, #{}} when PoolSize > 1 -> @@ -326,7 +326,7 @@ mk_client_opts( ], Config ), - Options#{ + mk_client_opt_password(Options#{ hosts => [HostPort], clientid => clientid(ResourceId, ClientScope, Config), connect_timeout => 30, @@ -334,7 +334,13 @@ mk_client_opts( force_ping => true, ssl => EnableSsl, ssl_opts => maps:to_list(maps:remove(enable, Ssl)) - }. + }). + +mk_client_opt_password(Options = #{password := Secret}) -> + %% TODO: Teach `emqtt` to accept 0-arity closures as passwords. + Options#{password := emqx_secret:unwrap(Secret)}; +mk_client_opt_password(Options) -> + Options. ms_to_s(Ms) -> erlang:ceil(Ms / 1000). diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector_schema.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector_schema.erl index 1dc3ca5f8..eb298c5ff 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector_schema.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector_schema.erl @@ -99,13 +99,9 @@ fields("server_configs") -> } )}, {password, - mk( - binary(), + emqx_schema_secret:mk( #{ - format => <<"password">>, - sensitive => true, - desc => ?DESC("password"), - converter => fun emqx_schema:password_converter/2 + desc => ?DESC("password") } )}, {clean_start, diff --git a/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_SUITE.erl b/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_SUITE.erl index 986a755d5..bde546bd0 100644 --- a/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_SUITE.erl +++ b/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_SUITE.erl @@ -21,13 +21,15 @@ -import(emqx_dashboard_api_test_helpers, [request/4, uri/1]). -include("emqx/include/emqx.hrl"). +-include("emqx/include/emqx_hooks.hrl"). +-include("emqx/include/asserts.hrl"). -include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). %% output functions -export([inspect/3]). --define(BRIDGE_CONF_DEFAULT, <<"bridges: {}">>). -define(TYPE_MQTT, <<"mqtt">>). -define(BRIDGE_NAME_INGRESS, <<"ingress_mqtt_bridge">>). -define(BRIDGE_NAME_EGRESS, <<"egress_mqtt_bridge">>). @@ -38,14 +40,18 @@ -define(EGRESS_REMOTE_TOPIC, "egress_remote_topic"). -define(EGRESS_LOCAL_TOPIC, "egress_local_topic"). --define(SERVER_CONF(Username), #{ +-define(SERVER_CONF, #{ + <<"type">> => ?TYPE_MQTT, <<"server">> => <<"127.0.0.1:1883">>, - <<"username">> => Username, - <<"password">> => <<"">>, <<"proto_ver">> => <<"v4">>, <<"ssl">> => #{<<"enable">> => false} }). +-define(SERVER_CONF(Username, Password), (?SERVER_CONF)#{ + <<"username">> => Username, + <<"password">> => Password +}). + -define(INGRESS_CONF, #{ <<"remote">> => #{ <<"topic">> => <>, @@ -129,43 +135,32 @@ suite() -> [{timetrap, {seconds, 30}}]. init_per_suite(Config) -> - _ = application:load(emqx_conf), - ok = emqx_common_test_helpers:start_apps( + Apps = emqx_cth_suite:start( [ - emqx_rule_engine, + emqx_conf, emqx_bridge, + emqx_rule_engine, emqx_bridge_mqtt, - emqx_dashboard + {emqx_dashboard, + "dashboard {" + "\n listeners.http { bind = 18083 }" + "\n default_username = connector_admin" + "\n default_password = public" + "\n }"} ], - fun set_special_configs/1 + #{work_dir => emqx_cth_suite:work_dir(Config)} ), - ok = emqx_common_test_helpers:load_config( - emqx_rule_engine_schema, - <<"rule_engine {rules {}}">> - ), - ok = emqx_common_test_helpers:load_config(emqx_bridge_schema, ?BRIDGE_CONF_DEFAULT), - Config. + [{suite_apps, Apps} | Config]. -end_per_suite(_Config) -> - emqx_common_test_helpers:stop_apps([ - emqx_dashboard, - emqx_bridge_mqtt, - emqx_bridge, - emqx_rule_engine - ]), - ok. - -set_special_configs(emqx_dashboard) -> - emqx_dashboard_api_test_helpers:set_default_config(<<"connector_admin">>); -set_special_configs(_) -> - ok. +end_per_suite(Config) -> + emqx_cth_suite:stop(?config(suite_apps, Config)). init_per_testcase(_, Config) -> - {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000), ok = snabbkaffe:start_trace(), Config. end_per_testcase(_, _Config) -> + ok = unhook_authenticate(), clear_resources(), snabbkaffe:stop(), ok. @@ -187,14 +182,86 @@ clear_resources() -> %%------------------------------------------------------------------------------ %% Testcases %%------------------------------------------------------------------------------ + +t_conf_bridge_authn_anonymous(_) -> + ok = hook_authenticate(), + {ok, 201, _Bridge} = request( + post, + uri(["bridges"]), + ?SERVER_CONF#{ + <<"name">> => <<"t_conf_bridge_anonymous">>, + <<"ingress">> => ?INGRESS_CONF#{<<"pool_size">> => 1} + } + ), + ?assertReceive( + {authenticate, #{username := undefined, password := undefined}} + ). + +t_conf_bridge_authn_password(_) -> + Username1 = <<"user1">>, + Password1 = <<"from-here">>, + ok = hook_authenticate(), + {ok, 201, _Bridge1} = request( + post, + uri(["bridges"]), + ?SERVER_CONF(Username1, Password1)#{ + <<"name">> => <<"t_conf_bridge_authn_password">>, + <<"ingress">> => ?INGRESS_CONF#{<<"pool_size">> => 1} + } + ), + ?assertReceive( + {authenticate, #{username := Username1, password := Password1}} + ). + +t_conf_bridge_authn_passfile(Config) -> + DataDir = ?config(data_dir, Config), + Username2 = <<"user2">>, + PasswordFilename = filename:join(DataDir, "password"), + Password2 = <<"from-there">>, + ok = hook_authenticate(), + {ok, 201, _Bridge2} = request( + post, + uri(["bridges"]), + ?SERVER_CONF(Username2, iolist_to_binary(["file://", PasswordFilename]))#{ + <<"name">> => <<"t_conf_bridge_authn_passfile">>, + <<"ingress">> => ?INGRESS_CONF#{<<"pool_size">> => 1} + } + ), + ?assertReceive( + {authenticate, #{username := Username2, password := Password2}} + ), + ?assertMatch( + {ok, 201, #{ + <<"status">> := <<"disconnected">>, + <<"status_reason">> := <<"#{msg => failed_to_read_secret_file", _/bytes>> + }}, + request_json( + post, + uri(["bridges"]), + ?SERVER_CONF(<<>>, <<"file://im/pretty/sure/theres/no/such/file">>)#{ + <<"name">> => <<"t_conf_bridge_authn_no_passfile">> + } + ) + ). + +hook_authenticate() -> + emqx_hooks:add('client.authenticate', {?MODULE, authenticate, [self()]}, ?HP_HIGHEST). + +unhook_authenticate() -> + emqx_hooks:del('client.authenticate', {?MODULE, authenticate}). + +authenticate(Credential, _, TestRunnerPid) -> + _ = TestRunnerPid ! {authenticate, Credential}, + ignore. + +%%------------------------------------------------------------------------------ + t_mqtt_conn_bridge_ingress(_) -> - User1 = <<"user1">>, %% create an MQTT bridge, using POST {ok, 201, Bridge} = request( post, uri(["bridges"]), - ServerConf = ?SERVER_CONF(User1)#{ - <<"type">> => ?TYPE_MQTT, + ServerConf = ?SERVER_CONF#{ <<"name">> => ?BRIDGE_NAME_INGRESS, <<"ingress">> => ?INGRESS_CONF } @@ -249,7 +316,6 @@ t_mqtt_conn_bridge_ingress(_) -> ok. t_mqtt_conn_bridge_ingress_full_context(_Config) -> - User1 = <<"user1">>, IngressConf = emqx_utils_maps:deep_merge( ?INGRESS_CONF, @@ -258,8 +324,7 @@ t_mqtt_conn_bridge_ingress_full_context(_Config) -> {ok, 201, _Bridge} = request( post, uri(["bridges"]), - ?SERVER_CONF(User1)#{ - <<"type">> => ?TYPE_MQTT, + ?SERVER_CONF#{ <<"name">> => ?BRIDGE_NAME_INGRESS, <<"ingress">> => IngressConf } @@ -297,8 +362,7 @@ t_mqtt_conn_bridge_ingress_shared_subscription(_) -> Ns = lists:seq(1, 10), BridgeName = atom_to_binary(?FUNCTION_NAME), BridgeID = create_bridge( - ?SERVER_CONF(<<>>)#{ - <<"type">> => ?TYPE_MQTT, + ?SERVER_CONF#{ <<"name">> => BridgeName, <<"ingress">> => #{ <<"pool_size">> => PoolSize, @@ -337,8 +401,7 @@ t_mqtt_conn_bridge_ingress_shared_subscription(_) -> t_mqtt_egress_bridge_ignores_clean_start(_) -> BridgeName = atom_to_binary(?FUNCTION_NAME), BridgeID = create_bridge( - ?SERVER_CONF(<<"user1">>)#{ - <<"type">> => ?TYPE_MQTT, + ?SERVER_CONF#{ <<"name">> => BridgeName, <<"egress">> => ?EGRESS_CONF, <<"clean_start">> => false @@ -366,8 +429,7 @@ t_mqtt_egress_bridge_ignores_clean_start(_) -> t_mqtt_conn_bridge_ingress_downgrades_qos_2(_) -> BridgeName = atom_to_binary(?FUNCTION_NAME), BridgeID = create_bridge( - ?SERVER_CONF(<<"user1">>)#{ - <<"type">> => ?TYPE_MQTT, + ?SERVER_CONF#{ <<"name">> => BridgeName, <<"ingress">> => emqx_utils_maps:deep_merge( ?INGRESS_CONF, @@ -392,9 +454,8 @@ t_mqtt_conn_bridge_ingress_downgrades_qos_2(_) -> ok. t_mqtt_conn_bridge_ingress_no_payload_template(_) -> - User1 = <<"user1">>, BridgeIDIngress = create_bridge( - ?SERVER_CONF(User1)#{ + ?SERVER_CONF#{ <<"type">> => ?TYPE_MQTT, <<"name">> => ?BRIDGE_NAME_INGRESS, <<"ingress">> => ?INGRESS_CONF_NO_PAYLOAD_TEMPLATE @@ -428,10 +489,8 @@ t_mqtt_conn_bridge_ingress_no_payload_template(_) -> t_mqtt_conn_bridge_egress(_) -> %% then we add a mqtt connector, using POST - User1 = <<"user1">>, BridgeIDEgress = create_bridge( - ?SERVER_CONF(User1)#{ - <<"type">> => ?TYPE_MQTT, + ?SERVER_CONF#{ <<"name">> => ?BRIDGE_NAME_EGRESS, <<"egress">> => ?EGRESS_CONF } @@ -473,11 +532,8 @@ t_mqtt_conn_bridge_egress(_) -> t_mqtt_conn_bridge_egress_no_payload_template(_) -> %% then we add a mqtt connector, using POST - User1 = <<"user1">>, - BridgeIDEgress = create_bridge( - ?SERVER_CONF(User1)#{ - <<"type">> => ?TYPE_MQTT, + ?SERVER_CONF#{ <<"name">> => ?BRIDGE_NAME_EGRESS, <<"egress">> => ?EGRESS_CONF_NO_PAYLOAD_TEMPLATE } @@ -520,11 +576,9 @@ t_mqtt_conn_bridge_egress_no_payload_template(_) -> ok. t_egress_custom_clientid_prefix(_Config) -> - User1 = <<"user1">>, BridgeIDEgress = create_bridge( - ?SERVER_CONF(User1)#{ + ?SERVER_CONF#{ <<"clientid_prefix">> => <<"my-custom-prefix">>, - <<"type">> => ?TYPE_MQTT, <<"name">> => ?BRIDGE_NAME_EGRESS, <<"egress">> => ?EGRESS_CONF } @@ -545,17 +599,14 @@ t_egress_custom_clientid_prefix(_Config) -> ok. t_mqtt_conn_bridge_ingress_and_egress(_) -> - User1 = <<"user1">>, BridgeIDIngress = create_bridge( - ?SERVER_CONF(User1)#{ - <<"type">> => ?TYPE_MQTT, + ?SERVER_CONF#{ <<"name">> => ?BRIDGE_NAME_INGRESS, <<"ingress">> => ?INGRESS_CONF } ), BridgeIDEgress = create_bridge( - ?SERVER_CONF(User1)#{ - <<"type">> => ?TYPE_MQTT, + ?SERVER_CONF#{ <<"name">> => ?BRIDGE_NAME_EGRESS, <<"egress">> => ?EGRESS_CONF } @@ -627,8 +678,7 @@ t_mqtt_conn_bridge_ingress_and_egress(_) -> t_ingress_mqtt_bridge_with_rules(_) -> BridgeIDIngress = create_bridge( - ?SERVER_CONF(<<"user1">>)#{ - <<"type">> => ?TYPE_MQTT, + ?SERVER_CONF#{ <<"name">> => ?BRIDGE_NAME_INGRESS, <<"ingress">> => ?INGRESS_CONF } @@ -712,8 +762,7 @@ t_ingress_mqtt_bridge_with_rules(_) -> t_egress_mqtt_bridge_with_rules(_) -> BridgeIDEgress = create_bridge( - ?SERVER_CONF(<<"user1">>)#{ - <<"type">> => ?TYPE_MQTT, + ?SERVER_CONF#{ <<"name">> => ?BRIDGE_NAME_EGRESS, <<"egress">> => ?EGRESS_CONF } @@ -789,10 +838,8 @@ t_egress_mqtt_bridge_with_rules(_) -> t_mqtt_conn_bridge_egress_reconnect(_) -> %% then we add a mqtt connector, using POST - User1 = <<"user1">>, BridgeIDEgress = create_bridge( - ?SERVER_CONF(User1)#{ - <<"type">> => ?TYPE_MQTT, + ?SERVER_CONF#{ <<"name">> => ?BRIDGE_NAME_EGRESS, <<"egress">> => ?EGRESS_CONF, <<"resource_opts">> => #{ @@ -897,10 +944,8 @@ t_mqtt_conn_bridge_egress_reconnect(_) -> ok. t_mqtt_conn_bridge_egress_async_reconnect(_) -> - User1 = <<"user1">>, BridgeIDEgress = create_bridge( - ?SERVER_CONF(User1)#{ - <<"type">> => ?TYPE_MQTT, + ?SERVER_CONF#{ <<"name">> => ?BRIDGE_NAME_EGRESS, <<"egress">> => ?EGRESS_CONF, <<"resource_opts">> => #{ @@ -1018,5 +1063,9 @@ request_bridge_metrics(BridgeID) -> {ok, 200, BridgeMetrics} = request(get, uri(["bridges", BridgeID, "metrics"]), []), emqx_utils_json:decode(BridgeMetrics). +request_json(Method, Url, Body) -> + {ok, Code, Response} = request(Method, Url, Body), + {ok, Code, emqx_utils_json:decode(Response)}. + request(Method, Url, Body) -> request(<<"connector_admin">>, Method, Url, Body). diff --git a/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_SUITE_data/password b/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_SUITE_data/password new file mode 100644 index 000000000..d68418fda --- /dev/null +++ b/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_SUITE_data/password @@ -0,0 +1 @@ +from-there diff --git a/apps/emqx_bridge_mysql/test/emqx_bridge_mysql_SUITE.erl b/apps/emqx_bridge_mysql/test/emqx_bridge_mysql_SUITE.erl index a34b65ede..98b957b19 100644 --- a/apps/emqx_bridge_mysql/test/emqx_bridge_mysql_SUITE.erl +++ b/apps/emqx_bridge_mysql/test/emqx_bridge_mysql_SUITE.erl @@ -21,7 +21,6 @@ "DEFAULT CHARSET=utf8MB4;" ). -define(SQL_DROP_TABLE, "DROP TABLE mqtt_test"). --define(SQL_DELETE, "DELETE from mqtt_test"). -define(SQL_SELECT, "SELECT payload FROM mqtt_test"). % DB defaults @@ -112,8 +111,8 @@ end_per_suite(_Config) -> ok. init_per_testcase(_Testcase, Config) -> + connect_and_drop_table(Config), connect_and_create_table(Config), - connect_and_clear_table(Config), delete_bridge(Config), snabbkaffe:start_trace(), Config. @@ -122,9 +121,7 @@ end_per_testcase(_Testcase, Config) -> ProxyHost = ?config(proxy_host, Config), ProxyPort = ?config(proxy_port, Config), emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), - connect_and_clear_table(Config), ok = snabbkaffe:stop(), - delete_bridge(Config), emqx_common_test_helpers:call_janitor(), ok. @@ -323,9 +320,6 @@ connect_and_create_table(Config) -> connect_and_drop_table(Config) -> query_direct_mysql(Config, ?SQL_DROP_TABLE). -connect_and_clear_table(Config) -> - query_direct_mysql(Config, ?SQL_DELETE). - connect_and_get_payload(Config) -> query_direct_mysql(Config, ?SQL_SELECT). @@ -777,28 +771,21 @@ t_table_removed(Config) -> Name = ?config(mysql_name, Config), BridgeType = ?config(mysql_bridge_type, Config), ResourceID = emqx_bridge_resource:resource_id(BridgeType, Name), - ?check_trace( - begin - connect_and_create_table(Config), - ?assertMatch({ok, _}, create_bridge(Config)), - ?retry( - _Sleep = 1_000, - _Attempts = 20, - ?assertEqual({ok, connected}, emqx_resource_manager:health_check(ResourceID)) - ), - connect_and_drop_table(Config), - Val = integer_to_binary(erlang:unique_integer()), - SentData = #{payload => Val, timestamp => 1668602148000}, - Timeout = 1000, - ?assertMatch( - {error, - {unrecoverable_error, - {1146, <<"42S02">>, <<"Table 'mqtt.mqtt_test' doesn't exist">>}}}, - sync_query_resource(Config, {send_message, SentData, [], Timeout}) - ), - ok - end, - [] + connect_and_create_table(Config), + ?assertMatch({ok, _}, create_bridge(Config)), + ?retry( + _Sleep = 1_000, + _Attempts = 20, + ?assertEqual({ok, connected}, emqx_resource_manager:health_check(ResourceID)) + ), + connect_and_drop_table(Config), + Val = integer_to_binary(erlang:unique_integer()), + SentData = #{payload => Val, timestamp => 1668602148000}, + Timeout = 1000, + ?assertMatch( + {error, + {unrecoverable_error, {1146, <<"42S02">>, <<"Table 'mqtt.mqtt_test' doesn't exist">>}}}, + sync_query_resource(Config, {send_message, SentData, [], Timeout}) ), ok. @@ -807,38 +794,31 @@ t_nested_payload_template(Config) -> BridgeType = ?config(mysql_bridge_type, Config), ResourceID = emqx_bridge_resource:resource_id(BridgeType, Name), Value = integer_to_binary(erlang:unique_integer()), - ?check_trace( - begin - connect_and_create_table(Config), - {ok, _} = create_bridge( - Config, - #{ - <<"sql">> => - "INSERT INTO mqtt_test(payload, arrived) " - "VALUES (${payload.value}, FROM_UNIXTIME(${timestamp}/1000))" - } - ), - {ok, #{<<"from">> := [Topic]}} = create_rule_and_action_http(Config), - ?retry( - _Sleep = 1_000, - _Attempts = 20, - ?assertEqual({ok, connected}, emqx_resource_manager:health_check(ResourceID)) - ), - %% send message via rule action - Payload = emqx_utils_json:encode(#{value => Value}), - Message = emqx_message:make(Topic, Payload), - {_, {ok, _}} = - ?wait_async_action( - emqx:publish(Message), - #{?snk_kind := mysql_connector_query_return}, - 10_000 - ), - ?assertEqual( - {ok, [<<"payload">>], [[Value]]}, - connect_and_get_payload(Config) - ), - ok - end, - [] + {ok, _} = create_bridge( + Config, + #{ + <<"sql">> => + "INSERT INTO mqtt_test(payload, arrived) " + "VALUES (${payload.value}, FROM_UNIXTIME(${timestamp}/1000))" + } + ), + {ok, #{<<"from">> := [Topic]}} = create_rule_and_action_http(Config), + ?retry( + _Sleep = 1_000, + _Attempts = 20, + ?assertEqual({ok, connected}, emqx_resource_manager:health_check(ResourceID)) + ), + %% send message via rule action + Payload = emqx_utils_json:encode(#{value => Value}), + Message = emqx_message:make(Topic, Payload), + {_, {ok, _}} = + ?wait_async_action( + emqx:publish(Message), + #{?snk_kind := mysql_connector_query_return}, + 10_000 + ), + ?assertEqual( + {ok, [<<"payload">>], [[Value]]}, + connect_and_get_payload(Config) ), ok. diff --git a/apps/emqx_bridge_oracle/test/emqx_bridge_oracle_SUITE.erl b/apps/emqx_bridge_oracle/test/emqx_bridge_oracle_SUITE.erl index 6b949b047..878ae2e1d 100644 --- a/apps/emqx_bridge_oracle/test/emqx_bridge_oracle_SUITE.erl +++ b/apps/emqx_bridge_oracle/test/emqx_bridge_oracle_SUITE.erl @@ -16,7 +16,6 @@ -define(APPS, [emqx_bridge, emqx_resource, emqx_rule_engine, emqx_oracle, emqx_bridge_oracle]). -define(SID, "XE"). -define(RULE_TOPIC, "mqtt/rule"). -% -define(RULE_TOPIC_BIN, <>). %%------------------------------------------------------------------------------ %% CT boilerplate @@ -33,9 +32,6 @@ groups() -> {plain, AllTCs} ]. -only_once_tests() -> - [t_create_via_http]. - init_per_suite(Config) -> Config. diff --git a/apps/emqx_bridge_pgsql/test/emqx_bridge_pgsql_SUITE.erl b/apps/emqx_bridge_pgsql/test/emqx_bridge_pgsql_SUITE.erl index 156d4bd16..722489ba6 100644 --- a/apps/emqx_bridge_pgsql/test/emqx_bridge_pgsql_SUITE.erl +++ b/apps/emqx_bridge_pgsql/test/emqx_bridge_pgsql_SUITE.erl @@ -183,31 +183,33 @@ pgsql_config(BridgeType, Config) -> end, QueryMode = ?config(query_mode, Config), TlsEnabled = ?config(enable_tls, Config), + %% NOTE: supplying password through a file here, to verify that it works. + Password = create_passfile(BridgeType, Config), ConfigString = io_lib:format( - "bridges.~s.~s {\n" - " enable = true\n" - " server = ~p\n" - " database = ~p\n" - " username = ~p\n" - " password = ~p\n" - " sql = ~p\n" - " resource_opts = {\n" - " request_ttl = 500ms\n" - " batch_size = ~b\n" - " query_mode = ~s\n" - " }\n" - " ssl = {\n" - " enable = ~w\n" - " }\n" - "}", + "bridges.~s.~s {" + "\n enable = true" + "\n server = ~p" + "\n database = ~p" + "\n username = ~p" + "\n password = ~p" + "\n sql = ~p" + "\n resource_opts = {" + "\n request_ttl = 500ms" + "\n batch_size = ~b" + "\n query_mode = ~s" + "\n }" + "\n ssl = {" + "\n enable = ~w" + "\n }" + "\n }", [ BridgeType, Name, Server, ?PGSQL_DATABASE, ?PGSQL_USERNAME, - ?PGSQL_PASSWORD, + Password, ?SQL_BRIDGE, BatchSize, QueryMode, @@ -216,6 +218,12 @@ pgsql_config(BridgeType, Config) -> ), {Name, parse_and_check(ConfigString, BridgeType, Name)}. +create_passfile(BridgeType, Config) -> + Filename = binary_to_list(BridgeType) ++ ".passfile", + Filepath = filename:join(?config(priv_dir, Config), Filename), + ok = file:write_file(Filepath, ?PGSQL_PASSWORD), + "file://" ++ Filepath. + parse_and_check(ConfigString, BridgeType, Name) -> {ok, RawConf} = hocon:binary(ConfigString, #{format => map}), hocon_tconf:check_plain(emqx_bridge_schema, RawConf, #{required => false, atom_key => false}), @@ -379,7 +387,9 @@ t_setup_via_http_api_and_publish(Config) -> QueryMode = ?config(query_mode, Config), PgsqlConfig = PgsqlConfig0#{ <<"name">> => Name, - <<"type">> => BridgeType + <<"type">> => BridgeType, + %% NOTE: using literal passwords with HTTP API requests. + <<"password">> => <> }, ?assertMatch( {ok, _}, diff --git a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.app.src b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.app.src index b012874f8..c9abebf8b 100644 --- a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.app.src +++ b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_pulsar, [ {description, "EMQX Pulsar Bridge"}, - {vsn, "0.1.7"}, + {vsn, "0.1.8"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.erl b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.erl index beb8452b2..c7b378617 100644 --- a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.erl +++ b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.erl @@ -170,21 +170,17 @@ fields(auth_basic) -> [ {username, mk(binary(), #{required => true, desc => ?DESC("auth_basic_username")})}, {password, - mk(binary(), #{ + emqx_schema_secret:mk(#{ required => true, - desc => ?DESC("auth_basic_password"), - sensitive => true, - converter => fun emqx_schema:password_converter/2 + desc => ?DESC("auth_basic_password") })} ]; fields(auth_token) -> [ {jwt, - mk(binary(), #{ + emqx_schema_secret:mk(#{ required => true, - desc => ?DESC("auth_token_jwt"), - sensitive => true, - converter => fun emqx_schema:password_converter/2 + desc => ?DESC("auth_token_jwt") })} ]; fields("get_" ++ Type) -> diff --git a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_impl_producer.erl b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_impl_producer.erl index 33ac83ee1..fed0142c5 100644 --- a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_impl_producer.erl +++ b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_impl_producer.erl @@ -78,7 +78,6 @@ query_mode(_Config) -> -spec on_start(resource_id(), config()) -> {ok, state()}. on_start(InstanceId, Config) -> #{ - authentication := _Auth, bridge_name := BridgeName, servers := Servers0, ssl := SSL @@ -263,12 +262,14 @@ conn_opts(#{authentication := none}) -> #{}; conn_opts(#{authentication := #{username := Username, password := Password}}) -> #{ - auth_data => iolist_to_binary([Username, <<":">>, Password]), + %% TODO: teach `pulsar` to accept 0-arity closures as passwords. + auth_data => iolist_to_binary([Username, <<":">>, emqx_secret:unwrap(Password)]), auth_method_name => <<"basic">> }; conn_opts(#{authentication := #{jwt := JWT}}) -> #{ - auth_data => JWT, + %% TODO: teach `pulsar` to accept 0-arity closures as passwords. + auth_data => emqx_secret:unwrap(JWT), auth_method_name => <<"token">> }. diff --git a/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq_connector.erl b/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq_connector.erl index 2af1c16c8..2e4074f79 100644 --- a/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq_connector.erl +++ b/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq_connector.erl @@ -74,7 +74,7 @@ fields(config) -> desc => ?DESC("username") } )}, - {password, fun emqx_connector_schema_lib:password_required/1}, + {password, emqx_connector_schema_lib:password_field(#{required => true})}, {pool_size, hoconsc:mk( typerefl:pos_integer(), @@ -196,7 +196,6 @@ on_start( #{ pool_size := PoolSize, payload_template := PayloadTemplate, - password := Password, delivery_mode := InitialDeliveryMode } = InitialConfig ) -> @@ -206,7 +205,6 @@ on_start( persistent -> 2 end, Config = InitialConfig#{ - password => emqx_secret:wrap(Password), delivery_mode => DeliveryMode }, ?SLOG(info, #{ @@ -242,13 +240,11 @@ on_start( ok -> {ok, State}; {error, Reason} -> - LogMessage = - #{ - msg => "rabbitmq_connector_start_failed", - error_reason => Reason, - config => emqx_utils:redact(Config) - }, - ?SLOG(info, LogMessage), + ?SLOG(info, #{ + msg => "rabbitmq_connector_start_failed", + error_reason => Reason, + config => emqx_utils:redact(Config) + }), {error, Reason} end. @@ -321,6 +317,7 @@ create_rabbitmq_connection_and_channel(Config) -> heartbeat := Heartbeat, wait_for_publish_confirmations := WaitForPublishConfirmations } = Config, + %% TODO: teach `amqp` to accept 0-arity closures as passwords. Password = emqx_secret:unwrap(WrappedPassword), SSLOptions = case maps:get(ssl, Config, #{}) of diff --git a/apps/emqx_bridge_rabbitmq/test/emqx_bridge_rabbitmq_connector_SUITE.erl b/apps/emqx_bridge_rabbitmq/test/emqx_bridge_rabbitmq_connector_SUITE.erl index 106a4d67b..689c39dc5 100644 --- a/apps/emqx_bridge_rabbitmq/test/emqx_bridge_rabbitmq_connector_SUITE.erl +++ b/apps/emqx_bridge_rabbitmq/test/emqx_bridge_rabbitmq_connector_SUITE.erl @@ -10,6 +10,7 @@ -include("emqx_connector.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("stdlib/include/assert.hrl"). +-include_lib("common_test/include/ct.hrl"). -include_lib("amqp_client/include/amqp_client.hrl"). %% This test SUITE requires a running RabbitMQ instance. If you don't want to @@ -26,6 +27,9 @@ rabbit_mq_host() -> rabbit_mq_port() -> 5672. +rabbit_mq_password() -> + <<"guest">>. + rabbit_mq_exchange() -> <<"test_exchange">>. @@ -45,12 +49,12 @@ init_per_suite(Config) -> ) of true -> - ok = emqx_common_test_helpers:start_apps([emqx_conf]), - ok = emqx_connector_test_helpers:start_apps([emqx_resource]), - {ok, _} = application:ensure_all_started(emqx_connector), - {ok, _} = application:ensure_all_started(amqp_client), + Apps = emqx_cth_suite:start( + [emqx_conf, emqx_connector, emqx_bridge_rabbitmq], + #{work_dir => emqx_cth_suite:work_dir(Config)} + ), ChannelConnection = setup_rabbit_mq_exchange_and_queue(), - [{channel_connection, ChannelConnection} | Config]; + [{channel_connection, ChannelConnection}, {suite_apps, Apps} | Config]; false -> case os:getenv("IS_CI") of "yes" -> @@ -106,13 +110,11 @@ end_per_suite(Config) -> connection := Connection, channel := Channel } = get_channel_connection(Config), - ok = emqx_common_test_helpers:stop_apps([emqx_conf]), - ok = emqx_connector_test_helpers:stop_apps([emqx_resource]), - _ = application:stop(emqx_connector), %% Close the channel ok = amqp_channel:close(Channel), %% Close the connection - ok = amqp_connection:close(Connection). + ok = amqp_connection:close(Connection), + ok = emqx_cth_suite:stop(?config(suite_apps, Config)). % %%------------------------------------------------------------------------------ % %% Testcases @@ -125,23 +127,31 @@ t_lifecycle(Config) -> Config ). +t_start_passfile(Config) -> + ResourceID = atom_to_binary(?FUNCTION_NAME), + PasswordFilename = filename:join(?config(priv_dir, Config), "passfile"), + ok = file:write_file(PasswordFilename, rabbit_mq_password()), + InitialConfig = rabbitmq_config(#{ + password => iolist_to_binary(["file://", PasswordFilename]) + }), + ?assertMatch( + #{status := connected}, + create_local_resource(ResourceID, check_config(InitialConfig)) + ), + ?assertEqual( + ok, + emqx_resource:remove_local(ResourceID) + ). + perform_lifecycle_check(ResourceID, InitialConfig, TestConfig) -> #{ channel := Channel } = get_channel_connection(TestConfig), - {ok, #{config := CheckedConfig}} = - emqx_resource:check_config(emqx_bridge_rabbitmq_connector, InitialConfig), - {ok, #{ + CheckedConfig = check_config(InitialConfig), + #{ state := #{poolname := PoolName} = State, status := InitialStatus - }} = - emqx_resource:create_local( - ResourceID, - ?CONNECTOR_RESOURCE_GROUP, - emqx_bridge_rabbitmq_connector, - CheckedConfig, - #{} - ), + } = create_local_resource(ResourceID, CheckedConfig), ?assertEqual(InitialStatus, connected), %% Instance should match the state and status of the just started resource {ok, ?CONNECTOR_RESOURCE_GROUP, #{ @@ -184,6 +194,21 @@ perform_lifecycle_check(ResourceID, InitialConfig, TestConfig) -> % %% Helpers % %%------------------------------------------------------------------------------ +check_config(Config) -> + {ok, #{config := CheckedConfig}} = + emqx_resource:check_config(emqx_bridge_rabbitmq_connector, Config), + CheckedConfig. + +create_local_resource(ResourceID, CheckedConfig) -> + {ok, Bridge} = emqx_resource:create_local( + ResourceID, + ?CONNECTOR_RESOURCE_GROUP, + emqx_bridge_rabbitmq_connector, + CheckedConfig, + #{} + ), + Bridge. + perform_query(PoolName, Channel) -> %% Send message to queue: ok = emqx_resource:query(PoolName, {query, test_data()}), @@ -216,16 +241,19 @@ receive_simple_test_message(Channel) -> end. rabbitmq_config() -> + rabbitmq_config(#{}). + +rabbitmq_config(Overrides) -> Config = #{ server => rabbit_mq_host(), port => 5672, username => <<"guest">>, - password => <<"guest">>, + password => rabbit_mq_password(), exchange => rabbit_mq_exchange(), routing_key => rabbit_mq_routing_key() }, - #{<<"config">> => Config}. + #{<<"config">> => maps:merge(Config, Overrides)}. test_data() -> #{<<"msg_field">> => <<"Hello">>}. diff --git a/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq.app.src b/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq.app.src index e158a2e46..38c00e7ee 100644 --- a/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq.app.src +++ b/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_rocketmq, [ {description, "EMQX Enterprise RocketMQ Bridge"}, - {vsn, "0.1.3"}, + {vsn, "0.1.4"}, {registered, []}, {applications, [kernel, stdlib, emqx_resource, rocketmq]}, {env, []}, diff --git a/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq_connector.erl b/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq_connector.erl index dbac88249..81045ade4 100644 --- a/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq_connector.erl +++ b/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq_connector.erl @@ -48,13 +48,8 @@ fields(config) -> binary(), #{default => <<>>, desc => ?DESC("access_key")} )}, - {secret_key, - mk( - binary(), - #{default => <<>>, desc => ?DESC("secret_key"), sensitive => true} - )}, - {security_token, - mk(binary(), #{default => <<>>, desc => ?DESC(security_token), sensitive => true})}, + {secret_key, emqx_schema_secret:mk(#{default => <<>>, desc => ?DESC("secret_key")})}, + {security_token, emqx_schema_secret:mk(#{default => <<>>, desc => ?DESC(security_token)})}, {sync_timeout, mk( emqx_schema:timeout_duration(), @@ -294,21 +289,19 @@ make_producer_opts( acl_info => emqx_secret:wrap(ACLInfo) }. -acl_info(<<>>, <<>>, <<>>) -> +acl_info(<<>>, _, _) -> #{}; -acl_info(AccessKey, SecretKey, <<>>) when is_binary(AccessKey), is_binary(SecretKey) -> - #{ +acl_info(AccessKey, SecretKey, SecurityToken) when is_binary(AccessKey) -> + Info = #{ access_key => AccessKey, - secret_key => SecretKey - }; -acl_info(AccessKey, SecretKey, SecurityToken) when - is_binary(AccessKey), is_binary(SecretKey), is_binary(SecurityToken) --> - #{ - access_key => AccessKey, - secret_key => SecretKey, - security_token => SecurityToken - }; + secret_key => emqx_maybe:define(emqx_secret:unwrap(SecretKey), <<>>) + }, + case emqx_maybe:define(emqx_secret:unwrap(SecurityToken), <<>>) of + <<>> -> + Info; + Token -> + Info#{security_token => Token} + end; acl_info(_, _, _) -> #{}. diff --git a/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver.app.src b/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver.app.src index 1664fee59..331f9c29f 100644 --- a/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver.app.src +++ b/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_sqlserver, [ {description, "EMQX Enterprise SQL Server Bridge"}, - {vsn, "0.1.4"}, + {vsn, "0.1.5"}, {registered, []}, {applications, [kernel, stdlib, emqx_resource, odbc]}, {env, []}, diff --git a/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver_connector.erl b/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver_connector.erl index 6db8c2877..a87e71e31 100644 --- a/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver_connector.erl +++ b/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver_connector.erl @@ -199,7 +199,7 @@ on_start( Options = [ {server, to_bin(Server)}, {username, Username}, - {password, emqx_secret:wrap(maps:get(password, Config, ""))}, + {password, maps:get(password, Config, emqx_secret:wrap(""))}, {driver, Driver}, {database, Database}, {pool_size, PoolSize} diff --git a/apps/emqx_bridge_sqlserver/test/emqx_bridge_sqlserver_SUITE.erl b/apps/emqx_bridge_sqlserver/test/emqx_bridge_sqlserver_SUITE.erl index 101ead838..62214cb5e 100644 --- a/apps/emqx_bridge_sqlserver/test/emqx_bridge_sqlserver_SUITE.erl +++ b/apps/emqx_bridge_sqlserver/test/emqx_bridge_sqlserver_SUITE.erl @@ -130,7 +130,9 @@ end_per_group(_Group, _Config) -> ok. init_per_suite(Config) -> - Config. + Passfile = filename:join(?config(priv_dir, Config), "passfile"), + ok = file:write_file(Passfile, <>), + [{sqlserver_passfile, Passfile} | Config]. end_per_suite(_Config) -> emqx_mgmt_api_test_util:end_suite(), @@ -193,7 +195,9 @@ t_setup_via_http_api_and_publish(Config) -> SQLServerConfig0 = ?config(sqlserver_config, Config), SQLServerConfig = SQLServerConfig0#{ <<"name">> => Name, - <<"type">> => BridgeType + <<"type">> => BridgeType, + %% NOTE: using literal password with HTTP API requests. + <<"password">> => <> }, ?assertMatch( {ok, _}, @@ -449,6 +453,7 @@ sqlserver_config(BridgeType, Config) -> Name = atom_to_binary(?MODULE), BatchSize = batch_size(Config), QueryMode = ?config(query_mode, Config), + Passfile = ?config(sqlserver_passfile, Config), ConfigString = io_lib:format( "bridges.~s.~s {\n" @@ -472,7 +477,7 @@ sqlserver_config(BridgeType, Config) -> Server, ?SQL_SERVER_DATABASE, ?SQL_SERVER_USERNAME, - ?SQL_SERVER_PASSWORD, + "file://" ++ Passfile, ?SQL_BRIDGE, ?SQL_SERVER_DRIVER, BatchSize, diff --git a/apps/emqx_bridge_syskeeper/BSL.txt b/apps/emqx_bridge_syskeeper/BSL.txt new file mode 100644 index 000000000..0acc0e696 --- /dev/null +++ b/apps/emqx_bridge_syskeeper/BSL.txt @@ -0,0 +1,94 @@ +Business Source License 1.1 + +Licensor: Hangzhou EMQ Technologies Co., Ltd. +Licensed Work: EMQX Enterprise Edition + The Licensed Work is (c) 2023 + Hangzhou EMQ Technologies Co., Ltd. +Additional Use Grant: Students and educators are granted right to copy, + modify, and create derivative work for research + or education. +Change Date: 2027-02-01 +Change License: Apache License, Version 2.0 + +For information about alternative licensing arrangements for the Software, +please contact Licensor: https://www.emqx.com/en/contact + +Notice + +The Business Source License (this document, or the “License”) is not an Open +Source license. However, the Licensed Work will eventually be made available +under an Open Source License, as stated in this License. + +License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved. +“Business Source License” is a trademark of MariaDB Corporation Ab. + +----------------------------------------------------------------------------- + +Business Source License 1.1 + +Terms + +The Licensor hereby grants you the right to copy, modify, create derivative +works, redistribute, and make non-production use of the Licensed Work. The +Licensor may make an Additional Use Grant, above, permitting limited +production use. + +Effective on the Change Date, or the fourth anniversary of the first publicly +available distribution of a specific version of the Licensed Work under this +License, whichever comes first, the Licensor hereby grants you rights under +the terms of the Change License, and the rights granted in the paragraph +above terminate. + +If your use of the Licensed Work does not comply with the requirements +currently in effect as described in this License, you must purchase a +commercial license from the Licensor, its affiliated entities, or authorized +resellers, or you must refrain from using the Licensed Work. + +All copies of the original and modified Licensed Work, and derivative works +of the Licensed Work, are subject to this License. This License applies +separately for each version of the Licensed Work and the Change Date may vary +for each version of the Licensed Work released by Licensor. + +You must conspicuously display this License on each original or modified copy +of the Licensed Work. If you receive the Licensed Work in original or +modified form from a third party, the terms and conditions set forth in this +License apply to your use of that work. + +Any use of the Licensed Work in violation of this License will automatically +terminate your rights under this License for the current and all other +versions of the Licensed Work. + +This License does not grant you any right in any trademark or logo of +Licensor or its affiliates (provided that you may use a trademark or logo of +Licensor as expressly required by this License). + +TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON +AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS, +EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND +TITLE. + +MariaDB hereby grants you permission to use this License’s text to license +your works, and to refer to it using the trademark “Business Source License”, +as long as you comply with the Covenants of Licensor below. + +Covenants of Licensor + +In consideration of the right to use this License’s text and the “Business +Source License” name and trademark, Licensor covenants to MariaDB, and to all +other recipients of the licensed work to be provided by Licensor: + +1. To specify as the Change License the GPL Version 2.0 or any later version, + or a license that is compatible with GPL Version 2.0 or a later version, + where “compatible” means that software provided under the Change License can + be included in a program with software provided under GPL Version 2.0 or a + later version. Licensor may specify additional Change Licenses without + limitation. + +2. To either: (a) specify an additional grant of rights to use that does not + impose any additional restriction on the right granted in this License, as + the Additional Use Grant; or (b) insert the text “None”. + +3. To specify a Change Date. + +4. Not to modify this License in any other way. diff --git a/apps/emqx_bridge_syskeeper/README.md b/apps/emqx_bridge_syskeeper/README.md new file mode 100644 index 000000000..328fd488e --- /dev/null +++ b/apps/emqx_bridge_syskeeper/README.md @@ -0,0 +1,30 @@ +# EMQX Syskeeper Bridge + +Nari Syskeeper 2000 is a one-way Physical Isolation Net Gap. + +The application is used to connect EMQX and Syskeeper. +Users can create a rule and quickly ingest IoT data to the Syskeeper by leveraging +[EMQX Rules](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html). + +# Documentation + +- Refer to [Rules engine](https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html) + for the EMQX rules engine introduction. + +# HTTP APIs + +- Several APIs are provided for bridge management, which includes create bridge, + update bridge, get bridge, stop or restart bridge and list bridges etc. + + Refer to [API Docs - Bridges](https://docs.emqx.com/en/enterprise/v5.0/admin/api-docs.html#tag/Bridges) + for more detailed information. + + +# Contributing + +Please see our [contributing.md](../../CONTRIBUTING.md). + + +# License + +EMQ Business Source License 1.1, refer to [LICENSE](BSL.txt). diff --git a/apps/emqx_bridge_syskeeper/doc/protocol_v1.md b/apps/emqx_bridge_syskeeper/doc/protocol_v1.md new file mode 100644 index 000000000..ca73c300d --- /dev/null +++ b/apps/emqx_bridge_syskeeper/doc/protocol_v1.md @@ -0,0 +1,370 @@ + +# Table of Contents + +1. [Packet Format](#orgb2a43d1) +2. [Common Header](#org5ca4c69) + 1. [Types](#org240efb3) + 2. [Shared Flags](#org804fcce) +3. [Handshake Packet](#org6a73ea8) +4. [Forward Packet](#org39c753e) + 1. [Flags](#org5177d26) + 2. [Payload](#orgb29cbd7) + 1. [Message Content map structure](#org75acfe6) +5. [Heartbeat Packet](#org388b69a) + + + + +# Packet Format + + + + + + + + + + + + + + + + + + +
+  bytes  + +   0   + +   1   + +   2   + +   3   + +         5         + +     6 .. end     +
+         + +     variable length     + +   common header   + +     payload      +
+ +The length of the remaining part(common header + payload) is indicated by the Length Header of each packet + + + + +# Common Header + + + + + + + + + + + + + + + + + + + +
+  bits  + +   0   + +   1   + +   2   + +   3   + +   4   + +   5   + +   6   + +   7   +
+        + +       packet type       + +      shared flags       +
+ + + + +## Types + + + + + + + + + + + + + + + + + + + +
+    type    + +    usage    +
+     0      + +  handshake  +
+     1      + +   forward   +
+     2      + +  heartbeat  +
+ + + + +## Shared Flags + +The usage of each bit is determined by the type of packet + + + + +# Handshake Packet + + + + + + + + + + + + + +
+  bytes  + +        0        + +        1        +
+         + +  common header  + +     version     +
+ + + + +# Forward Packet + + + + + + + + + + + + + + + + + + + + + + + + + +
+  bits  + +  0  + +  1  + +  2  + +  3  + +  4  + +  5  + +  6  + +   7   + +     ...     +
+       
+       
+        + 		+                
+   packet type  
+                 + 		+             + +  ACK  + +            
+   payload  
+             +
+   forward flags   +
+ + + + +## Flags + + + + + + + + + + + +
+  flag  + +                    usage                    +
+  ACK   + +       This packet need a ACK response       +
+ + + + +## Payload + + + + + + + + + + + + + + + + + +
+  bytes  + +   0   + +   ..    + +   n   + +  n+1  + +  ..   + +   x   +
+         + +   Content Length    + +  Message Content  +
+ +- Content length is a variable length number. +- Message content is a list in an opaque binary format whose element is a map structure + + + + +### Message Content map structure + + { + id: "0006081CCFF3D48F03C10000058B0000", // unique message id + qos: 1, + flags: {dup: false, retain: false}, + from: "clientid", + topic: "t/1", + payload: "hello, world", + timestamp: 1697786555281 + } + + + + +# Heartbeat Packet + + + + + + + + + + + +
+  bytes  + +        0        +
+         + +  common header  +
+ diff --git a/apps/emqx_bridge_syskeeper/doc/protocol_v1.org b/apps/emqx_bridge_syskeeper/doc/protocol_v1.org new file mode 100644 index 000000000..12d0fe850 --- /dev/null +++ b/apps/emqx_bridge_syskeeper/doc/protocol_v1.org @@ -0,0 +1,80 @@ +* Packet Format + +-------+-----+-----+-----+-----+-----------------+----------------+ + | bytes | 0 | 1 | 2 | 3 | 5 | 6 .. end | + +-------+-----+-----+-----+-----+-----------------+----------------+ + | | variable length | common header | payload | + +-------+-----------------------+-----------------+----------------+ + + The length of the remaining part(common header + payload) is indicated by the Length Header of each packet + +* Common Header + +------+-----+-----+-----+-----+-----+-----+-----+-----+ + | bits | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | + +------+-----+-----+-----+-----+-----+-----+-----+-----+ + | | packet type | shared flags | + +------+-----------------------+-----------------------+ +** Types + +----------+-----------+ + | type | usage | + +----------+-----------+ + | 0 | handshake | + +----------+-----------+ + | 1 | forward | + +----------+-----------+ + | 2 | heartbeat | + +----------+-----------+ +** Shared Flags + The usage of each bit is determined by the type of packet +* Handshake Packet + +-------+---------------+---------------+ + | bytes | 0 | 1 | + +-------+---------------+---------------+ + | | common header | version | + +-------+---------------+---------------+ +* Forward Packet + +------+---+---+---+---+---+---+---+-----+-----------+ + | bits | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | ... | + +------+---+---+---+---+---+---+---+-----+-----------+ + | | | | ACK | | + | | packet type +-----------+-----+ payload | + | | | forward flags | | + +------+---------------+-----------------+-----------+ + +** Flags + +------+-------------------------------------------+ + | flag | usage | + +------+-------------------------------------------+ + | ACK | This packet need a ACK response | + +------+-------------------------------------------+ + +** Payload + +-------+-----+-------+-----+-----+-----+-----+ + | bytes | 0 | .. | n | n+1 | .. | x | + +-------+-----+-------+-----+-----+-----+-----+ + | | Content Length | Message Content | + +-------+-------------------+-----------------+ + + + Content length is a variable length number. + + Message content is a list in an opaque binary format whose element is a map structure + +*** Message Content map structure + +#+begin_src json + { + id: "0006081CCFF3D48F03C10000058B0000", // unique message id + qos: 1, + flags: {dup: false, retain: false}, + from: "clientid", + topic: "t/1", + payload: "hello, world", + timestamp: 1697786555281 + } +#+end_src + +* Heartbeat Packet + + +-------+---------------+ + | bytes | 0 | + +-------+---------------+ + | | common header | + +-------+---------------+ diff --git a/apps/emqx_bridge_syskeeper/docker-ct b/apps/emqx_bridge_syskeeper/docker-ct new file mode 100644 index 000000000..80f0d394b --- /dev/null +++ b/apps/emqx_bridge_syskeeper/docker-ct @@ -0,0 +1 @@ +toxiproxy diff --git a/apps/emqx_bridge_syskeeper/include/emqx_bridge_syskeeper.hrl b/apps/emqx_bridge_syskeeper/include/emqx_bridge_syskeeper.hrl new file mode 100644 index 000000000..4e14fafb0 --- /dev/null +++ b/apps/emqx_bridge_syskeeper/include/emqx_bridge_syskeeper.hrl @@ -0,0 +1,15 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-ifndef(EMQX_BRIDGE_SYSKEEPER). +-define(EMQX_BRIDGE_SYSKEEPER, true). + +-define(TYPE_HANDSHAKE, 0). +-define(TYPE_FORWARD, 1). +-define(TYPE_HEARTBEAT, 2). + +-type packet_type() :: handshake | forward | heartbeat. +-type packet_data() :: none | binary() | [binary()]. +-type packet_type_val() :: ?TYPE_HANDSHAKE..?TYPE_HEARTBEAT. + +-endif. diff --git a/apps/emqx_bridge_syskeeper/rebar.config b/apps/emqx_bridge_syskeeper/rebar.config new file mode 100644 index 000000000..31879d9ce --- /dev/null +++ b/apps/emqx_bridge_syskeeper/rebar.config @@ -0,0 +1,6 @@ +%% -*- mode: erlang; -*- +{erl_opts, [debug_info]}. +{deps, [ {emqx_connector, {path, "../../apps/emqx_connector"}} + , {emqx_resource, {path, "../../apps/emqx_resource"}} + , {emqx_bridge, {path, "../../apps/emqx_bridge"}} + ]}. diff --git a/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper.app.src b/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper.app.src new file mode 100644 index 000000000..3c7995cb7 --- /dev/null +++ b/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper.app.src @@ -0,0 +1,13 @@ +{application, emqx_bridge_syskeeper, [ + {description, "EMQX Enterprise Data bridge for Syskeeper"}, + {vsn, "0.1.0"}, + {registered, []}, + {applications, [ + kernel, + stdlib, + emqx_resource + ]}, + {env, []}, + {modules, []}, + {links, []} +]}. diff --git a/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper.erl b/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper.erl new file mode 100644 index 000000000..55e3d08b9 --- /dev/null +++ b/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper.erl @@ -0,0 +1,146 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_bridge_syskeeper). + +-include_lib("typerefl/include/types.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). +-include_lib("emqx_bridge/include/emqx_bridge.hrl"). +-include_lib("emqx_resource/include/emqx_resource.hrl"). + +-import(hoconsc, [mk/2, enum/1, ref/2]). + +-export([ + bridge_v2_examples/1, + values/1 +]). + +-export([ + namespace/0, + roots/0, + fields/1, + desc/1 +]). + +%% ------------------------------------------------------------------------------------------------- +%% api +bridge_v2_examples(Method) -> + [ + #{ + <<"syskeeper_forwarder">> => #{ + summary => <<"Syskeeper Forwarder Bridge">>, + value => values(Method) + } + } + ]. + +values(get) -> + maps:merge( + #{ + status => <<"connected">>, + node_status => [ + #{ + node => <<"emqx@localhost">>, + status => <<"connected">> + } + ] + }, + values(post) + ); +values(post) -> + maps:merge( + #{ + name => <<"syskeeper_forwarder">>, + type => <<"syskeeper_forwarder">> + }, + values(put) + ); +values(put) -> + #{ + enable => true, + connector => <<"syskeeper_forwarder">>, + parameters => #{ + target_topic => <<"${topic}">>, + target_qos => <<"-1">>, + template => <<"${payload}">> + }, + resource_opts => #{ + worker_pool_size => 16 + } + }. + +%% ------------------------------------------------------------------------------------------------- +%% Hocon Schema Definitions +namespace() -> "syskeeper". + +roots() -> []. + +fields(config) -> + [ + {enable, mk(boolean(), #{desc => ?DESC("config_enable"), default => true})}, + {description, emqx_schema:description_schema()}, + {connector, + mk(binary(), #{ + desc => ?DESC(emqx_connector_schema, "connector_field"), required => true + })}, + {parameters, + mk( + ref(?MODULE, "parameters"), + #{required => true, desc => ?DESC("parameters")} + )}, + {local_topic, mk(binary(), #{required => false, desc => ?DESC(mqtt_topic)})}, + {resource_opts, + mk( + ref(?MODULE, "creation_opts"), + #{ + required => false, + default => #{}, + desc => ?DESC(emqx_resource_schema, <<"resource_opts">>) + } + )} + ]; +fields("parameters") -> + [ + {target_topic, + mk( + binary(), + #{desc => ?DESC("target_topic"), default => <<"${topic}">>} + )}, + {target_qos, + mk( + range(-1, 2), + #{desc => ?DESC("target_qos"), default => -1} + )}, + {template, + mk( + binary(), + #{desc => ?DESC("template"), default => <<"${payload}">>} + )} + ]; +fields("creation_opts") -> + emqx_resource_schema:create_opts([{request_ttl, #{default => infinity}}]); +fields("post") -> + [type_field(), name_field() | fields(config)]; +fields("put") -> + fields(config); +fields("get") -> + emqx_bridge_schema:status_fields() ++ fields("post"). + +desc(config) -> + ?DESC("desc_config"); +desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" -> + ["Configuration for Syskeeper using `", string:to_upper(Method), "` method."]; +desc("parameters") -> + ?DESC("parameters"); +desc("creation_opts" = Name) -> + emqx_resource_schema:desc(Name); +desc(_) -> + undefined. + +%% ------------------------------------------------------------------------------------------------- + +type_field() -> + {type, mk(enum([syskeeper_forwarder]), #{required => true, desc => ?DESC("desc_type")})}. + +name_field() -> + {name, mk(binary(), #{required => true, desc => ?DESC("desc_name")})}. diff --git a/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_client.erl b/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_client.erl new file mode 100644 index 000000000..18822886f --- /dev/null +++ b/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_client.erl @@ -0,0 +1,180 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_bridge_syskeeper_client). + +-behaviour(gen_server). + +%% API +-export([ + start_link/1, + forward/3, + heartbeat/2 +]). + +%% gen_server callbacks +-export([ + init/1, + handle_call/3, + handle_cast/2, + handle_info/2, + terminate/2, + code_change/3, + format_status/2 +]). + +-include("emqx_bridge_syskeeper.hrl"). + +-type state() :: #{ + ack_mode := need_ack | no_ack, + ack_timeout := timer:time(), + socket := undefined | inet:socket(), + frame_state := emqx_bridge_syskeeper_frame:state(), + last_error := undefined | tuple() +}. + +-type send_result() :: {ok, state()} | {error, term()}. + +%% ------------------------------------------------------------------------------------------------- +%% API +forward(Pid, Msg, Timeout) -> + call(Pid, {?FUNCTION_NAME, Msg}, Timeout). + +heartbeat(Pid, Timeout) -> + ok =:= call(Pid, ?FUNCTION_NAME, Timeout). + +%% ------------------------------------------------------------------------------------------------- +%% Starts Bridge which transfer data to Syskeeper + +start_link(Options) -> + gen_server:start_link(?MODULE, Options, []). + +%% ------------------------------------------------------------------------------------------------- +%%% gen_server callbacks + +%% Initialize syskeeper client +init(#{ack_timeout := AckTimeout, ack_mode := AckMode} = Options) -> + erlang:process_flag(trap_exit, true), + connect(Options, #{ + ack_timeout => AckTimeout, + ack_mode => AckMode, + socket => undefined, + last_error => undefined, + frame_state => emqx_bridge_syskeeper_frame:make_state_with_conf(Options) + }). + +handle_call({forward, Msgs}, _From, State) -> + Result = send_packet(forward, Msgs, State), + handle_reply_result(Result, State); +handle_call(heartbeat, _From, State) -> + Result = send_ack_packet(heartbeat, none, State), + handle_reply_result(Result, State); +handle_call(_Request, _From, State) -> + {reply, ok, State}. + +handle_cast(_Request, State) -> + {noreply, State}. + +handle_info({tcp_closed, _} = Reason, State) -> + {noreply, State#{socket := undefined, last_error := Reason}}; +handle_info({last_error, _, _} = Reason, State) -> + {noreply, State#{socket := undefined, last_error := Reason}}; +handle_info(_Info, State) -> + {noreply, State}. + +terminate(_Reason, #{socket := Socket} = _State) -> + close_socket(Socket), + ok. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +-spec format_status( + Opt :: normal | terminate, + Status :: list() +) -> Status :: term(). +format_status(_Opt, Status) -> + Status. + +%% ------------------------------------------------------------------------------------------------ +connect( + #{ + hostname := Host, + port := Port + }, + State +) -> + case + gen_tcp:connect(Host, Port, [ + {active, true}, + {mode, binary}, + {nodelay, true} + ]) + of + {ok, Socket} -> + send_ack_packet(handshake, none, State#{socket := Socket}); + {error, Reason} -> + {stop, Reason} + end. + +-spec send_ack_packet(packet_type(), packet_data(), state()) -> send_result(). +send_ack_packet(Type, Data, State) -> + send_packet(Type, Data, State, true). + +-spec send_packet(packet_type(), packet_data(), state()) -> send_result(). +send_packet(Type, Data, State) -> + send_packet(Type, Data, State, false). + +-spec send_packet(packet_type(), packet_data(), state(), boolean()) -> send_result(). +send_packet(_Type, _Data, #{socket := undefined, last_error := Reason}, _Force) -> + {error, Reason}; +send_packet(Type, Data, #{frame_state := FrameState} = State, Force) -> + Packet = emqx_bridge_syskeeper_frame:encode(Type, Data, FrameState), + case socket_send(Packet, State) of + ok -> + wait_ack(State, Force); + {error, _} = Error -> + Error + end. + +-spec socket_send(binary() | [binary()], state()) -> ok | {error, _Reason}. +socket_send(Bin, State) when is_binary(Bin) -> + socket_send([Bin], State); +socket_send(Bins, #{socket := Socket}) -> + Map = fun(Data) -> + Len = erlang:byte_size(Data), + VarLen = emqx_bridge_syskeeper_frame:serialize_variable_byte_integer(Len), + <> + end, + gen_tcp:send(Socket, lists:map(Map, Bins)). + +-spec wait_ack(state(), boolean()) -> send_result(). +wait_ack(#{ack_timeout := AckTimeout, ack_mode := AckMode} = State, Force) when + AckMode =:= need_ack; Force +-> + receive + {tcp, _Socket, <<16#FF>>} -> + {ok, State}; + {tcp_closed, _} = Reason -> + {error, Reason}; + {tcp_error, _, _} = Reason -> + {error, Reason} + after AckTimeout -> + {error, wait_ack_timeout} + end; +wait_ack(State, _Force) -> + {ok, State}. + +close_socket(undefined) -> + ok; +close_socket(Socket) -> + catch gen_tcp:close(Socket), + ok. + +call(Pid, Msg, Timeout) -> + gen_server:call(Pid, Msg, Timeout). + +handle_reply_result({ok, _}, State) -> + {reply, ok, State}; +handle_reply_result({error, Reason}, State) -> + {reply, {error, {recoverable_error, Reason}}, State#{last_error := Reason}}. diff --git a/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_connector.erl b/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_connector.erl new file mode 100644 index 000000000..c267ee521 --- /dev/null +++ b/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_connector.erl @@ -0,0 +1,347 @@ +%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_bridge_syskeeper_connector). + +-behaviour(emqx_resource). + +-include_lib("emqx_resource/include/emqx_resource.hrl"). +-include_lib("typerefl/include/types.hrl"). +-include_lib("emqx/include/logger.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). + +-export([roots/0, fields/1, desc/1, connector_examples/1]). + +%% `emqx_resource' API +-export([ + callback_mode/0, + query_mode/1, + on_start/2, + on_stop/2, + on_query/3, + on_batch_query/3, + on_get_status/2, + on_add_channel/4, + on_remove_channel/3, + on_get_channels/1, + on_get_channel_status/3 +]). + +-export([ + connect/1 +]). + +-import(hoconsc, [mk/2, enum/1, ref/2]). + +-define(SYSKEEPER_HOST_OPTIONS, #{ + default_port => 9092 +}). + +-define(EXTRA_CALL_TIMEOUT, 2000). + +%% ------------------------------------------------------------------------------------------------- +%% api +connector_examples(Method) -> + [ + #{ + <<"syskeeper_forwarder">> => #{ + summary => <<"Syskeeper Forwarder Connector">>, + value => values(Method) + } + } + ]. + +values(get) -> + maps:merge( + #{ + status => <<"connected">>, + node_status => [ + #{ + node => <<"emqx@localhost">>, + status => <<"connected">> + } + ] + }, + values(post) + ); +values(post) -> + maps:merge( + #{ + name => <<"syskeeper_forwarder">>, + type => <<"syskeeper_forwarder">> + }, + values(put) + ); +values(put) -> + #{ + enable => true, + server => <<"127.0.0.1:9092">>, + ack_mode => <<"no_ack">>, + ack_timeout => <<"10s">>, + pool_size => 16 + }. + +%% ------------------------------------------------------------------------------------------------- +%% Hocon schema +roots() -> + [{config, #{type => hoconsc:ref(?MODULE, config)}}]. + +fields(config) -> + [ + {enable, mk(boolean(), #{desc => ?DESC("config_enable"), default => true})}, + {description, emqx_schema:description_schema()}, + {server, server()}, + {ack_mode, + mk( + enum([need_ack, no_ack]), + #{desc => ?DESC(ack_mode), default => <<"no_ack">>} + )}, + {ack_timeout, + mk( + emqx_schema:timeout_duration_ms(), + #{desc => ?DESC(ack_timeout), default => <<"10s">>} + )}, + {pool_size, fun + (default) -> + 16; + (Other) -> + emqx_connector_schema_lib:pool_size(Other) + end} + ]; +fields("post") -> + [type_field(), name_field() | fields(config)]; +fields("put") -> + fields(config); +fields("get") -> + emqx_bridge_schema:status_fields() ++ fields("post"). + +desc(config) -> + ?DESC("desc_config"); +desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" -> + ["Configuration for Syskeeper Proxy using `", string:to_upper(Method), "` method."]; +desc(_) -> + undefined. + +server() -> + Meta = #{desc => ?DESC("server")}, + emqx_schema:servers_sc(Meta, ?SYSKEEPER_HOST_OPTIONS). + +type_field() -> + {type, mk(enum([syskeeper_forwarder]), #{required => true, desc => ?DESC("desc_type")})}. + +name_field() -> + {name, mk(binary(), #{required => true, desc => ?DESC("desc_name")})}. + +%% ------------------------------------------------------------------------------------------------- +%% `emqx_resource' API + +callback_mode() -> always_sync. + +query_mode(_) -> sync. + +on_start( + InstanceId, + #{ + server := Server, + pool_size := PoolSize, + ack_timeout := AckTimeout + } = Config +) -> + ?SLOG(info, #{ + msg => "starting_syskeeper_connector", + connector => InstanceId, + config => Config + }), + + HostCfg = emqx_schema:parse_server(Server, ?SYSKEEPER_HOST_OPTIONS), + + Options = [ + {options, + maps:merge( + HostCfg, + maps:with([ack_mode, ack_timeout], Config) + )}, + {pool_size, PoolSize} + ], + + State = #{ + pool_name => InstanceId, + ack_timeout => AckTimeout, + channels => #{} + }, + case emqx_resource_pool:start(InstanceId, ?MODULE, Options) of + ok -> + {ok, State}; + Error -> + Error + end. + +on_stop(InstanceId, _State) -> + ?SLOG(info, #{ + msg => "stopping_syskeeper_connector", + connector => InstanceId + }), + emqx_resource_pool:stop(InstanceId). + +on_query(InstanceId, {_MessageTag, _} = Query, State) -> + do_query(InstanceId, [Query], State); +on_query(_InstanceId, Query, _State) -> + {error, {unrecoverable_error, {invalid_request, Query}}}. + +%% we only support batch insert +on_batch_query(InstanceId, [{_MessageTag, _} | _] = Query, State) -> + do_query(InstanceId, Query, State); +on_batch_query(_InstanceId, Query, _State) -> + {error, {unrecoverable_error, {invalid_request, Query}}}. + +on_get_status(_InstanceId, #{pool_name := Pool, ack_timeout := AckTimeout}) -> + Health = emqx_resource_pool:health_check_workers( + Pool, {emqx_bridge_syskeeper_client, heartbeat, [AckTimeout + ?EXTRA_CALL_TIMEOUT]} + ), + status_result(Health). + +status_result(true) -> connected; +status_result(false) -> connecting; +status_result({error, _}) -> connecting. + +on_add_channel( + _InstanceId, + #{channels := Channels} = OldState, + ChannelId, + #{ + parameters := #{ + target_topic := TargetTopic, + target_qos := TargetQoS, + template := Template + } + } +) -> + case maps:is_key(ChannelId, Channels) of + true -> + {error, already_exists}; + _ -> + Channel = #{ + target_qos => TargetQoS, + target_topic => emqx_placeholder:preproc_tmpl(TargetTopic), + template => emqx_placeholder:preproc_tmpl(Template) + }, + Channels2 = Channels#{ChannelId => Channel}, + {ok, OldState#{channels => Channels2}} + end. + +on_remove_channel(_InstanceId, #{channels := Channels} = OldState, ChannelId) -> + Channels2 = maps:remove(ChannelId, Channels), + {ok, OldState#{channels => Channels2}}. + +on_get_channels(InstanceId) -> + emqx_bridge_v2:get_channels_for_connector(InstanceId). + +on_get_channel_status(_InstanceId, ChannelId, #{channels := Channels}) -> + case maps:is_key(ChannelId, Channels) of + true -> + connected; + _ -> + {error, not_exists} + end. + +%% ------------------------------------------------------------------------------------------------- +%% Helper fns + +do_query( + InstanceId, + Query, + #{pool_name := PoolName, ack_timeout := AckTimeout, channels := Channels} = State +) -> + ?TRACE( + "QUERY", + "syskeeper_connector_received", + #{connector => InstanceId, query => Query, state => State} + ), + + Result = + case try_render_message(Query, Channels) of + {ok, Msg} -> + ecpool:pick_and_do( + PoolName, + {emqx_bridge_syskeeper_client, forward, [Msg, AckTimeout + ?EXTRA_CALL_TIMEOUT]}, + no_handover + ); + Error -> + Error + end, + + case Result of + {error, Reason} -> + ?tp( + syskeeper_connector_query_return, + #{error => Reason} + ), + ?SLOG(error, #{ + msg => "syskeeper_connector_do_query_failed", + connector => InstanceId, + query => Query, + reason => Reason + }), + case Reason of + ecpool_empty -> + {error, {recoverable_error, Reason}}; + _ -> + Result + end; + _ -> + ?tp( + syskeeper_connector_query_return, + #{result => Result} + ), + Result + end. + +connect(Opts) -> + Options = proplists:get_value(options, Opts), + emqx_bridge_syskeeper_client:start_link(Options). + +try_render_message(Datas, Channels) -> + try_render_message(Datas, Channels, []). + +try_render_message([{MessageTag, Data} | T], Channels, Acc) -> + case maps:find(MessageTag, Channels) of + {ok, Channel} -> + case render_message(Data, Channel) of + {ok, Msg} -> + try_render_message(T, Channels, [Msg | Acc]); + Error -> + Error + end; + _ -> + {error, {unrecoverable_error, {invalid_message_tag, MessageTag}}} + end; +try_render_message([], _Channels, Acc) -> + {ok, lists:reverse(Acc)}. + +render_message(#{id := Id, qos := QoS, clientid := From} = Data, #{ + target_qos := TargetQoS, target_topic := TargetTopicTks, template := Template +}) -> + Msg = maps:with([qos, flags, topic, payload, timestamp], Data), + Topic = emqx_placeholder:proc_tmpl(TargetTopicTks, Msg), + {ok, Msg#{ + id => emqx_guid:from_hexstr(Id), + qos := + case TargetQoS of + -1 -> + QoS; + _ -> + TargetQoS + end, + from => From, + topic := Topic, + payload := format_data(Template, Msg) + }}; +render_message(Data, _Channel) -> + {error, {unrecoverable_error, {invalid_data, Data}}}. + +format_data([], Msg) -> + emqx_utils_json:encode(Msg); +format_data(Tokens, Msg) -> + emqx_placeholder:proc_tmpl(Tokens, Msg). diff --git a/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_frame.erl b/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_frame.erl new file mode 100644 index 000000000..d2f8febb9 --- /dev/null +++ b/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_frame.erl @@ -0,0 +1,163 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% @doc EMQ X Bridge Sysk Frame +%%-------------------------------------------------------------------- + +-module(emqx_bridge_syskeeper_frame). + +%% API +-export([ + versions/0, + current_version/0, + make_state_with_conf/1, + make_state/1, + encode/3, + parse/2, + parse_handshake/1 +]). + +-export([ + bool2int/1, + int2bool/1, + marshaller/1, + serialize_variable_byte_integer/1, + parse_variable_byte_integer/1 +]). + +-export_type([state/0, versions/0, handshake/0, forward/0, packet/0]). + +-include("emqx_bridge_syskeeper.hrl"). + +-type state() :: #{ + handler := atom(), + version := versions(), + ack => boolean() +}. + +-type versions() :: 1. + +-type handshake() :: #{type := handshake, version := versions()}. +-type forward() :: #{type := forward, ack := boolean(), messages := list(map())}. +-type heartbeat() :: #{type := heartbeat}. + +-type packet() :: + handshake() + | forward() + | heartbeat(). + +-callback version() -> versions(). +-callback encode(packet_type_val(), packet_data(), state()) -> binary(). +-callback parse(packet_type(), binary(), state()) -> packet(). + +-define(HIGHBIT, 2#10000000). +-define(LOWBITS, 2#01111111). +-define(MULTIPLIER_MAX, 16#200000). + +-export_type([packet_type/0]). + +%%------------------------------------------------------------------- +%%% API +%%------------------------------------------------------------------- +-spec versions() -> list(versions()). +versions() -> + [1]. + +-spec current_version() -> versions(). +current_version() -> + 1. + +-spec make_state_with_conf(map()) -> state(). +make_state_with_conf(#{ack_mode := Mode}) -> + State = make_state(current_version()), + State#{ack => Mode =:= need_ack}. + +-spec make_state(versions()) -> state(). +make_state(Version) -> + case lists:member(Version, versions()) of + true -> + Handler = erlang:list_to_existing_atom( + io_lib:format("emqx_bridge_syskeeper_frame_v~B", [Version]) + ), + #{ + handler => Handler, + version => Version + }; + _ -> + erlang:throw({unsupport_version, Version}) + end. + +-spec encode(packet_type(), term(), state()) -> binary(). +encode(Type, Data, #{handler := Handler} = State) -> + Handler:encode(packet_type_val(Type), Data, State). + +-spec parse(binary(), state()) -> _. +parse(<> = Bin, #{handler := Handler} = State) -> + Type = to_packet_type(TypeVal), + Handler:parse(Type, Bin, State). + +parse_handshake(Data) -> + State = make_state(1), + parse_handshake(Data, State). + +parse_handshake(Data, #{version := Version} = State) -> + case parse(Data, State) of + {ok, #{type := handshake, version := Version} = Shake} -> + {ok, {State, Shake}}; + {ok, #{type := handshake, version := NewVersion}} -> + State2 = make_state(NewVersion), + parse_handshake(Data, State2); + Error -> + Error + end. + +bool2int(true) -> + 1; +bool2int(_) -> + 0. + +int2bool(1) -> + true; +int2bool(_) -> + false. + +marshaller(Item) when is_binary(Item) -> + erlang:binary_to_term(Item); +marshaller(Item) -> + erlang:term_to_binary(Item). + +serialize_variable_byte_integer(N) when N =< ?LOWBITS -> + <<0:1, N:7>>; +serialize_variable_byte_integer(N) -> + <<1:1, (N rem ?HIGHBIT):7, (serialize_variable_byte_integer(N div ?HIGHBIT))/binary>>. + +parse_variable_byte_integer(Bin) -> + parse_variable_byte_integer(Bin, 1, 0). + +%%------------------------------------------------------------------- +%%% Internal functions +%%------------------------------------------------------------------- +to_packet_type(?TYPE_HANDSHAKE) -> + handshake; +to_packet_type(?TYPE_FORWARD) -> + forward; +to_packet_type(?TYPE_HEARTBEAT) -> + heartbeat. + +packet_type_val(handshake) -> + ?TYPE_HANDSHAKE; +packet_type_val(forward) -> + ?TYPE_FORWARD; +packet_type_val(heartbeat) -> + ?TYPE_HEARTBEAT. + +parse_variable_byte_integer(<<1:1, _Len:7, _Rest/binary>>, Multiplier, _Value) when + Multiplier > ?MULTIPLIER_MAX +-> + {error, malformed_variable_byte_integer}; +parse_variable_byte_integer(<<1:1, Len:7, Rest/binary>>, Multiplier, Value) -> + parse_variable_byte_integer(Rest, Multiplier * ?HIGHBIT, Value + Len * Multiplier); +parse_variable_byte_integer(<<0:1, Len:7, Rest/binary>>, Multiplier, Value) -> + {ok, Value + Len * Multiplier, Rest}; +parse_variable_byte_integer(<<>>, _Multiplier, _Value) -> + {error, incomplete}. diff --git a/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_frame_v1.erl b/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_frame_v1.erl new file mode 100644 index 000000000..200730659 --- /dev/null +++ b/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_frame_v1.erl @@ -0,0 +1,70 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% @doc EMQ X Bridge Sysk Frame version 1 +%%-------------------------------------------------------------------- + +-module(emqx_bridge_syskeeper_frame_v1). + +%% API +-export([ + version/0, + encode/3, + parse/3 +]). + +-behaviour(emqx_bridge_syskeeper_frame). + +-include("emqx_bridge_syskeeper.hrl"). + +-define(B2I(X), emqx_bridge_syskeeper_frame:bool2int((X))). +-define(I2B(X), emqx_bridge_syskeeper_frame:int2bool((X))). + +-import(emqx_bridge_syskeeper_frame, [ + serialize_variable_byte_integer/1, parse_variable_byte_integer/1, marshaller/1 +]). + +%%------------------------------------------------------------------- +%%% API +%%------------------------------------------------------------------- +version() -> + 1. + +encode(?TYPE_HANDSHAKE = Type, _, _) -> + Version = version(), + <>; +encode(?TYPE_FORWARD = Type, Messages, #{ack := Ack}) -> + encode_forward(Messages, Type, Ack); +encode(?TYPE_HEARTBEAT = Type, _, _) -> + <>. + +-dialyzer({nowarn_function, parse/3}). +parse(handshake, <<_:4, _:4, Version:8>>, _) -> + {ok, #{type => handshake, version => Version}}; +parse(forward, Bin, _) -> + parse_forward(Bin); +parse(heartbeat, <<_:4, _:4>>, _) -> + {ok, #{type => heartbeat}}. + +%%------------------------------------------------------------------- +%%% Internal functions +%%------------------------------------------------------------------- +encode_forward(Messages, Type, Ack) -> + AckVal = ?B2I(Ack), + Data = marshaller(Messages), + Len = erlang:byte_size(Data), + LenVal = serialize_variable_byte_integer(Len), + <>. + +parse_forward(<<_:4, AckVal:4, Bin/binary>>) -> + case parse_variable_byte_integer(Bin) of + {ok, Len, Rest} -> + <> = Rest, + {ok, #{ + type => forward, + ack => ?I2B(AckVal), + messages => marshaller(MsgBin) + }}; + Error -> + Error + end. diff --git a/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_proxy.erl b/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_proxy.erl new file mode 100644 index 000000000..1968022c1 --- /dev/null +++ b/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_proxy.erl @@ -0,0 +1,116 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_bridge_syskeeper_proxy). + +-include_lib("typerefl/include/types.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). +-include_lib("emqx_bridge/include/emqx_bridge.hrl"). +-include_lib("emqx_resource/include/emqx_resource.hrl"). + +-import(hoconsc, [mk/2, enum/1, ref/2]). + +-export([ + connector_examples/1, + values/1 +]). + +-export([ + namespace/0, + roots/0, + fields/1, + desc/1 +]). + +-define(SYSKEEPER_HOST_OPTIONS, #{ + default_port => 9092 +}). + +%% ------------------------------------------------------------------------------------------------- +%% api +connector_examples(Method) -> + [ + #{ + <<"syskeeper_proxy">> => #{ + summary => <<"Syskeeper Proxy Connector">>, + value => values(Method) + } + } + ]. + +values(get) -> + maps:merge( + #{ + status => <<"connected">>, + node_status => [ + #{ + node => <<"emqx@localhost">>, + status => <<"connected">> + } + ] + }, + values(post) + ); +values(post) -> + maps:merge( + #{ + name => <<"syskeeper_proxy">>, + type => <<"syskeeper_proxy">> + }, + values(put) + ); +values(put) -> + #{ + enable => true, + listen => <<"127.0.0.1:9092">>, + acceptors => 16, + handshake_timeout => <<"16s">> + }. + +%% ------------------------------------------------------------------------------------------------- +%% Hocon Schema Definitions +namespace() -> "connector_syskeeper_proxy". + +roots() -> []. + +fields(config) -> + [ + {enable, mk(boolean(), #{desc => ?DESC("config_enable"), default => true})}, + {description, emqx_schema:description_schema()}, + {listen, listen()}, + {acceptors, + mk( + non_neg_integer(), + #{desc => ?DESC("acceptors"), default => 16} + )}, + {handshake_timeout, + mk( + emqx_schema:timeout_duration_ms(), + #{desc => ?DESC(handshake_timeout), default => <<"10s">>} + )} + ]; +fields("post") -> + [type_field(), name_field() | fields(config)]; +fields("put") -> + fields(config); +fields("get") -> + emqx_bridge_schema:status_fields() ++ fields("post"). + +desc(config) -> + ?DESC("desc_config"); +desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" -> + ["Configuration for Syskeeper Proxy using `", string:to_upper(Method), "` method."]; +desc(_) -> + undefined. + +listen() -> + Meta = #{desc => ?DESC("listen")}, + emqx_schema:servers_sc(Meta, ?SYSKEEPER_HOST_OPTIONS). + +%% ------------------------------------------------------------------------------------------------- + +type_field() -> + {type, mk(enum([syskeeper_proxy]), #{required => true, desc => ?DESC("desc_type")})}. + +name_field() -> + {name, mk(binary(), #{required => true, desc => ?DESC("desc_name")})}. diff --git a/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_proxy_server.erl b/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_proxy_server.erl new file mode 100644 index 000000000..057d7579c --- /dev/null +++ b/apps/emqx_bridge_syskeeper/src/emqx_bridge_syskeeper_proxy_server.erl @@ -0,0 +1,279 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_bridge_syskeeper_proxy_server). + +-behaviour(gen_statem). + +-include_lib("emqx/include/logger.hrl"). + +-elvis([{elvis_style, invalid_dynamic_call, disable}]). + +%% `emqx_resource' API +-export([ + query_mode/1, + on_start/2, + on_stop/2, + on_get_status/2 +]). + +%% API +-export([start_link/3]). + +%% gen_statem callbacks +-export([callback_mode/0, init/1, terminate/3, code_change/4]). +-export([handle_event/4]). + +-type state() :: wait_ready | handshake | running. +-type data() :: #{ + transport := atom(), + socket := inet:socket(), + frame_state := + undefined + | emqx_bridge_sysk_frame:state(), + buffer := binary(), + conf := map() +}. + +-define(DEFAULT_PORT, 9092). + +%% ------------------------------------------------------------------------------------------------- +%% emqx_resource + +query_mode(_) -> + no_queries. + +on_start( + InstanceId, + #{ + listen := Server, + acceptors := Acceptors + } = Config +) -> + ?SLOG(info, #{ + msg => "starting_syskeeper_proxy_server", + connector => InstanceId, + config => Config + }), + + #{hostname := Host, port := Port} = emqx_schema:parse_server(Server, #{ + default_port => ?DEFAULT_PORT + }), + ListenOn = {Host, Port}, + + Options = [ + {acceptors, Acceptors}, + {tcp_options, [{mode, binary}, {reuseaddr, true}, {nodelay, true}]} + ], + MFArgs = {?MODULE, start_link, [maps:with([handshake_timeout], Config)]}, + ok = emqx_resource:allocate_resource(InstanceId, listen_on, ListenOn), + + case esockd:open(?MODULE, ListenOn, Options, MFArgs) of + {ok, _} -> + {ok, #{listen_on => ListenOn}}; + Error -> + Error + end. + +on_stop(InstanceId, _State) -> + ?SLOG(info, #{ + msg => "stopping_syskeeper_proxy_server", + connector => InstanceId + }), + case emqx_resource:get_allocated_resources(InstanceId) of + #{listen_on := ListenOn} -> + esockd:close(?MODULE, ListenOn); + _ -> + ok + end. + +on_get_status(_InstanceId, #{listen_on := ListenOn}) -> + try + _ = esockd:listener({?MODULE, ListenOn}), + connected + catch + _:_ -> + disconnected + end. + +%% ------------------------------------------------------------------------------------------------- +-spec start_link(atom(), inet:socket(), map()) -> + {ok, Pid :: pid()} + | ignore + | {error, Error :: term()}. +start_link(Transport, Socket, Conf) -> + gen_statem:start_link(?MODULE, [Transport, Socket, Conf], []). + +%% ------------------------------------------------------------------------------------------------- +%% gen_statem callbacks + +-spec callback_mode() -> gen_statem:callback_mode_result(). +callback_mode() -> handle_event_function. + +%% ------------------------------------------------------------------------------------------------- +-spec init(Args :: term()) -> + gen_statem:init_result(term()). +init([Transport, Socket, Conf]) -> + {ok, wait_ready, + #{ + transport => Transport, + socket => Socket, + conf => Conf, + buffer => <<>>, + frame_state => undefined + }, + {next_event, internal, wait_ready}}. + +handle_event(internal, wait_ready, wait_ready, Data) -> + wait_ready(Data); +handle_event(state_timeout, handshake_timeout, handshake, Data) -> + ?SLOG(info, #{ + msg => "syskeeper_proxy_server_handshake_timeout", + data => Data + }), + {stop, normal}; +handle_event(internal, try_parse, running, Data) -> + try_parse(running, Data); +handle_event(info, {tcp, _Socket, Bin}, State, Data) -> + try_parse(State, combine_buffer(Bin, Data)); +handle_event(info, {tcp_closed, _}, _State, _Data) -> + {stop, normal}; +handle_event(info, {tcp_error, Error, Reason}, _State, _Data) -> + ?SLOG(warning, #{ + msg => "syskeeper_proxy_server_tcp_error", + error => Error, + reason => Reason + }), + {stop, normal}; +handle_event(Event, Content, State, Data) -> + ?SLOG(warning, #{ + msg => "syskeeper_proxy_server_unexpected_event", + event => Event, + content => Content, + state => State, + data => Data + }), + keep_state_and_data. + +-spec terminate(Reason :: term(), State :: state(), Data :: data()) -> + any(). +terminate(_Reason, _State, _Data) -> + ok. + +code_change(_OldVsn, State, Data, _Extra) -> + {ok, State, Data}. + +%% ------------------------------------------------------------------------------------------------- +%%% Internal functions +send(#{transport := Transport, socket := Socket}, Bin) -> + Transport:send(Socket, Bin). + +ack(Data) -> + ack(Data, true). + +ack(Data, false) -> + send(Data, <<0>>); +ack(Data, true) -> + send(Data, <<16#FF>>). + +wait_ready( + #{ + transport := Transport, + socket := RawSocket, + conf := #{handshake_timeout := Timeout} + } = + Data +) -> + case Transport:wait(RawSocket) of + {ok, Socket} -> + Transport:setopts(Socket, [{active, true}]), + {next_state, handshake, + Data#{ + socket => Socket, + frame_state => undefined + }, + {state_timeout, Timeout, handshake_timeout}}; + {error, Reason} -> + ok = Transport:fast_close(RawSocket), + ?SLOG(error, #{ + msg => "syskeeper_proxy_server_listen_error", + transport => Transport, + reason => Reason + }), + {stop, Reason} + end. + +combine_buffer(Bin, #{buffer := Buffer} = Data) -> + Data#{buffer := <>}. + +try_parse(State, #{buffer := Bin} = Data) -> + case emqx_bridge_syskeeper_frame:parse_variable_byte_integer(Bin) of + {ok, Len, Rest} -> + case Rest of + <> -> + Data2 = Data#{buffer := Rest2}, + Result = parse(Payload, Data2), + handle_parse_result(Result, State, Data2); + _ -> + {keep_state, Data} + end; + {error, incomplete} -> + {keep_state, Data}; + {error, Reason} -> + ?SLOG(error, #{ + msg => "syskeeper_proxy_server_try_parse_error", + state => State, + data => Data, + reason => Reason + }), + {stop, parse_error} + end. + +%% maybe handshake +parse(Bin, #{frame_state := undefined}) -> + emqx_bridge_syskeeper_frame:parse_handshake(Bin); +parse(Bin, #{frame_state := State}) -> + emqx_bridge_syskeeper_frame:parse(Bin, State). + +do_forward(Ack, Messages, Data) -> + lists:foreach( + fun(Message) -> + Msg = emqx_message:from_map(Message#{headers => #{}, extra => #{}}), + _ = emqx_broker:safe_publish(Msg) + end, + Messages + ), + case Ack of + true -> + ack(Data); + _ -> + ok + end. + +handle_parse_result({ok, Msg}, State, Data) -> + handle_packet(Msg, State, Data); +handle_parse_result({error, Reason} = Error, State, Data) -> + handle_parse_error(Error, State, #{buffer := _Bin} = Data), + ?SLOG(error, #{ + msg => "syskeeper_proxy_server_parse_result_error", + state => State, + data => Data, + reason => Reason + }), + {stop, parse_error}. + +handle_parse_error(_, handshake, Data) -> + ack(Data, false); +handle_parse_error(_, _, _) -> + ok. + +handle_packet({FrameState, _Shake}, handshake, Data) -> + ack(Data), + {next_state, running, Data#{frame_state := FrameState}, {next_event, internal, try_parse}}; +handle_packet(#{type := forward, ack := Ack, messages := Messages}, running, Data) -> + do_forward(Ack, Messages, Data), + try_parse(running, Data); +handle_packet(#{type := heartbeat}, running, Data) -> + ack(Data), + try_parse(running, Data). diff --git a/apps/emqx_bridge_syskeeper/test/emqx_bridge_syskeeper_SUITE.erl b/apps/emqx_bridge_syskeeper/test/emqx_bridge_syskeeper_SUITE.erl new file mode 100644 index 000000000..54330ea37 --- /dev/null +++ b/apps/emqx_bridge_syskeeper/test/emqx_bridge_syskeeper_SUITE.erl @@ -0,0 +1,388 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_bridge_syskeeper_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +-define(HOST, "127.0.0.1"). +-define(PORT, 9092). +-define(ACK_TIMEOUT, 2000). +-define(HANDSHAKE_TIMEOUT, 10000). +-define(SYSKEEPER_NAME, <<"syskeeper">>). +-define(SYSKEEPER_PROXY_NAME, <<"syskeeper_proxy">>). +-define(BATCH_SIZE, 3). +-define(TOPIC, <<"syskeeper/message">>). + +%%------------------------------------------------------------------------------ +%% CT boilerplate +%%------------------------------------------------------------------------------ + +all() -> + [ + {group, lifecycle}, + {group, need_ack}, + {group, no_ack} + ]. + +groups() -> + TCs = emqx_common_test_helpers:all(?MODULE), + Lifecycle = [ + t_setup_proxy_via_config, + t_setup_proxy_via_http_api, + t_setup_forwarder_via_config, + t_setup_forwarder_via_http_api, + t_get_status + ], + Write = TCs -- Lifecycle, + BatchingGroups = [{group, with_batch}, {group, without_batch}], + [ + {need_ack, BatchingGroups}, + {no_ack, BatchingGroups}, + {with_batch, Write}, + {without_batch, Write}, + {lifecycle, Lifecycle} + ]. + +init_per_group(need_ack, Config) -> + [{ack_mode, need_ack} | Config]; +init_per_group(no_ack, Config) -> + [{ack_mode, no_ack} | Config]; +init_per_group(with_batch, Config0) -> + [{enable_batch, true} | Config0]; +init_per_group(without_batch, Config0) -> + [{enable_batch, false} | Config0]; +init_per_group(_Group, Config) -> + Config. + +end_per_group(_Group, _Config) -> + ok. + +init_per_suite(Config) -> + ok = emqx_common_test_helpers:start_apps([ + emqx_conf, + emqx_connector, + emqx_bridge, + emqx_bridge_syskeeper + ]), + _ = emqx_bridge_enterprise:module_info(), + emqx_mgmt_api_test_util:init_suite(), + Config. + +end_per_suite(_Config) -> + emqx_mgmt_api_test_util:end_suite(), + ok = emqx_common_test_helpers:stop_apps([ + emqx_bridge_syskeeper, emqx_bridge, emqx_connector, emqx_conf + ]). + +init_per_testcase(_Testcase, Config) -> + snabbkaffe:start_trace(), + Config. + +end_per_testcase(_Testcase, _Config) -> + ok = snabbkaffe:stop(), + delete_bridge(syskeeper_forwarder, ?SYSKEEPER_NAME), + delete_connectors(syskeeper_forwarder, ?SYSKEEPER_NAME), + delete_connectors(syskeeper_proxy, ?SYSKEEPER_PROXY_NAME), + ok. + +%%------------------------------------------------------------------------------ +%% Helper fns +%%------------------------------------------------------------------------------ +syskeeper_config(Config) -> + BatchSize = + case proplists:get_value(enable_batch, Config, false) of + true -> ?BATCH_SIZE; + false -> 1 + end, + ConfigString = + io_lib:format( + "actions.~s.~s {\n" + " enable = true\n" + " connector = ~ts\n" + " parameters = {\n" + " target_topic = \"${topic}\"\n" + " target_qos = -1\n" + " template = \"${payload}\"\n" + " },\n" + " resource_opts = {\n" + " request_ttl = 500ms\n" + " batch_size = ~b\n" + " }\n" + "}", + [ + syskeeper_forwarder, + ?SYSKEEPER_NAME, + ?SYSKEEPER_NAME, + BatchSize + ] + ), + {?SYSKEEPER_NAME, parse_bridge_and_check(ConfigString, syskeeper_forwarder, ?SYSKEEPER_NAME)}. + +syskeeper_connector_config(Config) -> + AckMode = proplists:get_value(ack_mode, Config, no_ack), + ConfigString = + io_lib:format( + "connectors.~s.~s {\n" + " enable = true\n" + " server = \"~ts\"\n" + " ack_mode = ~p\n" + " ack_timeout = ~p\n" + " pool_size = 1\n" + "}", + [ + syskeeper_forwarder, + ?SYSKEEPER_NAME, + server(), + AckMode, + ?ACK_TIMEOUT + ] + ), + {?SYSKEEPER_NAME, + parse_connectors_and_check(ConfigString, syskeeper_forwarder, ?SYSKEEPER_NAME)}. + +syskeeper_proxy_config(_Config) -> + ConfigString = + io_lib:format( + "connectors.~s.~s {\n" + " enable = true\n" + " listen = \"~ts\"\n" + " acceptors = 1\n" + " handshake_timeout = ~p\n" + "}", + [ + syskeeper_proxy, + ?SYSKEEPER_PROXY_NAME, + server(), + ?HANDSHAKE_TIMEOUT + ] + ), + {?SYSKEEPER_PROXY_NAME, + parse_connectors_and_check(ConfigString, syskeeper_proxy, ?SYSKEEPER_PROXY_NAME)}. + +parse_and_check(ConfigString, SchemaMod, RootKey, Type0, Name) -> + Type = to_bin(Type0), + {ok, RawConf} = hocon:binary(ConfigString, #{format => map}), + hocon_tconf:check_plain(SchemaMod, RawConf, #{required => false, atom_key => false}), + #{RootKey := #{Type := #{Name := Config}}} = RawConf, + Config. + +parse_bridge_and_check(ConfigString, BridgeType, Name) -> + parse_and_check(ConfigString, emqx_bridge_schema, <<"actions">>, BridgeType, Name). + +parse_connectors_and_check(ConfigString, ConnectorType, Name) -> + Config = parse_and_check( + ConfigString, emqx_connector_schema, <<"connectors">>, ConnectorType, Name + ), + emqx_utils_maps:safe_atom_key_map(Config). + +create_bridge(Type, Name, Conf) -> + emqx_bridge_v2:create(Type, Name, Conf). + +delete_bridge(Type, Name) -> + emqx_bridge_v2:remove(Type, Name). + +create_both_bridge(Config) -> + {ProxyName, ProxyConf} = syskeeper_proxy_config(Config), + {ConnectorName, ConnectorConf} = syskeeper_connector_config(Config), + {Name, Conf} = syskeeper_config(Config), + ?assertMatch( + {ok, _}, + create_connectors(syskeeper_proxy, ProxyName, ProxyConf) + ), + timer:sleep(1000), + ?assertMatch( + {ok, _}, + create_connectors(syskeeper_forwarder, ConnectorName, ConnectorConf) + ), + timer:sleep(1000), + ?assertMatch({ok, _}, create_bridge(syskeeper_forwarder, Name, Conf)). + +create_bridge_http(Params) -> + call_create_http("actions", Params). + +create_connectors_http(Params) -> + call_create_http("connectors", Params). + +call_create_http(Root, Params) -> + Path = emqx_mgmt_api_test_util:api_path([Root]), + AuthHeader = emqx_mgmt_api_test_util:auth_header_(), + case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params) of + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; + Error -> Error + end. + +create_connectors(Type, Name, Conf) -> + emqx_connector:create(Type, Name, Conf). + +delete_connectors(Type, Name) -> + emqx_connector:remove(Type, Name). + +send_message(_Config, Payload) -> + Name = ?SYSKEEPER_NAME, + BridgeType = syskeeper_forwarder, + emqx_bridge_v2:send_message(BridgeType, Name, Payload, #{}). + +to_bin(List) when is_list(List) -> + unicode:characters_to_binary(List, utf8); +to_bin(Atom) when is_atom(Atom) -> + erlang:atom_to_binary(Atom); +to_bin(Bin) when is_binary(Bin) -> + Bin. + +to_str(Atom) when is_atom(Atom) -> + erlang:atom_to_list(Atom). + +server() -> + erlang:iolist_to_binary(io_lib:format("~ts:~B", [?HOST, ?PORT])). + +make_message() -> + Message = emqx_message:make(?MODULE, ?TOPIC, ?SYSKEEPER_NAME), + Id = emqx_guid:to_hexstr(emqx_guid:gen()), + From = emqx_message:from(Message), + Msg = emqx_message:to_map(Message), + Msg#{id => Id, clientid => From}. + +receive_msg() -> + receive + {deliver, ?TOPIC, Msg} -> + {ok, Msg} + after 500 -> + {error, no_message} + end. + +%%------------------------------------------------------------------------------ +%% Testcases +%%------------------------------------------------------------------------------ +t_setup_proxy_via_config(Config) -> + {Name, Conf} = syskeeper_proxy_config(Config), + ?assertMatch( + {ok, _}, + create_connectors(syskeeper_proxy, Name, Conf) + ), + ?assertMatch( + X when is_pid(X), + esockd:listener({emqx_bridge_syskeeper_proxy_server, {?HOST, ?PORT}}) + ), + delete_connectors(syskeeper_proxy, Name), + ?assertError( + not_found, + esockd:listener({emqx_bridge_syskeeper_proxy_server, {?HOST, ?PORT}}) + ). + +t_setup_proxy_via_http_api(Config) -> + {Name, ProxyConf0} = syskeeper_proxy_config(Config), + ProxyConf = ProxyConf0#{ + <<"name">> => Name, + <<"type">> => syskeeper_proxy + }, + ?assertMatch( + {ok, _}, + create_connectors_http(ProxyConf) + ), + + ?assertMatch( + X when is_pid(X), + esockd:listener({emqx_bridge_syskeeper_proxy_server, {?HOST, ?PORT}}) + ), + + delete_connectors(syskeeper_proxy, Name), + + ?assertError( + not_found, + esockd:listener({emqx_bridge_syskeeper_proxy_server, {?HOST, ?PORT}}) + ). + +t_setup_forwarder_via_config(Config) -> + {ConnectorName, ConnectorConf} = syskeeper_connector_config(Config), + {Name, Conf} = syskeeper_config(Config), + ?assertMatch( + {ok, _}, + create_connectors(syskeeper_forwarder, ConnectorName, ConnectorConf) + ), + ?assertMatch({ok, _}, create_bridge(syskeeper_forwarder, Name, Conf)). + +t_setup_forwarder_via_http_api(Config) -> + {ConnectorName, ConnectorConf0} = syskeeper_connector_config(Config), + {Name, Conf0} = syskeeper_config(Config), + + ConnectorConf = ConnectorConf0#{ + <<"name">> => ConnectorName, + <<"type">> => syskeeper_forwarder + }, + + Conf = Conf0#{ + <<"name">> => Name, + <<"type">> => syskeeper_forwarder + }, + + ?assertMatch( + {ok, _}, + create_connectors_http(ConnectorConf) + ), + + ?assertMatch( + {ok, _}, + create_bridge_http(Conf) + ). + +t_get_status(Config) -> + create_both_bridge(Config), + ?assertMatch( + #{status := connected}, emqx_bridge_v2:health_check(syskeeper_forwarder, ?SYSKEEPER_NAME) + ), + delete_connectors(syskeeper_proxy, ?SYSKEEPER_PROXY_NAME), + ?retry( + _Sleep = 500, + _Attempts = 10, + ?assertMatch( + #{status := connecting}, + emqx_bridge_v2:health_check(syskeeper_forwarder, ?SYSKEEPER_NAME) + ) + ). + +t_write_failure(Config) -> + create_both_bridge(Config), + delete_connectors(syskeeper_proxy, ?SYSKEEPER_PROXY_NAME), + SentData = make_message(), + Result = + ?wait_async_action( + send_message(Config, SentData), + #{?snk_kind := buffer_worker_flush_ack}, + 2_000 + ), + ?assertMatch({{error, {resource_error, _}}, _}, Result). + +t_invalid_data(Config) -> + create_both_bridge(Config), + {_, {ok, #{result := Result}}} = + ?wait_async_action( + send_message(Config, #{}), + #{?snk_kind := buffer_worker_flush_ack}, + 2_000 + ), + ?assertMatch({error, {unrecoverable_error, {invalid_data, _}}}, Result). + +t_forward(Config) -> + emqx_broker:subscribe(?TOPIC), + create_both_bridge(Config), + SentData = make_message(), + {_, {ok, #{result := _Result}}} = + ?wait_async_action( + send_message(Config, SentData), + #{?snk_kind := buffer_worker_flush_ack}, + 2_000 + ), + ?retry( + 500, + 10, + ?assertMatch({ok, _}, receive_msg()) + ), + emqx_broker:unsubscribe(?TOPIC), + ok. diff --git a/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine.app.src b/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine.app.src index e363f2f9c..5375a6ba9 100644 --- a/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine.app.src +++ b/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_tdengine, [ {description, "EMQX Enterprise TDEngine Bridge"}, - {vsn, "0.1.5"}, + {vsn, "0.1.6"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine_connector.erl b/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine_connector.erl index dcef8506c..522007cbc 100644 --- a/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine_connector.erl +++ b/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine_connector.erl @@ -6,7 +6,6 @@ -behaviour(emqx_resource). --include_lib("emqx_resource/include/emqx_resource.hrl"). -include_lib("typerefl/include/types.hrl"). -include_lib("emqx/include/logger.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). @@ -48,8 +47,8 @@ adjust_fields(Fields) -> fun ({username, OrigUsernameFn}) -> {username, add_default_fn(OrigUsernameFn, <<"root">>)}; - ({password, OrigPasswordFn}) -> - {password, make_required_fn(OrigPasswordFn)}; + ({password, _}) -> + {password, emqx_connector_schema_lib:password_field(#{required => true})}; (Field) -> Field end, @@ -62,12 +61,6 @@ add_default_fn(OrigFn, Default) -> (Field) -> OrigFn(Field) end. -make_required_fn(OrigFn) -> - fun - (required) -> true; - (Field) -> OrigFn(Field) - end. - server() -> Meta = #{desc => ?DESC("server")}, emqx_schema:servers_sc(Meta, ?TD_HOST_OPTIONS). @@ -223,7 +216,10 @@ aggregate_query(BatchTks, BatchReqs, Acc) -> ). connect(Opts) -> - tdengine:start_link(Opts). + %% TODO: teach `tdengine` to accept 0-arity closures as passwords. + {value, {password, Secret}, OptsRest} = lists:keytake(password, 1, Opts), + NOpts = [{password, emqx_secret:unwrap(Secret)} | OptsRest], + tdengine:start_link(NOpts). query_opts(#{database := Database} = _Opts) -> [{db_name, Database}]. diff --git a/apps/emqx_conf/include/emqx_conf.hrl b/apps/emqx_conf/include/emqx_conf.hrl index 83737e746..eeaa7c09e 100644 --- a/apps/emqx_conf/include/emqx_conf.hrl +++ b/apps/emqx_conf/include/emqx_conf.hrl @@ -59,8 +59,7 @@ emqx_authn_http_schema, emqx_authn_jwt_schema, emqx_authn_scram_mnesia_schema, - emqx_authn_ldap_schema, - emqx_authn_ldap_bind_schema + emqx_authn_ldap_schema ]). -define(EE_AUTHN_PROVIDER_SCHEMA_MODS, [ diff --git a/apps/emqx_conf/src/emqx_conf_cli.erl b/apps/emqx_conf/src/emqx_conf_cli.erl index ddabdae95..fc00c7dc9 100644 --- a/apps/emqx_conf/src/emqx_conf_cli.erl +++ b/apps/emqx_conf/src/emqx_conf_cli.erl @@ -37,10 +37,15 @@ -define(AUDIT_MOD, audit). -define(UPDATE_READONLY_KEYS_PROHIBITED, "update_readonly_keys_prohibited"). +-dialyzer({no_match, [load/0]}). + load() -> emqx_ctl:register_command(?CLUSTER_CALL, {?MODULE, admins}, [hidden]), emqx_ctl:register_command(?CONF, {?MODULE, conf}, []), - emqx_ctl:register_command(?AUDIT_MOD, {?MODULE, audit}, [hidden]), + case emqx_release:edition() of + ee -> emqx_ctl:register_command(?AUDIT_MOD, {?MODULE, audit}, [hidden]); + ce -> ok + end, ok. unload() -> @@ -108,15 +113,14 @@ admins(_) -> emqx_ctl:usage(usage_sync()). audit(Level, From, Log) -> - Log1 = redact(Log#{time => logger:timestamp()}), - ?AUDIT(Level, From, Log1). + ?AUDIT(Level, redact(Log#{from => From})). -redact(Logs = #{cmd := admins, args := ["add", Username, _Password | Rest]}) -> - Logs#{args => ["add", Username, "******" | Rest]}; -redact(Logs = #{cmd := admins, args := ["passwd", Username, _Password]}) -> - Logs#{args => ["passwd", Username, "******"]}; -redact(Logs = #{cmd := license, args := ["update", _License]}) -> - Logs#{args => ["update", "******"]}; +redact(Logs = #{cmd := admins, args := [<<"add">>, Username, _Password | Rest]}) -> + Logs#{args => [<<"add">>, Username, <<"******">> | Rest]}; +redact(Logs = #{cmd := admins, args := [<<"passwd">>, Username, _Password]}) -> + Logs#{args => [<<"passwd">>, Username, <<"******">>]}; +redact(Logs = #{cmd := license, args := [<<"update">>, _License]}) -> + Logs#{args => [<<"update">>, "******"]}; redact(Logs) -> Logs. diff --git a/apps/emqx_connector/src/emqx_connector_schema_lib.erl b/apps/emqx_connector/src/emqx_connector_schema_lib.erl index 07e7fe375..76a06cb5a 100644 --- a/apps/emqx_connector/src/emqx_connector_schema_lib.erl +++ b/apps/emqx_connector/src/emqx_connector_schema_lib.erl @@ -23,14 +23,14 @@ pool_size/1, relational_db_fields/0, ssl_fields/0, - prepare_statement_fields/0 + prepare_statement_fields/0, + password_field/0, + password_field/1 ]). -export([ database/1, username/1, - password/1, - password_required/1, auto_reconnect/1 ]). @@ -66,10 +66,19 @@ relational_db_fields() -> %% See emqx_resource.hrl {pool_size, fun pool_size/1}, {username, fun username/1}, - {password, fun password/1}, + {password, password_field()}, {auto_reconnect, fun auto_reconnect/1} ]. +-spec password_field() -> hocon_schema:field_schema(). +password_field() -> + password_field(#{}). + +-spec password_field(#{atom() => _}) -> hocon_schema:field_schema(). +password_field(Overrides) -> + Base = #{desc => ?DESC("password")}, + emqx_schema_secret:mk(maps:merge(Base, Overrides)). + prepare_statement_fields() -> [{prepare_statement, fun prepare_statement/1}]. @@ -95,22 +104,6 @@ username(desc) -> ?DESC("username"); username(required) -> false; username(_) -> undefined. -password(type) -> binary(); -password(desc) -> ?DESC("password"); -password(required) -> false; -password(format) -> <<"password">>; -password(sensitive) -> true; -password(converter) -> fun emqx_schema:password_converter/2; -password(_) -> undefined. - -password_required(type) -> binary(); -password_required(desc) -> ?DESC("password"); -password_required(required) -> true; -password_required(format) -> <<"password">>; -password_required(sensitive) -> true; -password_required(converter) -> fun emqx_schema:password_converter/2; -password_required(_) -> undefined. - auto_reconnect(type) -> boolean(); auto_reconnect(desc) -> ?DESC("auto_reconnect"); auto_reconnect(default) -> true; diff --git a/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl b/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl index c8ec8e1be..19b9fa244 100644 --- a/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl +++ b/apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl @@ -25,6 +25,10 @@ resource_type(kafka_producer) -> %% We use AEH's Kafka interface. resource_type(azure_event_hub_producer) -> emqx_bridge_kafka_impl_producer; +resource_type(syskeeper_forwarder) -> + emqx_bridge_syskeeper_connector; +resource_type(syskeeper_proxy) -> + emqx_bridge_syskeeper_proxy_server; resource_type(Type) -> error({unknown_connector_type, Type}). @@ -56,6 +60,22 @@ connector_structs() -> desc => <<"Azure Event Hub Connector Config">>, required => false } + )}, + {syskeeper_forwarder, + mk( + hoconsc:map(name, ref(emqx_bridge_syskeeper_connector, config)), + #{ + desc => <<"Syskeeper Connector Config">>, + required => false + } + )}, + {syskeeper_proxy, + mk( + hoconsc:map(name, ref(emqx_bridge_syskeeper_proxy, config)), + #{ + desc => <<"Syskeeper Proxy Connector Config">>, + required => false + } )} ]. @@ -74,7 +94,9 @@ examples(Method) -> schema_modules() -> [ emqx_bridge_kafka, - emqx_bridge_azure_event_hub + emqx_bridge_azure_event_hub, + emqx_bridge_syskeeper_connector, + emqx_bridge_syskeeper_proxy ]. api_schemas(Method) -> @@ -82,7 +104,11 @@ api_schemas(Method) -> %% We need to map the `type' field of a request (binary) to a %% connector schema module. api_ref(emqx_bridge_kafka, <<"kafka_producer">>, Method ++ "_connector"), - api_ref(emqx_bridge_azure_event_hub, <<"azure_event_hub_producer">>, Method ++ "_connector") + api_ref( + emqx_bridge_azure_event_hub, <<"azure_event_hub_producer">>, Method ++ "_connector" + ), + api_ref(emqx_bridge_syskeeper_connector, <<"syskeeper_forwarder">>, Method), + api_ref(emqx_bridge_syskeeper_proxy, <<"syskeeper_proxy">>, Method) ]. api_ref(Module, Type, Method) -> diff --git a/apps/emqx_connector/src/schema/emqx_connector_schema.erl b/apps/emqx_connector/src/schema/emqx_connector_schema.erl index 22eb523be..e4308ac54 100644 --- a/apps/emqx_connector/src/schema/emqx_connector_schema.erl +++ b/apps/emqx_connector/src/schema/emqx_connector_schema.erl @@ -60,7 +60,9 @@ enterprise_fields_connectors() -> []. -endif. connector_type_to_bridge_types(kafka_producer) -> [kafka, kafka_producer]; -connector_type_to_bridge_types(azure_event_hub_producer) -> [azure_event_hub_producer]. +connector_type_to_bridge_types(azure_event_hub_producer) -> [azure_event_hub_producer]; +connector_type_to_bridge_types(syskeeper_forwarder) -> [syskeeper_forwarder]; +connector_type_to_bridge_types(syskeeper_proxy) -> []. actions_config_name() -> <<"actions">>. diff --git a/apps/emqx_dashboard/include/emqx_dashboard.hrl b/apps/emqx_dashboard/include/emqx_dashboard.hrl index 9013436e7..c41dbb71c 100644 --- a/apps/emqx_dashboard/include/emqx_dashboard.hrl +++ b/apps/emqx_dashboard/include/emqx_dashboard.hrl @@ -13,16 +13,9 @@ %% See the License for the specific language governing permissions and %% limitations under the License. %%-------------------------------------------------------------------- --define(ADMIN, emqx_admin). +-include("emqx_dashboard_rbac.hrl"). -%% TODO: -%% The predefined roles of the preliminary RBAC implementation, -%% these may be removed when developing the full RBAC feature. -%% In full RBAC feature, the role may be customised created and deleted, -%% a predefined configuration would replace these macros. --define(ROLE_VIEWER, <<"viewer">>). --define(ROLE_SUPERUSER, <<"administrator">>). --define(ROLE_DEFAULT, ?ROLE_SUPERUSER). +-define(ADMIN, emqx_admin). -define(BACKEND_LOCAL, local). -define(SSO_USERNAME(Backend, Name), {Backend, Name}). diff --git a/apps/emqx_dashboard/include/emqx_dashboard_rbac.hrl b/apps/emqx_dashboard/include/emqx_dashboard_rbac.hrl new file mode 100644 index 000000000..386ae8bea --- /dev/null +++ b/apps/emqx_dashboard/include/emqx_dashboard_rbac.hrl @@ -0,0 +1,33 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-ifndef(EMQX_DASHBOARD_RBAC). +-define(EMQX_DASHBOARD_RBAC, true). + +%% TODO: +%% The predefined roles of the preliminary RBAC implementation, +%% these may be removed when developing the full RBAC feature. +%% In full RBAC feature, the role may be customised created and deleted, +%% a predefined configuration would replace these macros. +-define(ROLE_VIEWER, <<"viewer">>). +-define(ROLE_SUPERUSER, <<"administrator">>). +-define(ROLE_DEFAULT, ?ROLE_SUPERUSER). + +-define(ROLE_API_VIEWER, <<"viewer">>). +-define(ROLE_API_SUPERUSER, <<"administrator">>). +-define(ROLE_API_PUBLISHER, <<"publisher">>). +-define(ROLE_API_DEFAULT, ?ROLE_API_SUPERUSER). + +-endif. diff --git a/apps/emqx_dashboard/src/emqx_dashboard.erl b/apps/emqx_dashboard/src/emqx_dashboard.erl index 4f9e34238..96f81ca84 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard.erl @@ -72,7 +72,7 @@ start_listeners(Listeners) -> base_path => emqx_dashboard_swagger:base_path(), modules => minirest_api:find_api_modules(apps()), authorization => Authorization, - log => fun emqx_dashboard_audit:log/1, + log => audit_log_fun(), security => [#{'basicAuth' => []}, #{'bearerAuth' => []}], swagger_global_spec => GlobalSpec, dispatch => dispatch(), @@ -210,9 +210,19 @@ filter_false(K, V, S) -> [{K, V} | S]. listener_name(Protocol) -> list_to_atom(atom_to_list(Protocol) ++ ":dashboard"). +-dialyzer({no_match, [audit_log_fun/0]}). + +audit_log_fun() -> + case emqx_release:edition() of + ee -> fun emqx_dashboard_audit:log/2; + ce -> undefined + end. + -if(?EMQX_RELEASE_EDITION =/= ee). + %% dialyzer complains about the `unauthorized_role' clause... --dialyzer({no_match, [authorize/1]}). +-dialyzer({no_match, [authorize/1, api_key_authorize/3]}). + -endif. authorize(Req) -> @@ -222,7 +232,7 @@ authorize(Req) -> {bearer, Token} -> case emqx_dashboard_admin:verify_token(Req, Token) of {ok, Username} -> - {ok, #{auth_type => jwt_token, username => Username}}; + {ok, #{auth_type => jwt_token, source => Username}}; {error, token_timeout} -> {401, 'TOKEN_TIME_OUT', <<"Token expired, get new token by POST /login">>}; {error, not_found} -> @@ -251,14 +261,16 @@ listeners() -> api_key_authorize(Req, Key, Secret) -> Path = cowboy_req:path(Req), - case emqx_mgmt_auth:authorize(Path, Key, Secret) of + case emqx_mgmt_auth:authorize(Path, Req, Key, Secret) of ok -> - {ok, #{auth_type => api_key, api_key => Key}}; + {ok, #{auth_type => api_key, source => Key}}; {error, <<"not_allowed">>} -> return_unauthorized( ?BAD_API_KEY_OR_SECRET, <<"Not allowed, Check api_key/api_secret">> ); + {error, unauthorized_role} -> + {403, 'UNAUTHORIZED_ROLE', ?API_KEY_NOT_ALLOW_MSG}; {error, _} -> return_unauthorized( ?BAD_API_KEY_OR_SECRET, diff --git a/apps/emqx_dashboard/src/emqx_dashboard_admin.erl b/apps/emqx_dashboard/src/emqx_dashboard_admin.erl index e9aac164b..c264a1b0f 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_admin.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_admin.erl @@ -207,8 +207,15 @@ add_user_(Username, Password, Role, Desc) -> description = Desc }, mnesia:write(Admin), + ?SLOG(info, #{msg => "dashboard_sso_user_added", username => Username, role => Role}), flatten_username(#{username => Username, role => Role, description => Desc}); [_] -> + ?SLOG(info, #{ + msg => "dashboard_sso_user_add_failed", + reason => "username_already_exists", + username => Username, + role => Role + }), mnesia:abort(<<"username_already_exist">>) end. @@ -416,7 +423,7 @@ ensure_role(Role) when is_binary(Role) -> -if(?EMQX_RELEASE_EDITION == ee). legal_role(Role) -> - emqx_dashboard_rbac:valid_role(Role). + emqx_dashboard_rbac:valid_dashboard_role(Role). role(Data) -> emqx_dashboard_rbac:role(Data). @@ -447,8 +454,10 @@ lookup_user(Backend, Username) when is_atom(Backend) -> -dialyzer({no_match, [add_user/4, update_user/3]}). +legal_role(?ROLE_DEFAULT) -> + ok; legal_role(_) -> - ok. + {error, <<"Role does not exist">>}. role(_) -> ?ROLE_DEFAULT. diff --git a/apps/emqx_dashboard/src/emqx_dashboard_audit.erl b/apps/emqx_dashboard/src/emqx_dashboard_audit.erl index cb5c0f42b..4b51b2cb0 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_audit.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_audit.erl @@ -17,30 +17,102 @@ -module(emqx_dashboard_audit). -include_lib("emqx/include/logger.hrl"). +-include_lib("emqx/include/http_api.hrl"). %% API --export([log/1]). +-export([log/2]). -log(Meta0) -> - #{req_start := ReqStart, req_end := ReqEnd, code := Code, method := Method} = Meta0, - Duration = erlang:convert_time_unit(ReqEnd - ReqStart, native, millisecond), - Level = level(Method, Code, Duration), - Username = maps:get(username, Meta0, <<"">>), - From = from(maps:get(auth_type, Meta0, "")), - Meta1 = maps:without([req_start, req_end], Meta0), - Meta2 = Meta1#{time => logger:timestamp(), duration_ms => Duration}, - Meta = emqx_utils:redact(Meta2), - ?AUDIT( - Level, - From, - Meta#{username => binary_to_list(Username), node => node()} - ), - ok. +%% filter high frequency events +-define(HIGH_FREQUENCY_REQUESTS, [ + <<"/publish">>, + <<"/clients/:clientid/subscribe">>, + <<"/clients/:clientid/unsubscribe">>, + <<"/publish/bulk">>, + <<"/clients/:clientid/unsubscribe/bulk">>, + <<"/clients/:clientid/subscribe/bulk">>, + <<"/clients/kickout/bulk">> +]). -from(jwt_token) -> "dashboard"; -from(_) -> "rest_api". +log(#{code := Code, method := Method} = Meta, Req) -> + %% Keep level/2 and log_meta/1 inside of this ?AUDIT macro + ?AUDIT(level(Method, Code), log_meta(Meta, Req)). -level(get, _Code, _) -> debug; -level(_, Code, _) when Code >= 200 andalso Code < 300 -> info; -level(_, Code, _) when Code >= 300 andalso Code < 400 -> warning; -level(_, Code, _) when Code >= 400 andalso Code < 500 -> error; -level(_, _, _) -> critical. +log_meta(Meta, Req) -> + #{operation_id := OperationId, method := Method} = Meta, + case + Method =:= get orelse + (lists:member(OperationId, ?HIGH_FREQUENCY_REQUESTS) andalso + ignore_high_frequency_request()) + of + true -> + undefined; + false -> + Meta1 = #{ + time => logger:timestamp(), + from => from(Meta), + source => source(Meta), + duration_ms => duration_ms(Meta), + source_ip => source_ip(Req), + operation_type => operation_type(Meta), + %% method for http filter api. + http_method => Method, + http_request => http_request(Meta), + http_status_code => maps:get(code, Meta), + operation_result => operation_result(Meta), + node => node() + }, + Meta2 = maps:without([req_start, req_end, method, headers, body, bindings, code], Meta), + emqx_utils:redact(maps:merge(Meta2, Meta1)) + end. + +duration_ms(#{req_start := ReqStart, req_end := ReqEnd}) -> + erlang:convert_time_unit(ReqEnd - ReqStart, native, millisecond). + +from(#{auth_type := jwt_token}) -> + dashboard; +from(#{auth_type := api_key}) -> + rest_api; +from(#{operation_id := <<"/login">>}) -> + dashboard; +from(#{code := Code} = Meta) when Code =:= 401 orelse Code =:= 403 -> + case maps:find(failure, Meta) of + {ok, #{code := 'BAD_API_KEY_OR_SECRET'}} -> rest_api; + {ok, #{code := 'UNAUTHORIZED_ROLE', message := ?API_KEY_NOT_ALLOW_MSG}} -> rest_api; + %% 'TOKEN_TIME_OUT' 'BAD_TOKEN' is dashboard code. + _ -> dashboard + end. + +source(#{source := Source}) -> Source; +source(#{operation_id := <<"/login">>, body := #{<<"username">> := Username}}) -> Username; +source(_Meta) -> <<"">>. + +source_ip(Req) -> + case cowboy_req:header(<<"x-forwarded-for">>, Req, undefined) of + undefined -> + {RemoteIP, _} = cowboy_req:peer(Req), + iolist_to_binary(inet:ntoa(RemoteIP)); + Addresses -> + hd(binary:split(Addresses, <<",">>)) + end. + +operation_type(Meta) -> + case maps:find(operation_id, Meta) of + {ok, OperationId} -> + lists:nth(2, binary:split(OperationId, <<"/">>, [global])); + _ -> + <<"unknown">> + end. + +http_request(Meta) -> + maps:with([method, headers, bindings, body], Meta). + +operation_result(#{failure := _}) -> failure; +operation_result(_) -> success. + +level(get, _Code) -> debug; +level(_, Code) when Code >= 200 andalso Code < 300 -> info; +level(_, Code) when Code >= 300 andalso Code < 400 -> warning; +level(_, Code) when Code >= 400 andalso Code < 500 -> error; +level(_, _) -> critical. + +ignore_high_frequency_request() -> + emqx_conf:get([log, audit, ignore_high_frequency_request], true). diff --git a/apps/emqx_dashboard_rbac/src/emqx_dashboard_rbac.erl b/apps/emqx_dashboard_rbac/src/emqx_dashboard_rbac.erl index 57132b65b..7b8ffef02 100644 --- a/apps/emqx_dashboard_rbac/src/emqx_dashboard_rbac.erl +++ b/apps/emqx_dashboard_rbac/src/emqx_dashboard_rbac.erl @@ -6,7 +6,12 @@ -include_lib("emqx_dashboard/include/emqx_dashboard.hrl"). --export([check_rbac/3, role/1, valid_role/1]). +-export([ + check_rbac/3, + role/1, + valid_dashboard_role/1, + valid_api_role/1 +]). -dialyzer({nowarn_function, role/1}). %%===================================================================== @@ -31,20 +36,35 @@ role(#?ADMIN{role = Role}) -> role([]) -> ?ROLE_SUPERUSER; role(#{role := Role}) -> + Role; +role(Role) when is_binary(Role) -> Role. -valid_role(Role) -> - case lists:member(Role, role_list()) of +valid_dashboard_role(Role) -> + valid_role(dashboard, Role). + +valid_api_role(Role) -> + valid_role(api, Role). + +%% =================================================================== + +valid_role(Type, Role) -> + case lists:member(Role, role_list(Type)) of true -> ok; _ -> {error, <<"Role does not exist">>} end. + %% =================================================================== check_rbac(?ROLE_SUPERUSER, _, _, _) -> true; check_rbac(?ROLE_VIEWER, <<"GET">>, _, _) -> true; +check_rbac(?ROLE_API_PUBLISHER, <<"POST">>, <<"/publish">>, _) -> + true; +check_rbac(?ROLE_API_PUBLISHER, <<"POST">>, <<"/publish/bulk">>, _) -> + true; %% everyone should allow to logout check_rbac(?ROLE_VIEWER, <<"POST">>, <<"/logout">>, _) -> true; @@ -58,5 +78,7 @@ check_rbac(?ROLE_VIEWER, <<"POST">>, <<"/users/", SubPath/binary>>, Username) -> check_rbac(_, _, _, _) -> false. -role_list() -> - [?ROLE_VIEWER, ?ROLE_SUPERUSER]. +role_list(dashboard) -> + [?ROLE_VIEWER, ?ROLE_SUPERUSER]; +role_list(api) -> + [?ROLE_API_VIEWER, ?ROLE_API_PUBLISHER, ?ROLE_API_SUPERUSER]. diff --git a/apps/emqx_dashboard_sso/rebar.config b/apps/emqx_dashboard_sso/rebar.config index 2691afbc1..874aca800 100644 --- a/apps/emqx_dashboard_sso/rebar.config +++ b/apps/emqx_dashboard_sso/rebar.config @@ -4,5 +4,5 @@ {deps, [ {emqx_ldap, {path, "../../apps/emqx_ldap"}}, {emqx_dashboard, {path, "../../apps/emqx_dashboard"}}, - {esaml, {git, "https://github.com/emqx/esaml", {tag, "v1.1.2"}}} + {esaml, {git, "https://github.com/emqx/esaml", {tag, "v1.1.3"}}} ]}. diff --git a/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_api.erl b/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_api.erl index c023ace51..830b50676 100644 --- a/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_api.erl +++ b/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_api.erl @@ -204,7 +204,7 @@ backend(get, #{bindings := #{backend := Type}}) -> undefined -> {404, #{code => ?BACKEND_NOT_FOUND, message => <<"Backend not found">>}}; Backend -> - {200, to_json(Backend)} + {200, to_redacted_json(Backend)} end; backend(put, #{bindings := #{backend := Backend}, body := Config}) -> ?SLOG(info, #{ @@ -264,9 +264,9 @@ valid_config(_, _, _) -> {error, invalid_config}. handle_backend_update_result({ok, #{backend := saml} = State}, _Config) -> - {200, to_json(maps:without([idp_meta, sp], State))}; + {200, to_redacted_json(maps:without([idp_meta, sp], State))}; handle_backend_update_result({ok, _State}, Config) -> - {200, to_json(Config)}; + {200, to_redacted_json(Config)}; handle_backend_update_result(ok, _) -> 204; handle_backend_update_result({error, not_exists}, _) -> @@ -278,9 +278,9 @@ handle_backend_update_result({error, Reason}, _) when is_binary(Reason) -> handle_backend_update_result({error, Reason}, _) -> {400, #{code => ?BAD_REQUEST, message => emqx_dashboard_sso:format(["Reason: ", Reason])}}. -to_json(Data) -> +to_redacted_json(Data) -> emqx_utils_maps:jsonable_map( - Data, + emqx_utils:redact(Data), fun(K, V) -> {K, emqx_utils_maps:binary_string(V)} end diff --git a/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_ldap.erl b/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_ldap.erl index 583f1d683..faa87b80e 100644 --- a/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_ldap.erl +++ b/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_ldap.erl @@ -96,7 +96,7 @@ parse_config(Config0) -> %% In this feature, the `bind_password` is fixed, so it should conceal from the swagger, %% but the connector still needs it, hence we should add it back here ensure_bind_password(Config) -> - Config#{bind_password => <<"${password}">>}. + Config#{method => #{type => bind, bind_password => <<"${password}">>}}. adjust_ldap_fields(Fields) -> lists:map(fun adjust_ldap_field/1, Fields). diff --git a/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_saml_api.erl b/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_saml_api.erl index ccc40e2c6..949938884 100644 --- a/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_saml_api.erl +++ b/apps/emqx_dashboard_sso/src/emqx_dashboard_sso_saml_api.erl @@ -95,6 +95,10 @@ sp_saml_callback(post, Req) -> State = #{enable := true} -> case (provider(saml)):callback(Req, State) of {redirect, Redirect} -> + ?SLOG(info, #{ + msg => "dashboard_saml_sso_login_successful", + redirect => "SAML login successful. Redirecting with LoginMeta." + }), Redirect; {error, Reason} -> ?SLOG(info, #{ diff --git a/apps/emqx_dashboard_sso/test/emqx_dashboard_sso_ldap_SUITE.erl b/apps/emqx_dashboard_sso/test/emqx_dashboard_sso_ldap_SUITE.erl index 8966ffca9..9e831b4d2 100644 --- a/apps/emqx_dashboard_sso/test/emqx_dashboard_sso_ldap_SUITE.erl +++ b/apps/emqx_dashboard_sso/test/emqx_dashboard_sso_ldap_SUITE.erl @@ -10,9 +10,11 @@ -include_lib("emqx_dashboard/include/emqx_dashboard.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). -include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). -define(LDAP_HOST, "ldap"). -define(LDAP_DEFAULT_PORT, 389). +-define(LDAP_PASSWORD, <<"public">>). -define(LDAP_USER, <<"viewer1">>). -define(LDAP_USER_PASSWORD, <<"viewer1">>). -define(LDAP_BASE_DN, <<"ou=dashboard,dc=emqx,dc=io">>). @@ -128,9 +130,19 @@ t_update({init, Config}) -> Config; t_update({'end', _Config}) -> ok; -t_update(_) -> +t_update(Config) -> Path = uri(["sso", "ldap"]), - {ok, 200, Result} = request(put, Path, ldap_config(#{<<"enable">> => <<"true">>})), + %% NOTE: this time verify that supplying password through file-based secret works. + PasswordFilename = filename:join([?config(priv_dir, Config), "passfile"]), + ok = file:write_file(PasswordFilename, ?LDAP_PASSWORD), + {ok, 200, Result} = request( + put, + Path, + ldap_config(#{ + <<"enable">> => <<"true">>, + <<"password">> => iolist_to_binary(["file://", PasswordFilename]) + }) + ), check_running([<<"ldap">>]), ?assertMatch(#{backend := <<"ldap">>, enable := true}, decode_json(Result)), ?assertMatch([#{backend := <<"ldap">>, enable := true}], get_sso()), @@ -287,7 +299,7 @@ ldap_config(Override) -> <<"base_dn">> => ?LDAP_BASE_DN, <<"filter">> => ?LDAP_FILTER_WITH_UID, <<"username">> => <<"cn=root,dc=emqx,dc=io">>, - <<"password">> => <<"public">>, + <<"password">> => ?LDAP_PASSWORD, <<"pool_size">> => 8 }, Override diff --git a/apps/emqx_enterprise/src/emqx_enterprise_schema.erl b/apps/emqx_enterprise/src/emqx_enterprise_schema.erl index 658666fc7..66af3206b 100644 --- a/apps/emqx_enterprise/src/emqx_enterprise_schema.erl +++ b/apps/emqx_enterprise/src/emqx_enterprise_schema.erl @@ -83,6 +83,24 @@ fields("log_audit_handler") -> desc => ?DESC(emqx_conf_schema, "log_file_handler_max_size"), importance => ?IMPORTANCE_MEDIUM } + )}, + {"max_filter_size", + hoconsc:mk( + range(10, 30000), + #{ + default => 5000, + desc => ?DESC(emqx_conf_schema, "audit_log_max_filter_limit"), + importance => ?IMPORTANCE_MEDIUM + } + )}, + {"ignore_high_frequency_request", + hoconsc:mk( + boolean(), + #{ + default => true, + desc => ?DESC(emqx_conf_schema, "audit_log_ignore_high_frequency_request"), + importance => ?IMPORTANCE_MEDIUM + } )} ] ++ CommonConfs1; fields(Name) -> diff --git a/apps/emqx_enterprise/test/emqx_enterprise_schema_SUITE.erl b/apps/emqx_enterprise/test/emqx_enterprise_schema_SUITE.erl index e2aece927..bf1f358ea 100644 --- a/apps/emqx_enterprise/test/emqx_enterprise_schema_SUITE.erl +++ b/apps/emqx_enterprise/test/emqx_enterprise_schema_SUITE.erl @@ -95,6 +95,8 @@ t_audit_log_conf(_Config) -> <<"enable">> => false, <<"level">> => <<"info">>, <<"path">> => <<"log/audit.log">>, + <<"ignore_high_frequency_request">> => true, + <<"max_filter_size">> => 5000, <<"rotation_count">> => 10, <<"rotation_size">> => <<"50MB">>, <<"time_offset">> => <<"system">> diff --git a/apps/emqx_exhook/priv/protos/exhook.proto b/apps/emqx_exhook/priv/protos/exhook.proto index 928e9b20b..e5d7b3606 100644 --- a/apps/emqx_exhook/priv/protos/exhook.proto +++ b/apps/emqx_exhook/priv/protos/exhook.proto @@ -460,8 +460,11 @@ message SubOpts { // The QoS level uint32 qos = 1; + // deprecated + reserved 2; + reserved "share"; // The group name for shared subscription - string share = 2; + // string share = 2; // The Retain Handling option (MQTT v5.0) // diff --git a/apps/emqx_exhook/src/emqx_exhook.app.src b/apps/emqx_exhook/src/emqx_exhook.app.src index 8a57249e9..79c34e36b 100644 --- a/apps/emqx_exhook/src/emqx_exhook.app.src +++ b/apps/emqx_exhook/src/emqx_exhook.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_exhook, [ {description, "EMQX Extension for Hook"}, - {vsn, "5.0.14"}, + {vsn, "5.0.15"}, {modules, []}, {registered, []}, {mod, {emqx_exhook_app, []}}, diff --git a/apps/emqx_exhook/src/emqx_exhook_handler.erl b/apps/emqx_exhook/src/emqx_exhook_handler.erl index b4358969d..2bcb91b12 100644 --- a/apps/emqx_exhook/src/emqx_exhook_handler.erl +++ b/apps/emqx_exhook/src/emqx_exhook_handler.erl @@ -143,7 +143,7 @@ on_client_authorize(ClientInfo, Action, Topic, Result) -> Req = #{ clientinfo => clientinfo(ClientInfo), type => Type, - topic => Topic, + topic => emqx_topic:maybe_format_share(Topic), result => Bool }, case @@ -191,15 +191,15 @@ on_session_created(ClientInfo, _SessInfo) -> on_session_subscribed(ClientInfo, Topic, SubOpts) -> Req = #{ clientinfo => clientinfo(ClientInfo), - topic => Topic, - subopts => maps:with([qos, share, rh, rap, nl], SubOpts) + topic => emqx_topic:maybe_format_share(Topic), + subopts => maps:with([qos, rh, rap, nl], SubOpts) }, cast('session.subscribed', Req). on_session_unsubscribed(ClientInfo, Topic, _SubOpts) -> Req = #{ clientinfo => clientinfo(ClientInfo), - topic => Topic + topic => emqx_topic:maybe_format_share(Topic) }, cast('session.unsubscribed', Req). @@ -413,7 +413,13 @@ enrich_header(Headers, Message) -> end. topicfilters(Tfs) when is_list(Tfs) -> - [#{name => Topic, qos => Qos} || {Topic, #{qos := Qos}} <- Tfs]. + GetQos = fun(SubOpts) -> + maps:get(qos, SubOpts, 0) + end, + [ + #{name => emqx_topic:maybe_format_share(Topic), qos => GetQos(SubOpts)} + || {Topic, SubOpts} <- Tfs + ]. ntoa({0, 0, 0, 0, 0, 16#ffff, AB, CD}) -> list_to_binary(inet_parse:ntoa({AB bsr 8, AB rem 256, CD bsr 8, CD rem 256})); diff --git a/apps/emqx_exhook/test/props/prop_exhook_hooks.erl b/apps/emqx_exhook/test/props/prop_exhook_hooks.erl index 2c9b5bb06..cf48fff80 100644 --- a/apps/emqx_exhook/test/props/prop_exhook_hooks.erl +++ b/apps/emqx_exhook/test/props/prop_exhook_hooks.erl @@ -546,8 +546,7 @@ subopts(SubOpts) -> qos => maps:get(qos, SubOpts, 0), rh => maps:get(rh, SubOpts, 0), rap => maps:get(rap, SubOpts, 0), - nl => maps:get(nl, SubOpts, 0), - share => maps:get(share, SubOpts, <<>>) + nl => maps:get(nl, SubOpts, 0) }. authresult_to_bool(AuthResult) -> diff --git a/apps/emqx_gateway/src/emqx_gateway_api.erl b/apps/emqx_gateway/src/emqx_gateway_api.erl index 61f29059f..ae2533f97 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api.erl @@ -93,10 +93,9 @@ gateways(get, Request) -> gateway(get, #{bindings := #{name := Name}}) -> try - GwName = gw_name(Name), - case emqx_gateway:lookup(GwName) of + case emqx_gateway:lookup(Name) of undefined -> - {200, #{name => GwName, status => unloaded}}; + {200, #{name => Name, status => unloaded}}; Gateway -> GwConf = emqx_gateway_conf:gateway(Name), GwInfo0 = emqx_gateway_utils:unix_ts_to_rfc3339( @@ -125,15 +124,14 @@ gateway(put, #{ }) -> GwConf = maps:without([<<"name">>], GwConf0), try - GwName = gw_name(Name), LoadOrUpdateF = - case emqx_gateway:lookup(GwName) of + case emqx_gateway:lookup(Name) of undefined -> fun emqx_gateway_conf:load_gateway/2; _ -> fun emqx_gateway_conf:update_gateway/2 end, - case LoadOrUpdateF(GwName, GwConf) of + case LoadOrUpdateF(Name, GwConf) of {ok, _} -> {204}; {error, Reason} -> @@ -148,12 +146,11 @@ gateway(put, #{ gateway_enable(put, #{bindings := #{name := Name, enable := Enable}}) -> try - GwName = gw_name(Name), - case emqx_gateway:lookup(GwName) of + case emqx_gateway:lookup(Name) of undefined -> return_http_error(404, <<"NOT FOUND">>); _Gateway -> - {ok, _} = emqx_gateway_conf:update_gateway(GwName, #{<<"enable">> => Enable}), + {ok, _} = emqx_gateway_conf:update_gateway(Name, #{<<"enable">> => Enable}), {204} end catch @@ -161,14 +158,6 @@ gateway_enable(put, #{bindings := #{name := Name, enable := Enable}}) -> return_http_error(404, <<"NOT FOUND">>) end. --spec gw_name(binary()) -> stomp | coap | lwm2m | mqttsn | exproto | no_return(). -gw_name(<<"stomp">>) -> stomp; -gw_name(<<"coap">>) -> coap; -gw_name(<<"lwm2m">>) -> lwm2m; -gw_name(<<"mqttsn">>) -> mqttsn; -gw_name(<<"exproto">>) -> exproto; -gw_name(_Else) -> throw(not_found). - %%-------------------------------------------------------------------- %% Swagger defines %%-------------------------------------------------------------------- @@ -249,7 +238,7 @@ params_gateway_name_in_path() -> [ {name, mk( - binary(), + hoconsc:enum(emqx_gateway_schema:gateway_names()), #{ in => path, desc => ?DESC(gateway_name_in_qs), @@ -390,7 +379,10 @@ fields(Gw) when Gw == mqttsn; Gw == coap; Gw == lwm2m; - Gw == exproto + Gw == exproto; + Gw == gbt32960; + Gw == ocpp; + Gw == jt808 -> [{name, mk(Gw, #{desc => ?DESC(gateway_name)})}] ++ convert_listener_struct(emqx_gateway_schema:gateway_schema(Gw)); @@ -399,7 +391,10 @@ fields(Gw) when Gw == update_mqttsn; Gw == update_coap; Gw == update_lwm2m; - Gw == update_exproto + Gw == update_exproto; + Gw == update_gbt32960; + Gw == update_ocpp; + Gw == update_jt808 -> "update_" ++ GwStr = atom_to_list(Gw), Gw1 = list_to_existing_atom(GwStr), @@ -408,14 +403,18 @@ fields(Listener) when Listener == tcp_listener; Listener == ssl_listener; Listener == udp_listener; - Listener == dtls_listener + Listener == dtls_listener; + Listener == ws_listener; + Listener == wss_listener -> Type = case Listener of tcp_listener -> tcp; ssl_listener -> ssl; udp_listener -> udp; - dtls_listener -> dtls + dtls_listener -> dtls; + ws_listener -> ws; + wss_listener -> wss end, [ {id, @@ -447,31 +446,30 @@ fields(gateway_stats) -> [{key, mk(binary(), #{})}]. schema_load_or_update_gateways_conf() -> + Names = emqx_gateway_schema:gateway_names(), emqx_dashboard_swagger:schema_with_examples( - hoconsc:union([ - ref(?MODULE, stomp), - ref(?MODULE, mqttsn), - ref(?MODULE, coap), - ref(?MODULE, lwm2m), - ref(?MODULE, exproto), - ref(?MODULE, update_stomp), - ref(?MODULE, update_mqttsn), - ref(?MODULE, update_coap), - ref(?MODULE, update_lwm2m), - ref(?MODULE, update_exproto) - ]), + hoconsc:union( + [ + ref(?MODULE, Name) + || Name <- + Names ++ + [ + erlang:list_to_existing_atom("update_" ++ erlang:atom_to_list(Name)) + || Name <- Names + ] + ] + ), examples_update_gateway_confs() ). schema_gateways_conf() -> emqx_dashboard_swagger:schema_with_examples( - hoconsc:union([ - ref(?MODULE, stomp), - ref(?MODULE, mqttsn), - ref(?MODULE, coap), - ref(?MODULE, lwm2m), - ref(?MODULE, exproto) - ]), + hoconsc:union( + [ + ref(?MODULE, Name) + || Name <- emqx_gateway_schema:gateway_names() + ] + ), examples_gateway_confs() ). @@ -502,14 +500,18 @@ listeners_schema(?R_REF(_Mod, tcp_udp_listeners)) -> ref(udp_listener), ref(dtls_listener) ]) - ). + ); +listeners_schema(?R_REF(_Mod, ws_listeners)) -> + hoconsc:array(hoconsc:union([ref(ws_listener), ref(wss_listener)])). listener_schema() -> hoconsc:union([ ref(?MODULE, tcp_listener), ref(?MODULE, ssl_listener), ref(?MODULE, udp_listener), - ref(?MODULE, dtls_listener) + ref(?MODULE, dtls_listener), + ref(?MODULE, ws_listener), + ref(?MODULE, wss_listener) ]). %%-------------------------------------------------------------------- @@ -756,6 +758,59 @@ examples_gateway_confs() -> } ] } + }, + gbt32960_gateway => + #{ + summary => <<"A simple GBT32960 gateway config">>, + value => + #{ + enable => true, + name => <<"gbt32960">>, + enable_stats => true, + mountpoint => <<"gbt32960/${clientid}">>, + retry_interval => <<"8s">>, + max_retry_times => 3, + message_queue_len => 10, + listeners => + [ + #{ + type => <<"tcp">>, + name => <<"default">>, + bind => <<"7325">>, + max_connections => 1024000, + max_conn_rate => 1000 + } + ] + } + }, + ocpp_gateway => + #{ + summary => <<"A simple OCPP gateway config">>, + vaule => + #{ + enable => true, + name => <<"ocpp">>, + enable_stats => true, + mountpoint => <<"ocpp/">>, + default_heartbeat_interval => <<"60s">>, + upstream => + #{ + topic => <<"cp/${cid}">>, + reply_topic => <<"cp/${cid}/reply">>, + error_topic => <<"cp/${cid}/error">> + }, + dnstream => #{topic => <<"cp/${cid}">>}, + message_format_checking => disable, + listeners => + [ + #{ + type => <<"ws">>, + name => <<"default">>, + bind => <<"33033">>, + max_connections => 1024000 + } + ] + } } }. @@ -854,5 +909,37 @@ examples_update_gateway_confs() -> handler => #{address => <<"http://127.0.0.1:9001">>} } + }, + gbt32960_gateway => + #{ + summary => <<"A simple GBT32960 gateway config">>, + value => + #{ + enable => true, + enable_stats => true, + mountpoint => <<"gbt32960/${clientid}">>, + retry_interval => <<"8s">>, + max_retry_times => 3, + message_queue_len => 10 + } + }, + ocpp_gateway => + #{ + summary => <<"A simple OCPP gateway config">>, + vaule => + #{ + enable => true, + enable_stats => true, + mountpoint => <<"ocpp/">>, + default_heartbeat_interval => <<"60s">>, + upstream => + #{ + topic => <<"cp/${cid}">>, + reply_topic => <<"cp/${cid}/reply">>, + error_topic => <<"cp/${cid}/error">> + }, + dnstream => #{topic => <<"cp/${cid}">>}, + message_format_checking => disable + } } }. diff --git a/apps/emqx_gateway/src/emqx_gateway_api_authn.erl b/apps/emqx_gateway/src/emqx_gateway_api_authn.erl index 539d65112..55672318a 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api_authn.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api_authn.erl @@ -327,7 +327,7 @@ params_gateway_name_in_path() -> [ {name, mk( - binary(), + hoconsc:enum(emqx_gateway_schema:gateway_names()), #{ in => path, desc => ?DESC(emqx_gateway_api, gateway_name_in_qs), diff --git a/apps/emqx_gateway/src/emqx_gateway_api_authn_user_import.erl b/apps/emqx_gateway/src/emqx_gateway_api_authn_user_import.erl index 30aeaf8fe..321b145ac 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api_authn_user_import.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api_authn_user_import.erl @@ -52,7 +52,7 @@ %%-------------------------------------------------------------------- api_spec() -> - emqx_dashboard_swagger:spec(?MODULE, #{check_schema => false}). + emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true, translate_body => true}). paths() -> [ @@ -157,7 +157,7 @@ params_gateway_name_in_path() -> [ {name, mk( - binary(), + hoconsc:enum(emqx_gateway_schema:gateway_names()), #{ in => path, desc => ?DESC(emqx_gateway_api, gateway_name_in_qs), diff --git a/apps/emqx_gateway/src/emqx_gateway_api_clients.erl b/apps/emqx_gateway/src/emqx_gateway_api_clients.erl index b698446b9..aedb4b0fa 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api_clients.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api_clients.erl @@ -33,7 +33,7 @@ ] ). -%% minirest/dashbaord_swagger behaviour callbacks +%% minirest/dashboard_swagger behaviour callbacks -export([ api_spec/0, paths/0, @@ -700,7 +700,7 @@ params_gateway_name_in_path() -> [ {name, mk( - binary(), + hoconsc:enum(emqx_gateway_schema:gateway_names()), #{ in => path, desc => ?DESC(emqx_gateway_api, gateway_name) diff --git a/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl b/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl index 046e23300..284576983 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl @@ -609,7 +609,7 @@ params_gateway_name_in_path() -> [ {name, mk( - binary(), + hoconsc:enum(emqx_gateway_schema:gateway_names()), #{ in => path, desc => ?DESC(emqx_gateway_api, gateway_name_in_qs), diff --git a/apps/emqx_gateway/src/emqx_gateway_cm.erl b/apps/emqx_gateway/src/emqx_gateway_cm.erl index 7df3b2552..2c8d708df 100644 --- a/apps/emqx_gateway/src/emqx_gateway_cm.erl +++ b/apps/emqx_gateway/src/emqx_gateway_cm.erl @@ -23,7 +23,7 @@ -behaviour(gen_server). --include("include/emqx_gateway.hrl"). +-include("emqx_gateway.hrl"). -include_lib("emqx/include/logger.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). diff --git a/apps/emqx_gateway/src/emqx_gateway_cm_registry.erl b/apps/emqx_gateway/src/emqx_gateway_cm_registry.erl index f5bede084..f7a72af5f 100644 --- a/apps/emqx_gateway/src/emqx_gateway_cm_registry.erl +++ b/apps/emqx_gateway/src/emqx_gateway_cm_registry.erl @@ -17,7 +17,7 @@ %% @doc The gateway connection registry -module(emqx_gateway_cm_registry). --include("include/emqx_gateway.hrl"). +-include("emqx_gateway.hrl"). -behaviour(gen_server). diff --git a/apps/emqx_gateway/src/emqx_gateway_ctx.erl b/apps/emqx_gateway/src/emqx_gateway_ctx.erl index 11ad55d3e..6df1a8aae 100644 --- a/apps/emqx_gateway/src/emqx_gateway_ctx.erl +++ b/apps/emqx_gateway/src/emqx_gateway_ctx.erl @@ -17,7 +17,7 @@ %% @doc The gateway instance context -module(emqx_gateway_ctx). --include("include/emqx_gateway.hrl"). +-include("emqx_gateway.hrl"). %% @doc The running context for a Connection/Channel process. %% diff --git a/apps/emqx_gateway/src/emqx_gateway_gw_sup.erl b/apps/emqx_gateway/src/emqx_gateway_gw_sup.erl index 0f7ff4ffc..345d94432 100644 --- a/apps/emqx_gateway/src/emqx_gateway_gw_sup.erl +++ b/apps/emqx_gateway/src/emqx_gateway_gw_sup.erl @@ -23,7 +23,7 @@ -behaviour(supervisor). --include("include/emqx_gateway.hrl"). +-include("emqx_gateway.hrl"). -export([start_link/1]). diff --git a/apps/emqx_gateway/src/emqx_gateway_http.erl b/apps/emqx_gateway/src/emqx_gateway_http.erl index e8f0e034f..677176acc 100644 --- a/apps/emqx_gateway/src/emqx_gateway_http.erl +++ b/apps/emqx_gateway/src/emqx_gateway_http.erl @@ -17,7 +17,7 @@ %% @doc Gateway Interface Module for HTTP-APIs -module(emqx_gateway_http). --include("include/emqx_gateway.hrl"). +-include("emqx_gateway.hrl"). -include_lib("emqx/include/logger.hrl"). -include_lib("emqx_auth/include/emqx_authn_chains.hrl"). @@ -160,10 +160,10 @@ cluster_gateway_status(GwName) -> max_connections_count(Config) -> Listeners = emqx_gateway_utils:normalize_config(Config), lists:foldl( - fun({_, _, _, SocketOpts, _}, Acc) -> + fun({_, _, _, Conf0}, Acc) -> emqx_gateway_utils:plus_max_connections( Acc, - proplists:get_value(max_connections, SocketOpts, 0) + maps:get(max_connections, Conf0, 0) ) end, 0, @@ -184,7 +184,7 @@ current_connections_count(GwName) -> get_listeners_status(GwName, Config) -> Listeners = emqx_gateway_utils:normalize_config(Config), lists:map( - fun({Type, LisName, ListenOn, _, _}) -> + fun({Type, LisName, ListenOn, _}) -> Name0 = listener_id(GwName, Type, LisName), Name = {Name0, ListenOn}, LisO = #{id => Name0, type => Type, name => LisName}, @@ -513,29 +513,23 @@ codestr(501) -> 'NOT_IMPLEMENTED'. fmtstr(Fmt, Args) -> lists:flatten(io_lib:format(Fmt, Args)). --spec with_authn(binary(), function()) -> any(). +-spec with_authn(atom(), function()) -> any(). with_authn(GwName0, Fun) -> with_gateway(GwName0, fun(GwName, _GwConf) -> Authn = emqx_gateway_http:authn(GwName), Fun(GwName, Authn) end). --spec with_listener_authn(binary(), binary(), function()) -> any(). +-spec with_listener_authn(atom(), binary(), function()) -> any(). with_listener_authn(GwName0, Id, Fun) -> with_gateway(GwName0, fun(GwName, _GwConf) -> Authn = emqx_gateway_http:authn(GwName, Id), Fun(GwName, Authn) end). --spec with_gateway(binary(), function()) -> any(). -with_gateway(GwName0, Fun) -> +-spec with_gateway(atom(), function()) -> any(). +with_gateway(GwName, Fun) -> try - GwName = - try - binary_to_existing_atom(GwName0) - catch - _:_ -> error(badname) - end, case emqx_gateway:lookup(GwName) of undefined -> return_http_error(404, "Gateway not loaded"); diff --git a/apps/emqx_gateway/src/emqx_gateway_insta_sup.erl b/apps/emqx_gateway/src/emqx_gateway_insta_sup.erl index 8dce8582d..2898298a7 100644 --- a/apps/emqx_gateway/src/emqx_gateway_insta_sup.erl +++ b/apps/emqx_gateway/src/emqx_gateway_insta_sup.erl @@ -19,7 +19,7 @@ -behaviour(gen_server). --include("include/emqx_gateway.hrl"). +-include("emqx_gateway.hrl"). -include_lib("emqx/include/logger.hrl"). %% APIs diff --git a/apps/emqx_gateway/src/emqx_gateway_registry.erl b/apps/emqx_gateway/src/emqx_gateway_registry.erl index 50aad9445..20a3e1c42 100644 --- a/apps/emqx_gateway/src/emqx_gateway_registry.erl +++ b/apps/emqx_gateway/src/emqx_gateway_registry.erl @@ -17,7 +17,7 @@ %% @doc The Registry Centre of Gateway -module(emqx_gateway_registry). --include("include/emqx_gateway.hrl"). +-include("emqx_gateway.hrl"). -behaviour(gen_server). diff --git a/apps/emqx_gateway/src/emqx_gateway_schema.erl b/apps/emqx_gateway/src/emqx_gateway_schema.erl index ed149d1f5..c84cf086b 100644 --- a/apps/emqx_gateway/src/emqx_gateway_schema.erl +++ b/apps/emqx_gateway/src/emqx_gateway_schema.erl @@ -45,12 +45,15 @@ ip_port/0 ]). -elvis([{elvis_style, dont_repeat_yourself, disable}]). +-elvis([{elvis_style, invalid_dynamic_call, disable}]). -export([namespace/0, roots/0, fields/1, desc/1, tags/0]). -export([proxy_protocol_opts/0]). --export([mountpoint/0, mountpoint/1, gateway_common_options/0, gateway_schema/1]). +-export([mountpoint/0, mountpoint/1, gateway_common_options/0, gateway_schema/1, gateway_names/0]). + +-export([ws_listener/0, wss_listener/0, ws_opts/2]). namespace() -> gateway. @@ -123,6 +126,10 @@ fields(ssl_listener) -> } )} ]; +fields(ws_listener) -> + ws_listener() ++ ws_opts(<<>>, <<>>); +fields(wss_listener) -> + wss_listener() ++ ws_opts(<<>>, <<>>); fields(udp_listener) -> [ %% some special configs for udp listener @@ -246,6 +253,134 @@ mountpoint(Default) -> } ). +ws_listener() -> + [ + {acceptors, sc(integer(), #{default => 16, desc => ?DESC(tcp_listener_acceptors)})} + ] ++ + tcp_opts() ++ + proxy_protocol_opts() ++ + common_listener_opts(). + +wss_listener() -> + ws_listener() ++ + [ + {ssl_options, + sc( + hoconsc:ref(emqx_schema, "listener_wss_opts"), + #{ + desc => ?DESC(ssl_listener_options), + validator => fun emqx_schema:validate_server_ssl_opts/1 + } + )} + ]. + +ws_opts(DefaultPath, DefaultSubProtocols) when + is_binary(DefaultPath), is_binary(DefaultSubProtocols) +-> + [ + {"path", + sc( + string(), + #{ + default => DefaultPath, + desc => ?DESC(fields_ws_opts_path) + } + )}, + {"piggyback", + sc( + hoconsc:enum([single, multiple]), + #{ + default => single, + desc => ?DESC(fields_ws_opts_piggyback) + } + )}, + {"compress", + sc( + boolean(), + #{ + default => false, + desc => ?DESC(fields_ws_opts_compress) + } + )}, + {"idle_timeout", + sc( + duration(), + #{ + default => <<"7200s">>, + desc => ?DESC(fields_ws_opts_idle_timeout) + } + )}, + {"max_frame_size", + sc( + hoconsc:union([infinity, integer()]), + #{ + default => infinity, + desc => ?DESC(fields_ws_opts_max_frame_size) + } + )}, + {"fail_if_no_subprotocol", + sc( + boolean(), + #{ + default => true, + desc => ?DESC(fields_ws_opts_fail_if_no_subprotocol) + } + )}, + {"supported_subprotocols", + sc( + emqx_schema:comma_separated_list(), + #{ + default => DefaultSubProtocols, + desc => ?DESC(fields_ws_opts_supported_subprotocols) + } + )}, + {"check_origin_enable", + sc( + boolean(), + #{ + default => false, + desc => ?DESC(fields_ws_opts_check_origin_enable) + } + )}, + {"allow_origin_absence", + sc( + boolean(), + #{ + default => true, + desc => ?DESC(fields_ws_opts_allow_origin_absence) + } + )}, + {"check_origins", + sc( + emqx_schema:comma_separated_binary(), + #{ + default => <<"http://localhost:18083, http://127.0.0.1:18083">>, + desc => ?DESC(fields_ws_opts_check_origins) + } + )}, + {"proxy_address_header", + sc( + string(), + #{ + default => <<"x-forwarded-for">>, + desc => ?DESC(fields_ws_opts_proxy_address_header) + } + )}, + {"proxy_port_header", + sc( + string(), + #{ + default => <<"x-forwarded-port">>, + desc => ?DESC(fields_ws_opts_proxy_port_header) + } + )}, + {"deflate_opts", + sc( + ref(emqx_schema, "deflate_opts"), + #{} + )} + ]. + common_listener_opts() -> [ {enable, @@ -324,7 +459,7 @@ proxy_protocol_opts() -> sc( duration(), #{ - default => <<"15s">>, + default => <<"3s">>, desc => ?DESC(tcp_listener_proxy_protocol_timeout) } )} @@ -333,13 +468,21 @@ proxy_protocol_opts() -> %%-------------------------------------------------------------------- %% dynamic schemas -%% FIXME: don't hardcode the gateway names -gateway_schema(stomp) -> emqx_stomp_schema:fields(stomp); -gateway_schema(mqttsn) -> emqx_mqttsn_schema:fields(mqttsn); -gateway_schema(coap) -> emqx_coap_schema:fields(coap); -gateway_schema(lwm2m) -> emqx_lwm2m_schema:fields(lwm2m); -gateway_schema(exproto) -> emqx_exproto_schema:fields(exproto). +gateway_schema(Name) -> + case emqx_gateway_utils:find_gateway_definition(Name) of + {ok, #{config_schema_module := SchemaMod}} -> + SchemaMod:fields(Name); + {error, _} = Error -> + throw(Error) + end. +gateway_names() -> + Definations = emqx_gateway_utils:find_gateway_definitions(), + [ + Name + || #{name := Name} = Defination <- Definations, + emqx_gateway_utils:check_gateway_edition(Defination) + ]. %%-------------------------------------------------------------------- %% helpers diff --git a/apps/emqx_gateway/src/emqx_gateway_sup.erl b/apps/emqx_gateway/src/emqx_gateway_sup.erl index 4e928bbf9..ffb7d9220 100644 --- a/apps/emqx_gateway/src/emqx_gateway_sup.erl +++ b/apps/emqx_gateway/src/emqx_gateway_sup.erl @@ -18,7 +18,7 @@ -behaviour(supervisor). --include("include/emqx_gateway.hrl"). +-include("emqx_gateway.hrl"). -export([start_link/0]). diff --git a/apps/emqx_gateway/src/emqx_gateway_utils.erl b/apps/emqx_gateway/src/emqx_gateway_utils.erl index 72751297b..ed3f10594 100644 --- a/apps/emqx_gateway/src/emqx_gateway_utils.erl +++ b/apps/emqx_gateway/src/emqx_gateway_utils.erl @@ -45,8 +45,10 @@ global_chain/1, listener_chain/3, find_gateway_definitions/0, + find_gateway_definition/1, plus_max_connections/2, - random_clientid/1 + random_clientid/1, + check_gateway_edition/1 ]). -export([stringfy/1]). @@ -80,6 +82,11 @@ max_mailbox_size => 32000 }). +-define(IS_ESOCKD_LISTENER(T), + T == tcp orelse T == ssl orelse T == udp orelse T == dtls +). +-define(IS_COWBOY_LISTENER(T), T == ws orelse T == wss). + -elvis([{elvis_style, god_modules, disable}]). -spec childspec(supervisor:worker(), Mod :: atom()) -> @@ -133,7 +140,7 @@ find_sup_child(Sup, ChildId) -> {ok, [pid()]} | {error, term()} when - ModCfg :: #{frame_mod := atom(), chann_mod := atom()}. + ModCfg :: #{frame_mod := atom(), chann_mod := atom(), connection_mod => atom()}. start_listeners(Listeners, GwName, Ctx, ModCfg) -> start_listeners(Listeners, GwName, Ctx, ModCfg, []). @@ -165,13 +172,12 @@ start_listeners([L | Ls], GwName, Ctx, ModCfg, Acc) -> start_listener( GwName, Ctx, - {Type, LisName, ListenOn, SocketOpts, Cfg}, + {Type, LisName, ListenOn, Cfg}, ModCfg ) -> ListenOnStr = emqx_listeners:format_bind(ListenOn), ListenerId = emqx_gateway_utils:listener_id(GwName, Type, LisName), - NCfg = maps:merge(Cfg, ModCfg), case start_listener( GwName, @@ -179,8 +185,8 @@ start_listener( Type, LisName, ListenOn, - SocketOpts, - NCfg + Cfg, + ModCfg ) of {ok, Pid} -> @@ -197,15 +203,69 @@ start_listener( emqx_gateway_utils:supervisor_ret({error, Reason}) end. -start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) -> +start_listener(GwName, Ctx, Type, LisName, ListenOn, Confs, ModCfg) when + ?IS_ESOCKD_LISTENER(Type) +-> Name = emqx_gateway_utils:listener_id(GwName, Type, LisName), - NCfg = Cfg#{ - ctx => Ctx, - listener => {GwName, Type, LisName} - }, - NSocketOpts = merge_default(Type, SocketOpts), - MFA = {emqx_gateway_conn, start_link, [NCfg]}, - do_start_listener(Type, Name, ListenOn, NSocketOpts, MFA). + SocketOpts = merge_default(Type, esockd_opts(Type, Confs)), + HighLevelCfgs0 = filter_out_low_level_opts(Type, Confs), + HighLevelCfgs = maps:merge( + HighLevelCfgs0, + ModCfg#{ + ctx => Ctx, + listener => {GwName, Type, LisName} + } + ), + ConnMod = maps:get(connection_mod, ModCfg, emqx_gateway_conn), + MFA = {ConnMod, start_link, [HighLevelCfgs]}, + do_start_listener(Type, Name, ListenOn, SocketOpts, MFA); +start_listener(GwName, Ctx, Type, LisName, ListenOn, Confs, ModCfg) when + ?IS_COWBOY_LISTENER(Type) +-> + Name = emqx_gateway_utils:listener_id(GwName, Type, LisName), + RanchOpts = ranch_opts(Type, ListenOn, Confs), + HighLevelCfgs0 = filter_out_low_level_opts(Type, Confs), + HighLevelCfgs = maps:merge( + HighLevelCfgs0, + ModCfg#{ + ctx => Ctx, + listener => {GwName, Type, LisName} + } + ), + WsOpts = ws_opts(Confs, HighLevelCfgs), + case Type of + ws -> cowboy:start_clear(Name, RanchOpts, WsOpts); + wss -> cowboy:start_tls(Name, RanchOpts, WsOpts) + end. + +filter_out_low_level_opts(Type, RawCfg = #{gw_conf := Conf0}) when ?IS_ESOCKD_LISTENER(Type) -> + EsockdKeys = [ + gw_conf, + bind, + acceptors, + max_connections, + max_conn_rate, + tcp_options, + ssl_options, + udp_options, + dtls_options + ], + Conf1 = maps:without(EsockdKeys, RawCfg), + maps:merge(Conf0, Conf1); +filter_out_low_level_opts(Type, RawCfg = #{gw_conf := Conf0}) when ?IS_COWBOY_LISTENER(Type) -> + CowboyKeys = [ + gw_conf, + bind, + acceptors, + max_connections, + max_conn_rate, + tcp_options, + ssl_options, + udp_options, + dtls_options + ], + Conf1 = maps:without(CowboyKeys, RawCfg), + maps:merge(Conf0, Conf1). merge_default(Udp, Options) -> {Key, Default} = @@ -244,8 +304,8 @@ stop_listeners(GwName, Listeners) -> lists:foreach(fun(L) -> stop_listener(GwName, L) end, Listeners). -spec stop_listener(GwName :: atom(), Listener :: tuple()) -> ok. -stop_listener(GwName, {Type, LisName, ListenOn, SocketOpts, Cfg}) -> - StopRet = stop_listener(GwName, Type, LisName, ListenOn, SocketOpts, Cfg), +stop_listener(GwName, {Type, LisName, ListenOn, Cfg}) -> + StopRet = stop_listener(GwName, Type, LisName, ListenOn, Cfg), ListenOnStr = emqx_listeners:format_bind(ListenOn), case StopRet of ok -> @@ -261,7 +321,7 @@ stop_listener(GwName, {Type, LisName, ListenOn, SocketOpts, Cfg}) -> end, StopRet. -stop_listener(GwName, Type, LisName, ListenOn, _SocketOpts, _Cfg) -> +stop_listener(GwName, Type, LisName, ListenOn, _Cfg) -> Name = emqx_gateway_utils:listener_id(GwName, Type, LisName), esockd:close(Name, ListenOn). @@ -378,8 +438,7 @@ stringfy(T) -> Type :: udp | tcp | ssl | dtls, Name :: atom(), ListenOn :: esockd:listen_on(), - SocketOpts :: esockd:option(), - Cfg :: map() + RawCfg :: map() }). normalize_config(RawConf) -> LisMap = maps:get(listeners, RawConf, #{}), @@ -391,14 +450,7 @@ normalize_config(RawConf) -> maps:fold( fun(Name, Confs, AccIn2) -> ListenOn = maps:get(bind, Confs), - SocketOpts = esockd_opts(Type, Confs), - RemainCfgs = maps:without( - [bind, tcp, ssl, udp, dtls] ++ - proplists:get_keys(SocketOpts), - Confs - ), - Cfg = maps:merge(Cfg0, RemainCfgs), - [{Type, Name, ListenOn, SocketOpts, Cfg} | AccIn2] + [{Type, Name, ListenOn, Confs#{gw_conf => Cfg0}} | AccIn2] end, [], Liss @@ -410,7 +462,7 @@ normalize_config(RawConf) -> ) ). -esockd_opts(Type, Opts0) -> +esockd_opts(Type, Opts0) when ?IS_ESOCKD_LISTENER(Type) -> Opts1 = maps:with( [ acceptors, @@ -425,37 +477,70 @@ esockd_opts(Type, Opts0) -> maps:to_list( case Type of tcp -> - Opts2#{tcp_options => sock_opts(tcp, Opts0)}; + Opts2#{tcp_options => sock_opts(tcp_options, Opts0)}; ssl -> Opts2#{ - tcp_options => sock_opts(tcp, Opts0), - ssl_options => ssl_opts(ssl, Opts0) + tcp_options => sock_opts(tcp_options, Opts0), + ssl_options => ssl_opts(ssl_options, Opts0) }; udp -> - Opts2#{udp_options => sock_opts(udp, Opts0)}; + Opts2#{udp_options => sock_opts(udp_options, Opts0)}; dtls -> Opts2#{ - udp_options => sock_opts(udp, Opts0), - dtls_options => ssl_opts(dtls, Opts0) + udp_options => sock_opts(udp_options, Opts0), + dtls_options => ssl_opts(dtls_options, Opts0) } end ). +sock_opts(Name, Opts) -> + maps:to_list( + maps:without( + [active_n, keepalive], + maps:get(Name, Opts, #{}) + ) + ). + ssl_opts(Name, Opts) -> Type = case Name of - ssl -> tls; - dtls -> dtls + ssl_options -> tls; + dtls_options -> dtls end, emqx_tls_lib:to_server_opts(Type, maps:get(Name, Opts, #{})). -sock_opts(Name, Opts) -> - maps:to_list( - maps:without( - [active_n], - maps:get(Name, Opts, #{}) - ) - ). +ranch_opts(Type, ListenOn, Opts) -> + NumAcceptors = maps:get(acceptors, Opts, 4), + MaxConnections = maps:get(max_connections, Opts, 1024), + SocketOpts1 = + case Type of + wss -> + sock_opts(tcp_options, Opts) ++ + proplists:delete(handshake_timeout, ssl_opts(ssl_options, Opts)); + ws -> + sock_opts(tcp_options, Opts) + end, + SocketOpts = ip_port(ListenOn) ++ proplists:delete(reuseaddr, SocketOpts1), + #{ + num_acceptors => NumAcceptors, + max_connections => MaxConnections, + handshake_timeout => maps:get(handshake_timeout, Opts, 15000), + socket_opts => SocketOpts + }. + +ws_opts(Opts, Conf) -> + ConnMod = maps:get(connection_mod, Conf, emqx_gateway_conn), + WsPaths = [ + {emqx_utils_maps:deep_get([websocket, path], Opts, "") ++ "/[...]", ConnMod, Conf} + ], + Dispatch = cowboy_router:compile([{'_', WsPaths}]), + ProxyProto = maps:get(proxy_protocol, Opts, false), + #{env => #{dispatch => Dispatch}, proxy_header => ProxyProto}. + +ip_port(Port) when is_integer(Port) -> + [{port, Port}]; +ip_port({Addr, Port}) -> + [{ip, Addr}, {port, Port}]. %%-------------------------------------------------------------------- %% Envs @@ -538,6 +623,32 @@ find_gateway_definitions() -> ) ). +-spec find_gateway_definition(atom()) -> {ok, map()} | {error, term()}. +find_gateway_definition(Name) -> + ensure_gateway_loaded(), + find_gateway_definition(Name, ignore_lib_apps(application:loaded_applications())). + +-dialyzer({no_match, [find_gateway_definition/2]}). +find_gateway_definition(Name, [App | T]) -> + Attrs = find_attrs(App, gateway), + SearchFun = fun({_App, _Mod, #{name := GwName}}) -> + GwName =:= Name + end, + case lists:search(SearchFun, Attrs) of + {value, {_App, _Mod, Defination}} -> + case check_gateway_edition(Defination) of + true -> + {ok, Defination}; + _ -> + {error, invalid_edition} + end; + false -> + find_gateway_definition(Name, T) + end; +find_gateway_definition(_Name, []) -> + {error, not_found}. + +-dialyzer({no_match, [gateways/1]}). gateways([]) -> []; gateways([ @@ -550,7 +661,20 @@ gateways([ }} | More ]) when is_atom(Name), is_atom(CbMod), is_atom(SchemaMod) -> - [Defination | gateways(More)]. + case check_gateway_edition(Defination) of + true -> + [Defination | gateways(More)]; + _ -> + gateways(More) + end. + +-if(?EMQX_RELEASE_EDITION == ee). +check_gateway_edition(_Defination) -> + true. +-else. +check_gateway_edition(Defination) -> + ce == maps:get(edition, Defination, ce). +-endif. find_attrs(App, Def) -> [ @@ -624,7 +748,9 @@ ensure_gateway_loaded() -> emqx_gateway_stomp, emqx_gateway_coap, emqx_gateway_lwm2m, - emqx_gateway_mqttsn + emqx_gateway_mqttsn, + emqx_gateway_gbt32960, + emqx_gateway_ocpp ] ). diff --git a/apps/emqx_gateway/test/emqx_gateway_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_SUITE.erl index 9e0beb8cd..2574db644 100644 --- a/apps/emqx_gateway/test/emqx_gateway_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_SUITE.erl @@ -74,13 +74,7 @@ end_per_testcase(_TestCase, _Config) -> %%-------------------------------------------------------------------- t_registered_gateway(_) -> - [ - {coap, #{cbkmod := emqx_gateway_coap}}, - {exproto, #{cbkmod := emqx_gateway_exproto}}, - {lwm2m, #{cbkmod := emqx_gateway_lwm2m}}, - {mqttsn, #{cbkmod := emqx_gateway_mqttsn}}, - {stomp, #{cbkmod := emqx_gateway_stomp}} - ] = emqx_gateway:registered_gateway(). + [{coap, #{cbkmod := emqx_gateway_coap}} | _] = emqx_gateway:registered_gateway(). t_load_unload_list_lookup(_) -> {ok, _} = emqx_gateway:load(?GWNAME, #{idle_timeout => 1000}), diff --git a/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl index 9cda5bc23..0b562e851 100644 --- a/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl @@ -96,10 +96,8 @@ t_gateways(_) -> ok. t_gateway(_) -> - {404, GwNotFoundReq1} = request(get, "/gateways/not_a_known_atom"), - assert_not_found(GwNotFoundReq1), - {404, GwNotFoundReq2} = request(get, "/gateways/undefined"), - assert_not_found(GwNotFoundReq2), + ?assertMatch({400, #{code := <<"BAD_REQUEST">>}}, request(get, "/gateways/not_a_known_atom")), + ?assertMatch({400, #{code := <<"BAD_REQUEST">>}}, request(get, "/gateways/undefined")), {204, _} = request(put, "/gateways/stomp", #{}), {200, StompGw} = request(get, "/gateways/stomp"), assert_fields_exist( @@ -110,7 +108,7 @@ t_gateway(_) -> {200, #{enable := true}} = request(get, "/gateways/stomp"), {204, _} = request(put, "/gateways/stomp", #{enable => false}), {200, #{enable := false}} = request(get, "/gateways/stomp"), - {404, _} = request(put, "/gateways/undefined", #{}), + ?assertMatch({400, #{code := <<"BAD_REQUEST">>}}, request(put, "/gateways/undefined", #{})), {400, _} = request(put, "/gateways/stomp", #{bad_key => "foo"}), ok. @@ -129,8 +127,14 @@ t_gateway_enable(_) -> {200, #{enable := NotEnable}} = request(get, "/gateways/stomp"), {204, _} = request(put, "/gateways/stomp/enable/" ++ atom_to_list(Enable), undefined), {200, #{enable := Enable}} = request(get, "/gateways/stomp"), - {404, _} = request(put, "/gateways/undefined/enable/true", undefined), - {404, _} = request(put, "/gateways/not_a_known_atom/enable/true", undefined), + ?assertMatch( + {400, #{code := <<"BAD_REQUEST">>}}, + request(put, "/gateways/undefined/enable/true", undefined) + ), + ?assertMatch( + {400, #{code := <<"BAD_REQUEST">>}}, + request(put, "/gateways/not_a_known_atom/enable/true", undefined) + ), {404, _} = request(put, "/gateways/coap/enable/true", undefined), ok. diff --git a/apps/emqx_gateway/test/emqx_gateway_auth_ct.erl b/apps/emqx_gateway/test/emqx_gateway_auth_ct.erl index 92bf95a69..215302105 100644 --- a/apps/emqx_gateway/test/emqx_gateway_auth_ct.erl +++ b/apps/emqx_gateway/test/emqx_gateway_auth_ct.erl @@ -45,7 +45,7 @@ -include_lib("common_test/include/ct.hrl"). -include_lib("emqx/include/emqx_placeholder.hrl"). --define(CALL(Msg), gen_server:call(?MODULE, {?FUNCTION_NAME, Msg})). +-define(CALL(Msg), gen_server:call(?MODULE, {?FUNCTION_NAME, Msg}, 15000)). -define(AUTHN_HTTP_PORT, 37333). -define(AUTHN_HTTP_PATH, "/auth"). diff --git a/apps/emqx_gateway/test/emqx_gateway_cli_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_cli_SUITE.erl index b2280bb20..f5be9ce14 100644 --- a/apps/emqx_gateway/test/emqx_gateway_cli_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_cli_SUITE.erl @@ -118,14 +118,8 @@ t_gateway_registry_usage(_) -> t_gateway_registry_list(_) -> emqx_gateway_cli:'gateway-registry'(["list"]), - ?assertEqual( - "Registered Name: coap, Callback Module: emqx_gateway_coap\n" - "Registered Name: exproto, Callback Module: emqx_gateway_exproto\n" - "Registered Name: lwm2m, Callback Module: emqx_gateway_lwm2m\n" - "Registered Name: mqttsn, Callback Module: emqx_gateway_mqttsn\n" - "Registered Name: stomp, Callback Module: emqx_gateway_stomp\n", - acc_print() - ). + %% TODO: assert it. + _ = acc_print(). t_gateway_usage(_) -> ?assertEqual( @@ -142,14 +136,8 @@ t_gateway_usage(_) -> t_gateway_list(_) -> emqx_gateway_cli:gateway(["list"]), - ?assertEqual( - "Gateway(name=coap, status=unloaded)\n" - "Gateway(name=exproto, status=unloaded)\n" - "Gateway(name=lwm2m, status=unloaded)\n" - "Gateway(name=mqttsn, status=unloaded)\n" - "Gateway(name=stomp, status=unloaded)\n", - acc_print() - ), + %% TODO: assert it. + _ = acc_print(), emqx_gateway_cli:gateway(["load", "mqttsn", ?CONF_MQTTSN]), ?assertEqual("ok\n", acc_print()), diff --git a/apps/emqx_gateway_coap/README.md b/apps/emqx_gateway_coap/README.md index 405366e89..653fd7433 100644 --- a/apps/emqx_gateway_coap/README.md +++ b/apps/emqx_gateway_coap/README.md @@ -5,7 +5,7 @@ with [Publish-Subscribe Broker for the CoAP](https://datatracker.ietf.org/doc/ht ## Quick Start -In EMQX 5.0, CoAP gateways can be configured and enabled through the Dashboard. +In EMQX 5.0, CoAP gateway can be configured and enabled through the Dashboard. It can also be enabled via the HTTP API or emqx.conf, e.g. In emqx.conf: diff --git a/apps/emqx_gateway_coap/rebar.config b/apps/emqx_gateway_coap/rebar.config index 3b070a72a..493ebe04f 100644 --- a/apps/emqx_gateway_coap/rebar.config +++ b/apps/emqx_gateway_coap/rebar.config @@ -1,3 +1,4 @@ +%% -*- mode: erlang -*- {erl_opts, [debug_info]}. {deps, [ {emqx, {path, "../emqx"}}, diff --git a/apps/emqx_gateway_coap/src/emqx_gateway_coap.app.src b/apps/emqx_gateway_coap/src/emqx_gateway_coap.app.src index 30c176139..10dd6efef 100644 --- a/apps/emqx_gateway_coap/src/emqx_gateway_coap.app.src +++ b/apps/emqx_gateway_coap/src/emqx_gateway_coap.app.src @@ -1,3 +1,4 @@ +%% -*- mode: erlang -*- {application, emqx_gateway_coap, [ {description, "CoAP Gateway"}, {vsn, "0.1.5"}, diff --git a/apps/emqx_gateway_exproto/rebar.config b/apps/emqx_gateway_exproto/rebar.config index 473fa9b67..aafbe4e13 100644 --- a/apps/emqx_gateway_exproto/rebar.config +++ b/apps/emqx_gateway_exproto/rebar.config @@ -1,3 +1,4 @@ +%% -*- mode: erlang -*- {erl_opts, [debug_info]}. {deps, [ {emqx, {path, "../emqx"}}, diff --git a/apps/emqx_gateway_exproto/src/emqx_gateway_exproto.app.src b/apps/emqx_gateway_exproto/src/emqx_gateway_exproto.app.src index 09622763b..890435d59 100644 --- a/apps/emqx_gateway_exproto/src/emqx_gateway_exproto.app.src +++ b/apps/emqx_gateway_exproto/src/emqx_gateway_exproto.app.src @@ -1,3 +1,4 @@ +%% -*- mode: erlang -*- {application, emqx_gateway_exproto, [ {description, "ExProto Gateway"}, {vsn, "0.1.4"}, diff --git a/apps/emqx_gateway_exproto/test/emqx_exproto_SUITE.erl b/apps/emqx_gateway_exproto/test/emqx_exproto_SUITE.erl index 1c4c7ba08..76e11ef00 100644 --- a/apps/emqx_gateway_exproto/test/emqx_exproto_SUITE.erl +++ b/apps/emqx_gateway_exproto/test/emqx_exproto_SUITE.erl @@ -636,18 +636,18 @@ close({dtls, Sock}) -> %% Server-Opts socketopts(tcp) -> - #{tcp => tcp_opts()}; + #{tcp_options => tcp_opts()}; socketopts(ssl) -> #{ - tcp => tcp_opts(), - ssl => ssl_opts() + tcp_options => tcp_opts(), + ssl_options => ssl_opts() }; socketopts(udp) -> - #{udp => udp_opts()}; + #{udp_options => udp_opts()}; socketopts(dtls) -> #{ - udp => udp_opts(), - dtls => dtls_opts() + udp_options => udp_opts(), + dtls_options => dtls_opts() }. tcp_opts() -> diff --git a/apps/emqx_gateway_gbt32960/BSL.txt b/apps/emqx_gateway_gbt32960/BSL.txt new file mode 100644 index 000000000..0acc0e696 --- /dev/null +++ b/apps/emqx_gateway_gbt32960/BSL.txt @@ -0,0 +1,94 @@ +Business Source License 1.1 + +Licensor: Hangzhou EMQ Technologies Co., Ltd. +Licensed Work: EMQX Enterprise Edition + The Licensed Work is (c) 2023 + Hangzhou EMQ Technologies Co., Ltd. +Additional Use Grant: Students and educators are granted right to copy, + modify, and create derivative work for research + or education. +Change Date: 2027-02-01 +Change License: Apache License, Version 2.0 + +For information about alternative licensing arrangements for the Software, +please contact Licensor: https://www.emqx.com/en/contact + +Notice + +The Business Source License (this document, or the “License”) is not an Open +Source license. However, the Licensed Work will eventually be made available +under an Open Source License, as stated in this License. + +License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved. +“Business Source License” is a trademark of MariaDB Corporation Ab. + +----------------------------------------------------------------------------- + +Business Source License 1.1 + +Terms + +The Licensor hereby grants you the right to copy, modify, create derivative +works, redistribute, and make non-production use of the Licensed Work. The +Licensor may make an Additional Use Grant, above, permitting limited +production use. + +Effective on the Change Date, or the fourth anniversary of the first publicly +available distribution of a specific version of the Licensed Work under this +License, whichever comes first, the Licensor hereby grants you rights under +the terms of the Change License, and the rights granted in the paragraph +above terminate. + +If your use of the Licensed Work does not comply with the requirements +currently in effect as described in this License, you must purchase a +commercial license from the Licensor, its affiliated entities, or authorized +resellers, or you must refrain from using the Licensed Work. + +All copies of the original and modified Licensed Work, and derivative works +of the Licensed Work, are subject to this License. This License applies +separately for each version of the Licensed Work and the Change Date may vary +for each version of the Licensed Work released by Licensor. + +You must conspicuously display this License on each original or modified copy +of the Licensed Work. If you receive the Licensed Work in original or +modified form from a third party, the terms and conditions set forth in this +License apply to your use of that work. + +Any use of the Licensed Work in violation of this License will automatically +terminate your rights under this License for the current and all other +versions of the Licensed Work. + +This License does not grant you any right in any trademark or logo of +Licensor or its affiliates (provided that you may use a trademark or logo of +Licensor as expressly required by this License). + +TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON +AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS, +EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND +TITLE. + +MariaDB hereby grants you permission to use this License’s text to license +your works, and to refer to it using the trademark “Business Source License”, +as long as you comply with the Covenants of Licensor below. + +Covenants of Licensor + +In consideration of the right to use this License’s text and the “Business +Source License” name and trademark, Licensor covenants to MariaDB, and to all +other recipients of the licensed work to be provided by Licensor: + +1. To specify as the Change License the GPL Version 2.0 or any later version, + or a license that is compatible with GPL Version 2.0 or a later version, + where “compatible” means that software provided under the Change License can + be included in a program with software provided under GPL Version 2.0 or a + later version. Licensor may specify additional Change Licenses without + limitation. + +2. To either: (a) specify an additional grant of rights to use that does not + impose any additional restriction on the right granted in this License, as + the Additional Use Grant; or (b) insert the text “None”. + +3. To specify a Change Date. + +4. Not to modify this License in any other way. diff --git a/apps/emqx_gateway_gbt32960/README.md b/apps/emqx_gateway_gbt32960/README.md new file mode 100644 index 000000000..779e7004c --- /dev/null +++ b/apps/emqx_gateway_gbt32960/README.md @@ -0,0 +1,24 @@ +# emqx_gbt32960 + +The GBT32960 Gateway is based on the GBT32960 specification. + +## Quick Start + +In EMQX 5.0, GBT32960 gateway can be configured and enabled through the Dashboard. + +It can also be enabled via the HTTP API or emqx.conf, e.g. In emqx.conf: + +```properties +gateway.gbt32960 { + + mountpoint = "gbt32960/${clientid}" + + listeners.tcp.default { + bind = 7325 + } +} +``` + +> Note: +> Configuring the gateway via emqx.conf requires changes on a per-node basis, +> but configuring it via Dashboard or the HTTP API will take effect across the cluster. diff --git a/apps/emqx_gateway_gbt32960/doc/Data_Exchange_Guide_CN.md b/apps/emqx_gateway_gbt32960/doc/Data_Exchange_Guide_CN.md new file mode 100644 index 000000000..8fdd77449 --- /dev/null +++ b/apps/emqx_gateway_gbt32960/doc/Data_Exchange_Guide_CN.md @@ -0,0 +1,743 @@ +# EMQX GBT/32960 网关 + +该文档定义了 **GBT/32960** 网关和 **EMQX** 之间数据交换的格式 + +约定: +- Payload 采用 Json 格式进行组装 +- Json Key 采用大驼峰格式命名 +- 使用车辆的 `vin` 值作为 `clientid` +- 默认挂载点为: gbt32960/${clientid} + +# Upstream +数据流向: Terminal -> GBT/32960 -> EMQX + +## 车辆登入 +Topic: gbt32960/${clientid}/upstream/vlogin + +```json +{ + "Cmd": 1, + "Encrypt": 1, + "Vin": "1G1BL52P7TR115520", + "Data": { + "ICCID": "12345678901234567890", + "Id": "C", + "Length": 1, + "Num": 1, + "Seq": 1, + "Time": { + "Day": 29, + "Hour": 12, + "Minute": 19, + "Month": 12, + "Second": 20, + "Year": 12 + } + } +} +``` + +其中 + +| 字段 | 类型 | 描述 | +| --------- | ------- | ------------------------------------------------------------ | +| `Cmd` | Integer | 命令单元； `1` 表示车辆登入 | +| `Encrypt` | Integer | 数据单元加密方式，`1` 表示不加密，`2` 数据经过 RSA 加密，`3` 数据经过 ASE128 算法加密；`254` 表示异常；`255` 表示无效；其他预留 | +| `Vin` | String | 唯一识别码，即车辆 VIN 码 | +| `Data` | Object | 数据单元， JSON 对象格式。 | + +车辆登入的数据单元格式为 + +| 字段 | 类型 | 描述 | +| -------- | ------- | ------------------------------------------------------------ | +| `Time` | Object | 数据采集时间，按年，月，日，时，分，秒，格式见示例。 | +| `Seq` | Integer | 登入流水号 | +| `ICCID` | String | 长度为20的字符串，SIM 卡的 ICCID 号 | +| `Num` | Integer | 可充电储能子系统数，有效值 0 ~ 250 | +| `Length` | Integer | 可充电储能系统编码长度，有效值 0 ~ 50 | +| `Id` | String | 可充电储能系统编码，长度为 "子系统数" 与 "编码长度" 值的乘积 | + +## 车辆登出 + +Topic: gbt32960/${clientid}/upstream/vlogout + +车辆登出的 `Cmd` 值为 4，其余字段含义与登入相同： + +```json +{ + "Cmd": 4, + "Encrypt": 1, + "Vin": "1G1BL52P7TR115520", + "Data": { + "Seq": 1, + "Time": { + "Day": 1, + "Hour": 2, + "Minute": 59, + "Month": 1, + "Second": 0, + "Year": 16 + } + } +} +``` + +## 实时信息上报 + +Topic: gbt32960/${clientid}/upstream/info + +> 不同信息类型上报，格式上只有 Infos 里面的对象属性不同，通过 `Type` 进行区分 +> Infos 为数组，代表车载终端每次报文可以上报多个信息 + +### 整车数据 + +```json +{ + "Cmd": 2, + "Encrypt": 1, + "Vin": "1G1BL52P7TR115520", + "Data": { + "Infos": [ + { + "AcceleratorPedal": 90, + "BrakePedal": 0, + "Charging": 1, + "Current": 15000, + "DC": 1, + "Gear": 5, + "Mileage": 999999, + "Mode": 1, + "Resistance": 6000, + "SOC": 50, + "Speed": 2000, + "Status": 1, + "Type": "Vehicle", + "Voltage": 5000 + } + ], + "Time": { + "Day": 1, + "Hour": 2, + "Minute": 59, + "Month": 1, + "Second": 0, + "Year": 16 + } + } +} +``` + + + +其中，整车信息字段含义如下： + +| 字段 | 类型 | 描述 | +| ------------ | ------- | ------------------------------------------------------------ | +| `Type` | String | 数据类型，`Vehicle` 表示该结构为整车信息 | +| `Status` | Integer | 车辆状态，`1` 表示启动状态；`2` 表示熄火；`3` 表示其状态；`254` 表示异常；`255` 表示无效 | +| `Charging` | Integer | 充电状态，`1` 表示停车充电；`2` 行驶充电；`3` 未充电状态；`4` 充电完成；`254` 表示异常；`255` 表示无效 | +| `Mode` | Integer | 运行模式，`1` 表示纯电；`2` 混动；`3` 燃油；`254` 表示异常；`255` 表示无效 | +| `Speed` | Integer | 车速，有效值 （ 0~ 2200，表示 0 km/h ~ 220 km/h），单位 0.1 km/h | +| `Mileage` | Integer | 累计里程，有效值 0 ~9,999,999（表示 0 km ~ 999,999.9 km），单位 0.1 km | +| `Voltage` | Integer | 总电压，有效值范围 0 ~10000（表示 0 V ~ 1000 V）单位 0.1 V | +| `Current` | Integer | 总电流，有效值 0 ~ 20000 （偏移量 1000，表示 -1000 A ~ +1000 A，单位 0.1 A | +| `SOC` | Integer | SOC，有效值 0 ~ 100（表示 0% ~ 100%） | +| `DC` | Integer | DC，`1` 工作；`2` 断开；`254` 表示异常；`255` 表示无效 | +| `Gear` | Integer | 档位，参考原协议的 表 A.1，此值为其转换为整数的值 | +| `Resistance` | Integer | 绝缘电阻，有效范围 0 ~ 60000（表示 0 k欧姆 ~ 60000 k欧姆） | + +### 驱动电机数据 + +```json +{ + "Cmd": 2, + "Encrypt": 1, + "Vin": "1G1BL52P7TR115520", + "Data": { + "Infos": [ + { + "Motors": [ + { + "CtrlTemp": 125, + "DCBusCurrent": 31203, + "InputVoltage": 30012, + "MotorTemp": 125, + "No": 1, + "Rotating": 30000, + "Status": 1, + "Torque": 25000 + }, + { + "CtrlTemp": 125, + "DCBusCurrent": 30200, + "InputVoltage": 32000, + "MotorTemp": 145, + "No": 2, + "Rotating": 30200, + "Status": 1, + "Torque": 25300 + } + ], + "Number": 2, + "Type": "DriveMotor" + } + ], + "Time": { + "Day": 1, + "Hour": 2, + "Minute": 59, + "Month": 1, + "Second": 0, + "Year": 16 + } + } +} +``` + +其中，驱动电机数据各个字段的含义是 + +| 字段 | 类型 | 描述 | +| -------- | ------- | ------------------------------ | +| `Type` | String | 数据类型，此处为 `DriveMotor` | +| `Number` | Integer | 驱动电机个数，有效值 1~253 | +| `Motors` | Array | 驱动电机数据列表 | + +驱动电机数据字段为： + +| 字段 | 类型 | 描述 | +| -------------- | -------- | ------------------------------------------------------------ | +| `No` | Integer | 驱动电机序号，有效值 1~253 | +| `Status` | Integer | 驱动电机状态，`1` 表示耗电；`2`发电；`3` 关闭状态；`4` 准备状态；`254` 表示异常；`255` 表示无效 | +| `CtrlTemp` | Integer | 驱动电机控制器温度，有效值 0~250（数值偏移 40°C，表示 -40°C ~ +210°C）单位 °C | +| `Rotating` | Interger | 驱动电机转速，有效值 0~65531（数值偏移 20000表示 -20000 r/min ~ 45531 r/min）单位 1 r/min | +| `Torque` | Integer | 驱动电机转矩，有效值 0~65531（数据偏移量 20000，表示 - 2000 N·m ~ 4553.1 N·m）单位 0.1 N·m | +| `MotorTemp` | Integer | 驱动电机温度，有效值 0~250（数据偏移量 40 °C，表示 -40°C ~ +210°C）单位 1°C | +| `InputVoltage` | Integer | 电机控制器输入电压，有效值 0~60000（表示 0V ~ 6000V）单位 0.1 V | +| `DCBusCurrent` | Interger | 电机控制器直流母线电流，有效值 0~20000（数值偏移 1000A，表示 -1000A ~ +1000 A）单位 0.1 A | + +### 燃料电池数据 + +```json +{ + "Cmd": 2, + "Encrypt": 1, + "Vin": "1G1BL52P7TR115520", + "Data": { + "Infos": [ + { + "CellCurrent": 12000, + "CellVoltage": 10000, + "DCStatus": 1, + "FuelConsumption": 45000, + "H_ConcSensorCode": 11, + "H_MaxConc": 35000, + "H_MaxPress": 500, + "H_MaxTemp": 12500, + "H_PressSensorCode": 12, + "H_TempProbeCode": 10, + "ProbeNum": 2, + "ProbeTemps": [120, 121], + "Type": "FuelCell" + } + ], + "Time": { + "Day": 1, + "Hour": 2, + "Minute": 59, + "Month": 1, + "Second": 0, + "Year": 16 + } + } +} +``` + +其中，燃料电池数据各个字段的含义是 + +| 字段 | 类型 | 描述 | +| ------------------- | ------- | ------------------------------------------------------------ | +| `Type` | String | 数据类型，此处为 `FuleCell` | +| `CellVoltage` | Integer | 燃料电池电压，有效值范围 0~20000（表示 0V ~ 2000V）单位 0.1 V | +| `CellCurrent` | Integer | 燃料电池电流，有效值范围 0~20000（表示 0A ~ +2000A）单位 0.1 A | +| `FuelConsumption` | Integer | 燃料消耗率，有效值范围 0~60000（表示 0kg/100km ~ 600 kg/100km) 单位 0.01 kg/100km | +| `ProbeNum` | Integer | 燃料电池探针总数，有效值范围 0~65531 | +| `ProbeTemps` | Array | 燃料电池每探针温度值 | +| `H_MaxTemp` | Integer | 氢系统最高温度，有效值 0~2400（偏移量40°C，表示 -40°C ~ 200°C）单位 0.1 °C | +| `H_TempProbeCode` | Integer | 氢系统最高温度探针代号，有效值 1~252 | +| `H_MaxConc` | Integer | 氢气最高浓度，有效值 0~60000（表示 0mg/kg ~ 50000 mg/kg）单位 1mg/kg | +| `H_ConcSensorCode` | Integer | 氢气最高浓度传感器代号，有效值 1~252 | +| `H_MaxPress` | Integer | 氢气最高压力，有效值 0~1000（表示 0 MPa ~ 100 MPa）最小单位 0.1 MPa | +| `H_PressSensorCode` | Integer | 氢气最高压力传感器代号，有效值 1~252 | +| `DCStatus` | Integer | 高压 DC/DC状态，`1` 表示工作；`2`断开 | + +### 发动机数据 + +```json +{ + "Cmd": 2, + "Encrypt": 1, + "Vin": "1G1BL52P7TR115520", + "Data": { + "Infos": [ + { + "CrankshaftSpeed": 2000, + "FuelConsumption": 200, + "Status": 1, + "Type": "Engine" + } + ], + "Time": { + "Day": 1, + "Hour": 22, + "Minute": 59, + "Month": 10, + "Second": 0, + "Year": 16 + } + } +} +``` + +其中，发动机数据各个字段的含义是 + +| 字段 | 类型 | 描述 | +| ----------------- | ------- | ------------------------------------------------------------ | +| `Type` | String | 数据类型，此处为 `Engine` | +| `Status` | Integer | 发动机状态，`1` 表示启动；`2` 关闭 | +| `CrankshaftSpeed` | Integer | 曲轴转速，有效值 0~60000（表示 0r/min ~ 60000r/min）单位 1r/min | +| `FuelConsumption` | Integer | 燃料消耗率，有效范围 0~60000（表示 0L/100km ~ 600L/100km）单位 0.01 L/100km | + + + +### 车辆位置数据 + +```json +{ + "Cmd": 2, + "Encrypt": 1, + "Vin": "1G1BL52P7TR115520", + "Data": { + "Infos": [ + { + "Latitude": 100, + "Longitude": 10, + "Status": 0, + "Type": "Location" + } + ], + "Time": { + "Day": 1, + "Hour": 22, + "Minute": 59, + "Month": 10, + "Second": 0, + "Year": 16 + } + } +} +``` + +其中，车辆位置数据各个字段的含义是 + +| 字段 | 类型 | 描述 | +| ----------- | ------- | ----------------------------------------------------- | +| `Type` | String | 数据类型，此处为 `Location` | +| `Status` | Integer | 定位状态，见原协议表15，此处为所有比特位的整型值 | +| `Longitude` | Integer | 经度，以度为单位的纬度值乘以 10^6，精确到百万分之一度 | +| `Latitude` | Integer | 纬度，以度为单位的纬度值乘以 10^6，精确到百万分之一度 | + + + +### 极值数据 + +```json +{ + "Cmd": 2, + "Encrypt": 1, + "Vin": "1G1BL52P7TR115520", + "Data": { + "Infos": [ + { + "MaxBatteryVoltage": 7500, + "MaxTemp": 120, + "MaxTempProbeNo": 12, + "MaxTempSubsysNo": 14, + "MaxVoltageBatteryCode": 10, + "MaxVoltageBatterySubsysNo": 12, + "MinBatteryVoltage": 2000, + "MinTemp": 40, + "MinTempProbeNo": 13, + "MinTempSubsysNo": 15, + "MinVoltageBatteryCode": 11, + "MinVoltageBatterySubsysNo": 13, + "Type": "Extreme" + } + ], + "Time": { + "Day": 30, + "Hour": 12, + "Minute": 22, + "Month": 5, + "Second": 59, + "Year": 17 + } + } +} +``` + +其中，极值数据各个字段的含义是 + +| 字段 | 类型 | 描述 | +| --------------------------- | ------- | ------------------------------------------------------------ | +| `Type` | String | 数据类型，此处为 `Extreme` | +| `MaxVoltageBatterySubsysNo` | Integer | 最高电压电池子系统号，有效值 1~250 | +| `MaxVoltageBatteryCode` | Integer | 最高电压电池单体代号，有效值 1~250 | +| `MaxBatteryVoltage` | Integer | 电池单体电压最高值，有效值 0~15000（表示 0V~15V）单位 0.001V | +| `MinVoltageBatterySubsysNo` | Integer | 最低电压电池子系统号，有效值 1~250 | +| `MinVoltageBatteryCode` | Integer | 最低电压电池单体代号，有效值 1~250 | +| `MinBatteryVoltage` | Integer | 电池单体电压最低值，有效值 0~15000（表示 0V~15V）单位 0.001V | +| `MaxTempSubsysNo` | Integer | 最高温度子系统号，有效值 1~250 | +| `MaxTempProbeNo` | Integer | 最高温度探针序号，有效值 1~250 | +| `MaxTemp` | Integer | 最高温度值，有效值范围 0~250（偏移量40，表示 -40°C ~ +210°C） | +| `MinTempSubsysNo` | Integer | 最低温度子系统号，有效值 1~250 | +| `MinTempProbeNo` | Integer | 最低温度探针序号，有效值 1~250 | +| `MinTemp` | Integer | 最低温度值，有效值范围 0~250（偏移量40，表示 -40°C ~ +210°C） | + + + +### 报警数据 + +```json +{ + "Cmd": 2, + "Encrypt": 1, + "Vin": "1G1BL52P7TR115520", + "Data": { + "Infos": [ + { + "FaultChargeableDeviceNum": 1, + "FaultChargeableDeviceList": ["00C8"], + "FaultDriveMotorNum": 0, + "FaultDriveMotorList": [], + "FaultEngineNum": 1, + "FaultEngineList": ["006F"], + "FaultOthersNum": 0, + "FaultOthersList": [], + "GeneralAlarmFlag": 3, + "MaxAlarmLevel": 1, + "Type": "Alarm" + } + ], + "Time": { + "Day": 20, + "Hour": 22, + "Minute": 23, + "Month": 12, + "Second": 59, + "Year": 17 + } + } +} +``` + +其中，报警数据各个字段的含义是 + +| 字段 | 类型 | 描述 | +| --------------------------- | ------- | ------------------------------------------------------------ | +| `Type` | String | 数据类型，此处为 `Alarm` | +| `MaxAlarmLevel` | Integer | 最高报警等级，有效值范围 0~3，`0` 表示无故障，`1` 表示 `1` 级故障 | +| `GeneralAlarmFlag` | Integer | 通用报警标志位，见原协议表 18 | +| `FaultChargeableDeviceNum` | Integer | 可充电储能装置故障总数，有效值 0~252 | +| `FaultChargeableDeviceList` | Array | 可充电储能装置故障代码列表 | +| `FaultDriveMotorNum` | Integer | 驱动电机故障总数，有效置范围 0 ~252 | +| `FaultDriveMotorList` | Array | 驱动电机故障代码列表 | +| `FaultEngineNum` | Integer | 发动机故障总数，有效值范围 0~252 | +| `FaultEngineList` | Array | 发动机故障代码列表 | +| `FaultOthersNum` | Integer | 其他故障总数 | +| `FaultOthersList` | Array | 其他故障代码列表 | + + + +### 可充电储能装置电压数据 + +```json +{ + "Cmd": 2, + "Encrypt": 1, + "Vin": "1G1BL52P7TR115520", + "Data": { + "Infos": [ + { + "Number": 2, + "SubSystems": [ + { + "CellsTotal": 2, + "CellsVoltage": [5000], + "ChargeableCurrent": 10000, + "ChargeableSubsysNo": 1, + "ChargeableVoltage": 5000, + "FrameCellsCount": 1, + "FrameCellsIndex": 0 + }, + { + "CellsTotal": 2, + "CellsVoltage": [5001], + "ChargeableCurrent": 10001, + "ChargeableSubsysNo": 2, + "ChargeableVoltage": 5001, + "FrameCellsCount": 1, + "FrameCellsIndex": 1 + } + ], + "Type": "ChargeableVoltage" + } + ], + "Time": { + "Day": 1, + "Hour": 22, + "Minute": 59, + "Month": 10, + "Second": 0, + "Year": 16 + } + } +} +``` + + + +其中，字段定义如下 + +| 字段 | 类型 | 描述 | +| ----------- | ------- | ------------------------------------ | +| `Type` | String | 数据类型，此处为 `ChargeableVoltage` | +| `Number` | Integer | 可充电储能子系统个数，有效范围 1~250 | +| `SubSystem` | Object | 可充电储能子系统电压信息列表 | + +可充电储能子系统电压信息数据格式： + +| 字段 | 类型 | 描述 | +| -------------------- | ------- | ------------------------------------------------------------ | +| `ChargeableSubsysNo` | Integer | 可充电储能子系统号，有效值范围，1~250 | +| `ChargeableVoltage` | Integer | 可充电储能装置电压，有效值范围，0~10000（表示 0V ~ 1000V）单位 0.1 V | +| `ChargeableCurrent` | Integer | 可充电储能装置电流，有效值范围，0~20000（数值偏移量 1000A，表示 -1000A ~ +1000A）单位 0.1 A | +| `CellsTotal` | Integer | 单体电池总数，有效值范围 1~65531 | +| `FrameCellsIndex` | Integer | 本帧起始电池序号，当本帧单体个数超过 200 时，应该拆分多个帧进行传输，有效值范围 1~65531 | +| `FrameCellsCount` | Integer | 本帧单体电池总数，有效值范围 1~200 | +| `CellsVoltage` | Array | 单体电池电压，有效值范围 0~60000（表示 0V ~ 60.000V）单位 0.001V | + + + +### 可充电储能装置温度数据 + +```json +{ + "Cmd": 2, + "Encrypt": 1, + "Vin": "1G1BL52P7TR115520", + "Data": { + "Infos": [ + { + "Number": 2, + "SubSystems": [ + { + "ChargeableSubsysNo": 1, + "ProbeNum": 10, + "ProbesTemp": [0, 0, 0, 0, 0, 0, 0, 0, 19, 136] + }, + { + "ChargeableSubsysNo": 2, + "ProbeNum": 1, + "ProbesTemp": [100] + } + ], + "Type": "ChargeableTemp" + } + ], + "Time": { + "Day": 1, + "Hour": 22, + "Minute": 59, + "Month": 10, + "Second": 0, + "Year": 16 + } + } +} +``` +其中，数据格式为： + +| 字段 | 类型 | 描述 | +| ------------ | ------- | --------------------------------- | +| `Type` | String | 数据类型，此处为 `ChargeableTemp` | +| `Number` | Integer | 可充电储能子系统温度信息列表长度 | +| `SubSystems` | Object | 可充电储能子系统温度信息列表 | + +可充电储能子系统温度信息格式为 + +| 字段 | 类型 | 描述 | +| -------------------- | -------- | ------------------------------------ | +| `ChargeableSubsysNo` | Ineteger | 可充电储能子系统号，有效值 1~250 | +| `ProbeNum` | Integer | 可充电储能温度探针个数 | +| `ProbesTemp` | Array | 可充电储能子系统各温度探针温度值列表 | + + + +## 数据补发 + +Topic: gbt32960/${clientid}/upstream/reinfo + +**数据格式: 略** (与实时数据上报相同) + +# Downstream + +> 请求数据流向: EMQX -> GBT/32960 -> Terminal + +> 应答数据流向: Terminal -> GBT/32960 -> EMQX + +下行主题: gbt32960/${clientid}/dnstream +上行应答主题: gbt32960/${clientid}/upstream/response + +## 参数查询 + + + +**Req:** + +```json +{ + "Action": "Query", + "Total": 2, + "Ids": ["0x01", "0x02"] +} +``` + +| 字段 | 类型 | 描述 | +| -------- | ------- | -------------------------------------------------- | +| `Action` | String | 下发命令类型，此处为 `Query` | +| `Total` | Integer | 查询参数总数 | +| `Ids` | Array | 需查询参数的 ID 列表，具体 ID 含义见原协议 表 B.10 | + +**Response:** + +```json +{ + "Cmd": 128, + "Encrypt": 1, + "Vin": "1G1BL52P7TR115520", + "Data": { + "Total": 2, + "Params": [ + {"0x01": 6000}, + {"0x02": 10} + ], + "Time": { + "Day": 2, + "Hour": 11, + "Minute": 12, + "Month": 2, + "Second": 12, + "Year": 17 + } + } +} +``` + + + +## 参数设置 + +**Req:** +```json +{ + "Action": "Setting", + "Total": 2, + "Params": [{"0x01": 5000}, + {"0x02": 200}] +} +``` + +| 字段 | 类型 | 描述 | +| -------- | ------- | ------------------------------ | +| `Action` | String | 下发命令类型，此处为 `Setting` | +| `Total` | Integer | 设置参数总数 | +| `Params` | Array | 需设置参数的 ID 和 值 | + +**Response:** + +```json +// fixme? 终端是按照这种方式返回? +{ + "Cmd": 129, + "Encrypt": 1, + "Vin": "1G1BL52P7TR115520", + "Data": { + "Total": 2, + "Params": [ + {"0x01": 5000}, + {"0x02": 200} + ], + "Time": { + "Day": 2, + "Hour": 11, + "Minute": 12, + "Month": 2, + "Second": 12, + "Year": 17 + } + } +} +``` + +## 终端控制 +**命令的不同, 参数不同; 无参数时为空** + +远程升级: +**Req:** + +```json +{ + "Action": "Control", + "Command": "0x01", + "Param": { + "DialingName": "hz203", + "Username": "user001", + "Password": "password01", + "Ip": "192.168.199.1", + "Port": 8080, + "ManufacturerId": "BMWA", + "HardwareVer": "1.0.0", + "SoftwareVer": "1.0.0", + "UpgradeUrl": "ftp://emqtt.io/ftp/server", + "Timeout": 10 + } +} +``` + +| 字段 | 类型 | 描述 | +| --------- | ------- | ------------------------------ | +| `Action` | String | 下发命令类型，此处为 `Control` | +| `Command` | Integer | 下发指令 ID,见原协议表 B.15 | +| `Param` | Object | 命令参数 | + +列表 + +车载终端关机: + +```json +{ + "Action": "Control", + "Command": "0x02" +} +``` + +... + +车载终端报警: +```json +{ + "Action": "Control", + "Command": "0x06", + "Param": {"Level": 0, "Message": "alarm message"} +} +``` diff --git a/apps/emqx_gateway_gbt32960/doc/Data_Exchange_Guide_EN.md b/apps/emqx_gateway_gbt32960/doc/Data_Exchange_Guide_EN.md new file mode 100644 index 000000000..98f7db30b --- /dev/null +++ b/apps/emqx_gateway_gbt32960/doc/Data_Exchange_Guide_EN.md @@ -0,0 +1,744 @@ +# EMQX GBT/32960 Gateway + +This document defines the format of the data exchange internal the **GBT/32960** gateway and the **EMQX**. + +Conventions: +- Payloads are assembled in Json format +- Json Keys are named in big hump format +- Use the `vin` value of the vehicle as the `clientid`. +- The default mountpoint is: `gbt32960/${clientid}` + +# Upstream +Data flow: Terminal -> GBT/32960 Gateway -> EMQX + +## Vehicle Login +Topic: gbt32960/${clientid}/upstream/vlogin + +```json +{ + "Cmd": 1, + "Encrypt": 1, + "Vin": "1G1BL52P7TR115520", + "Data": { + "ICCID": "12345678901234567890", + "Id": "C", + "Length": 1, + "Num": 1, + "Seq": 1, + "Time": { + "Day": 29, + "Hour": 12, + "Minute": 19, + "Month": 12, + "Second": 20, + "Year": 12 + } + } +} +``` + +definition: + +| Field | Type | Description | +| --------- | ------- | ------------------------------------------------------------ | +| `Cmd` | Integer | Command, `1` :: vehicle login | +| `Encrypt` | Integer | Data encryption method, `1` :: no encryption; `2` :: RSA; `3` :: ASE128; `254` :: abnormal; `255` :: invalid; other reserved | +| `Vin` | String | Unique identifier，namely, the vehicle VIN code | +| `Data` | Object | The JSON format data | + +The data unit format for vehicle login is: + +| Field | Type | Description | +| -------- | ------- | ------------------------------------------------------------ | +| `Time` | Object | Data collection time, in year, month, day, hour, minute, second. See the example for the format | +| `Seq` | Integer | Login sequence number | +| `ICCID` | String | String of length 20, the ICCID number of the SIM card | +| `Num` | Integer | Number of rechargeable energy storage subsystems, valid values 0 ~ 250 | +| `Length` | Integer | Rechargeable energy storage system encoding length, valid value 0 ~ 50 | +| `Id` | String | Rechargeable energy storage system code, the length is the product of the `Num` and the `Length` value | + +## Vehicle logout + +Topic: gbt32960/${clientid}/upstream/vlogout + +The `Cmd` value of vehicle logout is 4, and the meaning of the other fields is the same as that of the login. + +```json +{ + "Cmd": 4, + "Encrypt": 1, + "Vin": "1G1BL52P7TR115520", + "Data": { + "Seq": 1, + "Time": { + "Day": 1, + "Hour": 2, + "Minute": 59, + "Month": 1, + "Second": 0, + "Year": 16 + } + } +} +``` + +## Real-time information reporting + +Topic: gbt32960/${clientid}/upstream/info + +> When reporting messages of different information types, only the object attribute is different, which are distinguished by the `Type` field. +> Infos is an array, which means that the vehicle terminal can report multiple information in each message + +### Vehicle data + +```json +{ + "Cmd": 2, + "Encrypt": 1, + "Vin": "1G1BL52P7TR115520", + "Data": { + "Infos": [ + { + "AcceleratorPedal": 90, + "BrakePedal": 0, + "Charging": 1, + "Current": 15000, + "DC": 1, + "Gear": 5, + "Mileage": 999999, + "Mode": 1, + "Resistance": 6000, + "SOC": 50, + "Speed": 2000, + "Status": 1, + "Type": "Vehicle", + "Voltage": 5000 + } + ], + "Time": { + "Day": 1, + "Hour": 2, + "Minute": 59, + "Month": 1, + "Second": 0, + "Year": 16 + } + } +} +``` + + + +The meaning of the vehicle information field is as follows: + +| Field | Type | Description | +| ------------ | ------- | ------------------------------------------------------------ | +| `Type` | String | Data type, `Vehicle` :: this is the vehicle information | +| `Status` | Integer | Vehicle status, `1` :: started; `2` :: stalled; `3` :: others; `254` :: abnormal; `255` :: invalid | +| `Charging` | Integer | Charging status, `1` :: parking and charging; `2` :: driving and charging; `3` :: not charging; `4` :: charging is completed; `254` :: abnormal; `255` :: invalid | +| `Mode` | Integer | Operating mode, `1` :: pure electric; `2` :: hybrid; `3` :: fuel; `254` :: abnormal; `255` :: invalid | +| `Speed` | Integer | Vehicle speed, valid value (0~ 2200, indicating 0 km/h ~ 220 km/h), unit 0.1 km/h | +| `Mileage` | Integer | Accumulated mileage, valid value 0 ~9,999,999 (representing 0 km ~ 999,999.9 km), unit 0.1 km | +| `Voltage` | Integer | Total voltage, valid value 0 ~10000 (representing 0 V ~ 1000 V) unit 0.1 V | +| `Current` | Integer | Total current, valid value 0 ~ 20000 (offset 1000, :: -1000 A ~ +1000 A) unit 0.1 A | +| `SOC` | Integer | SOC, valid values 0 ~ 100 (representing 0% ~ 100%) | +| `DC` | Integer | DC, `1` works; `2` disconnects; `254` :: abnormal; `255` :: invalid | +| `Gear` | Integer | Gear, refer to Table A.1 of the original protocol, this value is converted into an integer | +| `Resistance` | Integer | Insulation resistance, valid range 0 ~ 60000 (representing 0 k ohm ~ 60000 k ohm) | + + + +### Drive motor data + +```json +{ + "Cmd": 2, + "Encrypt": 1, + "Vin": "1G1BL52P7TR115520", + "Data": { + "Infos": [ + { + "Motors": [ + { + "CtrlTemp": 125, + "DCBusCurrent": 31203, + "InputVoltage": 30012, + "MotorTemp": 125, + "No": 1, + "Rotating": 30000, + "Status": 1, + "Torque": 25000 + }, + { + "CtrlTemp": 125, + "DCBusCurrent": 30200, + "InputVoltage": 32000, + "MotorTemp": 145, + "No": 2, + "Rotating": 30200, + "Status": 1, + "Torque": 25300 + } + ], + "Number": 2, + "Type": "DriveMotor" + } + ], + "Time": { + "Day": 1, + "Hour": 2, + "Minute": 59, + "Month": 1, + "Second": 0, + "Year": 16 + } + } +} +``` + +The meaning of each field of the drive motor data is: + +| Field | Type | Description | +| -------- | ------- | ------------------------------- | +| `Type` | String | Data type, here is `DriveMotor` | +| `Number` | Integer | Number of drive motors, valid value 1~253 | +| `Motors` | Array | Drive motor data list | + +The drive motor data fields are: + +| Field | Type | Description | +| -------------- | -------- | --------------------------------------------------------------------------------------------------------------------| +| `No` | Integer | Drive motor serial number, valid value 1 ~ 253 | +| `Status` | Integer | Drive motor status, `1` :: consuming; `2` producing; `3` closed; `4` ready; `254` :: abnormal; `255` :: invalid | +| `CtrlTemp` | Integer | Drive motor controller temperature, valid value 0 ~ 250 (value offset 40°C, indicating -40°C ~ +210°C) unit °C | +| `Rotating` | Interger | Drive motor speed, valid value 0 ~ 65531 (numeric offset 20000 :: -20000 r/min ~ 45531 r/min) unit 1 r/min | +| `Torque` | Integer | Drive motor torque, valid value 0 ~ 65531 (data offset 20000, represents - 2000 N·m ~ 4553.1 N·m) unit 0.1 N·m | +| `MotorTemp` | Integer | Drive motor temperature, valid value 0 ~ 250 (data offset 40 °C, represents -40°C ~ +210°C) unit 1°C | +| `InputVoltage` | Integer | Motor controller input voltage, valid value 0 ~ 60000 (representing 0V ~ 6000V) unit 0.1 V | +| `DCBusCurrent` | Interger | Motor controller DC bus current, valid value 0 ~ 20000 (value offset 1000A, indicating -1000A ~ +1000 A) unit 0.1 A | + + +### Fuel cell data + +```json +{ + "Cmd": 2, + "Encrypt": 1, + "Vin": "1G1BL52P7TR115520", + "Data": { + "Infos": [ + { + "CellCurrent": 12000, + "CellVoltage": 10000, + "DCStatus": 1, + "FuelConsumption": 45000, + "H_ConcSensorCode": 11, + "H_MaxConc": 35000, + "H_MaxPress": 500, + "H_MaxTemp": 12500, + "H_PressSensorCode": 12, + "H_TempProbeCode": 10, + "ProbeNum": 2, + "ProbeTemps": [120, 121], + "Type": "FuelCell" + } + ], + "Time": { + "Day": 1, + "Hour": 2, + "Minute": 59, + "Month": 1, + "Second": 0, + "Year": 16 + } + } +} +``` + +The meaning of each field of fuel cell data is + +| Field | Type | Description | +| ------------------- | ------- | ---------------------------------------------------------------------------------------------------------------------| +| `Type` | String | Data type, here is `FuleCell` | +| `CellVoltage` | Integer | Fuel cell voltage, valid value range 0~20000 (representing 0V ~ 2000V) unit 0.1 V | +| `CellCurrent` | Integer | Fuel cell current, valid value range 0~20000 (representing 0A ~ +2000A) unit 0.1 A | +| `FuelConsumption` | Integer | Fuel consumption rate, valid value range 0~60000 (representing 0kg/100km ~ 600 kg/100km) unit 0.01 kg/100km | +| `ProbeNum` | Integer | Total number of fuel cell probes, valid value range 0~65531 | +| `ProbeTemps` | Array | Fuel cell temperature value per probe | +| `H_MaxTemp` | Integer | Maximum temperature of the hydrogen system, effective value 0~2400 (offset 40°C, indicating -40°C ~ 200°C) unit 0.1 °C | +| `H_TempProbeCode` | Integer | Hydrogen system maximum temperature probe code, valid value 1~252 | +| `H_MaxConc` | Integer | Maximum hydrogen concentration, valid value 0~60000 (representing 0mg/kg ~ 50000 mg/kg) unit 1mg/kg | +| `H_ConcSensorCode` | Integer | Hydrogen maximum concentration sensor code, valid value 1~252 | +| `H_MaxPress` | Integer | Maximum pressure of hydrogen, valid value 0~1000 (representing 0 MPa ~ 100 MPa) minimum unit 0.1 MPa | +| `H_PressSensorCode` | Integer | Hydrogen maximum pressure sensor code, valid value 1~252 | +| `DCStatus` | Integer | High voltage DC/DC status, `1` :: working; `2` :: disconnected | + +### Engine data + +```json +{ + "Cmd": 2, + "Encrypt": 1, + "Vin": "1G1BL52P7TR115520", + "Data": { + "Infos": [ + { + "CrankshaftSpeed": 2000, + "FuelConsumption": 200, + "Status": 1, + "Type": "Engine" + } + ], + "Time": { + "Day": 1, + "Hour": 22, + "Minute": 59, + "Month": 10, + "Second": 0, + "Year": 16 + } + } +} +``` + +The meaning of each field of the engine data is + +| Field | Type | Description | +| ------------------ | ------- | ------------------------------------------------------------------------------------------------| +| `Type` | String | Data type, here is `Engine` | +| `Status` | Integer | Engine status, `1` :: started; `2` :: shutdown | +| `CrankshaftSpeed` | Integer | Crankshaft speed, valid value 0~60000 (representing 0r/min ~ 60000r/min) unit 1r/min | +| `FuelConsumption` | Integer | Fuel consumption rate, valid range 0~60000 (representing 0L/100km ~ 600L/100km) unit 0.01 L/100km | + + + +### Vehicle location data + +```json +{ + "Cmd": 2, + "Encrypt": 1, + "Vin": "1G1BL52P7TR115520", + "Data": { + "Infos": [ + { + "Latitude": 100, + "Longitude": 10, + "Status": 0, + "Type": "Location" + } + ], + "Time": { + "Day": 1, + "Hour": 22, + "Minute": 59, + "Month": 10, + "Second": 0, + "Year": 16 + } + } +} +``` + +The meaning of each field of the vehicle location data is: + +| Field | Type | Description | +| ----------- | ------- | ------------------------------------------------------------------------------------------------| +| `Type` | String | Data type, here is `Location` | +| `Status` | Integer | Positioning status, see table 15 of original protocol, here is the integer value of all bits | +| `Longitude` | Integer | Longitude, latitude value in degrees multiplied by 10^6 to the nearest millionth of a degree | +| `Latitude` | Integer | Latitude, the latitude value in degrees multiplied by 10^6 to the nearest millionth of a degree | + + + +### Maximum value data + +```json +{ + "Cmd": 2, + "Encrypt": 1, + "Vin": "1G1BL52P7TR115520", + "Data": { + "Infos": [ + { + "MaxBatteryVoltage": 7500, + "MaxTemp": 120, + "MaxTempProbeNo": 12, + "MaxTempSubsysNo": 14, + "MaxVoltageBatteryCode": 10, + "MaxVoltageBatterySubsysNo": 12, + "MinBatteryVoltage": 2000, + "MinTemp": 40, + "MinTempProbeNo": 13, + "MinTempSubsysNo": 15, + "MinVoltageBatteryCode": 11, + "MinVoltageBatterySubsysNo": 13, + "Type": "Extreme" + } + ], + "Time": { + "Day": 30, + "Hour": 12, + "Minute": 22, + "Month": 5, + "Second": 59, + "Year": 17 + } + } +} +``` + +Among them, the meaning of each field of extreme value data is + +| Field | Type | Description | +| --------------------------- | ------- | -------------------------------------------------------------------------------------------------| +| `Type` | String | Data type, here is `Extreme` | +| `MaxVoltageBatterySubsysNo` | Integer | Maximum voltage battery subsystem number, valid value 1~250 | +| `MaxVoltageBatteryCode` | Integer | Maximum voltage battery cell code, valid value 1~250 | +| `MaxBatteryVoltage` | Integer | Maximum value of the battery cell voltage, valid value 0~15000 (representing 0V ~ 15V) unit 0.001V | +| `MinVoltageBatterySubsysNo` | Integer | Minimum voltage battery subsystem number, valid value 1~250 | +| `MinVoltageBatteryCode` | Integer | Minimum voltage battery cell code, valid value 1~250 | +| `MinBatteryVoltage` | Integer | Minimum value of battery cell voltage, valid value 0~15000 (representing 0V ~ 15V) unit 0.001V | +| `MaxTempSubsysNo` | Integer | Maximum temperature subsystem number, valid value 1~250 | +| `MaxTempProbeNo` | Integer | Maximum temperature probe serial number, valid value 1~250 | +| `MaxTemp` | Integer | Maximum temperature value, valid value range 0~250 (offset 40, representing -40°C ~ +210°C) | +| `MinTempSubsysNo` | Integer | Minimum temperature subsystem number, valid value 1~250 | +| `MinTempProbeNo` | Integer | Minimum temperature probe serial number, valid value 1~250 | +| `MinTemp` | Integer | Minimum temperature value, valid value range 0~250 (offset 40, representing -40°C ~ +210°C) | + + +### Alarm data + +```json +{ + "Cmd": 2, + "Encrypt": 1, + "Vin": "1G1BL52P7TR115520", + "Data": { + "Infos": [ + { + "FaultChargeableDeviceNum": 1, + "FaultChargeableDeviceList": ["00C8"], + "FaultDriveMotorNum": 0, + "FaultDriveMotorList": [], + "FaultEngineNum": 1, + "FaultEngineList": ["006F"], + "FaultOthersNum": 0, + "FaultOthersList": [], + "GeneralAlarmFlag": 3, + "MaxAlarmLevel": 1, + "Type": "Alarm" + } + ], + "Time": { + "Day": 20, + "Hour": 22, + "Minute": 23, + "Month": 12, + "Second": 59, + "Year": 17 + } + } +} +``` + +The meaning of each field of the alarm data is: + +| Field | Type | Description | +| --------------------------- | ------- | -------------------------------------------------------------------------------------------| +| `Type` | String | Data type, here is `Alarm` | +| `MaxAlarmLevel` | Integer | The maximum alarm level, valid value range is 0~3, `0` :: no fault, `1` :: `1` level fault | +| `GeneralAlarmFlag` | Integer | General alarm flag, see original protocol table 18 | +| `FaultChargeableDeviceNum` | Integer | Total number of rechargeable energy storage device faults, valid value 0~252 | +| `FaultChargeableDeviceList` | Array | Rechargeable energy storage device fault code list | +| `FaultDriveMotorNum` | Integer | Total number of drive motor faults, valid setting range 0 ~252 | +| `FaultDriveMotorList` | Array | Drive motor fault code list | +| `FaultEngineNum` | Integer | Total number of engine faults, valid value range 0~252 | +| `FaultEngineList` | Array | Engine fault code list | +| `FaultOthersNum` | Integer | Total number of other faults | +| `FaultOthersList` | Array | Other fault code list | + + + +### Rechargeable energy storage device voltage data + +```json +{ + "Cmd": 2, + "Encrypt": 1, + "Vin": "1G1BL52P7TR115520", + "Data": { + "Infos": [ + { + "Number": 2, + "SubSystems": [ + { + "CellsTotal": 2, + "CellsVoltage": [5000], + "ChargeableCurrent": 10000, + "ChargeableSubsysNo": 1, + "ChargeableVoltage": 5000, + "FrameCellsCount": 1, + "FrameCellsIndex": 0 + }, + { + "CellsTotal": 2, + "CellsVoltage": [5001], + "ChargeableCurrent": 10001, + "ChargeableSubsysNo": 2, + "ChargeableVoltage": 5001, + "FrameCellsCount": 1, + "FrameCellsIndex": 1 + } + ], + "Type": "ChargeableVoltage" + } + ], + "Time": { + "Day": 1, + "Hour": 22, + "Minute": 59, + "Month": 10, + "Second": 0, + "Year": 16 + } + } +} +``` + + + +The fields are defined as follows: + +| Field | Type | Description | +| ----------- | ------- | --------------------------------------------------------------------| +| `Type` | String | Data type, here is `ChargeableVoltage` | +| `Number` | Integer | Number of rechargeable energy storage subsystems, valid range 1~250 | +| `SubSystem` | Object | Rechargeable energy storage subsystem voltage information list | + +Rechargeable energy storage subsystem voltage information data format: + +| Field | Type | Description | +| -------------------- | ------- | -----------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `ChargeableSubsysNo` | Integer | Rechargeable energy storage subsystem number, valid value range, 1~250 | +| `ChargeableVoltage` | Integer | Rechargeable energy storage device voltage, valid value range, 0~10000 (representing 0V ~ 1000V) unit 0.1 V | +| `ChargeableCurrent` | Integer | Rechargeable energy storage device current, valid value range, 0~20000 (value offset 1000A, indicating -1000A ~ +1000A) unit 0.1 A | +| `CellsTotal` | Integer | Total number of cells, valid value range 1~65531 | +| `FrameCellsIndex` | Integer | The serial number of the cell at the beginning of this frame, when the number of single cells in this frame exceeds 200, it should be split into multiple frames for transmission, valid value range 1~65531 | +| `FrameCellsCount` | Integer | The total number of cells in this frame, valid value range 1~200 | +| `CellsVoltage` | Array | Cells voltage, valid value range 0~60000 (representing 0V ~ 60.000V) unit 0.001V | + + + +### Rechargeable energy storage device temperature data + +```json +{ + "Cmd": 2, + "Encrypt": 1, + "Vin": "1G1BL52P7TR115520", + "Data": { + "Infos": [ + { + "Number": 2, + "SubSystems": [ + { + "ChargeableSubsysNo": 1, + "ProbeNum": 10, + "ProbesTemp": [0, 0, 0, 0, 0, 0, 0, 0, 19, 136] + }, + { + "ChargeableSubsysNo": 2, + "ProbeNum": 1, + "ProbesTemp": [100] + } + ], + "Type": "ChargeableTemp" + } + ], + "Time": { + "Day": 1, + "Hour": 22, + "Minute": 59, + "Month": 10, + "Second": 0, + "Year": 16 + } + } +} +``` +The data format is: + +| Field | Type | Description | +| ------------ | ------- | --------------------------------------------------------------------------| +| `Type` | String | Data type, here is `ChargeableTemp` | +| `Number` | Integer | Rechargeable energy storage subsystem temperature information list length | +| `SubSystems` | Object | Rechargeable energy storage subsystem temperature information list | + +The rechargeable energy storage subsystem temperature information format is: + +| Field | Type | Description | +| -------------------- | -------- | --------------------------------------------------------------------------------------------------| +| `ChargeableSubsysNo` | Ineteger | Rechargeable energy storage subsystem number, valid value 1~250 | +| `ProbeNum` | Integer | Number of rechargeable energy storage temperature probes | +| `ProbesTemp` | Array | List of temperature values of each temperature probe of the rechargeable energy storage subsystem | + + + +## Data reissue + +Topic: gbt32960/${clientid}/upstream/reinfo + +**Data format: omitted** (same as real-time data reporting) + +# Downstream + +> Request data flow direction: EMQX -> GBT/32960 Gateway -> Terminal + +> Response data flow: Terminal -> GBT/32960 Gateway -> EMQX + +Downstream topic: gbt32960/${clientid}/dnstream +Upstream response topic: gbt32960/${clientid}/upstream/response + +## Parameters query + + + +**Req:** + +```json +{ + "Action": "Query", + "Total": 2, + "Ids": ["0x01", "0x02"] +} +``` + +| Field | Type | Description | +| -------- | ------- | ----------------------------------------------------------------------------------------------------| +| `Action` | String | The type of downstream command, here is `Query` | +| `Total` | Integer | Total number of query parameters | +| `Ids` | Array | List of IDs that need to be queried. For specific ID meanings, see the original protocol Table B.10 | + +**Response:** + +```json +{ + "Cmd": 128, + "Encrypt": 1, + "Vin": "1G1BL52P7TR115520", + "Data": { + "Total": 2, + "Params": [ + {"0x01": 6000}, + {"0x02": 10} + ], + "Time": { + "Day": 2, + "Hour": 11, + "Minute": 12, + "Month": 2, + "Second": 12, + "Year": 17 + } + } +} +``` + + + +## Parameters setting + +**Req:** +```json +{ + "Action": "Setting", + "Total": 2, + "Params": [{"0x01": 5000}, + {"0x02": 200}] +} +``` + +| Field | Type | Description | +| -------- | ------- | ----------------------------------------------| +| `Action` | String | Type of downstream command, here is `Setting` | +| `Total` | Integer | Set the total number of parameters | +| `Params` | Array | ID and value of parameters to be set | + +**Response:** + +```json +{ + "Cmd": 129, + "Encrypt": 1, + "Vin": "1G1BL52P7TR115520", + "Data": { + "Total": 2, + "Params": [ + {"0x01": 5000}, + {"0x02": 200} + ], + "Time": { + "Day": 2, + "Hour": 11, + "Minute": 12, + "Month": 2, + "Second": 12, + "Year": 17 + } + } +} +``` + +## Terminal control +**Different commands have different parameters; if there are no parameters, it will be empty** + +Remote Upgrade: +**Req:** + +```json +{ + "Action": "Control", + "Command": "0x01", + "Param": { + "DialingName": "hz203", + "Username": "user001", + "Password": "password01", + "Ip": "192.168.199.1", + "Port": 8080, + "ManufacturerId": "BMWA", + "HardwareVer": "1.0.0", + "SoftwareVer": "1.0.0", + "UpgradeUrl": "ftp://emqtt.io/ftp/server", + "Timeout": 10 + } +} +``` + +| Field | Type | Description | +| --------- | ------- | ----------------------------------------------------| +| `Action` | String | Type of command issued, here is `Control` | +| `Command` | Integer | Issued command ID, see original protocol table B.15 | +| `Param` | Object | Command parameters | + +The example list: + +Shut down the vehicle terminal: + +```json +{ + "Action": "Control", + "Command": "0x02" +} +``` + +... + +Vehicle terminal alarm: +```json +{ + "Action": "Control", + "Command": "0x06", + "Param": {"Level": 0, "Message": "alarm message"} +} +``` diff --git a/apps/emqx_gateway_gbt32960/include/emqx_gbt32960.hrl b/apps/emqx_gateway_gbt32960/include/emqx_gbt32960.hrl new file mode 100644 index 000000000..2649f3f98 --- /dev/null +++ b/apps/emqx_gateway_gbt32960/include/emqx_gbt32960.hrl @@ -0,0 +1,80 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-record(frame, {cmd, ack, vin, encrypt, length, data, check, rawdata}). + +-type frame() :: #frame{}. + +-define(CMD(CmdType), #frame{ + cmd = CmdType, + ack = ?ACK_IS_CMD +}). + +-define(CMD(CmdType, Data), #frame{ + cmd = CmdType, + data = Data, + ack = ?ACK_IS_CMD +}). + +-define(IS_ACK_CODE(C), + (C == ?ACK_SUCCESS orelse + C == ?ACK_ERROR orelse + C == ?ACK_VIN_REPEAT) +). + +%%-------------------------------------------------------------------- +%% CMD Feilds +%%-------------------------------------------------------------------- +-define(CMD_VIHECLE_LOGIN, 16#01). +-define(CMD_INFO_REPORT, 16#02). +-define(CMD_INFO_RE_REPORT, 16#03). +-define(CMD_VIHECLE_LOGOUT, 16#04). +-define(CMD_PLATFORM_LOGIN, 16#05). +-define(CMD_PLATFORM_LOGOUT, 16#06). +-define(CMD_HEARTBEAT, 16#07). +-define(CMD_SCHOOL_TIME, 16#08). +% 0x09~0x7F: Reserved by upstream system +% 0x80~0x82: Reserved by terminal data +-define(CMD_PARAM_QUERY, 16#80). +-define(CMD_PARAM_SETTING, 16#81). +-define(CMD_TERMINAL_CTRL, 16#82). + +% 0x83~0xBF: Reserved by downstream system +% 0xC0~0xFE: Customized data for Platform Exchange Protocol + +%%-------------------------------------------------------------------- +%% ACK Feilds +%%-------------------------------------------------------------------- +-define(ACK_SUCCESS, 16#01). +-define(ACK_ERROR, 16#02). +-define(ACK_VIN_REPEAT, 16#03). +-define(ACK_IS_CMD, 16#FE). + +%%-------------------------------------------------------------------- +%% Encrypt Feilds +%%-------------------------------------------------------------------- +-define(ENCRYPT_NONE, 16#01). +-define(ENCRYPT_RSA, 16#02). +-define(ENCRYPT_AES128, 16#03). +-define(ENCRYPT_ABNORMAL, 16#FE). +-define(ENCRYPT_INVAILD, 16#FF). + +%%-------------------------------------------------------------------- +%% Info Type Flags +%%-------------------------------------------------------------------- +-define(INFO_TYPE_VEHICLE, 16#01). +-define(INFO_TYPE_DRIVE_MOTOR, 16#02). +-define(INFO_TYPE_FUEL_CELL, 16#03). +-define(INFO_TYPE_ENGINE, 16#04). +-define(INFO_TYPE_LOCATION, 16#05). +-define(INFO_TYPE_EXTREME, 16#06). +-define(INFO_TYPE_ALARM, 16#07). +-define(INFO_TYPE_CHARGEABLE_VOLTAGE, 16#08). +-define(INFO_TYPE_CHARGEABLE_TEMP, 16#09). +% 0x0A~0x2F: Customized data for Platform Exchange Protocol +% 0x30~0x7F: Reserved +% 0x80~0xFE: Customized by user + +-define(DEFAULT_MOUNTPOINT, <<"gbt32960/${clientid}/">>). +-define(DEFAULT_DOWNLINK_TOPIC, <<"dnstream">>). diff --git a/apps/emqx_gateway_gbt32960/rebar.config b/apps/emqx_gateway_gbt32960/rebar.config new file mode 100644 index 000000000..456746d25 --- /dev/null +++ b/apps/emqx_gateway_gbt32960/rebar.config @@ -0,0 +1,7 @@ +%% -*- mode: erlang -*- +{erl_opts, [debug_info]}. +{deps, [ + {emqx, {path, "../../apps/emqx"}}, + {emqx_utils, {path, "../emqx_utils"}}, + {emqx_gateway, {path, "../../apps/emqx_gateway"}} +]}. diff --git a/apps/emqx_gateway_gbt32960/src/emqx_gateway_gbt32960.app.src b/apps/emqx_gateway_gbt32960/src/emqx_gateway_gbt32960.app.src new file mode 100644 index 000000000..0ed2dca39 --- /dev/null +++ b/apps/emqx_gateway_gbt32960/src/emqx_gateway_gbt32960.app.src @@ -0,0 +1,11 @@ +%% -*- mode: erlang -*- +{application, emqx_gateway_gbt32960, [ + {description, "GBT32960 Gateway"}, + {vsn, "0.1.0"}, + {registered, []}, + {applications, [kernel, stdlib, emqx, emqx_gateway]}, + {env, []}, + {modules, []}, + {licenses, ["BSL"]}, + {links, []} +]}. diff --git a/apps/emqx_gateway_gbt32960/src/emqx_gateway_gbt32960.erl b/apps/emqx_gateway_gbt32960/src/emqx_gateway_gbt32960.erl new file mode 100644 index 000000000..e4bcd4969 --- /dev/null +++ b/apps/emqx_gateway_gbt32960/src/emqx_gateway_gbt32960.erl @@ -0,0 +1,98 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +%% @doc The GBT32960 Gateway implement +-module(emqx_gateway_gbt32960). + +-include_lib("emqx/include/logger.hrl"). +-include_lib("emqx_gateway/include/emqx_gateway.hrl"). + +%% define a gateway named gbt32960 +-gateway(#{ + name => gbt32960, + callback_module => ?MODULE, + config_schema_module => emqx_gbt32960_schema, + edition => ee +}). + +%% callback_module must implement the emqx_gateway_impl behaviour +-behaviour(emqx_gateway_impl). + +%% callback for emqx_gateway_impl +-export([ + on_gateway_load/2, + on_gateway_update/3, + on_gateway_unload/2 +]). + +-import( + emqx_gateway_utils, + [ + normalize_config/1, + start_listeners/4, + stop_listeners/2 + ] +). + +%%-------------------------------------------------------------------- +%% emqx_gateway_impl callbacks +%%-------------------------------------------------------------------- + +on_gateway_load( + _Gateway = #{ + name := GwName, + config := Config + }, + Ctx +) -> + Listeners = normalize_config(Config), + ModCfg = #{ + frame_mod => emqx_gbt32960_frame, + chann_mod => emqx_gbt32960_channel + }, + case + start_listeners( + Listeners, GwName, Ctx, ModCfg + ) + of + {ok, ListenerPids} -> + %% FIXME: How to throw an exception to interrupt the restart logic ? + %% FIXME: Assign ctx to GwState + {ok, ListenerPids, _GwState = #{ctx => Ctx}}; + {error, {Reason, Listener}} -> + throw( + {badconf, #{ + key => listeners, + value => Listener, + reason => Reason + }} + ) + end. + +on_gateway_update(Config, Gateway, GwState = #{ctx := Ctx}) -> + GwName = maps:get(name, Gateway), + try + %% XXX: 1. How hot-upgrade the changes ??? + %% XXX: 2. Check the New confs first before destroy old state??? + on_gateway_unload(Gateway, GwState), + on_gateway_load(Gateway#{config => Config}, Ctx) + catch + Class:Reason:Stk -> + logger:error( + "Failed to update ~ts; " + "reason: {~0p, ~0p} stacktrace: ~0p", + [GwName, Class, Reason, Stk] + ), + {error, Reason} + end. + +on_gateway_unload( + _Gateway = #{ + name := GwName, + config := Config + }, + _GwState +) -> + Listeners = normalize_config(Config), + stop_listeners(GwName, Listeners). diff --git a/apps/emqx_gateway_gbt32960/src/emqx_gbt32960_channel.erl b/apps/emqx_gateway_gbt32960/src/emqx_gbt32960_channel.erl new file mode 100644 index 000000000..5cb65f104 --- /dev/null +++ b/apps/emqx_gateway_gbt32960/src/emqx_gbt32960_channel.erl @@ -0,0 +1,864 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_gbt32960_channel). +-behaviour(emqx_gateway_channel). + +-include("emqx_gbt32960.hrl"). +-include_lib("emqx/include/types.hrl"). +-include_lib("emqx/include/logger.hrl"). +-include_lib("emqx/include/emqx.hrl"). +-include_lib("emqx/include/emqx_mqtt.hrl"). + +-export([ + info/1, + info/2, + stats/1 +]). + +-export([ + init/2, + handle_in/2, + handle_deliver/2, + handle_timeout/3, + terminate/2, + set_conn_state/2 +]). + +-export([ + handle_call/3, + handle_cast/2, + handle_info/2 +]). + +-record(channel, { + %% Context + ctx :: emqx_gateway_ctx:context(), + %% ConnInfo + conninfo :: emqx_types:conninfo(), + %% ClientInfo + clientinfo :: emqx_types:clientinfo(), + %% Session + session :: undefined | map(), + %% Keepalive + keepalive :: maybe(emqx_keepalive:keepalive()), + %% Conn State + conn_state :: conn_state(), + %% Timers + timers :: #{atom() => undefined | disabled | reference()}, + %% Inflight + inflight :: emqx_inflight:inflight(), + %% Message Queue + mqueue :: queue:queue(), + retx_interval, + retx_max_times, + max_mqueue_len +}). + +-type conn_state() :: idle | connecting | connected | disconnected. + +-type channel() :: #channel{}. + +-type reply() :: + {outgoing, emqx_types:packet()} + | {outgoing, [emqx_types:packet()]} + | {event, conn_state() | updated} + | {close, Reason :: atom()}. + +-type replies() :: reply() | [reply()]. + +-define(TIMER_TABLE, #{ + alive_timer => keepalive, + retry_timer => retry_delivery +}). + +-define(INFO_KEYS, [conninfo, conn_state, clientinfo, session, will_msg]). + +-dialyzer({nowarn_function, init/2}). + +%%-------------------------------------------------------------------- +%% Info, Attrs and Caps +%%-------------------------------------------------------------------- + +%% @doc Get infos of the channel. +-spec info(channel()) -> emqx_types:infos(). +info(Channel) -> + maps:from_list(info(?INFO_KEYS, Channel)). + +-spec info(list(atom()) | atom(), channel()) -> term(). +info(Keys, Channel) when is_list(Keys) -> + [{Key, info(Key, Channel)} || Key <- Keys]; +info(ctx, #channel{ctx = Ctx}) -> + Ctx; +info(conninfo, #channel{conninfo = ConnInfo}) -> + ConnInfo; +info(zone, #channel{clientinfo = #{zone := Zone}}) -> + Zone; +info(clientid, #channel{clientinfo = #{clientid := ClientId}}) -> + ClientId; +info(clientinfo, #channel{clientinfo = ClientInfo}) -> + ClientInfo; +info(session, _) -> + #{}; +info(conn_state, #channel{conn_state = ConnState}) -> + ConnState; +info(keepalive, #channel{keepalive = undefined}) -> + undefined; +info(keepalive, #channel{keepalive = Keepalive}) -> + emqx_keepalive:info(Keepalive); +info(will_msg, _) -> + undefined. + +-spec stats(channel()) -> emqx_types:stats(). +stats(#channel{inflight = Inflight, mqueue = Queue}) -> + %% XXX: A fake stats for managed by emqx_management + [ + {subscriptions_cnt, 1}, + {subscriptions_max, 0}, + {inflight_cnt, emqx_inflight:size(Inflight)}, + {inflight_max, emqx_inflight:max_size(Inflight)}, + {mqueue_len, queue:len(Queue)}, + {mqueue_max, 0}, + {mqueue_dropped, 0}, + {next_pkt_id, 0}, + {awaiting_rel_cnt, 0}, + {awaiting_rel_max, 0} + ]. + +set_conn_state(ConnState, Channel) -> + Channel#channel{conn_state = ConnState}. + +%%-------------------------------------------------------------------- +%% Init the Channel +%%-------------------------------------------------------------------- + +init( + ConnInfo = #{ + peername := {PeerHost, _Port}, + sockname := {_Host, SockPort} + }, + Options +) -> + % TODO: init rsa_key from user input + Peercert = maps:get(peercert, ConnInfo, undefined), + Mountpoint = maps:get(mountpoint, Options, ?DEFAULT_MOUNTPOINT), + ListenerId = + case maps:get(listener, Options, undefined) of + undefined -> undefined; + {GwName, Type, LisName} -> emqx_gateway_utils:listener_id(GwName, Type, LisName) + end, + EnableAuthn = maps:get(enable_authn, Options, true), + + ClientInfo = setting_peercert_infos( + Peercert, + #{ + zone => default, + listener => ListenerId, + protocol => gbt32960, + peerhost => PeerHost, + sockport => SockPort, + clientid => undefined, + username => undefined, + is_bridge => false, + is_superuser => false, + enable_authn => EnableAuthn, + mountpoint => Mountpoint + } + ), + + Ctx = maps:get(ctx, Options), + + #{ + retry_interval := RetxInterv, + max_retry_times := RetxMaxTime, + message_queue_len := MessageQueueLen + } = Options, + + #channel{ + ctx = Ctx, + conninfo = ConnInfo, + clientinfo = ClientInfo, + inflight = emqx_inflight:new(1), + mqueue = queue:new(), + timers = #{}, + conn_state = idle, + retx_interval = RetxInterv, + retx_max_times = RetxMaxTime, + max_mqueue_len = MessageQueueLen + }. + +setting_peercert_infos(NoSSL, ClientInfo) when + NoSSL =:= nossl; + NoSSL =:= undefined +-> + ClientInfo; +setting_peercert_infos(Peercert, ClientInfo) -> + {DN, CN} = {esockd_peercert:subject(Peercert), esockd_peercert:common_name(Peercert)}, + ClientInfo#{dn => DN, cn => CN}. + +%%-------------------------------------------------------------------- +%% Handle incoming packet +%%-------------------------------------------------------------------- +-spec handle_in(frame() | {frame_error, any()}, channel()) -> + {ok, channel()} + | {ok, replies(), channel()} + | {shutdown, Reason :: term(), channel()} + | {shutdown, Reason :: term(), replies(), channel()}. + +handle_in( + Frame = ?CMD(?CMD_VIHECLE_LOGIN), + Channel +) -> + case + emqx_utils:pipeline( + [ + fun enrich_clientinfo/2, + fun enrich_conninfo/2, + fun set_log_meta/2, + %% TODO: How to implement the banned in the gateway instance? + %, fun check_banned/2 + fun auth_connect/2 + ], + Frame, + Channel#channel{conn_state = connecting} + ) + of + {ok, _NPacket, NChannel} -> + process_connect(Frame, ensure_connected(NChannel)); + {error, ReasonCode, NChannel} -> + log(warning, #{msg => "login_failed", reason => ReasonCode}, NChannel), + shutdown(ReasonCode, NChannel) + end; +handle_in(_Frame, Channel = #channel{conn_state = ConnState}) when + ConnState =/= connected +-> + shutdown(protocol_error, Channel); +handle_in(Frame = ?CMD(?CMD_INFO_REPORT), Channel) -> + _ = upstreaming(Frame, Channel), + {ok, Channel}; +handle_in(Frame = ?CMD(?CMD_INFO_RE_REPORT), Channel) -> + _ = upstreaming(Frame, Channel), + {ok, Channel}; +handle_in(Frame = ?CMD(?CMD_VIHECLE_LOGOUT), Channel) -> + %% XXX: unsubscribe gbt32960/dnstream/${vin}? + _ = upstreaming(Frame, Channel), + {ok, Channel}; +handle_in(Frame = ?CMD(?CMD_PLATFORM_LOGIN), Channel) -> + #{ + <<"Username">> := _Username, + <<"Password">> := _Password + } = Frame#frame.data, + %% TODO: + _ = upstreaming(Frame, Channel), + {ok, Channel}; +handle_in(Frame = ?CMD(?CMD_PLATFORM_LOGOUT), Channel) -> + %% TODO: + _ = upstreaming(Frame, Channel), + {ok, Channel}; +handle_in(Frame = ?CMD(?CMD_HEARTBEAT), Channel) -> + handle_out({?ACK_SUCCESS, Frame}, Channel); +handle_in(Frame = ?CMD(?CMD_SCHOOL_TIME), Channel) -> + %% TODO: How verify this request + handle_out({?ACK_SUCCESS, Frame}, Channel); +handle_in(Frame = #frame{cmd = Cmd}, Channel = #channel{inflight = Inflight}) -> + {Outgoings, NChannel} = dispatch_frame(Channel#channel{inflight = ack_frame(Cmd, Inflight)}), + _ = upstreaming(Frame, NChannel), + {ok, [{outgoing, Outgoings}], NChannel}; +handle_in(Frame, Channel) -> + log(warning, #{msg => "unexcepted_frame", frame => Frame}, Channel), + {ok, Channel}. + +%%-------------------------------------------------------------------- +%% Handle out +%%-------------------------------------------------------------------- + +handle_out({AckCode, Frame}, Channel) when + ?IS_ACK_CODE(AckCode) +-> + {ok, [{outgoing, ack(AckCode, Frame)}], Channel}. + +handle_out({AckCode, Frame}, Outgoings, Channel) when ?IS_ACK_CODE(AckCode) -> + {ok, [{outgoing, ack(AckCode, Frame)} | Outgoings], Channel}. + +%%-------------------------------------------------------------------- +%% Handle Delivers from broker to client +%%-------------------------------------------------------------------- +-spec handle_deliver(list(emqx_types:deliver()), channel()) -> + {ok, channel()} + | {ok, replies(), channel()}. + +handle_deliver( + Messages0, + Channel = #channel{ + clientinfo = #{clientid := ClientId, mountpoint := Mountpoint}, + mqueue = Queue, + max_mqueue_len = MaxQueueLen + } +) -> + Messages = lists:map( + fun({deliver, _, M}) -> + emqx_mountpoint:unmount(Mountpoint, M) + end, + Messages0 + ), + case MaxQueueLen - queue:len(Queue) of + N when N =< 0 -> + discard_downlink_messages(Messages, Channel), + {ok, Channel}; + N -> + {NMessages, Dropped} = split_by_pos(Messages, N), + log(debug, #{msg => "enqueue_messages", messages => NMessages}, Channel), + metrics_inc('messages.delivered', Channel, erlang:length(NMessages)), + discard_downlink_messages(Dropped, Channel), + Frames = msgs2frame(NMessages, ClientId, Channel), + NQueue = lists:foldl(fun(F, Q) -> queue:in(F, Q) end, Queue, Frames), + {Outgoings, NChannel} = dispatch_frame(Channel#channel{mqueue = NQueue}), + {ok, [{outgoing, Outgoings}], NChannel} + end. + +split_by_pos(L, Pos) -> + split_by_pos(L, Pos, []). + +split_by_pos([], _, A1) -> + {lists:reverse(A1), []}; +split_by_pos(L, 0, A1) -> + {lists:reverse(A1), L}; +split_by_pos([E | L], N, A1) -> + split_by_pos(L, N - 1, [E | A1]). + +msgs2frame(Messages, Vin, Channel) -> + lists:filtermap( + fun(#message{payload = Payload}) -> + case emqx_utils_json:safe_decode(Payload, [return_maps]) of + {ok, Maps} -> + case msg2frame(Maps, Vin) of + {error, Reason} -> + log( + debug, + #{ + msg => "convert_message_to_frame_error", + reason => Reason, + data => Maps + }, + Channel + ), + false; + Frame -> + {true, Frame} + end; + {error, Reason} -> + log(error, #{msg => "json_decode_error", reason => Reason}, Channel), + false + end + end, + Messages + ). + +%%-------------------------------------------------------------------- +%% Handle call +%%-------------------------------------------------------------------- + +-spec handle_call(Req :: term(), From :: term(), channel()) -> + {reply, Reply :: term(), channel()} + | {reply, Reply :: term(), replies(), channel()} + | {shutdown, Reason :: term(), Reply :: term(), channel()} + | {shutdown, Reason :: term(), Reply :: term(), frame(), channel()}. + +handle_call(kick, _From, Channel) -> + Channel1 = ensure_disconnected(kicked, Channel), + disconnect_and_shutdown(kicked, ok, Channel1); +handle_call(discard, _From, Channel) -> + disconnect_and_shutdown(discarded, ok, Channel); +handle_call(Req, _From, Channel) -> + log(error, #{msg => "unexpected_call", call => Req}, Channel), + reply(ignored, Channel). + +%%-------------------------------------------------------------------- +%% Handle cast +%%-------------------------------------------------------------------- + +-spec handle_cast(Req :: term(), channel()) -> + ok | {ok, channel()} | {shutdown, Reason :: term(), channel()}. +handle_cast(_Req, Channel) -> + {ok, Channel}. + +%%-------------------------------------------------------------------- +%% Handle info +%%-------------------------------------------------------------------- + +-spec handle_info(Info :: term(), channel()) -> + ok | {ok, channel()} | {shutdown, Reason :: term(), channel()}. + +handle_info({sock_closed, Reason}, Channel = #channel{conn_state = idle}) -> + shutdown(Reason, Channel); +handle_info({sock_closed, Reason}, Channel = #channel{conn_state = connecting}) -> + shutdown(Reason, Channel); +handle_info( + {sock_closed, Reason}, + Channel = + #channel{ + conn_state = connected + } +) -> + NChannel = ensure_disconnected(Reason, Channel), + shutdown(Reason, NChannel); +handle_info({sock_closed, Reason}, Channel = #channel{conn_state = disconnected}) -> + log(error, #{msg => "unexpected_sock_closed", reason => Reason}, Channel), + {ok, Channel}; +handle_info(Info, Channel) -> + log(error, #{msg => "unexpected_info}", info => Info}, Channel), + {ok, Channel}. + +%%-------------------------------------------------------------------- +%% Handle timeout +%%-------------------------------------------------------------------- + +-spec handle_timeout(reference(), Msg :: term(), channel()) -> + {ok, channel()} + | {ok, replies(), channel()} + | {shutdown, Reason :: term(), channel()}. + +handle_timeout( + _TRef, + {keepalive, _StatVal}, + Channel = #channel{keepalive = undefined} +) -> + {ok, Channel}; +handle_timeout( + _TRef, + {keepalive, _StatVal}, + Channel = #channel{conn_state = disconnected} +) -> + {ok, Channel}; +handle_timeout( + _TRef, + {keepalive, StatVal}, + Channel = #channel{keepalive = Keepalive} +) -> + case emqx_keepalive:check(StatVal, Keepalive) of + {ok, NKeepalive} -> + NChannel = Channel#channel{keepalive = NKeepalive}, + {ok, reset_timer(alive_timer, NChannel)}; + {error, timeout} -> + shutdown(keepalive_timeout, Channel) + end; +handle_timeout( + _TRef, + retry_delivery, + Channel = #channel{inflight = Inflight, retx_interval = RetxInterv} +) -> + case emqx_inflight:is_empty(Inflight) of + true -> + {ok, clean_timer(retry_timer, Channel)}; + false -> + Frames = emqx_inflight:to_list(Inflight), + {Outgoings, NInflight} = retry_delivery( + Frames, erlang:system_time(millisecond), RetxInterv, Inflight, [] + ), + {Outgoings2, NChannel} = dispatch_frame(Channel#channel{inflight = NInflight}), + {ok, [{outgoing, Outgoings ++ Outgoings2}], reset_timer(retry_timer, NChannel)} + end; +handle_timeout(_TRef, Msg, Channel) -> + log(error, #{msg => "unexpected_timeout", content => Msg}, Channel), + {ok, Channel}. + +%%-------------------------------------------------------------------- +%% Ensure timers +%%-------------------------------------------------------------------- + +ensure_timer(Name, Channel = #channel{timers = Timers}) -> + TRef = maps:get(Name, Timers, undefined), + Time = interval(Name, Channel), + case TRef == undefined andalso Time > 0 of + true -> ensure_timer(Name, Time, Channel); + %% Timer disabled or exists + false -> Channel + end. + +ensure_timer(Name, Time, Channel = #channel{timers = Timers}) -> + log(debug, #{msg => "start_timer", name => Name, time => Time}, Channel), + Msg = maps:get(Name, ?TIMER_TABLE), + TRef = emqx_utils:start_timer(Time, Msg), + Channel#channel{timers = Timers#{Name => TRef}}. + +reset_timer(Name, Channel) -> + ensure_timer(Name, clean_timer(Name, Channel)). + +clean_timer(Name, Channel = #channel{timers = Timers}) -> + Channel#channel{timers = maps:remove(Name, Timers)}. + +interval(alive_timer, #channel{keepalive = KeepAlive}) -> + emqx_keepalive:info(interval, KeepAlive); +interval(retry_timer, #channel{retx_interval = RetxIntv}) -> + RetxIntv. + +%%-------------------------------------------------------------------- +%% Terminate +%%-------------------------------------------------------------------- + +terminate(Reason, #channel{ + ctx = Ctx, + session = Session, + clientinfo = ClientInfo +}) -> + run_hooks(Ctx, 'session.terminated', [ClientInfo, Reason, Session]). + +%%-------------------------------------------------------------------- +%% Ensure connected + +enrich_clientinfo( + Packet, + Channel = #channel{ + clientinfo = ClientInfo + } +) -> + {ok, NPacket, NClientInfo} = emqx_utils:pipeline( + [ + fun maybe_assign_clientid/2, + %% FIXME: CALL After authentication successfully + fun fix_mountpoint/2 + ], + Packet, + ClientInfo + ), + {ok, NPacket, Channel#channel{clientinfo = NClientInfo}}. + +enrich_conninfo( + _Packet, + Channel = #channel{ + conninfo = ConnInfo, + clientinfo = ClientInfo + } +) -> + #{clientid := ClientId, username := Username} = ClientInfo, + NConnInfo = ConnInfo#{ + proto_name => <<"GBT32960">>, + proto_ver => <<"">>, + clean_start => true, + keepalive => 0, + expiry_interval => 0, + conn_props => #{}, + receive_maximum => 0, + clientid => ClientId, + username => Username + }, + {ok, Channel#channel{conninfo = NConnInfo}}. + +set_log_meta(_Packet, #channel{clientinfo = #{clientid := ClientId}}) -> + emqx_logger:set_metadata_clientid(ClientId), + ok. + +auth_connect( + _Packet, + Channel = #channel{ + ctx = Ctx, + clientinfo = ClientInfo + } +) -> + #{ + clientid := ClientId, + username := Username + } = ClientInfo, + case emqx_gateway_ctx:authenticate(Ctx, ClientInfo) of + {ok, NClientInfo} -> + {ok, Channel#channel{clientinfo = NClientInfo}}; + {error, Reason} -> + ?SLOG(warning, #{ + msg => "client_login_failed", + clientid => ClientId, + username => Username, + reason => Reason + }), + {error, Reason} + end. + +ensure_connected( + Channel = #channel{ + ctx = Ctx, + conninfo = ConnInfo, + clientinfo = ClientInfo + } +) -> + NConnInfo = ConnInfo#{connected_at => erlang:system_time(millisecond)}, + ok = run_hooks(Ctx, 'client.connected', [ClientInfo, NConnInfo]), + Channel#channel{ + conninfo = NConnInfo, + conn_state = connected + }. + +process_connect( + Frame, + Channel = #channel{ + ctx = Ctx, + conninfo = ConnInfo, + clientinfo = ClientInfo + } +) -> + SessFun = fun(_, _) -> #{} end, + case + emqx_gateway_ctx:open_session( + Ctx, + true, + ClientInfo, + ConnInfo, + SessFun + ) + of + {ok, #{session := Session}} -> + NChannel = Channel#channel{session = Session}, + subscribe_downlink(?DEFAULT_DOWNLINK_TOPIC, Channel), + _ = upstreaming(Frame, NChannel), + %% XXX: connection_accepted is not defined by stomp protocol + _ = run_hooks(Ctx, 'client.connack', [ConnInfo, connection_accepted, #{}]), + handle_out({?ACK_SUCCESS, Frame}, [{event, connected}], NChannel); + {error, Reason} -> + log( + error, + #{ + msg => "failed_to_open_session", + reason => Reason + }, + Channel + ), + shutdown(Reason, Channel) + end. + +maybe_assign_clientid(#frame{vin = Vin}, ClientInfo) -> + {ok, ClientInfo#{clientid => Vin, username => Vin}}. + +fix_mountpoint(_Packet, #{mountpoint := undefined}) -> + ok; +fix_mountpoint(_Packet, ClientInfo = #{mountpoint := Mountpoint}) -> + %% TODO: Enrich the variable replacement???? + %% i.e: ${ClientInfo.auth_result.productKey} + Mountpoint1 = emqx_mountpoint:replvar(Mountpoint, ClientInfo), + {ok, ClientInfo#{mountpoint := Mountpoint1}}. + +%%-------------------------------------------------------------------- +%% Ensure disconnected + +ensure_disconnected( + Reason, + Channel = #channel{ + ctx = Ctx, + conninfo = ConnInfo, + clientinfo = ClientInfo + } +) -> + NConnInfo = ConnInfo#{disconnected_at => erlang:system_time(millisecond)}, + ok = run_hooks( + Ctx, + 'client.disconnected', + [ClientInfo, Reason, NConnInfo] + ), + Channel#channel{conninfo = NConnInfo, conn_state = disconnected}. + +%%-------------------------------------------------------------------- +%% Helper functions +%%-------------------------------------------------------------------- + +run_hooks(Ctx, Name, Args) -> + emqx_gateway_ctx:metrics_inc(Ctx, Name), + emqx_hooks:run(Name, Args). + +reply(Reply, Channel) -> + {reply, Reply, Channel}. + +shutdown(Reason, Channel) -> + {shutdown, Reason, Channel}. + +shutdown(Reason, Reply, Channel) -> + {shutdown, Reason, Reply, Channel}. + +disconnect_and_shutdown(Reason, Reply, Channel) -> + shutdown(Reason, Reply, Channel). + +retry_delivery([], _Now, _Interval, Inflight, Acc) -> + {lists:reverse(Acc), Inflight}; +retry_delivery([{Key, {_Frame, 0, _}} | Frames], Now, Interval, Inflight, Acc) -> + %% todo log(error, "has arrived max re-send times, drop ~p", [Frame]), + NInflight = emqx_inflight:delete(Key, Inflight), + retry_delivery(Frames, Now, Interval, NInflight, Acc); +retry_delivery([{Key, {Frame, RetxCount, Ts}} | Frames], Now, Interval, Inflight, Acc) -> + Diff = Now - Ts, + case Diff >= Interval of + true -> + NInflight = emqx_inflight:update(Key, {Frame, RetxCount - 1, Now}, Inflight), + retry_delivery(Frames, Now, Interval, NInflight, [Frame | Acc]); + _ -> + retry_delivery(Frames, Now, Interval, Inflight, Acc) + end. + +upstreaming( + Frame, Channel = #channel{clientinfo = #{mountpoint := Mountpoint, clientid := ClientId}} +) -> + {Topic, Payload} = transform(Frame, Mountpoint), + log(debug, #{msg => "upstreaming_to_topic", topic => Topic, payload => Payload}, Channel), + emqx:publish(emqx_message:make(ClientId, ?QOS_1, Topic, Payload)). + +transform(Frame = ?CMD(Cmd), Mountpoint) -> + Suffix = + case Cmd of + ?CMD_VIHECLE_LOGIN -> <<"upstream/vlogin">>; + ?CMD_INFO_REPORT -> <<"upstream/info">>; + ?CMD_INFO_RE_REPORT -> <<"upstream/reinfo">>; + ?CMD_VIHECLE_LOGOUT -> <<"upstream/vlogout">>; + ?CMD_PLATFORM_LOGIN -> <<"upstream/plogin">>; + ?CMD_PLATFORM_LOGOUT -> <<"upstream/plogout">>; + %CMD_HEARTBEAT, CMD_SCHOOL_TIME ... + _ -> <<"upstream/transparent">> + end, + Topic = emqx_mountpoint:mount(Mountpoint, Suffix), + Payload = to_json(Frame), + {Topic, Payload}; +transform(Frame = #frame{ack = Ack}, Mountpoint) when + ?IS_ACK_CODE(Ack) +-> + Topic = emqx_mountpoint:mount(Mountpoint, <<"upstream/response">>), + Payload = to_json(Frame), + {Topic, Payload}. + +to_json(#frame{cmd = Cmd, vin = Vin, encrypt = Encrypt, data = Data}) -> + emqx_utils_json:encode(#{'Cmd' => Cmd, 'Vin' => Vin, 'Encrypt' => Encrypt, 'Data' => Data}). + +ack(Code, Frame = #frame{data = Data, ack = ?ACK_IS_CMD}) -> + % PROTO: Update time & ack feilds only + Frame#frame{ack = Code, data = Data#{<<"Time">> => gentime()}}. + +ack_frame(Key, Inflight) -> + case emqx_inflight:contain(Key, Inflight) of + true -> emqx_inflight:delete(Key, Inflight); + false -> Inflight + end. + +dispatch_frame( + Channel = #channel{ + mqueue = Queue, + inflight = Inflight, + retx_max_times = RetxMax + } +) -> + case emqx_inflight:is_full(Inflight) orelse queue:is_empty(Queue) of + true -> + {[], Channel}; + false -> + {{value, Frame}, NewQueue} = queue:out(Queue), + + log(debug, #{msg => "delivery", frame => Frame}, Channel), + + NewInflight = emqx_inflight:insert( + Frame#frame.cmd, {Frame, RetxMax, erlang:system_time(millisecond)}, Inflight + ), + NChannel = Channel#channel{mqueue = NewQueue, inflight = NewInflight}, + {[Frame], ensure_timer(retry_timer, NChannel)} + end. + +gentime() -> + {Year, Mon, Day} = date(), + {Hour, Min, Sec} = time(), + Year1 = list_to_integer(string:substr(integer_to_list(Year), 3, 2)), + #{ + <<"Year">> => Year1, + <<"Month">> => Mon, + <<"Day">> => Day, + <<"Hour">> => Hour, + <<"Minute">> => Min, + <<"Second">> => Sec + }. + +%%-------------------------------------------------------------------- +%% Message to frame +%%-------------------------------------------------------------------- + +msg2frame(#{<<"Action">> := <<"Query">>, <<"Total">> := Total, <<"Ids">> := Ids}, Vin) -> + % Ids = [<<"0x01">>, <<"0x02">>] --> [1, 2] + Data = #{ + <<"Time">> => gentime(), + <<"Total">> => Total, + <<"Ids">> => lists:map(fun hexstring_to_byte/1, Ids) + }, + #frame{ + cmd = ?CMD_PARAM_QUERY, ack = ?ACK_IS_CMD, vin = Vin, encrypt = ?ENCRYPT_NONE, data = Data + }; +msg2frame(#{<<"Action">> := <<"Setting">>, <<"Total">> := Total, <<"Params">> := Params}, Vin) -> + % Params = [#{<<"0x01">> := 5000}, #{<<"0x02">> := 400}] + % Params1 = [#{1 := 5000}, #{2 := 400}] + Params1 = lists:foldr( + fun(M, Acc) -> + [{K, V}] = maps:to_list(M), + [#{hexstring_to_byte(K) => V} | Acc] + end, + [], + Params + ), + Data = #{<<"Time">> => gentime(), <<"Total">> => Total, <<"Params">> => Params1}, + #frame{ + cmd = ?CMD_PARAM_SETTING, ack = ?ACK_IS_CMD, vin = Vin, encrypt = ?ENCRYPT_NONE, data = Data + }; +msg2frame(Data = #{<<"Action">> := <<"Control">>, <<"Command">> := Command}, Vin) -> + Param = maps:get(<<"Param">>, Data, <<>>), + Data1 = #{ + <<"Time">> => gentime(), + <<"Command">> => hexstring_to_byte(Command), + <<"Param">> => Param + }, + #frame{ + cmd = ?CMD_TERMINAL_CTRL, + ack = ?ACK_IS_CMD, + vin = Vin, + encrypt = ?ENCRYPT_NONE, + data = Data1 + }; +msg2frame(_Data, _Vin) -> + {error, unsupproted}. + +hexstring_to_byte(S) when is_binary(S) -> + hexstring_to_byte(binary_to_list(S)); +hexstring_to_byte("0x" ++ S) -> + tune_byte(list_to_integer(S, 16)); +hexstring_to_byte(S) -> + tune_byte(list_to_integer(S)). + +tune_byte(I) when I =< 16#FF -> I; +tune_byte(_) -> exit(invalid_byte). + +discard_downlink_messages([], _Channel) -> + ok; +discard_downlink_messages(Messages, Channel) -> + log( + error, + #{ + msg => "discard_new_downlink_messages", + reason => + "Discard new downlink messages due to that too" + " many messages are waiting their ACKs.", + messages => Messages + }, + Channel + ), + metrics_inc('delivery.dropped', Channel, erlang:length(Messages)). + +log(Level, Meta, #channel{clientinfo = #{clientid := ClientId, username := Username}} = _Channel) -> + ?SLOG(Level, Meta#{clientid => ClientId, username => Username}). + +metrics_inc(Name, #channel{ctx = Ctx}, Oct) -> + emqx_gateway_ctx:metrics_inc(Ctx, Name, Oct). + +subscribe_downlink( + Topic, + #channel{ + ctx = Ctx, + clientinfo = + ClientInfo = + #{ + clientid := ClientId, + mountpoint := Mountpoint + } + } +) -> + {ParsedTopic, SubOpts0} = emqx_topic:parse(Topic), + SubOpts = maps:merge(emqx_gateway_utils:default_subopts(), SubOpts0), + MountedTopic = emqx_mountpoint:mount(Mountpoint, ParsedTopic), + _ = emqx_broker:subscribe(MountedTopic, ClientId, SubOpts), + run_hooks(Ctx, 'session.subscribed', [ClientInfo, MountedTopic, SubOpts]). diff --git a/apps/emqx_gateway_gbt32960/src/emqx_gbt32960_frame.erl b/apps/emqx_gateway_gbt32960/src/emqx_gbt32960_frame.erl new file mode 100644 index 000000000..f4b679711 --- /dev/null +++ b/apps/emqx_gateway_gbt32960/src/emqx_gbt32960_frame.erl @@ -0,0 +1,808 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_gbt32960_frame). + +-behaviour(emqx_gateway_frame). + +-include("emqx_gbt32960.hrl"). +-include_lib("emqx/include/logger.hrl"). + +%% emqx_gateway_frame callbacks +-export([ + initial_parse_state/1, + serialize_opts/0, + serialize_pkt/2, + parse/2, + format/1, + type/1, + is_message/1 +]). + +-define(FLAG, 1 / binary). +-define(BYTE, 8 / big - integer). +-define(WORD, 16 / big - integer). +-define(DWORD, 32 / big - integer). +%% CMD: 1, ACK: 1, VIN: 17, Enc: 1, Len: 2 +-define(HEADER_SIZE, 22). + +-define(IS_RESPONSE(Ack), + Ack == ?ACK_SUCCESS orelse + Ack == ?ACK_ERROR orelse + Ack == ?ACK_VIN_REPEAT +). + +-type phase() :: search_heading | parse. + +-type parser_state() :: #{ + data := binary(), + phase := phase() +}. + +-ifdef(TEST). +-export([serialize/1]). +-endif. + +%%-------------------------------------------------------------------- +%% Init a Parser +%%-------------------------------------------------------------------- + +-spec initial_parse_state(map()) -> parser_state(). +initial_parse_state(_) -> + #{data => <<>>, phase => search_heading}. + +-spec serialize_opts() -> emqx_gateway_frame:serialize_options(). +serialize_opts() -> + #{}. + +%%-------------------------------------------------------------------- +%% Parse Message +%%-------------------------------------------------------------------- +parse(Bin, State) -> + case enter_parse(Bin, State) of + {ok, Message, Rest} -> + {ok, Message, Rest, State#{data => <<>>, phase => search_heading}}; + {error, Error} -> + {error, Error}; + {more_data_follow, Partial} -> + {more, State#{data => Partial, phase => parse}} + end. + +enter_parse(Bin, #{phase := search_heading}) -> + case search_heading(Bin) of + {ok, Rest} -> + parse_msg(Rest); + Error -> + Error + end; +enter_parse(Bin, #{data := Data}) -> + parse_msg(<>). + +search_heading(<<16#23, 16#23, Rest/binary>>) -> + {ok, Rest}; +search_heading(<<_, Rest/binary>>) -> + search_heading(Rest); +search_heading(<<>>) -> + {error, invalid_frame}. + +parse_msg(Binary) -> + case byte_size(Binary) >= ?HEADER_SIZE of + true -> + {Frame, Rest2} = parse_header(Binary), + case byte_size(Rest2) >= Frame#frame.length + 1 of + true -> parse_body(Rest2, Frame); + false -> {more_data_follow, Binary} + end; + false -> + {more_data_follow, Binary} + end. + +parse_header(<> = Binary) -> + Check = cal_check(Binary, ?HEADER_SIZE, undefined), + { + #frame{cmd = Cmd, ack = Ack, vin = VIN, encrypt = Encrypt, length = Length, check = Check}, + Rest2 + }. + +parse_body(Binary, Frame = #frame{length = Length, check = OldCheck, encrypt = Encrypt}) -> + <> = Binary, + Check = cal_check(Binary, Length, OldCheck), + case CheckByte == Check of + true -> + RawData = decipher(Data, Encrypt), + {ok, Frame#frame{data = parse_data(Frame, RawData), rawdata = RawData}, Rest}; + false -> + {error, frame_check_error} + end. + +% Algo: ?ENCRYPT_NONE, ENCRYPT_RSA, ENCRYPT_AES128 +decipher(Data, _Algo) -> + % TODO: decypher data + Data. + +% Algo: ?ENCRYPT_NONE, ENCRYPT_RSA, ENCRYPT_AES128 +encipher(Data, _Algo) -> + % TODO: encipher data + Data. + +parse_data( + #frame{cmd = ?CMD_VIHECLE_LOGIN}, + <> +) -> + #{ + <<"Time">> => #{ + <<"Year">> => Year, + <<"Month">> => Month, + <<"Day">> => Day, + <<"Hour">> => Hour, + <<"Minute">> => Minute, + <<"Second">> => Second + }, + <<"Seq">> => Seq, + <<"ICCID">> => ICCID, + <<"Num">> => Num, + <<"Length">> => Length, + <<"Id">> => Id + }; +parse_data( + #frame{cmd = ?CMD_INFO_REPORT}, + <> +) -> + #{ + <<"Time">> => #{ + <<"Year">> => Year, + <<"Month">> => Month, + <<"Day">> => Day, + <<"Hour">> => Hour, + <<"Minute">> => Minute, + <<"Second">> => Second + }, + <<"Infos">> => parse_info(Infos, []) + }; +parse_data( + #frame{cmd = ?CMD_INFO_RE_REPORT}, + <> +) -> + #{ + <<"Time">> => #{ + <<"Year">> => Year, + <<"Month">> => Month, + <<"Day">> => Day, + <<"Hour">> => Hour, + <<"Minute">> => Minute, + <<"Second">> => Second + }, + <<"Infos">> => parse_info(Infos, []) + }; +parse_data( + #frame{cmd = ?CMD_VIHECLE_LOGOUT}, + <> +) -> + #{ + <<"Time">> => #{ + <<"Year">> => Year, + <<"Month">> => Month, + <<"Day">> => Day, + <<"Hour">> => Hour, + <<"Minute">> => Minute, + <<"Second">> => Second + }, + <<"Seq">> => Seq + }; +parse_data( + #frame{cmd = ?CMD_PLATFORM_LOGIN}, + <> +) -> + #{ + <<"Time">> => #{ + <<"Year">> => Year, + <<"Month">> => Month, + <<"Day">> => Day, + <<"Hour">> => Hour, + <<"Minute">> => Minute, + <<"Second">> => Second + }, + <<"Seq">> => Seq, + <<"Username">> => Username, + <<"Password">> => Password, + <<"Encrypt">> => Encrypt + }; +parse_data( + #frame{cmd = ?CMD_PLATFORM_LOGOUT}, + <> +) -> + #{ + <<"Time">> => #{ + <<"Year">> => Year, + <<"Month">> => Month, + <<"Day">> => Day, + <<"Hour">> => Hour, + <<"Minute">> => Minute, + <<"Second">> => Second + }, + <<"Seq">> => Seq + }; +parse_data(#frame{cmd = ?CMD_HEARTBEAT}, <<>>) -> + #{}; +parse_data(#frame{cmd = ?CMD_SCHOOL_TIME}, <<>>) -> + #{}; +parse_data( + #frame{cmd = ?CMD_PARAM_QUERY}, + <> +) -> + %% XXX: need check ACK filed? + #{ + <<"Time">> => #{ + <<"Year">> => Year, + <<"Month">> => Month, + <<"Day">> => Day, + <<"Hour">> => Hour, + <<"Minute">> => Minute, + <<"Second">> => Second + }, + <<"Total">> => Total, + <<"Params">> => parse_params(Rest) + }; +parse_data( + #frame{cmd = ?CMD_PARAM_SETTING}, + <> +) -> + ?SLOG(debug, #{msg => "rest", data => Rest}), + #{ + <<"Time">> => #{ + <<"Year">> => Year, + <<"Month">> => Month, + <<"Day">> => Day, + <<"Hour">> => Hour, + <<"Minute">> => Minute, + <<"Second">> => Second + }, + <<"Total">> => Total, + <<"Params">> => parse_params(Rest) + }; +parse_data( + #frame{cmd = ?CMD_TERMINAL_CTRL}, + <> +) -> + #{ + <<"Time">> => #{ + <<"Year">> => Year, + <<"Month">> => Month, + <<"Day">> => Day, + <<"Hour">> => Hour, + <<"Minute">> => Minute, + <<"Second">> => Second + }, + <<"Command">> => Command, + <<"Param">> => parse_ctrl_param(Command, Rest) + }; +parse_data(Frame, Data) -> + ?SLOG(error, #{msg => "invalid_frame", frame => Frame, data => Data}), + error(invalid_frame). + +%%-------------------------------------------------------------------- +%% Parse Report Data Info +%%-------------------------------------------------------------------- + +parse_info(<<>>, Acc) -> + lists:reverse(Acc); +parse_info(<>, Acc) -> + <> = Body, + parse_info(Rest, [ + #{ + <<"Type">> => <<"Vehicle">>, + <<"Status">> => Status, + <<"Charging">> => Charging, + <<"Mode">> => Mode, + <<"Speed">> => Speed, + <<"Mileage">> => Mileage, + <<"Voltage">> => Voltage, + <<"Current">> => Current, + <<"SOC">> => SOC, + <<"DC">> => DC, + <<"Gear">> => Gear, + <<"Resistance">> => Resistance, + <<"AcceleratorPedal">> => AcceleratorPedal, + <<"BrakePedal">> => BrakePedal + } + | Acc + ]); +parse_info(<>, Acc) -> + % 12 is packet len of per drive motor + Len = Number * 12, + <> = Rest, + parse_info(Rest1, [ + #{ + <<"Type">> => <<"DriveMotor">>, + <<"Number">> => Number, + <<"Motors">> => parse_drive_motor(Bodys, []) + } + | Acc + ]); +parse_info(<>, Acc) -> + <> = + Rest, + + <> = Rest1, + + <> = Rest2, + parse_info(Rest3, [ + #{ + <<"Type">> => <<"FuelCell">>, + <<"CellVoltage">> => CellVoltage, + <<"CellCurrent">> => CellCurrent, + <<"FuelConsumption">> => FuelConsumption, + <<"ProbeNum">> => ProbeNum, + <<"ProbeTemps">> => binary_to_list(ProbeTemps), + <<"H_MaxTemp">> => HMaxTemp, + <<"H_TempProbeCode">> => HTempProbeCode, + <<"H_MaxConc">> => HMaxConc, + <<"H_ConcSensorCode">> => HConcSensorCode, + <<"H_MaxPress">> => HMaxPress, + <<"H_PressSensorCode">> => HPressSensorCode, + <<"DCStatus">> => DCStatus + } + | Acc + ]); +parse_info( + <>, + Acc +) -> + parse_info(Rest, [ + #{ + <<"Type">> => <<"Engine">>, + <<"Status">> => Status, + <<"CrankshaftSpeed">> => CrankshaftSpeed, + <<"FuelConsumption">> => FuelConsumption + } + | Acc + ]); +parse_info( + <>, Acc +) -> + parse_info(Rest, [ + #{ + <<"Type">> => <<"Location">>, + <<"Status">> => Status, + <<"Longitude">> => Longitude, + <<"Latitude">> => Latitude + } + | Acc + ]); +parse_info(<>, Acc) -> + <> = Body, + + parse_info(Rest, [ + #{ + <<"Type">> => <<"Extreme">>, + <<"MaxVoltageBatterySubsysNo">> => MaxVoltageBatterySubsysNo, + <<"MaxVoltageBatteryCode">> => MaxVoltageBatteryCode, + <<"MaxBatteryVoltage">> => MaxBatteryVoltage, + <<"MinVoltageBatterySubsysNo">> => MinVoltageBatterySubsysNo, + <<"MinVoltageBatteryCode">> => MinVoltageBatteryCode, + <<"MinBatteryVoltage">> => MinBatteryVoltage, + <<"MaxTempSubsysNo">> => MaxTempSubsysNo, + <<"MaxTempProbeNo">> => MaxTempProbeNo, + <<"MaxTemp">> => MaxTemp, + <<"MinTempSubsysNo">> => MinTempSubsysNo, + <<"MinTempProbeNo">> => MinTempProbeNo, + <<"MinTemp">> => MinTemp + } + | Acc + ]); +parse_info(<>, Acc) -> + <> = + Rest, + N1 = FaultChargeableDeviceNum * 4, + <> = Rest1, + N2 = FaultDriveMotorNum * 4, + <> = Rest2, + N3 = FaultEngineNum * 4, + <> = Rest3, + N4 = FaultOthersNum * 4, + <> = Rest4, + parse_info(Rest5, [ + #{ + <<"Type">> => <<"Alarm">>, + <<"MaxAlarmLevel">> => MaxAlarmLevel, + <<"GeneralAlarmFlag">> => GeneralAlarmFlag, + <<"FaultChargeableDeviceNum">> => FaultChargeableDeviceNum, + <<"FaultChargeableDeviceList">> => tune_fault_codelist(FaultChargeableDeviceList), + <<"FaultDriveMotorNum">> => FaultDriveMotorNum, + <<"FaultDriveMotorList">> => tune_fault_codelist(FaultDriveMotorList), + <<"FaultEngineNum">> => FaultEngineNum, + <<"FaultEngineList">> => tune_fault_codelist(FaultEngineList), + <<"FaultOthersNum">> => FaultOthersNum, + <<"FaultOthersList">> => tune_fault_codelist(FaultOthersList) + } + | Acc + ]); +parse_info(<>, Acc) -> + {Rest1, SubSystems} = parse_chargeable_voltage(Rest, Number, []), + parse_info(Rest1, [ + #{ + <<"Type">> => <<"ChargeableVoltage">>, + <<"Number">> => Number, + <<"SubSystems">> => SubSystems + } + | Acc + ]); +parse_info(<>, Acc) -> + {Rest1, SubSystems} = parse_chargeable_temp(Rest, Number, []), + parse_info(Rest1, [ + #{ + <<"Type">> => <<"ChargeableTemp">>, + <<"Number">> => Number, + <<"SubSystems">> => SubSystems + } + | Acc + ]); +parse_info(Rest, Acc) -> + ?SLOG(error, #{msg => "invalid_info_feild", rest => Rest, acc => Acc}), + error(invalid_info_feild). + +parse_drive_motor(<<>>, Acc) -> + lists:reverse(Acc); +parse_drive_motor( + <>, + Acc +) -> + parse_drive_motor(Rest, [ + #{ + <<"No">> => No, + <<"Status">> => Status, + <<"CtrlTemp">> => CtrlTemp, + <<"Rotating">> => Rotating, + <<"Torque">> => Torque, + <<"MotorTemp">> => MotorTemp, + <<"InputVoltage">> => InputVoltage, + <<"DCBusCurrent">> => DCBusCurrent + } + | Acc + ]). + +parse_chargeable_voltage(Rest, 0, Acc) -> + {Rest, lists:reverse(Acc)}; +parse_chargeable_voltage( + <>, + Num, + Acc +) -> + Len = FrameCellsCount * 2, + <> = Rest, + parse_chargeable_voltage(Rest1, Num - 1, [ + #{ + <<"ChargeableSubsysNo">> => ChargeableSubsysNo, + <<"ChargeableVoltage">> => ChargeableVoltage, + <<"ChargeableCurrent">> => ChargeableCurrent, + <<"CellsTotal">> => CellsTotal, + <<"FrameCellsIndex">> => FrameCellsIndex, + <<"FrameCellsCount">> => FrameCellsCount, + <<"CellsVoltage">> => tune_voltage(CellsVoltage) + } + | Acc + ]). + +parse_chargeable_temp(Rest, 0, Acc) -> + {Rest, lists:reverse(Acc)}; +parse_chargeable_temp(<>, Num, Acc) -> + <> = Rest, + parse_chargeable_temp(Rest1, Num - 1, [ + #{ + <<"ChargeableSubsysNo">> => ChargeableSubsysNo, + <<"ProbeNum">> => ProbeNum, + <<"ProbesTemp">> => binary_to_list(ProbesTemp) + } + | Acc + ]). +tune_fault_codelist(<<>>) -> + []; +tune_fault_codelist(Data) -> + lists:flatten([list_to_binary(io_lib:format("~4.16.0B", [X])) || <> <= Data]). + +tune_voltage(Bin) -> tune_voltage_(Bin, []). +tune_voltage_(<<>>, Acc) -> lists:reverse(Acc); +tune_voltage_(<>, Acc) -> tune_voltage_(Rest, [V | Acc]). + +parse_params(Bin) -> parse_params_(Bin, []). +parse_params_(<<>>, Acc) -> + lists:reverse(Acc); +parse_params_(<<16#01, Val:?WORD, Rest/binary>>, Acc) -> + parse_params_(Rest, [#{<<"0x01">> => Val} | Acc]); +parse_params_(<<16#02, Val:?WORD, Rest/binary>>, Acc) -> + parse_params_(Rest, [#{<<"0x02">> => Val} | Acc]); +parse_params_(<<16#03, Val:?WORD, Rest/binary>>, Acc) -> + parse_params_(Rest, [#{<<"0x03">> => Val} | Acc]); +parse_params_(<<16#04, Val:?BYTE, Rest/binary>>, Acc) -> + parse_params_(Rest, [#{<<"0x04">> => Val} | Acc]); +parse_params_(<<16#05, Rest/binary>>, Acc) -> + case [V || #{<<"0x04">> := V} <- Acc] of + [Len] -> + <> = Rest, + parse_params_(Rest1, [#{<<"0x05">> => Val} | Acc]); + _ -> + ?SLOG(error, #{ + msg => "invalid_data", reason => "cmd_0x04 must appear ahead of cmd_0x05" + }), + lists:reverse(Acc) + end; +parse_params_(<<16#06, Val:?WORD, Rest/binary>>, Acc) -> + parse_params_(Rest, [#{<<"0x06">> => Val} | Acc]); +parse_params_(<<16#07, Val:5/binary, Rest/binary>>, Acc) -> + parse_params_(Rest, [#{<<"0x07">> => Val} | Acc]); +parse_params_(<<16#08, Val:5/binary, Rest/binary>>, Acc) -> + parse_params_(Rest, [#{<<"0x08">> => Val} | Acc]); +parse_params_(<<16#09, Val:?BYTE, Rest/binary>>, Acc) -> + parse_params_(Rest, [#{<<"0x09">> => Val} | Acc]); +parse_params_(<<16#0A, Val:?WORD, Rest/binary>>, Acc) -> + parse_params_(Rest, [#{<<"0x0A">> => Val} | Acc]); +parse_params_(<<16#0B, Val:?WORD, Rest/binary>>, Acc) -> + parse_params_(Rest, [#{<<"0x0B">> => Val} | Acc]); +parse_params_(<<16#0C, Val:?BYTE, Rest/binary>>, Acc) -> + parse_params_(Rest, [#{<<"0x0C">> => Val} | Acc]); +parse_params_(<<16#0D, Val:?BYTE, Rest/binary>>, Acc) -> + parse_params_(Rest, [#{<<"0x0D">> => Val} | Acc]); +parse_params_(<<16#0E, Rest/binary>>, Acc) -> + case [V || #{<<"0x0D">> := V} <- Acc] of + [Len] -> + <> = Rest, + parse_params_(Rest1, [#{<<"0x0E">> => Val} | Acc]); + _ -> + ?SLOG(error, #{ + msg => "invalid_data", reason => "cmd_0x0D must appear ahead of cmd_0x0E" + }), + lists:reverse(Acc) + end; +parse_params_(<<16#0F, Val:?WORD, Rest/binary>>, Acc) -> + parse_params_(Rest, [#{<<"0x0F">> => Val} | Acc]); +parse_params_(<<16#10, Val:?BYTE, Rest/binary>>, Acc) -> + parse_params_(Rest, [#{<<"0x10">> => Val} | Acc]); +parse_params_(Cmd, Acc) -> + ?SLOG(error, #{msg => "unexcepted_param_identifier", cmd => Cmd}), + lists:reverse(Acc). + +parse_ctrl_param(16#01, Param) -> + parse_upgrade_feild(Param); +parse_ctrl_param(16#02, _) -> + <<>>; +parse_ctrl_param(16#03, _) -> + <<>>; +parse_ctrl_param(16#04, _) -> + <<>>; +parse_ctrl_param(16#05, _) -> + <<>>; +parse_ctrl_param(16#06, <>) -> + #{<<"Level">> => Level, <<"Message">> => Msg}; +parse_ctrl_param(16#07, _) -> + <<>>; +parse_ctrl_param(Cmd, Param) -> + ?SLOG(error, #{msg => "unexcepted_param", param => Param, cmd => Cmd}), + <<>>. + +parse_upgrade_feild(Param) -> + [ + DialingName, + Username, + Password, + <<0, 0, I1, I2, I3, I4>>, + <>, + ManufacturerId, + HardwareVer, + SoftwareVer, + UpgradeUrl, + <> + ] = re:split(Param, ";", [{return, binary}]), + + #{ + <<"DialingName">> => DialingName, + <<"Username">> => Username, + <<"Password">> => Password, + <<"Ip">> => list_to_binary(inet:ntoa({I1, I2, I3, I4})), + <<"Port">> => Port, + <<"ManufacturerId">> => ManufacturerId, + <<"HardwareVer">> => HardwareVer, + <<"SoftwareVer">> => SoftwareVer, + <<"UpgradeUrl">> => UpgradeUrl, + <<"Timeout">> => Timeout + }. + +%%-------------------------------------------------------------------- +%% serialize_pkt +%%-------------------------------------------------------------------- +serialize_pkt(Frame, _Opts) -> + serialize(Frame). + +serialize(#frame{cmd = Cmd, ack = Ack, vin = Vin, encrypt = Encrypt, data = Data, rawdata = RawData}) -> + Encrypted = encipher(serialize_data(Cmd, Ack, RawData, Data), Encrypt), + Len = byte_size(Encrypted), + Stream = <>, + Crc = cal_check(Stream, byte_size(Stream), undefined), + <<"##", Stream/binary, Crc:?BYTE>>. + +serialize_data(?CMD_PARAM_QUERY, ?ACK_IS_CMD, _, #{ + <<"Time">> := Time, + <<"Total">> := Total, + <<"Ids">> := Ids +}) when length(Ids) == Total -> + T = tune_time(Time), + Ids1 = tune_ids(Ids), + <>; +serialize_data(?CMD_PARAM_SETTING, ?ACK_IS_CMD, _, #{ + <<"Time">> := Time, + <<"Total">> := Total, + <<"Params">> := Params +}) when length(Params) == Total -> + T = tune_time(Time), + Params1 = tune_params(Params), + <>; +serialize_data(?CMD_TERMINAL_CTRL, ?ACK_IS_CMD, _, #{ + <<"Time">> := Time, + <<"Command">> := Cmd, + <<"Param">> := Param +}) -> + T = tune_time(Time), + Param1 = tune_ctrl_param(Cmd, Param), + <>; +serialize_data(_Cmd, Ack, RawData, #{<<"Time">> := Time}) when ?IS_RESPONSE(Ack) -> + Rest = + case byte_size(RawData) > 6 of + false -> <<>>; + true -> binary:part(RawData, 6, byte_size(RawData) - 6) + end, + T = tune_time(Time), + <>. + +tune_time(#{ + <<"Year">> := Year, + <<"Month">> := Month, + <<"Day">> := Day, + <<"Hour">> := Hour, + <<"Minute">> := Min, + <<"Second">> := Sec +}) -> + <>. + +tune_ids(Ids) -> + lists:foldr( + fun + (Id, Acc) when is_integer(Id) -> + <>; + (Id, Acc) when is_binary(Id) -> + <> + end, + <<>>, + Ids + ). + +tune_params(Params) -> + tune_params_(lists:reverse(Params), <<>>). + +tune_params_([], Bin) -> + Bin; +tune_params_([#{16#01 := Val} | Rest], Bin) -> + tune_params_(Rest, <<16#01:?BYTE, Val:?WORD, Bin/binary>>); +tune_params_([#{16#02 := Val} | Rest], Bin) -> + tune_params_(Rest, <<16#02:?BYTE, Val:?WORD, Bin/binary>>); +tune_params_([#{16#03 := Val} | Rest], Bin) -> + tune_params_(Rest, <<16#03:?BYTE, Val:?WORD, Bin/binary>>); +tune_params_([#{16#04 := Val} | Rest], Bin) -> + {Val05, Rest1} = take_param(16#05, Rest), + tune_params_(Rest1, <<16#04:?BYTE, Val:?BYTE, 16#05, Val05:Val/binary, Bin/binary>>); +tune_params_([#{16#05 := Val} | Rest], Bin) -> + tune_params_(Rest ++ [#{16#05 => Val}], Bin); +tune_params_([#{16#06 := Val} | Rest], Bin) -> + tune_params_(Rest, <<16#06:?BYTE, Val:?WORD, Bin/binary>>); +tune_params_([#{16#07 := Val} | Rest], Bin) when byte_size(Val) == 5 -> + tune_params_(Rest, <<16#07:?BYTE, Val/binary, Bin/binary>>); +tune_params_([#{16#08 := Val} | Rest], Bin) when byte_size(Val) == 5 -> + tune_params_(Rest, <<16#08:?BYTE, Val/binary, Bin/binary>>); +tune_params_([#{16#09 := Val} | Rest], Bin) -> + tune_params_(Rest, <<16#09:?BYTE, Val:?BYTE, Bin/binary>>); +tune_params_([#{16#0A := Val} | Rest], Bin) -> + tune_params_(Rest, <<16#0A:?BYTE, Val:?WORD, Bin/binary>>); +tune_params_([#{16#0B := Val} | Rest], Bin) -> + tune_params_(Rest, <<16#0B:?BYTE, Val:?WORD, Bin/binary>>); +tune_params_([#{16#0C := Val} | Rest], Bin) -> + tune_params_(Rest, <<16#0C:?BYTE, Val:?BYTE, Bin/binary>>); +tune_params_([#{16#0D := Val} | Rest], Bin) -> + {Val0E, Rest1} = take_param(16#0E, Rest), + tune_params_(Rest1, <<16#0D:?BYTE, Val:?BYTE, 16#0E, Val0E:Val/binary, Bin/binary>>); +tune_params_([#{16#0E := Val} | Rest], Bin) -> + tune_params_(Rest ++ [#{16#0E => Val}], Bin); +tune_params_([#{16#0F := Val} | Rest], Bin) -> + tune_params_(Rest, <<16#0F:?BYTE, Val:?WORD, Bin/binary>>); +tune_params_([#{16#10 := Val} | Rest], Bin) -> + tune_params_(Rest, <<16#10:?BYTE, Val:?BYTE, Bin/binary>>). + +tune_ctrl_param(16#00, _) -> + <<>>; +tune_ctrl_param(16#01, Param) -> + tune_upgrade_feild(Param); +tune_ctrl_param(16#02, _) -> + <<>>; +tune_ctrl_param(16#03, _) -> + <<>>; +tune_ctrl_param(16#04, _) -> + <<>>; +tune_ctrl_param(16#05, _) -> + <<>>; +tune_ctrl_param(16#06, #{<<"Level">> := Level, <<"Message">> := Msg}) -> + <>; +tune_ctrl_param(16#07, _) -> + <<>>; +tune_ctrl_param(Cmd, Param) -> + ?SLOG(error, #{msg => "unexcepted_cmd", cmd => Cmd, param => Param}), + <<>>. + +tune_upgrade_feild(Param) -> + TuneBin = fun + (Bin, Len) when is_binary(Bin), byte_size(Bin) =:= Len -> Bin; + (undefined, _) -> undefined; + (Bin, _) -> error({invalid_param_length, Bin}) + end, + TuneWrd = fun + (Val) when is_integer(Val), Val < 65535 -> <>; + (undefined) -> undefined; + (_) -> error(invalid_param_word_value) + end, + TuneAdr = fun + (Ip) when is_binary(Ip) -> + {ok, {I1, I2, I3, I4}} = inet:parse_address(binary_to_list(Ip)), + <<0, 0, I1, I2, I3, I4>>; + (undefined) -> + undefined; + (_) -> + error(invalid_ip_address) + end, + L = [ + maps:get(<<"DialingName">>, Param, undefined), + maps:get(<<"Username">>, Param, undefined), + maps:get(<<"Password">>, Param, undefined), + TuneAdr(maps:get(<<"Ip">>, Param, undefined)), + TuneWrd(maps:get(<<"Port">>, Param, undefined)), + TuneBin(maps:get(<<"ManufacturerId">>, Param, undefined), 4), + TuneBin(maps:get(<<"HardwareVer">>, Param, undefined), 5), + TuneBin(maps:get(<<"SoftwareVer">>, Param, undefined), 5), + maps:get(<<"UpgradeUrl">>, Param, undefined), + TuneWrd(maps:get(<<"Timeout">>, Param, undefined)) + ], + list_to_binary([I || I <- lists:join(";", L), I /= undefined]). + +take_param(K, Params) -> + V = search_param(K, Params), + {V, Params -- [#{K => V}]}. + +search_param(16#05, [#{16#05 := V} | _]) -> V; +search_param(16#0E, [#{16#0E := V} | _]) -> V; +search_param(K, [_ | Rest]) -> search_param(K, Rest). + +cal_check(_, 0, Check) -> Check; +cal_check(<>, Size, undefined) -> cal_check(Rest, Size - 1, C); +cal_check(<>, Size, Check) -> cal_check(Rest, Size - 1, Check bxor C). + +format(Msg) -> + io_lib:format("~p", [Msg]). + +type(_) -> + %% TODO: + gbt32960. + +is_message(#frame{}) -> + %% TODO: + true; +is_message(_) -> + false. diff --git a/apps/emqx_gateway_gbt32960/src/emqx_gbt32960_schema.erl b/apps/emqx_gateway_gbt32960/src/emqx_gbt32960_schema.erl new file mode 100644 index 000000000..743c74e70 --- /dev/null +++ b/apps/emqx_gateway_gbt32960/src/emqx_gbt32960_schema.erl @@ -0,0 +1,56 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_gbt32960_schema). + +-include("emqx_gbt32960.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). +-include_lib("typerefl/include/types.hrl"). + +%% config schema provides +-export([fields/1, desc/1]). + +fields(gbt32960) -> + [ + {mountpoint, emqx_gateway_schema:mountpoint(?DEFAULT_MOUNTPOINT)}, + {retry_interval, + sc( + emqx_schema:duration_ms(), + #{ + default => <<"8s">>, + desc => ?DESC(retry_interval) + } + )}, + {max_retry_times, + sc( + non_neg_integer(), + #{ + default => 3, + desc => ?DESC(max_retry_times) + } + )}, + {message_queue_len, + sc( + non_neg_integer(), + #{ + default => 10, + desc => ?DESC(message_queue_len) + } + )}, + {listeners, sc(ref(emqx_gateway_schema, tcp_listeners), #{desc => ?DESC(tcp_listeners)})} + ] ++ emqx_gateway_schema:gateway_common_options(). + +desc(gbt32960) -> + "The GBT-32960 gateway"; +desc(_) -> + undefined. + +%%-------------------------------------------------------------------- +%% internal functions + +sc(Type, Meta) -> + hoconsc:mk(Type, Meta). + +ref(Mod, Field) -> + hoconsc:ref(Mod, Field). diff --git a/apps/emqx_gateway_gbt32960/test/emqx_gbt32960_SUITE.erl b/apps/emqx_gateway_gbt32960/test/emqx_gbt32960_SUITE.erl new file mode 100644 index 000000000..56184fc5f --- /dev/null +++ b/apps/emqx_gateway_gbt32960/test/emqx_gbt32960_SUITE.erl @@ -0,0 +1,1445 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_gbt32960_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include("emqx_gbt32960.hrl"). +-include_lib("emqx/include/emqx.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +-define(BYTE, 8 / big - integer). +-define(WORD, 16 / big - integer). +-define(DWORD, 32 / big - integer). + +-define(PORT, 7325). +-define(LOGT(Format, Args), ct:pal("TEST_SUITE: " ++ Format, Args)). + +-define(CONF_DEFAULT, << + "\n" + "gateway.gbt32960 {\n" + " retry_interval = \"1s\"\n" + " listeners.tcp.default {\n" + " bind = 7325\n" + " }\n" + "}\n" +>>). + +all() -> + emqx_common_test_helpers:all(?MODULE). + +init_per_suite(Config) -> + application:load(emqx_gateway_gbt32960), + Apps = emqx_cth_suite:start( + [ + {emqx_conf, ?CONF_DEFAULT}, + emqx_gateway, + emqx_auth, + emqx_management, + {emqx_dashboard, "dashboard.listeners.http { enable = true, bind = 18083 }"} + ], + #{work_dir => emqx_cth_suite:work_dir(Config)} + ), + emqx_common_test_http:create_default_app(), + [{suite_apps, Apps} | Config]. + +end_per_suite(Config) -> + emqx_common_test_http:delete_default_app(), + emqx_cth_suite:stop(?config(suite_apps, Config)), + ok. + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% helper functions %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +encode(Cmd, Vin, Data) -> + encode(Cmd, ?ACK_IS_CMD, Vin, ?ENCRYPT_NONE, Data). + +encode(Cmd, Ack, Vin, Data) -> + encode(Cmd, Ack, Vin, ?ENCRYPT_NONE, Data). + +encode(Cmd, Ack, Vin, Encrypt, Data) -> + Size = byte_size(Data), + S1 = <>, + Crc = make_crc(S1, undefined), + Stream = <<"##", S1/binary, Crc:8>>, + ?LOGT("encode a packet=~p", [binary_to_hex_string(Stream)]), + Stream. + +make_crc(<<>>, Xor) -> Xor; +make_crc(<>, undefined) -> make_crc(Rest, C); +make_crc(<>, Xor) -> make_crc(Rest, C bxor Xor). + +make_time() -> + {Year, Mon, Day} = date(), + {Hour, Min, Sec} = time(), + Year1 = list_to_integer(string:substr(integer_to_list(Year), 3, 2)), + <>. + +binary_to_hex_string(Data) -> + lists:flatten([io_lib:format("~2.16.0B ", [X]) || <> <= Data]). + +to_json(#frame{cmd = Cmd, vin = Vin, encrypt = Encrypt, data = Data}) -> + emqx_utils_json:encode(#{'Cmd' => Cmd, 'Vin' => Vin, 'Encrypt' => Encrypt, 'Data' => Data}). + +get_published_msg() -> + receive + {deliver, _Topic, #message{topic = Topic, payload = Payload}} -> + {Topic, Payload} + after 5000 -> + error(timeout) + end. + +get_subscriptions() -> + lists:map(fun({_, Topic}) -> Topic end, ets:tab2list(emqx_subscription)). + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%% test cases %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +login_first() -> + emqx:subscribe("gbt32960/+/upstream/#"), + + % + % send VEHICLE LOGIN + % + Time = <<12, 12, 29, 12, 19, 20>>, + Data = <