diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 91aab2dcd..73a5812e8 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -71,6 +71,7 @@ includes() -> , "emqx_bridge_mqtt" , "emqx_modules" , "emqx_management" + , "emqx_gateway" ]. -endif. diff --git a/apps/emqx_exhook/src/emqx_exhook_app.erl b/apps/emqx_exhook/src/emqx_exhook_app.erl index 4e00340d8..2988be6d2 100644 --- a/apps/emqx_exhook/src/emqx_exhook_app.erl +++ b/apps/emqx_exhook/src/emqx_exhook_app.erl @@ -88,7 +88,7 @@ init_hooks_cnter() -> try _ = ets:new(?CNTER, [named_table, public]), ok catch - exit:badarg:_ -> + error:badarg:_ -> ok end. diff --git a/apps/emqx_gateway/.gitignore b/apps/emqx_gateway/.gitignore new file mode 100644 index 000000000..71ab0135c --- /dev/null +++ b/apps/emqx_gateway/.gitignore @@ -0,0 +1,20 @@ +.rebar3 +_* +.eunit +*.o +*.beam +*.plt +*.swp +*.swo +.erlang.cookie +ebin +log +erl_crash.dump +.rebar +logs +_build +.idea +*.iml +rebar3.crashdump +*~ +rebar.lock diff --git a/apps/emqx_gateway/LICENSE b/apps/emqx_gateway/LICENSE new file mode 100644 index 000000000..1f15def74 --- /dev/null +++ b/apps/emqx_gateway/LICENSE @@ -0,0 +1,191 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + Copyright 2021, JianBo He . + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/apps/emqx_gateway/Makefile b/apps/emqx_gateway/Makefile new file mode 100644 index 000000000..b2a54f7dd --- /dev/null +++ b/apps/emqx_gateway/Makefile @@ -0,0 +1,28 @@ +## shallow clone for speed + +REBAR_GIT_CLONE_OPTIONS += --depth 1 +export REBAR_GIT_CLONE_OPTIONS + +REBAR = rebar3 +all: compile + +compile: + $(REBAR) compile + +clean: distclean + +ct: + $(REBAR) as test ct -v + +eunit: + $(REBAR) as test eunit + +xref: + $(REBAR) xref + +cover: + $(REBAR) cover + +distclean: + @rm -rf _build + @rm -f data/app.*.config data/vm.*.args rebar.lock diff --git a/apps/emqx_gateway/README.md b/apps/emqx_gateway/README.md new file mode 100644 index 000000000..0f95f286a --- /dev/null +++ b/apps/emqx_gateway/README.md @@ -0,0 +1,309 @@ +# emqx_gateway + +***This is a very early prototype application*** for Gateway in EMQ X Broker 5.0 + +## Concept + + EMQ X Gateway Managment + - Gateway-Registry (or Gateway Type) + - *Load + - *UnLoad + - *List + + - Gateway + - *Create + - *Delete + - *Update + - *Stop-And-Start + - *Hot-Upgrade + - *Satrt/Enable + - *Stop/Disable + - Listener + +## ROADMAP + +Gateway v0.1: Management support + +Gateway v0.2: Conn/Frame/Protocol Template + +### Compatible with EMQ X + +> Why we need to compatible + +1. Authentication +2. Hooks/Event system +3. Messages Mode & Rule Engine +4. Cluster registration +5. Metrics & Statistic + +> How to do it + +> + +### User Interface + +#### Configurations + +```hocon +gateway { + + ## ... some confs for top scope + .. + ## End. + + ## Gateway Instances + + lwm2m[.name] { + + ## variable support + mountpoint: lwm2m/%e/ + + lifetime_min: 1s + lifetime_max: 86400s + #qmode_time_window: 22 + #auto_observe: off + + #update_msg_publish_condition: contains_object_list + + xml_dir: {{ platform_etc_dir }}/lwm2m_xml + + clientinfo_override: { + username: ${register.opts.uname} + password: ${register.opts.passwd} + clientid: ${epn} + } + + #authenticator: allow_anonymous + authenticator: [ + { + type: auth-http + method: post + //?? how to generate clientinfo ?? + params: $client.credential + } + ] + + translator: { + downlink: "dn/#" + uplink: { + notify: "up/notify" + response: "up/resp" + register: "up/resp" + update: "up/reps" + } + } + + %% ?? listener.$type.name ?? + listener.udp[.name] { + listen_on: 0.0.0.0:5683 + max_connections: 1024000 + max_conn_rate: 1000 + ## ?? udp keepalive in socket level ??? + #keepalive: + ## ?? udp proxy-protocol in socket level ??? + #proxy_protocol: on + #proxy_timeout: 30s + recbuf: 2KB + sndbuf: 2KB + buffer: 2KB + tune_buffer: off + #access: allow all + read_packets: 20 + } + + listener.dtls[.name] { + listen_on: 0.0.0.0:5684 + ... + } + } + + ## The CoAP Gateway + coap[.name] { + + #enable_stats: on + + authenticator: [ + ... + ] + + listener.udp[.name] { + ... + } + + listener.dtls[.name] { + ... + } +} + + ## The Stomp Gateway + stomp[.name] { + + allow_anonymous: true + + default_user.login: guest + default_user.passcode: guest + + frame.max_headers: 10 + frame.max_header_length: 1024 + frame.max_body_length: 8192 + + listener.tcp[.name] { + ... + } + + listener.ssl[.name] { + ... + } + } + + exproto[.name] { + + proto_name: DL-648 + + authenticators: [...] + + adapter: { + type: grpc + options: { + listen_on: 9100 + } + } + + handler: { + type: grpc + options: { + url: + } + } + + listener.tcp[.name] { + ... + } + } + + ## ============================ Enterpise gateways + + ## The JT/T 808 Gateway + jtt808[.name] { + + idle_timeout: 30s + enable_stats: on + max_packet_size: 8192 + + clientinfo_override: { + clientid: $phone + username: xxx + password: xxx + } + + authenticator: [ + { + type: auth-http + method: post + params: $clientinfo.credential + } + ] + + translator: { + subscribe: [jt808/%c/dn] + publish: [jt808/%c/up] + } + + listener.tcp[.name] { + ... + } + + listener.ssl[.name] { + ... + } + } + + gbt32960[.name] { + + frame.max_length: 8192 + retx_interval: 8s + retx_max_times: 3 + message_queue_len: 10 + + authenticators: [...] + + translator: { + ## upstream + login: gbt32960/${vin}/upstream/vlogin + logout: gbt32960/${vin}/upstream/vlogout + informing: gbt32960/${vin}/upstream/info + reinforming: gbt32960/${vin}/upstream/reinfo + ## downstream + downstream: gbt32960/${vin}/dnstream + response: gbt32960/${vin}/upstream/response + } + + listener.tcp[.name] { + ... + } + + listener.ssl[.name] { + ... + } + } + + privtcp[.name] { + + max_packet_size: 65535 + idle_timeout: 15s + + enable_stats: on + + force_gc_policy: 1000|1MB + force_shutdown_policy: 8000|800MB + + translator: { + up_topic: tcp/%c/up + dn_topic: tcp/%c/dn + } + + listener.tcp[.name]: { + ... + } + } +} +``` + +#### CLI + +##### Gateway + +```bash +## List all started gateway and gateway-instance +emqx_ctl gateway list +emqx_ctl gateway lookup +emqx_ctl gateway stop +emqx_ctl gateway start + +emqx_ctl gateway-registry re-searching +emqx_ctl gateway-registry list + +emqx_ctl gateway-clients list +emqx_ctl gateway-clients show +emqx_ctl gateway-clients kick + +## Banned ?? +emqx_ctl gateway-banned + +## Metrics +emqx_ctl gateway-metrics [] +``` + +#### Mangement by HTTP-API/Dashboard/ + +#### How to integrate a protocol to your platform + +### Develop your protocol gateway + +There are 3 way to create your protocol gateway for EMQ X 5.0: + +1. Use Erlang to create a new emqx plugin to handle all of protocol packets (same as v5.0 before) + +2. Based on the emqx-gateway-impl-bhvr and emqx-gateway + +3. Use the gRPC Gateway diff --git a/apps/emqx_gateway/etc/emqx_gateway.conf b/apps/emqx_gateway/etc/emqx_gateway.conf new file mode 100644 index 000000000..ab5b52143 --- /dev/null +++ b/apps/emqx_gateway/etc/emqx_gateway.conf @@ -0,0 +1,30 @@ +##-------------------------------------------------------------------- +## EMQ X Gateway configurations +##-------------------------------------------------------------------- + +## TODO: + +emqx_gateway: { + stomp.1: { + frame: { + max_headers: 10 + max_headers_length: 1024 + max_body_length: 8192 + } + + clientinfo_override: { + username: "${Packet.headers.login}" + password: "${Packet.headers.passcode}" + } + + authenticator: allow_anonymous + + listener.tcp.1: { + bind: 61613 + acceptors: 16 + max_connections: 1024000 + max_conn_rate: 1000 + active_n: 100 + } + } +} diff --git a/apps/emqx_gateway/include/emqx_gateway.hrl b/apps/emqx_gateway/include/emqx_gateway.hrl new file mode 100644 index 000000000..35fad7f23 --- /dev/null +++ b/apps/emqx_gateway/include/emqx_gateway.hrl @@ -0,0 +1,35 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2017-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-ifndef(EMQX_GATEWAY_HRL). +-define(EMQX_GATEWAY_HRL, 1). + +-type instance_id() :: atom(). +-type gateway_type() :: atom(). + +%% @doc The Gateway Instace defination +-type instance() :: + #{ id := instance_id() + , type := gateway_type() + , name := binary() + , descr => binary() | undefined + %% Appears only in creating or detailed info + , rawconf => map() + %% Appears only in getting instance status/info + , status => stopped | running + }. + +-endif. diff --git a/apps/emqx_gateway/rebar.config b/apps/emqx_gateway/rebar.config new file mode 100644 index 000000000..71fc61330 --- /dev/null +++ b/apps/emqx_gateway/rebar.config @@ -0,0 +1,7 @@ +{erl_opts, [debug_info]}. +{deps, []}. + +{shell, [ + % {config, "config/sys.config"}, + {apps, [emqx_gateway]} +]}. diff --git a/apps/emqx_gateway/src/bhvrs/emqx_gateway_conn.erl b/apps/emqx_gateway/src/bhvrs/emqx_gateway_conn.erl new file mode 100644 index 000000000..7392f0b20 --- /dev/null +++ b/apps/emqx_gateway/src/bhvrs/emqx_gateway_conn.erl @@ -0,0 +1,22 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% @doc The behavior abstrat for TCP based gateway conn +%% +-module(emqx_gateway_conn). + +%% TODO: Gateway v0.2 + diff --git a/apps/emqx_gateway/src/bhvrs/emqx_gateway_impl.erl b/apps/emqx_gateway/src/bhvrs/emqx_gateway_impl.erl new file mode 100644 index 000000000..9726dad02 --- /dev/null +++ b/apps/emqx_gateway/src/bhvrs/emqx_gateway_impl.erl @@ -0,0 +1,49 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_gateway_impl). + +-include("include/emqx_gateway.hrl"). + +-type state() :: map(). +-type reason() :: any(). + +%% @doc +-callback init(Options :: list()) -> {error, reason()} | {ok, GwState :: state()}. + +%% @doc +-callback on_insta_create(Insta :: instance(), + Ctx :: emqx_gateway_ctx:context(), + GwState :: state() + ) + -> {error, reason()} + | {ok, [GwInstaPid :: pid()], GwInstaState :: state()} + | {ok, [Childspec :: supervisor:child_spec()], GwInstaState :: state()}. + +%% @doc +-callback on_insta_update(NewInsta :: instance(), + OldInsta :: instance(), + GwInstaState :: state(), + GwState :: state()) + -> ok + | {ok, [GwInstaPid :: pid()], GwInstaState :: state()} + | {ok, [Childspec :: supervisor:child_spec()], GwInstaState :: state()} + | {error, reason()}. + +%% @doc +-callback on_insta_destroy(Insta :: instance(), + GwInstaState :: state(), + GwState :: state()) -> ok. diff --git a/apps/emqx_gateway/src/emqx_gateway.app.src b/apps/emqx_gateway/src/emqx_gateway.app.src new file mode 100644 index 000000000..287d710eb --- /dev/null +++ b/apps/emqx_gateway/src/emqx_gateway.app.src @@ -0,0 +1,11 @@ +{application, emqx_gateway, + [{description, "The Gateway management application"}, + {vsn, "0.1.0"}, + {registered, []}, + {mod, {emqx_gateway_app, []}}, + {applications, [kernel, stdlib]}, + {env, []}, + {modules, []}, + {licenses, ["Apache 2.0"]}, + {links, []} + ]}. diff --git a/apps/emqx_gateway/src/emqx_gateway.erl b/apps/emqx_gateway/src/emqx_gateway.erl new file mode 100644 index 000000000..f6c12ad53 --- /dev/null +++ b/apps/emqx_gateway/src/emqx_gateway.erl @@ -0,0 +1,84 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_gateway). + +-include("include/emqx_gateway.hrl"). + +%% APIs +-export([ registered_gateway/0 + , create/4 + , remove/1 + , lookup/1 + , update/1 + , start/1 + , stop/1 + , list/0 + ]). + +-spec registered_gateway() -> + [{gateway_type(), emqx_gateway_registry:descriptor()}]. +registered_gateway() -> + emqx_gateway_registry:list(). + +%%-------------------------------------------------------------------- +%% Gateway Instace APIs + +-spec list() -> [instance()]. +list() -> + lists:append(lists:map( + fun({_, Insta}) -> Insta end, + emqx_gateway_sup:list_gateway_insta() + )). + +-spec create(gateway_type(), binary(), binary(), map()) + -> {ok, pid()} + | {error, any()}. +create(Type, Name, Descr, RawConf) -> + Insta = #{ id => clacu_insta_id(Type, Name) + , type => Type + , name => Name + , descr => Descr + , rawconf => RawConf + }, + emqx_gateway_sup:create_gateway_insta(Insta). + +-spec remove(instance_id()) -> ok | {error, any()}. +remove(InstaId) -> + emqx_gateway_sup:remove_gateway_insta(InstaId). + +-spec lookup(instance_id()) -> instance() | undefined. +lookup(InstaId) -> + emqx_gateway_sup:lookup_gateway_insta(InstaId). + +-spec update(instance()) -> ok | {error, any()}. +update(NewInsta) -> + emqx_gateway_sup:update_gateway_insta(NewInsta). + +-spec start(instance_id()) -> ok | {error, any()}. +start(InstaId) -> + emqx_gateway_sup:start_gateway_insta(InstaId). + +-spec stop(instance_id()) -> ok | {error, any()}. +stop(InstaId) -> + emqx_gateway_sup:stop_gateway_insta(InstaId). + +%%-------------------------------------------------------------------- +%% Internal funcs +%%-------------------------------------------------------------------- + +clacu_insta_id(Type, Name) when is_binary(Name) -> + list_to_atom(lists:concat([Type, "#", binary_to_list(Name)])). diff --git a/apps/emqx_gateway/src/emqx_gateway_app.erl b/apps/emqx_gateway/src/emqx_gateway_app.erl new file mode 100644 index 000000000..230776b73 --- /dev/null +++ b/apps/emqx_gateway/src/emqx_gateway_app.erl @@ -0,0 +1,93 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_gateway_app). + +-behaviour(application). + +-include_lib("emqx/include/logger.hrl"). + +-emqx_plugin(?MODULE). + +-logger_header("[Gateway]"). + +-export([start/2, stop/1]). + +start(_StartType, _StartArgs) -> + {ok, Sup} = emqx_gateway_sup:start_link(), + emqx_gateway_cli:load(), + load_default_gateway_applications(), + create_gateway_by_default(), + {ok, Sup}. + +stop(_State) -> + emqx_gateway_cli:unload(), + ok. + +%%-------------------------------------------------------------------- +%% Internal funcs + +load_default_gateway_applications() -> + Apps = gateway_type_searching(), + ?LOG(info, "Starting the default gateway types: ~p", [Apps]), + lists:foreach(fun load/1, Apps). + +gateway_type_searching() -> + %% FIXME: Hardcoded apps + [emqx_stomp_impl]. + +load(Mod) -> + try + Mod:load(), + ?LOG(info, "Load ~s gateway application successfully!", [Mod]) + catch + Class : Reason -> + ?LOG(error, "Load ~s gateway application failed: {~p, ~p}", + [Mod, Class, Reason]) + end. + +create_gateway_by_default() -> + create_gateway_by_default(zipped_confs()). + +create_gateway_by_default([]) -> + ok; +create_gateway_by_default([{Type, Name, Confs}|More]) -> + case emqx_gateway_registry:lookup(Type) of + undefined -> + ?LOG(error, "Skip to start ~p#~p: not_registred_type", + [Type, Name]); + _ -> + case emqx_gateway:create(Type, + atom_to_binary(Name, utf8), + <<>>, + Confs) of + {ok, _} -> + ?LOG(debug, "Start ~p#~p successfully!", [Type, Name]); + {error, Reason} -> + ?LOG(error, "Start ~p#~p failed: ~0p", + [Type, Name, Reason]) + end + end, + create_gateway_by_default(More). + +zipped_confs() -> + All = maps:to_list(emqx_config:get([emqx_gateway])), + lists:append(lists:foldr( + fun({Type, Gws}, Acc) -> + {Names, Confs} = lists:unzip(maps:to_list(Gws)), + Types = [ Type || _ <- lists:seq(1, length(Names))], + [lists:zip3(Types, Names, Confs) | Acc] + end, [], All)). diff --git a/apps/emqx_gateway/src/emqx_gateway_cli.erl b/apps/emqx_gateway/src/emqx_gateway_cli.erl new file mode 100644 index 000000000..beb3e5eae --- /dev/null +++ b/apps/emqx_gateway/src/emqx_gateway_cli.erl @@ -0,0 +1,201 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% @doc The Command-Line-Interface module for Gateway Application +-module(emqx_gateway_cli). + +-export([ load/0 + , unload/0 + ]). + +-export([ gateway/1 + , 'gateway-registry'/1 + , 'gateway-clients'/1 + , 'gateway-metrics'/1 + %, 'gateway-banned'/1 + ]). + +-spec load() -> ok. +load() -> + Cmds = [Fun || {Fun, _} <- ?MODULE:module_info(exports), is_cmd(Fun)], + lists:foreach(fun(Cmd) -> emqx_ctl:register_command(Cmd, {?MODULE, Cmd}, []) end, Cmds). + +-spec unload() -> ok. +unload() -> + Cmds = [Fun || {Fun, _} <- ?MODULE:module_info(exports), is_cmd(Fun)], + lists:foreach(fun(Cmd) -> emqx_ctl:unregister_command(Cmd) end, Cmds). + +is_cmd(Fun) -> + not lists:member(Fun, [init, load, module_info]). + + +%%-------------------------------------------------------------------- +%% Cmds + +gateway(["list"]) -> + lists:foreach(fun(#{id := InstaId, name := Name, type := Type}) -> + %% FIXME: Get the real running status + emqx_ctl:print("Gateway(~s, name=~s, type=~s, status=running~n", + [InstaId, Name, Type]) + end, emqx_gateway:list()); + +gateway(["lookup", GatewayInstaId]) -> + case emqx_gateway:lookup(GatewayInstaId) of + undefined -> + emqx_ctl:print("undefined"); + Info -> + emqx_ctl:print("~p~n", [Info]) + end; + +gateway(["stop", GatewayInstaId]) -> + case emqx_gateway:stop(GatewayInstaId) of + ok -> + emqx_ctl:print("ok"); + {error, Reason} -> + emqx_ctl:print("Error: ~p~n", [Reason]) + end; + +gateway(["start", GatewayInstaId]) -> + case emqx_gateway:start(GatewayInstaId) of + ok -> + emqx_ctl:print("ok"); + {error, Reason} -> + emqx_ctl:print("Error: ~p~n", [Reason]) + end; + +gateway(_) -> + %% TODO: create/remove APIs + emqx_ctl:usage([ {"gateway list", + "List all created gateway instances"} + , {"gateway lookup ", + "Looup a gateway detailed informations"} + , {"gateway stop ", + "Stop a gateway instance and release all resources"} + , {"gateway start ", + "Start a gateway instance"} + ]). + +'gateway-registry'(["list"]) -> + lists:foreach( + fun({GwType, #{cbkmod := CbMod}}) -> + emqx_ctl:print("Registered Type: ~s, Callback Module: ~s~n", [GwType, CbMod]) + end, + emqx_gateway_registry:list()); + +'gateway-registry'(_) -> + emqx_ctl:usage([ {"gateway-registry list", + "List all registered gateway types"} + ]). + +'gateway-clients'(["list", Type]) -> + InfoTab = emqx_gateway_cm:tabname(info, Type), + dump(InfoTab, client); + +'gateway-clients'(["lookup", Type, ClientId]) -> + ChanTab = emqx_gateway_cm:tabname(chan, Type), + case ets:lookup(ChanTab, bin(ClientId)) of + [] -> emqx_ctl:print("Not Found.~n"); + [Chann] -> + InfoTab = emqx_gateway_cm:tabname(info, Type), + [ChannInfo] = ets:lookup(InfoTab, Chann), + print({client, ChannInfo}) + end; + +'gateway-clients'(["kick", Type, ClientId]) -> + case emqx_gateway_cm:kick_session(Type, bin(ClientId)) of + ok -> emqx_ctl:print("ok~n"); + _ -> emqx_ctl:print("Not Found.~n") + end; + +'gateway-clients'(_) -> + emqx_ctl:usage([ {"gateway-clients list ", + "List all clients for a type of gateway"} + , {"gateway-clients lookup ", + "Lookup the Client Info for specified client"} + , {"gateway-clients kick ", + "Kick out a client"} + ]). + +'gateway-metrics'([GatewayType]) -> + Tab = emqx_gateway_metrics:tabname(GatewayType), + case ets:info(Tab) of + undefined -> + emqx_ctl:print("Bad Gateway Tyep.~n"); + _ -> + lists:foreach( + fun({K, V}) -> + emqx_ctl:print("~-30s: ~w~n", [K, V]) + end, lists:sort(ets:tab2list(Tab))) + end; + +'gateway-metrics'(_) -> + emqx_ctl:usage([ {"gateway-metrics ", + "List all metrics for a type of gateway"} + ]). + +%%-------------------------------------------------------------------- +%% Internal funcs +%%-------------------------------------------------------------------- + +bin(S) -> iolist_to_binary(S). + +dump(Table, Tag) -> + dump(Table, Tag, ets:first(Table), []). + +dump(_Table, _, '$end_of_table', Result) -> + lists:reverse(Result); + +dump(Table, Tag, Key, Result) -> + PrintValue = [print({Tag, Record}) || Record <- ets:lookup(Table, Key)], + dump(Table, Tag, ets:next(Table, Key), [PrintValue | Result]). + +print({client, {_, Infos, Stats}}) -> + ClientInfo = maps:get(clientinfo, Infos, #{}), + ConnInfo = maps:get(conninfo, Infos, #{}), + _Session = maps:get(session, Infos, #{}), + SafeGet = fun(K, M) -> maps:get(K, M, undefined) end, + StatsGet = fun(K) -> proplists:get_value(K, Stats, 0) end, + + ConnectedAt = SafeGet(connected_at, ConnInfo), + InfoKeys = [clientid, username, peername, clean_start, keepalive, + subscriptions_cnt, send_msg, connected, created_at, connected_at], + Info = #{ clientid => SafeGet(clientid, ClientInfo), + username => SafeGet(username, ClientInfo), + peername => SafeGet(peername, ConnInfo), + clean_start => SafeGet(clean_start, ConnInfo), + keepalive => SafeGet(keepalive, ConnInfo), + subscriptions_cnt => StatsGet(subscriptions_cnt), + send_msg => StatsGet(send_msg), + connected => SafeGet(conn_state, ClientInfo) == connected, + created_at => ConnectedAt, + connected_at => ConnectedAt + }, + + emqx_ctl:print("Client(~s, username=~s, peername=~s, " + "clean_start=~s, keepalive=~w, " + "subscriptions=~w, delivered_msgs=~w, " + "connected=~s, created_at=~w, connected_at=~w)~n", + [format(K, maps:get(K, Info)) || K <- InfoKeys]). + +format(_, undefined) -> + undefined; + +format(peername, {IPAddr, Port}) -> + IPStr = emqx_mgmt_util:ntoa(IPAddr), + io_lib:format("~s:~p", [IPStr, Port]); + +format(_, Val) -> + Val. diff --git a/apps/emqx_gateway/src/emqx_gateway_cm.erl b/apps/emqx_gateway/src/emqx_gateway_cm.erl new file mode 100644 index 000000000..f8ca18c1a --- /dev/null +++ b/apps/emqx_gateway/src/emqx_gateway_cm.erl @@ -0,0 +1,447 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% @doc The gateway connection management +-module(emqx_gateway_cm). + +-behaviour(gen_server). + +-include("include/emqx_gateway.hrl"). +-include_lib("emqx/include/logger.hrl"). + +-logger_header("[PGW-CM]"). + +%% APIs +-export([start_link/1]). + +-export([ open_session/5 + , kick_session/2 + , kick_session/3 + , register_channel/4 + , unregister_channel/2 + , insert_channel_info/4 + , set_chan_info/3 + , set_chan_info/4 + , get_chan_info/2 + , get_chan_info/3 + , set_chan_stats/3 + , set_chan_stats/4 + , get_chan_stats/2 + , get_chan_stats/3 + , connection_closed/2 + ]). + +%% Internal funcs for getting tabname by GatewayId +-export([cmtabs/1, tabname/2]). + +%% gen_server callbacks +-export([ init/1 + , handle_call/3 + , handle_cast/2 + , handle_info/2 + , terminate/2 + , code_change/3 + ]). + +-record(state, { + type :: atom(), %% Gateway Id + locker :: pid(), %% ClientId Locker for CM + registry :: pid(), %% ClientId Registry server + chan_pmon :: emqx_pmon:pmon() + }). + +-define(T_TAKEOVER, 15000). + +%%-------------------------------------------------------------------- +%% APIs +%%-------------------------------------------------------------------- + +%% XXX: Options for cm process +start_link(Options) -> + Type = proplists:get_value(type, Options), + gen_server:start_link({local, procname(Type)}, ?MODULE, Options, []). + +procname(Type) -> + list_to_atom(lists:concat([emqx_gateway_, Type, '_cm'])). + +-spec cmtabs(Type :: atom()) -> {ChanTab :: atom(), + ConnTab :: atom(), + ChannInfoTab :: atom()}. +cmtabs(Type) -> + { tabname(chan, Type) %% Client Tabname; Record: {ClientId, Pid} + , tabname(conn, Type) %% Client ConnMod; Recrod: {{ClientId, Pid}, ConnMod} + , tabname(info, Type) %% ClientInfo Tabname; Record: {{ClientId, Pid}, ClientInfo, ClientStats} + }. + +tabname(chan, Type) -> + list_to_atom(lists:concat([emqx_gateway_, Type, '_channel'])); +tabname(conn, Type) -> + list_to_atom(lists:concat([emqx_gateway_, Type, '_channel_conn'])); +tabname(info, Type) -> + list_to_atom(lists:concat([emqx_gateway_, Type, '_channel_info'])). + +lockername(Type) -> + list_to_atom(lists:concat([emqx_gateway_, Type, '_locker'])). + +-spec register_channel(atom(), binary(), pid(), emqx_types:conninfo()) -> ok. +register_channel(Type, ClientId, ChanPid, #{conn_mod := ConnMod}) when is_pid(ChanPid) -> + Chan = {ClientId, ChanPid}, + true = ets:insert(tabname(chan, Type), Chan), + true = ets:insert(tabname(conn, Type), {Chan, ConnMod}), + ok = emqx_gateway_cm_registry:register_channel(Type, Chan), + cast(procname(Type), {registered, Chan}). + +%% @doc Unregister a channel. +-spec unregister_channel(atom(), emqx_types:clientid()) -> ok. +unregister_channel(Type, ClientId) when is_binary(ClientId) -> + true = do_unregister_channel(Type, {ClientId, self()}, cmtabs(Type)), + ok. + +%% @doc Insert/Update the channel info and stats +-spec insert_channel_info(atom(), + emqx_types:clientid(), + emqx_types:infos(), + emqx_types:stats()) -> ok. +insert_channel_info(Type, ClientId, Info, Stats) -> + Chan = {ClientId, self()}, + true = ets:insert(tabname(info, Type), {Chan, Info, Stats}), + %%?tp(debug, insert_channel_info, #{client_id => ClientId}), + ok. + +%% @doc Get info of a channel. +-spec get_chan_info(gateway_type(), emqx_types:clientid()) + -> emqx_types:infos() | undefined. +get_chan_info(Type, ClientId) -> + with_channel(Type, ClientId, + fun(ChanPid) -> + get_chan_info(Type, ClientId, ChanPid) + end). + +-spec get_chan_info(gateway_type(), emqx_types:clientid(), pid()) + -> emqx_types:infos() | undefined. +get_chan_info(Type, ClientId, ChanPid) when node(ChanPid) == node() -> + Chan = {ClientId, ChanPid}, + try ets:lookup_element(tabname(info, Type), Chan, 2) + catch + error:badarg -> undefined + end; +get_chan_info(Type, ClientId, ChanPid) -> + rpc_call(node(ChanPid), get_chan_info, [Type, ClientId, ChanPid]). + +%% @doc Update infos of the channel. +-spec set_chan_info(gateway_type(), + emqx_types:clientid(), + emqx_types:infos()) -> boolean(). +set_chan_info(Type, ClientId, Infos) -> + set_chan_info(Type, ClientId, self(), Infos). + +-spec set_chan_info(gateway_type(), + emqx_types:clientid(), + pid(), + emqx_types:infos()) -> boolean(). +set_chan_info(Type, ClientId, ChanPid, Infos) when node(ChanPid) == node() -> + Chan = {ClientId, ChanPid}, + try ets:update_element(tabname(info, Type), Chan, {2, Infos}) + catch + error:badarg -> false + end; +set_chan_info(Type, ClientId, ChanPid, Infos) -> + rpc_call(node(ChanPid), set_chan_info, [Type, ClientId, ChanPid, Infos]). + +%% @doc Get channel's stats. +-spec get_chan_stats(gateway_type(), emqx_types:clientid()) + -> emqx_types:stats() | undefined. +get_chan_stats(Type, ClientId) -> + with_channel(Type, ClientId, + fun(ChanPid) -> + get_chan_stats(Type, ClientId, ChanPid) + end). + +-spec get_chan_stats(gateway_type(), emqx_types:clientid(), pid()) + -> emqx_types:stats() | undefined. +get_chan_stats(Type, ClientId, ChanPid) when node(ChanPid) == node() -> + Chan = {ClientId, ChanPid}, + try ets:lookup_element(tabname(info, Type), Chan, 3) + catch + error:badarg -> undefined + end; +get_chan_stats(Type, ClientId, ChanPid) -> + rpc_call(node(ChanPid), get_chan_stats, [Type, ClientId, ChanPid]). + +-spec set_chan_stats(gateway_type(), + emqx_types:clientid(), + emqx_types:stats()) -> boolean(). +set_chan_stats(Type, ClientId, Stats) -> + set_chan_stats(Type, ClientId, self(), Stats). + +-spec set_chan_stats(gateway_type(), + emqx_types:clientid(), + pid(), + emqx_types:stats()) -> boolean(). +set_chan_stats(Type, ClientId, ChanPid, Stats) when node(ChanPid) == node() -> + Chan = {ClientId, self()}, + try ets:update_element(tabname(info, Type), Chan, {3, Stats}) + catch + error:badarg -> false + end; +set_chan_stats(Type, ClientId, ChanPid, Stats) -> + rpc_call(node(ChanPid), set_chan_stats, [Type, ClientId, ChanPid, Stats]). + +-spec connection_closed(gateway_type(), emqx_types:clientid()) -> true. +connection_closed(Type, ClientId) -> + %% XXX: Why we need to delete conn_mod tab ??? + Chan = {ClientId, self()}, + ets:delete_object(tabname(conn, Type), Chan). + +-spec open_session(Type :: atom(), CleanStart :: boolean(), + ClientInfo :: emqx_types:clientinfo(), + ConnInfo :: emqx_types:conninfo(), + CreateSessionFun :: function()) + -> {ok, #{session := map(), + present := boolean(), + pendings => list() + }} + | {error, any()}. + +open_session(Type, true = _CleanStart, ClientInfo, ConnInfo, CreateSessionFun) -> + Self = self(), + ClientId = maps:get(clientid, ClientInfo), + Fun = fun(_) -> + ok = discard_session(Type, ClientId), + Session = create_session(Type, + ClientInfo, + ConnInfo, + CreateSessionFun + ), + register_channel(Type, ClientId, Self, ConnInfo), + {ok, #{session => Session, present => false}} + end, + locker_trans(Type, ClientId, Fun); + +open_session(_Type, false = _CleanStart, + _ClientInfo, _ConnInfo, _CreateSessionFun) -> + {error, not_supported_now}. + +%% @private +create_session(_Type, ClientInfo, ConnInfo, CreateSessionFun) -> + try + Session = emqx_gateway_utils:apply( + CreateSessionFun, + [ClientInfo, ConnInfo] + ), + %% TODO: v0.2 session metrics & hooks + %ok = emqx_metrics:inc('session.created'), + %ok = emqx_hooks:run('session.created', [ClientInfo, emqx_session:info(Session)]), + Session + catch + Class : Reason : Stk -> + ?LOG(error, "Failed to create a session: ~p, ~p " + "Stacktrace:~0p", [Class, Reason, Stk]), + throw(Reason) + end. + +%% @doc Discard all the sessions identified by the ClientId. +-spec discard_session(Type :: atom(), binary()) -> ok. +discard_session(Type, ClientId) when is_binary(ClientId) -> + case lookup_channels(Type, ClientId) of + [] -> ok; + ChanPids -> lists:foreach(fun(Pid) -> do_discard_session(Type, ClientId, Pid) end, ChanPids) + end. + +%% @private +do_discard_session(Type, ClientId, Pid) -> + try + discard_session(Type, ClientId, Pid) + catch + _ : noproc -> % emqx_ws_connection: call + %?tp(debug, "session_already_gone", #{pid => Pid}), + ok; + _ : {noproc, _} -> % emqx_connection: gen_server:call + %?tp(debug, "session_already_gone", #{pid => Pid}), + ok; + _ : {{shutdown, _}, _} -> + %?tp(debug, "session_already_shutdown", #{pid => Pid}), + ok; + _ : _Error : _St -> + %?tp(error, "failed_to_discard_session", + % #{pid => Pid, reason => Error, stacktrace=>St}) + ok + end. + +%% @private +discard_session(Type, ClientId, ChanPid) when node(ChanPid) == node() -> + case get_chann_conn_mod(Type, ClientId, ChanPid) of + undefined -> ok; + ConnMod when is_atom(ConnMod) -> + ConnMod:call(ChanPid, discard, ?T_TAKEOVER) + end; + +%% @private +discard_session(Type, ClientId, ChanPid) -> + rpc_call(node(ChanPid), discard_session, [Type, ClientId, ChanPid]). + +-spec kick_session(gateway_type(), emqx_types:clientid()) + -> {error, any()} + | ok. +kick_session(Type, ClientId) -> + case lookup_channels(Type, ClientId) of + [] -> {error, not_found}; + [ChanPid] -> + kick_session(Type, ClientId, ChanPid); + ChanPids -> + [ChanPid|StalePids] = lists:reverse(ChanPids), + ?LOG(error, "More than one channel found: ~p", [ChanPids]), + lists:foreach(fun(StalePid) -> + catch discard_session(Type, ClientId, StalePid) + end, StalePids), + kick_session(Type, ClientId, ChanPid) + end. + +kick_session(Type, ClientId, ChanPid) when node(ChanPid) == node() -> + case get_chan_info(Type, ClientId, ChanPid) of + #{conninfo := #{conn_mod := ConnMod}} -> + ConnMod:call(ChanPid, kick, ?T_TAKEOVER); + undefined -> + {error, not_found} + end; + +kick_session(Type, ClientId, ChanPid) -> + rpc_call(node(ChanPid), kick_session, [Type, ClientId, ChanPid]). + +with_channel(Type, ClientId, Fun) -> + case lookup_channels(Type, ClientId) of + [] -> undefined; + [Pid] -> Fun(Pid); + Pids -> Fun(lists:last(Pids)) + end. + +%% @doc Lookup channels. +-spec(lookup_channels(atom(), emqx_types:clientid()) -> list(pid())). +lookup_channels(Type, ClientId) -> + emqx_gateway_cm_registry:lookup_channels(Type, ClientId). + +get_chann_conn_mod(Type, ClientId, ChanPid) when node(ChanPid) == node() -> + Chan = {ClientId, ChanPid}, + try [ConnMod] = ets:lookup_element(tabname(conn, Type), Chan, 2), ConnMod + catch + error:badarg -> undefined + end; +get_chann_conn_mod(Type, ClientId, ChanPid) -> + rpc_call(node(ChanPid), get_chann_conn_mod, [Type, ClientId, ChanPid]). + +%% Locker + +locker_trans(_Type, undefined, Fun) -> + Fun([]); +locker_trans(Type, ClientId, Fun) -> + Locker = lockername(Type), + case locker_lock(Locker, ClientId) of + {true, Nodes} -> + try Fun(Nodes) after locker_unlock(Locker, ClientId) end; + {false, _Nodes} -> + {error, client_id_unavailable} + end. + +locker_lock(Locker, ClientId) -> + ekka_locker:acquire(Locker, ClientId, quorum). + +locker_unlock(Locker, ClientId) -> + ekka_locker:release(Locker, ClientId, quorum). + +%% @private +rpc_call(Node, Fun, Args) -> + case rpc:call(Node, ?MODULE, Fun, Args) of + {badrpc, Reason} -> error(Reason); + Res -> Res + end. + +cast(Name, Msg) -> + gen_server:cast(Name, Msg). + +%%-------------------------------------------------------------------- +%% gen_server callbacks +%%-------------------------------------------------------------------- + +init(Options) -> + Type = proplists:get_value(type, Options), + + TabOpts = [public, {write_concurrency, true}], + + {ChanTab, ConnTab, InfoTab} = cmtabs(Type), + ok = emqx_tables:new(ChanTab, [bag, {read_concurrency, true}|TabOpts]), + ok = emqx_tables:new(ConnTab, [bag | TabOpts]), + ok = emqx_tables:new(InfoTab, [set, compressed | TabOpts]), + + %% Start link cm-registry process + %% XXX: Should I hang it under a higher level supervisor? + {ok, Registry} = emqx_gateway_cm_registry:start_link(Type), + + %% Start locker process + {ok, Locker} = ekka_locker:start_link(lockername(Type)), + + %% Interval update stats + %% TODO: v0.2 + %ok = emqx_stats:update_interval(chan_stats, fun ?MODULE:stats_fun/0), + + {ok, #state{type = Type, + locker = Locker, + registry = Registry, + chan_pmon = emqx_pmon:new()}}. + +handle_call(_Request, _From, State) -> + Reply = ok, + {reply, Reply, State}. + +handle_cast({registered, {ClientId, ChanPid}}, State = #state{chan_pmon = PMon}) -> + PMon1 = emqx_pmon:monitor(ChanPid, ClientId, PMon), + {noreply, State#state{chan_pmon = PMon1}}; + +handle_cast(_Msg, State) -> + {noreply, State}. + +handle_info({'DOWN', _MRef, process, Pid, _Reason}, + State = #state{type = Type, chan_pmon = PMon}) -> + ChanPids = [Pid | emqx_misc:drain_down(10000)], %% XXX: Fixed BATCH_SIZE + {Items, PMon1} = emqx_pmon:erase_all(ChanPids, PMon), + + CmTabs = cmtabs(Type), + ok = emqx_pool:async_submit(fun do_unregister_channel_task/3, [Items, Type, CmTabs]), + {noreply, State#state{chan_pmon = PMon1}}; + +handle_info(_Info, State) -> + {noreply, State}. + +terminate(_Reason, _State) -> + ok. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +do_unregister_channel_task(Items, Type, CmTabs) -> + lists:foreach( + fun({ChanPid, ClientId}) -> + do_unregister_channel(Type, {ClientId, ChanPid}, CmTabs) + end, Items). + +%%-------------------------------------------------------------------- +%% Internal funcs +%%-------------------------------------------------------------------- + +do_unregister_channel(Type, Chan, {ChanTab, ConnTab, InfoTab}) -> + ok = emqx_gateway_cm_registry:unregister_channel(Type, Chan), + true = ets:delete(ConnTab, Chan), + true = ets:delete(InfoTab, Chan), + ets:delete_object(ChanTab, Chan). diff --git a/apps/emqx_gateway/src/emqx_gateway_cm_registry.erl b/apps/emqx_gateway/src/emqx_gateway_cm_registry.erl new file mode 100644 index 000000000..4275fdf3e --- /dev/null +++ b/apps/emqx_gateway/src/emqx_gateway_cm_registry.erl @@ -0,0 +1,141 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% @doc The gateway connection registry +-module(emqx_gateway_cm_registry). + +-behaviour(gen_server). + +-logger_header("[PGW-CM-Registy]"). + +-export([start_link/1]). + +%% XXX: needless +%-export([is_enabled/0]). + +-export([ register_channel/2 + , unregister_channel/2 + ]). + +-export([lookup_channels/2]). + +%% gen_server callbacks +-export([ init/1 + , handle_call/3 + , handle_cast/2 + , handle_info/2 + , terminate/2 + , code_change/3 + ]). + +-define(LOCK, {?MODULE, cleanup_down}). + +-record(channel, {chid, pid}). + +%% @doc Start the global channel registry. +-spec(start_link(atom()) -> gen_server:startlink_ret()). +start_link(Type) -> + gen_server:start_link(?MODULE, [Type], []). + +-spec tabname(atom()) -> atom(). +tabname(Type) -> + list_to_atom(lists:concat([emqx_gateway_, Type, '_channel_registry'])). + +%%-------------------------------------------------------------------- +%% APIs +%%-------------------------------------------------------------------- + +%% @doc Register a global channel. +-spec register_channel(atom(), binary() | {binary(), pid()}) -> ok. +register_channel(Type, ClientId) when is_binary(ClientId) -> + register_channel(Type, {ClientId, self()}); + +register_channel(Type, {ClientId, ChanPid}) when is_binary(ClientId), is_pid(ChanPid) -> + ekka_mnesia:dirty_write(tabname(Type), record(ClientId, ChanPid)). + +%% @doc Unregister a global channel. +-spec unregister_channel(atom(), binary() | {binary(), pid()}) -> ok. +unregister_channel(Type, ClientId) when is_binary(ClientId) -> + unregister_channel(Type, {ClientId, self()}); + +unregister_channel(Type, {ClientId, ChanPid}) when is_binary(ClientId), is_pid(ChanPid) -> + ekka_mnesia:dirty_delete_object(tabname(Type), record(ClientId, ChanPid)). + +%% @doc Lookup the global channels. +-spec lookup_channels(atom(), binary()) -> list(pid()). +lookup_channels(Type, ClientId) -> + [ChanPid || #channel{pid = ChanPid} <- mnesia:dirty_read(tabname(Type), ClientId)]. + +record(ClientId, ChanPid) -> + #channel{chid = ClientId, pid = ChanPid}. + +%%-------------------------------------------------------------------- +%% gen_server callbacks +%%-------------------------------------------------------------------- + +init([Type]) -> + Tab = tabname(Type), + ok = ekka_mnesia:create_table(Tab, [ + {type, bag}, + {ram_copies, [node()]}, + {record_name, channel}, + {attributes, record_info(fields, channel)}, + {storage_properties, [{ets, [{read_concurrency, true}, + {write_concurrency, true}]}]}]), + ok = ekka_mnesia:copy_table(Tab, ram_copies), + %%ok = ekka_rlog:wait_for_shards([?CM_SHARD], infinity), + ok = ekka:monitor(membership), + {ok, #{type => Type}}. + +handle_call(Req, _From, State) -> + logger:error("Unexpected call: ~p", [Req]), + {reply, ignored, State}. + +handle_cast(Msg, State) -> + logger:error("Unexpected cast: ~p", [Msg]), + {noreply, State}. + +handle_info({membership, {mnesia, down, Node}}, State = #{type := Type}) -> + Tab = tabname(Type), + global:trans({?LOCK, self()}, + fun() -> + %% FIXME: The shard name should be fixed later + ekka_mnesia:transaction(?MODULE, fun cleanup_channels/2, [Node, Tab]) + end), + {noreply, State}; + +handle_info({membership, _Event}, State) -> + {noreply, State}; + +handle_info(Info, State) -> + logger:error("Unexpected info: ~p", [Info]), + {noreply, State}. + +terminate(_Reason, _State) -> + ok. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- + +cleanup_channels(Node, Tab) -> + Pat = [{#channel{pid = '$1', _ = '_'}, [{'==', {node, '$1'}, Node}], ['$_']}], + lists:foreach(fun(Chan) -> + mnesia:delete_object(Tab, Chan, write) + end, mnesia:select(Tab, Pat, write)). diff --git a/apps/emqx_gateway/src/emqx_gateway_ctx.erl b/apps/emqx_gateway/src/emqx_gateway_ctx.erl new file mode 100644 index 000000000..5dadf2aca --- /dev/null +++ b/apps/emqx_gateway/src/emqx_gateway_ctx.erl @@ -0,0 +1,148 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% @doc The gateway instance context +-module(emqx_gateway_ctx). + +-include("include/emqx_gateway.hrl"). + +-logger_header(["PGW-Ctx"]). + +%% @doc The running context for a Connection/Channel process. +%% +%% The `Context` encapsulates a complex structure of contextual information. +%% It is convenient to use it directly in Channel/Connection to read +%% configuration, register devices and other common operations. +%% +-type context() :: + #{ %% Gateway Instance ID + instid := instance_id() + %% Gateway ID + , type := gateway_type() + %% Autenticator + , auth := allow_anonymous | emqx_authentication:chain_id() + %% The ConnectionManager PID + , cm := pid() + }. + +%% Authentication circle +-export([ authenticate/2 + , open_session/5 + , insert_channel_info/4 + , set_chan_info/3 + , set_chan_stats/3 + , connection_closed/2 + ]). + +%% Message circle +-export([ authorize/4 + % Needless for pub/sub + %, publish/3 + %, subscribe/4 + ]). + +%% Metrics & Stats +-export([ metrics_inc/2 + , metrics_inc/3 + ]). + +%%-------------------------------------------------------------------- +%% Authentication circle + +%% @doc Authenticate whether the client has access to the Broker. +-spec authenticate(context(), emqx_types:clientinfo()) + -> {ok, emqx_types:clientinfo()} + | {error, any()}. +authenticate(_Ctx = #{auth := allow_anonymous}, ClientInfo) -> + {ok, ClientInfo#{anonymous => true}}; +authenticate(_Ctx = #{auth := ChainId}, ClientInfo0) -> + ClientInfo = ClientInfo0#{ + zone => undefined, + chain_id => ChainId + }, + case emqx_access_control:authenticate(ClientInfo) of + {ok, AuthResult} -> + {ok, mountpoint(maps:merge(ClientInfo, AuthResult))}; + {error, Reason} -> + {error, Reason} + end. + +%% @doc Register the session to the cluster. +%% +%% This function should be called after the client has authenticated +%% successfully so that the client can be managed in the cluster. +-spec open_session(context(), boolean(), emqx_types:clientinfo(), + emqx_types:conninfo(), function()) + -> {ok, #{session := any(), + present := boolean(), + pendings => list() + }} + | {error, any()}. +open_session(Ctx, false, ClientInfo, ConnInfo, CreateSessionFun) -> + logger:warning("clean_start=false is not supported now, " + "fallback to clean_start mode"), + open_session(Ctx, true, ClientInfo, ConnInfo, CreateSessionFun); + +open_session(_Ctx = #{type := Type}, + CleanStart, ClientInfo, ConnInfo, CreateSessionFun) -> + emqx_gateway_cm:open_session(Type, CleanStart, + ClientInfo, ConnInfo, CreateSessionFun). + +-spec insert_channel_info(context(), + emqx_types:clientid(), + emqx_types:infos(), + emqx_types:stats()) -> ok. +insert_channel_info(_Ctx = #{type := Type}, ClientId, Infos, Stats) -> + emqx_gateway_cm:insert_channel_info(Type, ClientId, Infos, Stats). + +%% @doc Set the Channel Info to the ConnectionManager for this client +-spec set_chan_info(context(), + emqx_types:clientid(), + emqx_types:infos()) -> boolean(). +set_chan_info(_Ctx = #{type := Type}, ClientId, Infos) -> + emqx_gateway_cm:set_chan_info(Type, ClientId, Infos). + +-spec set_chan_stats(context(), + emqx_types:clientid(), + emqx_types:stats()) -> boolean(). +set_chan_stats(_Ctx = #{type := Type}, ClientId, Stats) -> + emqx_gateway_cm:set_chan_stats(Type, ClientId, Stats). + +-spec connection_closed(context(), emqx_types:clientid()) -> boolean(). +connection_closed(_Ctx = #{type := Type}, ClientId) -> + emqx_gateway_cm:connection_closed(Type, ClientId). + +-spec authorize(context(), emqx_types:clientinfo(), + emqx_types:pubsub(), emqx_types:topic()) + -> allow | deny. +authorize(_Ctx, ClientInfo, PubSub, Topic) -> + emqx_access_control:authorize(ClientInfo, PubSub, Topic). + +metrics_inc(_Ctx = #{type := Type}, Name) -> + emqx_gateway_metrics:inc(Type, Name). + +metrics_inc(_Ctx = #{type := Type}, Name, Oct) -> + emqx_gateway_metrics:inc(Type, Name, Oct). + +%%-------------------------------------------------------------------- +%% Internal funcs +%%-------------------------------------------------------------------- + +mountpoint(ClientInfo = #{mountpoint := undefined}) -> + ClientInfo; +mountpoint(ClientInfo = #{mountpoint := MountPoint}) -> + MountPoint1 = emqx_mountpoint:replvar(MountPoint, ClientInfo), + ClientInfo#{mountpoint := MountPoint1}. diff --git a/apps/emqx_gateway/src/emqx_gateway_gw_sup.erl b/apps/emqx_gateway/src/emqx_gateway_gw_sup.erl new file mode 100644 index 000000000..21ad30c0d --- /dev/null +++ b/apps/emqx_gateway/src/emqx_gateway_gw_sup.erl @@ -0,0 +1,135 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% @doc The Gateway Top supervisor. +-module(emqx_gateway_gw_sup). + +-behaviour(supervisor). + +-include("include/emqx_gateway.hrl"). + +-export([start_link/1]). + +-export([ create_insta/3 + , remove_insta/2 + , update_insta/2 + , start_insta/2 + , stop_insta/2 + , list_insta/1 + ]). + +%% Supervisor callbacks +-export([init/1]). + +%%-------------------------------------------------------------------- +%% APIs +%%-------------------------------------------------------------------- + +start_link(Type) -> + supervisor:start_link({local, Type}, ?MODULE, [Type]). + +-spec create_insta(pid(), instance(), map()) -> {ok, GwInstaPid :: pid()} | {error, any()}. +create_insta(Sup, Insta = #{id := InstaId}, GwDscrptr) -> + case emqx_gateway_utils:find_sup_child(Sup, InstaId) of + {ok, _GwInstaPid} -> {error, alredy_existed}; + false -> + %% XXX: More instances options to it? + %% + Ctx = ctx(Sup, InstaId), + %% + ChildSpec = emqx_gateway_utils:childspec( + InstaId, + worker, + emqx_gateway_insta_sup, + [Insta, Ctx, GwDscrptr] + ), + emqx_gateway_utils:supervisor_ret( + supervisor:start_child(Sup, ChildSpec) + ) + end. + +-spec remove_insta(pid(), InstaId :: atom()) -> ok | {error, any()}. +remove_insta(Sup, InstaId) -> + case emqx_gateway_utils:find_sup_child(Sup, InstaId) of + false -> ok; + {ok, _GwInstaPid} -> + ok = supervisor:terminate_child(Sup, InstaId), + ok = supervisor:delete_child(Sup, InstaId) + end. + +-spec update_insta(pid(), NewInsta :: instance()) -> ok | {error, any()}. +update_insta(Sup, NewInsta = #{id := InstaId}) -> + case emqx_gateway_utils:find_sup_child(Sup, InstaId) of + false -> {error, not_found}; + {ok, GwInstaPid} -> + emqx_gateway_insta_sup:update(GwInstaPid, NewInsta) + end. + +-spec start_insta(pid(), atom()) -> ok | {error, any()}. +start_insta(Sup, InstaId) -> + case emqx_gateway_utils:find_sup_child(Sup, InstaId) of + false -> {error, not_found}; + {ok, GwInstaPid} -> + emqx_gateway_insta_sup:enable(GwInstaPid) + end. + +-spec stop_insta(pid(), atom()) -> ok | {error, any()}. +stop_insta(Sup, InstaId) -> + case emqx_gateway_utils:find_sup_child(Sup, InstaId) of + false -> {error, not_found}; + {ok, GwInstaPid} -> + emqx_gateway_insta_sup:disable(GwInstaPid) + end. + +-spec list_insta(pid()) -> [instance()]. +list_insta(Sup) -> + lists:filtermap( + fun({InstaId, GwInstaPid, _Type, _Mods}) -> + is_gateway_insta_id(InstaId) + andalso {true, emqx_gateway_insta_sup:info(GwInstaPid)} + end, supervisor:which_children(Sup)). + +%% Supervisor callback + +%% @doc Initialize Top Supervisor for a Protocol +init([Type]) -> + SupFlags = #{ strategy => one_for_one + , intensity => 10 + , period => 60 + }, + CmOpts = [{type, Type}], + CM = emqx_gateway_utils:childspec(worker, emqx_gateway_cm, [CmOpts]), + Metrics = emqx_gateway_utils:childspec(worker, emqx_gateway_metrics, [Type]), + {ok, {SupFlags, [CM, Metrics]}}. + +%%-------------------------------------------------------------------- +%% Internal funcs +%%-------------------------------------------------------------------- + +ctx(Sup, InstaId) -> + {_, Type} = erlang:process_info(Sup, registered_name), + {ok, CM} = emqx_gateway_utils:find_sup_child(Sup, emqx_gateway_cm), + #{ instid => InstaId + , type => Type + , cm => CM + }. + +is_gateway_insta_id(emqx_gateway_cm) -> + false; +is_gateway_insta_id(emqx_gateway_metrics) -> + false; +is_gateway_insta_id(_Id) -> + true. diff --git a/apps/emqx_gateway/src/emqx_gateway_insta_sup.erl b/apps/emqx_gateway/src/emqx_gateway_insta_sup.erl new file mode 100644 index 000000000..9f21f0e05 --- /dev/null +++ b/apps/emqx_gateway/src/emqx_gateway_insta_sup.erl @@ -0,0 +1,312 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% @doc The gateway instance management +-module(emqx_gateway_insta_sup). + +-behaviour(gen_server). + +-include("include/emqx_gateway.hrl"). + +-logger_header("[PGW-Insta-Sup]"). + +%% APIs +-export([ start_link/3 + , info/1 + , disable/1 + , enable/1 + , update/2 + ]). + +%% gen_server callbacks +-export([ init/1 + , handle_call/3 + , handle_cast/2 + , handle_info/2 + , terminate/2 + , code_change/3 + ]). + +-record(state, { + insta :: instance(), + ctx :: emqx_gateway_ctx:context(), + status :: stopped | running, + child_pids :: [pid()], + insta_state :: emqx_gateway_impl:state() | undefined + }). + +%%-------------------------------------------------------------------- +%% APIs +%%-------------------------------------------------------------------- + +start_link(Insta, Ctx, GwDscrptr) -> + gen_server:start_link( + {local, ?MODULE}, + ?MODULE, + [Insta, Ctx, GwDscrptr], + [] + ). + +-spec info(pid()) -> instance(). +info(Pid) -> + gen_server:call(Pid, info). + +%% @doc Stop instance +-spec disable(pid()) -> ok | {error, any()}. +disable(Pid) -> + call(Pid, disable). + +%% @doc Start instance +-spec enable(pid()) -> ok | {error, any()}. +enable(Pid) -> + call(Pid, enable). + +%% @doc Update the gateway instance configurations +-spec update(pid(), instance()) -> ok | {error, any()}. +update(Pid, NewInsta) -> + call(Pid, {update, NewInsta}). + +call(Pid, Req) -> + gen_server:call(Pid, Req, 5000). + +%%-------------------------------------------------------------------- +%% gen_server callbacks +%%-------------------------------------------------------------------- + +init([Insta, Ctx0, _GwDscrptr]) -> + process_flag(trap_exit, true), + #{rawconf := RawConf} = Insta, + Ctx = do_init_context(RawConf, Ctx0), + State = #state{ + insta = Insta, + ctx = Ctx, + child_pids = [], + status = stopped + }, + case cb_insta_create(State) of + {error, _Reason} -> + do_deinit_context(Ctx), + %% XXX: Return Reason?? + {stop, create_gateway_instance_failed}; + {ok, NState} -> + {ok, NState} + end. + +do_init_context(RawConf, Ctx) -> + Auth = case maps:get(authenticator, RawConf, allow_anonymous) of + allow_anonymous -> allow_anonymous; + Funcs when is_list(Funcs) -> + create_authenticator_for_gateway_insta(Funcs) + end, + Ctx#{auth => Auth}. + +do_deinit_context(Ctx) -> + cleanup_authenticator_for_gateway_insta(maps:get(auth, Ctx)), + ok. + +handle_call(info, _From, State = #state{insta = Insta}) -> + {reply, Insta, State}; + +handle_call(disable, _From, State = #state{status = Status}) -> + case Status of + running -> + case cb_insta_destroy(State) of + {ok, NState} -> + {reply, ok, NState}; + {error, Reason} -> + {reply, {error, Reason}, State} + end; + _ -> + {reply, {error, already_stopped}, State} + end; + +handle_call(enable, _From, State = #state{status = Status}) -> + case Status of + stopped -> + case cb_insta_create(State) of + {error, Reason} -> + {reply, {error, Reason}, State}; + {ok, NState} -> + {reply, ok, NState} + end; + _ -> + {reply, {error, already_started}, State} + end; + +%% Stopped -> update +handle_call({update, NewInsta}, _From, State = #state{insta = Insta, + status = stopped}) -> + case maps:get(id, NewInsta, undefined) == maps:get(id, Insta, undefined) of + true -> + {reply, ok, State#state{insta = NewInsta}}; + false -> + {reply, {error, bad_instan_id}, State} + end; + +%% Running -> update +handle_call({update, NewInsta}, _From, State = #state{insta = Insta, + status = running}) -> + case maps:get(id, NewInsta, undefined) == maps:get(id, Insta, undefined) of + true -> + case cb_insta_update(NewInsta, State) of + {ok, NState} -> + {reply, ok, NState}; + {error, Reason} -> + {reply, {error, Reason}, State} + end; + false -> + {reply, {error, bad_instan_id}, State} + end; + +handle_call(_Request, _From, State) -> + Reply = ok, + {reply, Reply, State}. + +handle_cast(_Msg, State) -> + {noreply, State}. + +handle_info({'EXIT', Pid, Reason}, State = #state{child_pids = Pids}) -> + case lists:member(Pid, Pids) of + true -> + logger:error("Child process ~p exited: ~0p.", [Pid, Reason]), + case Pids -- [Pid]of + [] -> + logger:error("All child process exited!"), + {noreply, State#state{status = stopped, + child_pids = [], + insta_state = undefined}}; + RemainPids -> + {noreply, State#state{child_pids = RemainPids}} + end; + _ -> + logger:error("Unknown process exited ~p:~0p", [Pid, Reason]), + {noreply, State} + end; + +handle_info(Info, State) -> + logger:warning("Unexcepted info: ~p", [Info]), + {noreply, State}. + +terminate(_Reason, State = #state{ctx = Ctx, child_pids = Pids}) -> + %% Cleanup instances + %% Step1. Destory instance + Pids /= [] andalso (_ = cb_insta_destroy(State)), + %% Step2. Delete authenticator resources + _ = do_deinit_context(Ctx), + ok. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%-------------------------------------------------------------------- +%% Internal funcs +%%-------------------------------------------------------------------- + +create_authenticator_for_gateway_insta(_Funcs) -> + todo. + +cleanup_authenticator_for_gateway_insta(allow_anonymouse) -> + ok; +cleanup_authenticator_for_gateway_insta(_ChainId) -> + todo. + +cb_insta_destroy(State = #state{insta = Insta = #{type := Type}, + insta_state = InstaState}) -> + try + #{cbkmod := CbMod, + state := GwState} = emqx_gateway_registry:lookup(Type), + CbMod:on_insta_destroy(Insta, InstaState, GwState), + {ok, State#state{child_pids = [], + insta_state = undefined, + status = stopped}} + catch + Class : Reason : Stk -> + logger:error("Destroy instance (~0p, ~0p, _) crashed: " + "{~p, ~p}, stacktrace: ~0p", + [Insta, InstaState, + Class, Reason, Stk]), + {error, {Class, Reason, Stk}} + end. + +cb_insta_create(State = #state{insta = Insta = #{type := Type}, + ctx = Ctx}) -> + try + #{cbkmod := CbMod, + state := GwState} = emqx_gateway_registry:lookup(Type), + case CbMod:on_insta_create(Insta, Ctx, GwState) of + {error, Reason} -> throw({callback_return_error, Reason}); + {ok, InstaPidOrSpecs, InstaState} -> + ChildPids = start_child_process(InstaPidOrSpecs), + {ok, State#state{ + status = running, + child_pids = ChildPids, + insta_state = InstaState + }} + end + catch + Class : Reason1 : Stk -> + logger:error("Create instance (~0p, ~0p, _) crashed: " + "{~p, ~p}, stacktrace: ~0p", + [Insta, Ctx, + Class, Reason1, Stk]), + {error, {Class, Reason1, Stk}} + end. + +cb_insta_update(NewInsta, + State = #state{insta = Insta = #{type := Type}, + ctx = Ctx, + insta_state = GwInstaState}) -> + try + #{cbkmod := CbMod, + state := GwState} = emqx_gateway_registry:lookup(Type), + case CbMod:on_insta_update(NewInsta, Insta, GwInstaState, GwState) of + {error, Reason} -> throw({callback_return_error, Reason}); + {ok, InstaPidOrSpecs, InstaState} -> + %% XXX: Hot-upgrade ??? + ChildPids = start_child_process(InstaPidOrSpecs), + {ok, State#state{ + status = running, + child_pids = ChildPids, + insta_state = InstaState + }} + end + catch + Class : Reason1 : Stk -> + logger:error("Update instance (~0p, ~0p, ~0p, _) crashed: " + "{~p, ~p}, stacktrace: ~0p", + [NewInsta, Insta, Ctx, + Class, Reason1, Stk]), + {error, {Class, Reason1, Stk}} + end. + +start_child_process([Indictor|_] = InstaPidOrSpecs) -> + case erlang:is_pid(Indictor) of + true -> + InstaPidOrSpecs; + _ -> + do_start_child_process(InstaPidOrSpecs) + end. + +do_start_child_process(ChildSpecs) when is_list(ChildSpecs) -> + lists:map(fun do_start_child_process/1, ChildSpecs); + +do_start_child_process(_ChildSpec = #{start := {M, F, A}}) -> + case erlang:apply(M, F, A) of + {ok, Pid} -> + Pid; + {error, Reason} -> + throw({start_child_process, Reason}) + end. diff --git a/apps/emqx_gateway/src/emqx_gateway_metrics.erl b/apps/emqx_gateway/src/emqx_gateway_metrics.erl new file mode 100644 index 000000000..04b711d0a --- /dev/null +++ b/apps/emqx_gateway/src/emqx_gateway_metrics.erl @@ -0,0 +1,101 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_gateway_metrics). + +-behaviour(gen_server). + +-include("include/emqx_gateway.hrl"). + +-logger_header("[PGW-Metrics]"). + +%% APIs +-export([start_link/1]). + +-export([ inc/2 + , inc/3 + , dec/2 + , dec/3 + ]). + +%% gen_server callbacks +-export([ init/1 + , handle_call/3 + , handle_cast/2 + , handle_info/2 + , terminate/2 + , code_change/3 + ]). + +-export([tabname/1]). + +-record(state, {}). + +%%-------------------------------------------------------------------- +%% APIs +%%-------------------------------------------------------------------- + +start_link(Type) -> + gen_server:start_link({local, ?MODULE}, ?MODULE, [Type], []). + +-spec inc(gateway_type(), atom()) -> ok. +inc(Type, Name) -> + inc(Type, Name, 1). + +-spec inc(gateway_type(), atom(), integer()) -> ok. +inc(Type, Name, Oct) -> + ets:update_counter(tabname(Type), Name, {2, Oct}, {Name, 0}), + ok. + +-spec dec(gateway_type(), atom()) -> ok. +dec(Type, Name) -> + inc(Type, Name, -1). + +-spec dec(gateway_type(), atom(), non_neg_integer()) -> ok. +dec(Type, Name, Oct) -> + inc(Type, Name, -Oct). + +tabname(Type) -> + list_to_atom(lists:concat([emqx_gateway_, Type, '_metrics'])). + +%%-------------------------------------------------------------------- +%% gen_server callbacks +%%-------------------------------------------------------------------- + +init([Type]) -> + TabOpts = [public, {write_concurrency, true}], + ok = emqx_tables:new(tabname(Type), [set|TabOpts]), + {ok, #state{}}. + +handle_call(_Request, _From, State) -> + Reply = ok, + {reply, Reply, State}. + +handle_cast(_Msg, State) -> + {noreply, State}. + +handle_info(_Info, State) -> + {noreply, State}. + +terminate(_Reason, _State) -> + ok. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%-------------------------------------------------------------------- +%% Internal funcs +%%-------------------------------------------------------------------- diff --git a/apps/emqx_gateway/src/emqx_gateway_registry.erl b/apps/emqx_gateway/src/emqx_gateway_registry.erl new file mode 100644 index 000000000..a100636cf --- /dev/null +++ b/apps/emqx_gateway/src/emqx_gateway_registry.erl @@ -0,0 +1,163 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% @doc The Registry Centre of Gateway Type +-module(emqx_gateway_registry). + +-include("include/emqx_gateway.hrl"). + +-logger_header("[PGW-Registry]"). + +-behavior(gen_server). + +%% APIs for Impl. +-export([ load/3 + , unload/1 + ]). + +-export([ list/0 + , lookup/1 + ]). + +%% APIs +-export([start_link/0]). + +%% gen_server callbacks +-export([ init/1 + , handle_call/3 + , handle_cast/2 + , handle_info/2 + , terminate/2 + , code_change/3 + ]). + +-record(state, { + loaded = #{} :: #{ gateway_type() => descriptor() } + }). + +%%-------------------------------------------------------------------- +%% APIs +%%-------------------------------------------------------------------- + +start_link() -> + gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). + +%%-------------------------------------------------------------------- +%% Mgmt +%%-------------------------------------------------------------------- + +-type registry_options() :: [registry_option()]. + +-type registry_option() :: {cbkmod, atom()}. + +-type gateway_options() :: list(). + +-type descriptor() :: #{ cbkmod := atom() + , rgopts := registry_options() + , gwopts := gateway_options() + , state => any() + }. + +-spec load(gateway_type(), registry_options(), gateway_options()) -> ok | {error, any()}. +load(Type, RgOpts, GwOpts) -> + CbMod = proplists:get_value(cbkmod, RgOpts, Type), + Dscrptr = #{ cbkmod => CbMod + , rgopts => RgOpts + , gwopts => GwOpts + }, + call({load, Type, Dscrptr}). + +-spec unload(gateway_type()) -> ok | {error, any()}. +unload(Type) -> + %% TODO: Checking ALL INSTACE HAS STOPPED + call({unload, Type}). + +%% TODO: +%unload(Type, Force) -> +% call({unload, Type, Froce}). + +%% @doc Return all registered protocol gateway implementation +-spec list() -> [{gateway_type(), descriptor()}]. +list() -> + call(all). + +-spec lookup(gateway_type()) -> undefined | descriptor(). +lookup(Type) -> + call({lookup, Type}). + +call(Req) -> + gen_server:call(?MODULE, Req, 5000). + +%%-------------------------------------------------------------------- +%% gen_server callbacks +%%-------------------------------------------------------------------- + +init([]) -> + %% TODO: Metrics ??? + process_flag(trap_exit, true), + {ok, #state{loaded = #{}}}. + +handle_call({load, Type, Dscrptr}, _From, State = #state{loaded = Gateways}) -> + case maps:get(Type, Gateways, notfound) of + notfound -> + try + GwOpts = maps:get(gwopts, Dscrptr), + CbMod = maps:get(cbkmod, Dscrptr), + {ok, GwState} = CbMod:init(GwOpts), + NDscrptr = maps:put(state, GwState, Dscrptr), + NGateways = maps:put(Type, NDscrptr, Gateways), + {reply, ok, State#state{loaded = NGateways}} + catch + Class : Reason : Stk -> + logger:error("Load ~s crashed {~p, ~p}; stacktrace: ~0p", + [Type, Class, Reason, Stk]), + {reply, {error, {Class, Reason}}, State} + end; + _ -> + {reply, {error, already_existed}, State} + end; + +handle_call({unload, Type}, _From, State = #state{loaded = Gateways}) -> + case maps:get(Type, Gateways, undefined) of + undefined -> + {reply, ok, State}; + _ -> + emqx_gateway_sup:stop_all_suptree(Type), + {reply, ok, State#state{loaded = maps:remove(Type, Gateways)}} + end; + +handle_call(all, _From, State = #state{loaded = Gateways}) -> + {reply, maps:to_list(Gateways), State}; + +handle_call({lookup, Type}, _From, State = #state{loaded = Gateways}) -> + Reply = maps:get(Type, Gateways, undefined), + {reply, Reply, State}; + +handle_call(Req, _From, State) -> + logger:error("Unexpected call: ~0p", [Req]), + {reply, ok, State}. + +handle_cast(_Msg, State) -> + {noreply, State}. + +handle_info(_Info, State) -> + {noreply, State}. + +terminate(_Reason, _State) -> + ok. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. diff --git a/apps/emqx_gateway/src/emqx_gateway_schema.erl b/apps/emqx_gateway/src/emqx_gateway_schema.erl new file mode 100644 index 000000000..8f05582c7 --- /dev/null +++ b/apps/emqx_gateway/src/emqx_gateway_schema.erl @@ -0,0 +1,178 @@ +-module(emqx_gateway_schema). + +-dialyzer(no_return). +-dialyzer(no_match). +-dialyzer(no_contracts). +-dialyzer(no_unused). +-dialyzer(no_fail_call). + +-include_lib("typerefl/include/types.hrl"). + +-type flag() :: true | false. +-type duration() :: integer(). +-type bytesize() :: integer(). +-type comma_separated_list() :: list(). +-type ip_port() :: tuple(). + +-typerefl_from_string({flag/0, emqx_schema, to_flag}). +-typerefl_from_string({duration/0, emqx_schema, to_duration}). +-typerefl_from_string({bytesize/0, emqx_schema, to_bytesize}). +-typerefl_from_string({comma_separated_list/0, emqx_schema, to_comma_separated_list}). +-typerefl_from_string({ip_port/0, emqx_schema, to_ip_port}). + +-behaviour(hocon_schema). + +-reflect_type([ flag/0 + , duration/0 + , bytesize/0 + , comma_separated_list/0 + , ip_port/0 + ]). + +-export([structs/0 , fields/1]). +-export([t/1, t/3, t/4, ref/1]). + +structs() -> ["emqx_gateway"]. + +fields("emqx_gateway") -> + [{stomp, t(ref(stomp))}]; + +fields(stomp) -> + [{"$id", t(ref(stomp_structs))}]; + +fields(stomp_structs) -> + [ {frame, t(ref(stomp_frame))} + , {clientinfo_override, t(ref(clientinfo_override))} + , {authenticator, t(union([allow_anonymous]))} + , {listener, t(ref(listener))} + ]; + +fields(stomp_frame) -> + [ {max_headers, t(integer(), undefined, 10)} + , {max_headers_length, t(integer(), undefined, 1024)} + , {max_body_length, t(integer(), undefined, 8192)} + ]; + +fields(clientinfo_override) -> + [ {username, t(string())} + , {password, t(string())} + , {clientid, t(string())} + ]; + +fields(listener) -> + [ {tcp, t(ref(tcp_listener))} + , {ssl, t(ref(ssl_listener))} + ]; + +fields(tcp_listener) -> + [ {"$name", t(ref(tcp_listener_settings))}]; + +fields(ssl_listener) -> + [ {"$name", t(ref(ssl_listener_settings))}]; + +fields(listener_settings) -> + %[ {"bind", t(union(ip_port(), integer()))} + [ {bind, t(integer())} + , {acceptors, t(integer(), undefined, 8)} + , {max_connections, t(integer(), undefined, 1024)} + , {max_conn_rate, t(integer())} + , {active_n, t(integer(), undefined, 100)} + %, {zone, t(string())} + %, {rate_limit, t(comma_separated_list())} + , {access, t(ref(access))} + , {proxy_protocol, t(flag())} + , {proxy_protocol_timeout, t(duration())} + , {backlog, t(integer(), undefined, 1024)} + , {send_timeout, t(duration(), undefined, "15s")} + , {send_timeout_close, t(flag(), undefined, true)} + , {recbuf, t(bytesize())} + , {sndbuf, t(bytesize())} + , {buffer, t(bytesize())} + , {high_watermark, t(bytesize(), undefined, "1MB")} + , {tune_buffer, t(flag())} + , {nodelay, t(boolean())} + , {reuseaddr, t(boolean())} + ]; + +fields(tcp_listener_settings) -> + [ + %% some special confs for tcp listener + ] ++ fields(listener_settings); + +fields(ssl_listener_settings) -> + [ + %% some special confs for ssl listener + ] ++ + ssl(undefined, #{handshake_timeout => "15s" + , depth => 10 + , reuse_sessions => true}) ++ fields(listener_settings); + +fields(access) -> + [ {"$id", #{type => string(), + nullable => true}}]; + +fields(ExtraField) -> + Mod = list_to_atom(ExtraField++"_schema"), + Mod:fields(ExtraField). + +%translations() -> []. +% +%translations(_) -> []. + +%%-------------------------------------------------------------------- +%% Helpers + +%% types + +t(Type) -> #{type => Type}. + +t(Type, Mapping, Default) -> + hoconsc:t(Type, #{mapping => Mapping, default => Default}). + +t(Type, Mapping, Default, OverrideEnv) -> + hoconsc:t(Type, #{ mapping => Mapping + , default => Default + , override_env => OverrideEnv + }). + +ref(Field) -> + hoconsc:ref(?MODULE, Field). + +%% utils + +%% generate a ssl field. +%% ssl("emqx", #{"verify" => verify_peer}) will return +%% [ {"cacertfile", t(string(), "emqx.cacertfile", undefined)} +%% , {"certfile", t(string(), "emqx.certfile", undefined)} +%% , {"keyfile", t(string(), "emqx.keyfile", undefined)} +%% , {"verify", t(union(verify_peer, verify_none), "emqx.verify", verify_peer)} +%% , {"server_name_indication", "emqx.server_name_indication", undefined)} +%% ... +ssl(Mapping, Defaults) -> + M = fun (Field) -> + case (Mapping) of + undefined -> undefined; + _ -> Mapping ++ "." ++ Field + end end, + D = fun (Field) -> maps:get(list_to_atom(Field), Defaults, undefined) end, + [ {"enable", t(flag(), M("enable"), D("enable"))} + , {"cacertfile", t(string(), M("cacertfile"), D("cacertfile"))} + , {"certfile", t(string(), M("certfile"), D("certfile"))} + , {"keyfile", t(string(), M("keyfile"), D("keyfile"))} + , {"verify", t(union(verify_peer, verify_none), M("verify"), D("verify"))} + , {"fail_if_no_peer_cert", t(boolean(), M("fail_if_no_peer_cert"), D("fail_if_no_peer_cert"))} + , {"secure_renegotiate", t(flag(), M("secure_renegotiate"), D("secure_renegotiate"))} + , {"reuse_sessions", t(flag(), M("reuse_sessions"), D("reuse_sessions"))} + , {"honor_cipher_order", t(flag(), M("honor_cipher_order"), D("honor_cipher_order"))} + , {"handshake_timeout", t(duration(), M("handshake_timeout"), D("handshake_timeout"))} + , {"depth", t(integer(), M("depth"), D("depth"))} + , {"password", hoconsc:t(string(), #{mapping => M("key_password"), + default => D("key_password"), + sensitive => true + })} + , {"dhfile", t(string(), M("dhfile"), D("dhfile"))} + , {"server_name_indication", t(union(disable, string()), M("server_name_indication"), + D("server_name_indication"))} + , {"tls_versions", t(comma_separated_list(), M("tls_versions"), D("tls_versions"))} + , {"ciphers", t(comma_separated_list(), M("ciphers"), D("ciphers"))} + , {"psk_ciphers", t(comma_separated_list(), M("ciphers"), D("ciphers"))}]. diff --git a/apps/emqx_gateway/src/emqx_gateway_sup.erl b/apps/emqx_gateway/src/emqx_gateway_sup.erl new file mode 100644 index 000000000..d56b27e52 --- /dev/null +++ b/apps/emqx_gateway/src/emqx_gateway_sup.erl @@ -0,0 +1,194 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_gateway_sup). + +-behaviour(supervisor). + +-include("include/emqx_gateway.hrl"). + +-export([start_link/0]). + +%% Gateway Instance APIs +-export([ create_gateway_insta/1 + , remove_gateway_insta/1 + , lookup_gateway_insta/1 + , update_gateway_insta/1 + , start_gateway_insta/1 + , stop_gateway_insta/1 + , list_gateway_insta/1 + , list_gateway_insta/0 + ]). + +%% Gateway APs +-export([ list_started_gateway/0 + , stop_all_suptree/1 + ]). + +%% supervisor callbacks +-export([init/1]). + +%%-------------------------------------------------------------------- +%% APIs +%%-------------------------------------------------------------------- + +start_link() -> + supervisor:start_link({local, ?MODULE}, ?MODULE, []). + +-spec create_gateway_insta(instance()) -> {ok, pid()} | {error, any()}. +create_gateway_insta(Insta = #{type := Type}) -> + case emqx_gateway_registry:lookup(Type) of + undefined -> {error, {unknown_gateway_id, Type}}; + GwDscrptr -> + {ok, GwSup} = ensure_gateway_suptree_ready(gatewayid(Type)), + emqx_gateway_gw_sup:create_insta(GwSup, Insta, GwDscrptr) + end. + +-spec remove_gateway_insta(instance_id()) -> ok | {error, any()}. +remove_gateway_insta(InstaId) -> + case search_gateway_insta_proc(InstaId) of + {ok, {GwSup, _}} -> + emqx_gateway_gw_sup:remove_insta(GwSup, InstaId); + _ -> + ok + end. + +-spec lookup_gateway_insta(instance_id()) -> instance() | undefined. +lookup_gateway_insta(InstaId) -> + case search_gateway_insta_proc(InstaId) of + {ok, {_, GwInstaPid}} -> + emqx_gateway_insta_sup:info(GwInstaPid); + _ -> + undefined + end. + +-spec update_gateway_insta(instance()) + -> ok + | {error, any()}. +update_gateway_insta(NewInsta = #{type := Type}) -> + case emqx_gateway_utils:find_sup_child(?MODULE, gatewayid(Type)) of + {ok, GwSup} -> + emqx_gateway_gw_sup:update_insta(GwSup, NewInsta); + _ -> {error, not_found} + end. + +start_gateway_insta(InstaId) -> + case search_gateway_insta_proc(InstaId) of + {ok, {GwSup, _}} -> + emqx_gateway_gw_sup:start_insta(GwSup, InstaId); + _ -> {error, not_found} + end. + +-spec stop_gateway_insta(instance_id()) -> ok | {error, any()}. +stop_gateway_insta(InstaId) -> + case search_gateway_insta_proc(InstaId) of + {ok, {GwSup, _}} -> + emqx_gateway_gw_sup:stop_insta(GwSup, InstaId); + _ -> {error, not_found} + end. + +-spec list_gateway_insta(gateway_type()) -> {ok, [instance()]} | {error, any()}. +list_gateway_insta(Type) -> + case emqx_gateway_utils:find_sup_child(?MODULE, gatewayid(Type)) of + {ok, GwSup} -> + {ok, emqx_gateway_gw_sup:list_insta(GwSup)}; + _ -> {error, not_found} + end. + +-spec list_gateway_insta() -> [{gateway_type(), instance()}]. +list_gateway_insta() -> + lists:map( + fun(SupId) -> + Instas = emqx_gateway_gw_sup:list_insta(SupId), + {SupId, Instas} + end, list_started_gateway()). + +-spec list_started_gateway() -> [gateway_type()]. +list_started_gateway() -> + started_gateway_type(). + +-spec stop_all_suptree(atom()) -> ok. +stop_all_suptree(Type) -> + case lists:keyfind(Type, 1, supervisor:which_children(?MODULE)) of + false -> ok; + _ -> + _ = supervisor:terminate_child(?MODULE, Type), + _ = supervisor:delete_child(?MODULE, Type), + ok + end. + +%% Supervisor callback + +init([]) -> + SupFlags = #{ strategy => one_for_one + , intensity => 10 + , period => 60 + }, + ChildSpecs = [ emqx_gateway_utils:childspec(worker, emqx_gateway_registry) + ], + {ok, {SupFlags, ChildSpecs}}. + +%%-------------------------------------------------------------------- +%% Internal funcs +%%-------------------------------------------------------------------- + +gatewayid(Type) -> + list_to_atom(lists:concat([Type])). + +ensure_gateway_suptree_ready(Type) -> + case lists:keyfind(Type, 1, supervisor:which_children(?MODULE)) of + false -> + ChildSpec = emqx_gateway_utils:childspec( + Type, + supervisor, + emqx_gateway_gw_sup, + [Type] + ), + emqx_gateway_utils:supervisor_ret( + supervisor:start_child(?MODULE, ChildSpec) + ); + {_Id, Pid, _Type, _Mods} -> + {ok, Pid} + end. + +search_gateway_insta_proc(InstaId) -> + search_gateway_insta_proc(InstaId, started_gateway_pid()). + +search_gateway_insta_proc(_InstaId, []) -> + {error, not_found}; +search_gateway_insta_proc(InstaId, [SupPid|More]) -> + case emqx_gateway_utils:find_sup_child(SupPid, InstaId) of + {ok, InstaPid} -> {ok, {SupPid, InstaPid}}; + _ -> + search_gateway_insta_proc(InstaId, More) + end. + +started_gateway_type() -> + lists:filtermap( + fun({Id, _, _, _}) -> + is_a_gateway_id(Id) andalso {true, Id} + end, supervisor:which_children(?MODULE)). + +started_gateway_pid() -> + lists:filtermap( + fun({Id, Pid, _, _}) -> + is_a_gateway_id(Id) andalso {true, Pid} + end, supervisor:which_children(?MODULE)). + +is_a_gateway_id(Id) -> + Id /= emqx_gateway_registry. + + diff --git a/apps/emqx_gateway/src/emqx_gateway_utils.erl b/apps/emqx_gateway/src/emqx_gateway_utils.erl new file mode 100644 index 000000000..184c3ff87 --- /dev/null +++ b/apps/emqx_gateway/src/emqx_gateway_utils.erl @@ -0,0 +1,163 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% @doc Utils funcs for emqx-gateway +-module(emqx_gateway_utils). + +-export([ childspec/2 + , childspec/3 + , childspec/4 + , supervisor_ret/1 + , find_sup_child/2 + ]). + +-export([ apply/2 + ]). + +-export([ normalize_rawconf/1 + ]). + +%% Common Envs +-export([ active_n/1 + , ratelimit/1 + , frame_options/1 + , init_gc_state/1 + , stats_timer/1 + , idle_timeout/1 + , oom_policy/1 + ]). + +-define(ACTIVE_N, 100). +-define(DEFAULT_IDLE_TIMEOUT, 30000). + +-spec childspec(supervisor:worker(), Mod :: atom()) + -> supervisor:child_spec(). +childspec(Type, Mod) -> + childspec(Mod, Type, Mod, []). + +-spec childspec(supervisor:worker(), Mod :: atom(), Args :: list()) + -> supervisor:child_spec(). +childspec(Type, Mod, Args) -> + childspec(Mod, Type, Mod, Args). + +-spec childspec(atom(), supervisor:worker(), Mod :: atom(), Args :: list()) + -> supervisor:child_spec(). +childspec(Id, Type, Mod, Args) -> + #{ id => Id + , start => {Mod, start_link, Args} + , type => Type + }. + +-spec supervisor_ret(supervisor:startchild_ret()) + -> {ok, pid()} + | {error, supervisor:startchild_err()}. +supervisor_ret({ok, Pid, _Info}) -> {ok, Pid}; +supervisor_ret(Ret) -> Ret. + +-spec find_sup_child(Sup :: pid() | atom(), ChildId :: supervisor:child_id()) + -> false + | {ok, pid()}. +find_sup_child(Sup, ChildId) -> + case lists:keyfind(ChildId, 1, supervisor:which_children(Sup)) of + false -> false; + {_Id, Pid, _Type, _Mods} -> {ok, Pid} + end. + +apply({M, F, A}, A2) when is_atom(M), + is_atom(M), + is_list(A), + is_list(A2) -> + erlang:apply(M, F, A ++ A2); +apply({F, A}, A2) when is_function(F), + is_list(A), + is_list(A2) -> + erlang:apply(F, A ++ A2); +apply(F, A2) when is_function(F), + is_list(A2) -> + erlang:apply(F, A2). + +-type listener() :: #{}. + +-type rawconf() :: + #{ clientinfo_override => #{} + , authenticators := #{} + , listeners => listener() + , atom() => any() + }. + +-spec normalize_rawconf(rawconf()) + -> list({ Type :: udp | tcp | ssl | dtls + , ListenOn :: esockd:listen_on() + , SocketOpts :: esockd:option() + , Cfg :: map() + }). +normalize_rawconf(RawConf = #{listener := LisMap}) -> + Cfg0 = maps:without([listener], RawConf), + lists:append(maps:fold(fun(Type, Liss, AccIn1) -> + Listeners = + maps:fold(fun(_Name, Confs, AccIn2) -> + ListenOn = maps:get(bind, Confs), + SocketOpts = esockd:parse_opt(maps:to_list(Confs)), + RemainCfgs = maps:without( + [bind] ++ proplists:get_keys(SocketOpts), + Confs), + Cfg = maps:merge(Cfg0, RemainCfgs), + [{Type, ListenOn, SocketOpts, Cfg}|AccIn2] + end, [], Liss), + [Listeners|AccIn1] + end, [], LisMap)). + +%%-------------------------------------------------------------------- +%% Envs + +active_n(Options) -> + maps:get( + active_n, + maps:get(listener, Options, #{active_n => ?ACTIVE_N}), + ?ACTIVE_N + ). + +-spec idle_timeout(map()) -> pos_integer(). +idle_timeout(Options) -> + maps:get(idle_timeout, Options, ?DEFAULT_IDLE_TIMEOUT). + +-spec ratelimit(map()) -> esockd_rate_limit:config() | undefined. +ratelimit(Options) -> + maps:get(ratelimit, Options, undefined). + +-spec frame_options(map()) -> map(). +frame_options(Options) -> + maps:get(frame, Options, #{}). + +-spec init_gc_state(map()) -> emqx_gc:gc_state() | undefined. +init_gc_state(Options) -> + emqx_misc:maybe_apply(fun emqx_gc:init/1, force_gc_policy(Options)). + +-spec force_gc_policy(map()) -> emqx_gc:opts() | undefined. +force_gc_policy(Options) -> + maps:get(force_gc_policy, Options, undefined). + +-spec oom_policy(map()) -> emqx_types:oom_policy(). +oom_policy(Options) -> + maps:get(force_shutdown_policy, Options). + +-spec stats_timer(map()) -> undefined | disabled. +stats_timer(Options) -> + case enable_stats(Options) of true -> undefined; false -> disabled end. + +-spec enable_stats(map()) -> boolean(). +enable_stats(Options) -> + maps:get(enable_stats, Options, true). diff --git a/apps/emqx_stomp/README.md b/apps/emqx_gateway/src/stomp/README.md similarity index 90% rename from apps/emqx_stomp/README.md rename to apps/emqx_gateway/src/stomp/README.md index ec841b1e6..c5736a755 100644 --- a/apps/emqx_stomp/README.md +++ b/apps/emqx_gateway/src/stomp/README.md @@ -1,13 +1,12 @@ -emqx-stomp -========== +# emqx-stomp + The plugin adds STOMP 1.0/1.1/1.2 protocol supports to the EMQ X broker. The STOMP clients could PubSub to the MQTT clients. -Configuration -------------- +## Configuration etc/emqx_stomp.conf @@ -58,20 +57,17 @@ stomp.frame.max_header_length = 1024 stomp.frame.max_body_length = 8192 ``` -Load the Plugin ---------------- +## Load the Plugin ``` ./bin/emqx_ctl plugins load emqx_stomp ``` -License -------- +## License Apache License Version 2.0 -Author ------- +## Author EMQ X Team. diff --git a/apps/emqx_gateway/src/stomp/emqx_stomp_channel.erl b/apps/emqx_gateway/src/stomp/emqx_stomp_channel.erl new file mode 100644 index 000000000..322baa120 --- /dev/null +++ b/apps/emqx_gateway/src/stomp/emqx_stomp_channel.erl @@ -0,0 +1,978 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_stomp_channel). + +-include("src/stomp/include/emqx_stomp.hrl"). +-include_lib("emqx/include/emqx.hrl"). +-include_lib("emqx/include/logger.hrl"). + +-logger_header("[Stomp-Proto]"). + +-import(proplists, [get_value/2, get_value/3]). + +%% API +-export([ info/1 + , info/2 + , stats/1 + ]). + +-export([ init/2 + , handle_in/2 + , handle_out/3 + , handle_deliver/2 + , handle_timeout/3 + , terminate/2 + , set_conn_state/2 + ]). + +-export([ handle_call/2 + , handle_info/2 + ]). + +%% for trans callback +-export([ handle_recv_send_frame/2 + , handle_recv_ack_frame/2 + , handle_recv_nack_frame/2 + ]). + +-record(channel, { + %% Context + ctx :: emqx_gateway_ctx:context(), + %% Stomp Connection Info + conninfo :: emqx_types:conninfo(), + %% Stomp Client Info + clientinfo :: emqx_types:clientinfo(), + %% ClientInfo override specs + clientinfo_override :: map(), + %% Connection Channel + conn_state :: conn_state(), + %% Heartbeat + heartbeat :: emqx_stomp_heartbeat:heartbeat(), + %% Subscriptions + subscriptions = [], + %% Timer + timers :: #{atom() => disable | undefined | reference()}, + %% Transaction + transaction :: #{binary() => list()} + }). + +-type(channel() :: #channel{}). + +-type(conn_state() :: idle | connecting | connected | disconnected). + +-type(reply() :: {outgoing, stomp_frame()} + | {outgoing, [stomp_frame()]} + | {event, conn_state()|updated} + | {close, Reason :: atom()}). + +-type(replies() :: emqx_stomp_frame:packet() | reply() | [reply()]). + +-define(TIMER_TABLE, #{ + incoming_timer => incoming, + outgoing_timer => outgoing, + clean_trans_timer => clean_trans + }). + +-define(TRANS_TIMEOUT, 60000). + +-define(DEFAULT_OVERRIDE, + #{ clientid => <<"">> %% Generate clientid by default + , username => <<"${Packet.headers.login}">> + , password => <<"${Packet.headers.passcode}">> + }). + +-define(INFO_KEYS, [conninfo, conn_state, clientinfo, session, will_msg]). + +-dialyzer({nowarn_function, [init/2,enrich_conninfo/2,ensure_connected/1, + process_connect/1,handle_in/2,handle_info/2, + ensure_disconnected/2,reverse_heartbeats/1, + negotiate_version/2]}). + +%%-------------------------------------------------------------------- +%% Init the channel +%%-------------------------------------------------------------------- + +%% @doc Init protocol +init(ConnInfo = #{peername := {PeerHost, _}, + sockname := {_, SockPort}}, Option) -> + Peercert = maps:get(peercert, ConnInfo, undefined), + Mountpoint = maps:get(mountpoint, Option, undefined), + ClientInfo = setting_peercert_infos( + Peercert, + #{ zone => undefined + , protocol => stomp + , peerhost => PeerHost + , sockport => SockPort + , clientid => undefined + , username => undefined + , is_bridge => false + , is_superuser => false + , mountpoint => Mountpoint + } + ), + + Ctx = maps:get(ctx, Option), + Override = maps:merge(?DEFAULT_OVERRIDE, + maps:get(clientinfo_override, Option, #{}) + ), + #channel{ ctx = Ctx + , conninfo = ConnInfo + , clientinfo = ClientInfo + , clientinfo_override = Override + , timers = #{} + , transaction = #{} + }. + +setting_peercert_infos(NoSSL, ClientInfo) + when NoSSL =:= nossl; + NoSSL =:= undefined -> + ClientInfo; +setting_peercert_infos(Peercert, ClientInfo) -> + {DN, CN} = {esockd_peercert:subject(Peercert), + esockd_peercert:common_name(Peercert)}, + ClientInfo#{dn => DN, cn => CN}. + +-spec info(channel()) -> emqx_types:infos(). +info(Channel) -> + maps:from_list(info(?INFO_KEYS, Channel)). + +-spec(info(list(atom())|atom(), channel()) -> term()). +info(Keys, Channel) when is_list(Keys) -> + [{Key, info(Key, Channel)} || Key <- Keys]; + +info(conninfo, #channel{conninfo = ConnInfo}) -> + ConnInfo; +info(conn_state, #channel{conn_state = ConnState}) -> + ConnState; +info(clientinfo, #channel{clientinfo = ClientInfo}) -> + ClientInfo; +info(session, _) -> + #{}; +info(will_msg, _) -> + undefined; +info(clientid, #channel{clientinfo = #{clientid := ClientId}}) -> + ClientId; +info(ctx, #channel{ctx = Ctx}) -> + Ctx. + +stats(_Channel) -> + []. + +set_conn_state(ConnState, Channel) -> + Channel#channel{conn_state = ConnState}. + +enrich_conninfo(_Packet, + Channel = #channel{conninfo = ConnInfo}) -> + %% XXX: How enrich more infos? + NConnInfo = ConnInfo#{ proto_name => <<"STOMP">> + , proto_ver => undefined + , clean_start => true + , keepalive => 0 + , expiry_interval => 0 + }, + {ok, Channel#channel{conninfo = NConnInfo}}. + +run_conn_hooks(Packet, Channel = #channel{ctx = Ctx, + conninfo = ConnInfo}) -> + %% XXX: Assign headers of Packet to ConnProps + ConnProps = #{}, + case run_hooks(Ctx, 'client.connect', [ConnInfo], ConnProps) of + Error = {error, _Reason} -> Error; + _NConnProps -> + {ok, Packet, Channel} + end. + +negotiate_version(#stomp_frame{headers = Headers}, + Channel = #channel{conninfo = ConnInfo}) -> + %% XXX: + case do_negotiate_version(header(<<"accept-version">>, Headers)) of + {ok, Version} -> + {ok, Channel#channel{conninfo = ConnInfo#{proto_ver => Version}}}; + {error, Reason}-> + {error, Reason} + end. + +enrich_clientinfo(Packet, + Channel = #channel{ + conninfo = ConnInfo, + clientinfo = ClientInfo0, + clientinfo_override = Override}) -> + ClientInfo = write_clientinfo( + feedvar(Override, Packet, ConnInfo, ClientInfo0), + ClientInfo0 + ), + {ok, NPacket, NClientInfo} = emqx_misc:pipeline( + [ fun maybe_assign_clientid/2 + , fun parse_heartbeat/2 + %% FIXME: CALL After authentication successfully + , fun fix_mountpoint/2 + ], Packet, ClientInfo + ), + {ok, NPacket, Channel#channel{clientinfo = NClientInfo}}. + +feedvar(Override, Packet, ConnInfo, ClientInfo) -> + Envs = #{ 'ConnInfo' => ConnInfo + , 'ClientInfo' => ClientInfo + , 'Packet' => connect_packet_to_map(Packet) + }, + maps:map(fun(_K, V) -> + Tokens = emqx_rule_utils:preproc_tmpl(V), + emqx_rule_utils:proc_tmpl(Tokens, Envs) + end, Override). + +connect_packet_to_map(#stomp_frame{headers = Headers}) -> + #{headers => maps:from_list(Headers)}. + +write_clientinfo(Override, ClientInfo) -> + Override1 = maps:with([username, password, clientid], Override), + maps:merge(ClientInfo, Override1). + +maybe_assign_clientid(_Packet, ClientInfo = #{clientid := ClientId}) + when ClientId == undefined; + ClientId == <<>> -> + {ok, ClientInfo#{clientid => emqx_guid:to_base62(emqx_guid:gen())}}; + +maybe_assign_clientid(_Packet, ClientInfo) -> + {ok, ClientInfo}. + +parse_heartbeat(#stomp_frame{headers = Headers}, ClientInfo) -> + Heartbeat0 = header(<<"heart-beat">>, Headers, <<"0,0">>), + CxCy = re:split(Heartbeat0, <<",">>, [{return, list}]), + Heartbeat = list_to_tuple([list_to_integer(S) || S <- CxCy]), + {ok, ClientInfo#{heartbeat => Heartbeat}}. + +fix_mountpoint(_Packet, #{mountpoint := undefined}) -> ok; +fix_mountpoint(_Packet, ClientInfo = #{mountpoint := Mountpoint}) -> + %% TODO: Enrich the varibale replacement???? + %% i.e: ${ClientInfo.auth_result.productKey} + Mountpoint1 = emqx_mountpoint:replvar(Mountpoint, ClientInfo), + {ok, ClientInfo#{mountpoint := Mountpoint1}}. + +set_log_meta(_Packet, #channel{clientinfo = #{clientid := ClientId}}) -> + emqx_logger:set_metadata_clientid(ClientId), + ok. + +auth_connect(_Packet, Channel = #channel{ctx = Ctx, + clientinfo = ClientInfo}) -> + #{clientid := ClientId, + username := Username} = ClientInfo, + case emqx_gateway_ctx:authenticate(Ctx, ClientInfo) of + {ok, NClientInfo} -> + {ok, Channel#channel{clientinfo = NClientInfo}}; + {error, Reason} -> + ?LOG(warning, "Client ~s (Username: '~s') login failed for ~0p", + [ClientId, Username, Reason]), + {error, Reason} + end. + +ensure_connected(Channel = #channel{ + ctx = Ctx, + conninfo = ConnInfo, + clientinfo = ClientInfo}) -> + NConnInfo = ConnInfo#{connected_at => erlang:system_time(millisecond)}, + ok = run_hooks(Ctx, 'client.connected', [ClientInfo, NConnInfo]), + Channel#channel{conninfo = NConnInfo, + conn_state = connected + }. + +process_connect(Channel = #channel{ + ctx = Ctx, + conninfo = ConnInfo, + clientinfo = ClientInfo + }) -> + SessFun = fun(_,_) -> #{} end, + case emqx_gateway_ctx:open_session( + Ctx, + true, + ClientInfo, + ConnInfo, + SessFun + ) of + {ok, _Sess} -> %% The stomp protocol doesn't have session + #{proto_ver := Version} = ConnInfo, + #{heartbeat := Heartbeat} = ClientInfo, + Headers = [{<<"version">>, Version}, + {<<"heart-beat">>, reverse_heartbeats(Heartbeat)}], + handle_out(connected, Headers, Channel); + {error, Reason} -> + ?LOG(error, "Failed to open session du to ~p", [Reason]), + Headers = [{<<"version">>, <<"1.0,1.1,1.2">>}, + {<<"content-type">>, <<"text/plain">>}], + handle_out(connerr, {Headers, undefined, <<"Not Authenticated">>}, Channel) + end. + +%%-------------------------------------------------------------------- +%% Handle incoming packet +%%-------------------------------------------------------------------- + +-spec handle_in(emqx_types:packet(), channel()) + -> {ok, channel()} + | {ok, replies(), channel()} + | {shutdown, Reason :: term(), channel()} + | {shutdown, Reason :: term(), replies(), channel()}. + +handle_in(Frame = ?PACKET(?CMD_STOMP), Channel) -> + handle_in(Frame#stomp_frame{command = <<"CONNECT">>}, Channel); + +handle_in(?PACKET(?CMD_CONNECT), + Channel = #channel{conn_state = connected}) -> + {error, unexpected_connect, Channel}; + +handle_in(Packet = ?PACKET(?CMD_CONNECT), Channel) -> + case emqx_misc:pipeline( + [ fun enrich_conninfo/2 + , fun run_conn_hooks/2 + , fun negotiate_version/2 + , fun enrich_clientinfo/2 + , fun set_log_meta/2 + %% TODO: How to implement the banned in the gateway instance? + %, fun check_banned/2 + , fun auth_connect/2 + ], Packet, Channel#channel{conn_state = connecting}) of + {ok, _NPacket, NChannel} -> + process_connect(ensure_connected(NChannel)); + {error, ReasonCode, NChannel} -> + ErrMsg = io_lib:format("Login Failed: ~s", [ReasonCode]), + handle_out(connerr, {[], undefined, ErrMsg}, NChannel) + end; + +handle_in(Frame = ?PACKET(?CMD_SEND, Headers), + Channel = #channel{ + ctx = Ctx, + clientinfo = ClientInfo + }) -> + Topic = header(<<"destination">>, Headers), + case emqx_gateway_ctx:authorize(Ctx, ClientInfo, publish, Topic) of + deny -> + handle_out(error, {receipt_id(Headers), "ACL Deny"}, Channel); + allow -> + case header(<<"transaction">>, Headers) of + undefined -> + handle_recv_send_frame(Frame, Channel); + TxId -> + add_action(TxId, {fun ?MODULE:handle_recv_send_frame/2, [Frame]}, receipt_id(Headers), Channel) + end + end; + +handle_in(?PACKET(?CMD_SUBSCRIBE, Headers), + Channel = #channel{ + ctx = Ctx, + subscriptions = Subs, + clientinfo = ClientInfo = #{mountpoint := Mountpoint} + }) -> + SubId = header(<<"id">>, Headers), + Topic = header(<<"destination">>, Headers), + Ack = header(<<"ack">>, Headers, <<"auto">>), + + MountedTopic = emqx_mountpoint:mount(Mountpoint, Topic), + + case lists:keyfind(SubId, 1, Subs) of + {SubId, MountedTopic, Ack} -> + maybe_outgoing_receipt(receipt_id(Headers), Channel); + {SubId, _OtherTopic, _OtherAck} -> + %% FIXME: + ?LOG(error, "Conflicts with subscribed topics ~s, id: ~s", + [_OtherTopic, SubId]), + ErrMsg = "Conflict subscribe id ", + handle_out(error, {receipt_id(Headers), ErrMsg}, Channel); + false -> + case emqx_gateway_ctx:authorize(Ctx, ClientInfo, subscribe, Topic) of + deny -> + handle_out(error, {receipt_id(Headers), "ACL Deny"}, Channel); + allow -> + _ = emqx_broker:subscribe(MountedTopic), + maybe_outgoing_receipt( + receipt_id(Headers), + Channel#channel{subscriptions = [{SubId, MountedTopic, Ack} | Subs]} + ) + end + end; + +handle_in(?PACKET(?CMD_UNSUBSCRIBE, Headers), + Channel = #channel{subscriptions = Subs}) -> + SubId = header(<<"id">>, Headers), + {ok, NChannel} = case lists:keyfind(SubId, 1, Subs) of + {SubId, Topic, _Ack} -> + ok = emqx_broker:unsubscribe(Topic), + {ok, Channel#channel{subscriptions = lists:keydelete(SubId, 1, Subs)}}; + false -> + {ok, Channel} + end, + handle_out(receipt, receipt_id(Headers), NChannel); + +%% XXX: How to ack a frame ??? +handle_in(Frame = ?PACKET(?CMD_ACK, Headers), Channel) -> + case header(<<"transaction">>, Headers) of + undefined -> handle_recv_ack_frame(Frame, Channel); + TxId -> add_action(TxId, {fun ?MODULE:handle_recv_ack_frame/2, [Frame]}, receipt_id(Headers), Channel) + end; + +%% NACK +%% id:12345 +%% transaction:tx1 +%% +%% ^@ +handle_in(Frame = ?PACKET(?CMD_NACK, Headers), Channel) -> + case header(<<"transaction">>, Headers) of + undefined -> handle_recv_nack_frame(Frame, Channel); + TxId -> add_action(TxId, {fun ?MODULE:handle_recv_nack_frame/2, [Frame]}, receipt_id(Headers), Channel) + end; + +%% The transaction header is REQUIRED, and the transaction identifier +%% will be used for SEND, COMMIT, ABORT, ACK, and NACK frames to bind +%% them to the named transaction. +%% +%% BEGIN +%% transaction:tx1 +%% +%% ^@ +handle_in(?PACKET(?CMD_BEGIN, Headers), + Channel = #channel{transaction = Trans}) -> + TxId = header(<<"transaction">>, Headers), + case maps:get(TxId, Trans, undefined) of + undefined -> + StartedAt = erlang:system_time(millisecond), + NChannel = ensure_clean_trans_timer( + Channel#channel{ + transaction = Trans#{TxId => {StartedAt, []}}} + ), + handle_out(receipt, receipt_id(Headers), NChannel); + _ -> + ErrMsg = ["Transaction ", TxId, " already started"], + handle_out(error, {receipt_id(Headers), ErrMsg}, Channel) + end; + +%% COMMIT +%% transaction:tx1 +%% +%% ^@ +handle_in(?PACKET(?CMD_COMMIT, Headers), Channel) -> + with_transaction(Headers, Channel, fun(TxId, Actions) -> + Chann0 = remove_trans(TxId, Channel), + case trans_pipeline(lists:reverse(Actions), [], Chann0) of + {ok, Outgoings, Chann1} -> + maybe_outgoing_receipt(receipt_id(Headers), Outgoings, Chann1); + {error, Reason} -> + %% FIXME: atomic for transaction ?? + ErrMsg = io_lib:format("Execute transaction ~s falied: ~0p", + [TxId, Reason] + ), + handle_out(error, {receipt_id(Headers), ErrMsg}, Chann0) + end + end); + +%% ABORT +%% transaction:tx1 +%% +%% ^@ +handle_in(?PACKET(?CMD_ABORT, Headers), + Channel = #channel{transaction = Trans}) -> + with_transaction(Headers, Channel, fun(Id, _Actions) -> + NChannel = Channel#channel{transaction = maps:remove(Id, Trans)}, + handle_out(receipt, receipt_id(Headers), NChannel) + end); + +handle_in(?PACKET(?CMD_DISCONNECT, Headers), Channel) -> + shutdown_with_recepit(normal, receipt_id(Headers), Channel); + +handle_in({frame_error, Reason}, Channel = #channel{conn_state = _ConnState}) -> + ?LOG(error, "Unexpected frame error: ~p", [Reason]), + shutdown(Reason, Channel). + +with_transaction(Headers, Channel = #channel{transaction = Trans}, Fun) -> + Id = header(<<"transaction">>, Headers), + ReceiptId = receipt_id(Headers), + case maps:get(Id, Trans, undefined) of + {_, Actions} -> + Fun(Id, Actions); + _ -> + ErrMsg = ["Transaction ", Id, " not found"], + handle_out(error, {ReceiptId, ErrMsg}, Channel) + end. + +remove_trans(Id, Channel = #channel{transaction = Trans}) -> + Channel#channel{transaction = maps:remove(Id, Trans)}. + +trans_pipeline([], Outgoings, Channel) -> + {ok, Outgoings, Channel}; + +trans_pipeline([{Func, Args}|More], Outgoings, Channel) -> + case erlang:apply(Func, Args ++ [Channel]) of + {ok, NChannel} -> + trans_pipeline(More, Outgoings, NChannel); + {ok, Outgoings1, NChannel} -> + trans_pipeline(More, Outgoings ++ Outgoings1, NChannel); + {error, Reason} -> + {error, Reason, Channel} + end. + +%%-------------------------------------------------------------------- +%% Handle outgoing packet +%%-------------------------------------------------------------------- + +-spec(handle_out(atom(), term(), channel()) + -> {ok, channel()} + | {ok, replies(), channel()} + | {shutdown, Reason :: term(), channel()} + | {shutdown, Reason :: term(), replies(), channel()}). + +handle_out(connerr, {Headers, ReceiptId, ErrMsg}, Channel) -> + Frame = error_frame(Headers, ReceiptId, ErrMsg), + shutdown(ErrMsg, Frame, Channel); + +handle_out(error, {ReceiptId, ErrMsg}, Channel) -> + Frame = error_frame(ReceiptId, ErrMsg), + {ok, Frame, Channel}; + +handle_out(connected, Headers, Channel = #channel{ + ctx = Ctx, + conninfo = ConnInfo + }) -> + %% XXX: connection_accepted is not defined by stomp protocol + _ = run_hooks(Ctx, 'client.connack', [ConnInfo, connection_accepted, []]), + Replies = [{outgoing, connected_frame(Headers)}, + {event, connected} + ], + {ok, Replies, ensure_heartbeart_timer(Channel)}; + +handle_out(receipt, undefined, Channel) -> + {ok, Channel}; +handle_out(receipt, ReceiptId, Channel) -> + Frame = receipt_frame(ReceiptId), + {ok, Frame, Channel}. + +%%-------------------------------------------------------------------- +%% Handle call +%%-------------------------------------------------------------------- + +-spec(handle_call(Req :: term(), channel()) + -> {reply, Reply :: term(), channel()} + | {shutdown, Reason :: term(), Reply :: term(), channel()} + | {shutdown, Reason :: term(), Reply :: term(), emqx_types:packet(), channel()}). +handle_call(kick, Channel) -> + NChannel = ensure_disconnected(kicked, Channel), + shutdown_and_reply(kicked, ok, NChannel); + +handle_call(discard, Channel) -> + shutdown_and_reply(discarded, ok, Channel); + +%% XXX: No Session Takeover +%handle_call({takeover, 'begin'}, Channel = #channel{session = Session}) -> +% reply(Session, Channel#channel{takeover = true}); +% +%handle_call({takeover, 'end'}, Channel = #channel{session = Session, +% pendings = Pendings}) -> +% ok = emqx_session:takeover(Session), +% %% TODO: Should not drain deliver here (side effect) +% Delivers = emqx_misc:drain_deliver(), +% AllPendings = lists:append(Delivers, Pendings), +% shutdown_and_reply(takeovered, AllPendings, Channel); + +handle_call(list_acl_cache, Channel) -> + {reply, emqx_acl_cache:list_acl_cache(), Channel}; + +%% XXX: No Quota Now +% handle_call({quota, Policy}, Channel) -> +% Zone = info(zone, Channel), +% Quota = emqx_limiter:init(Zone, Policy), +% reply(ok, Channel#channel{quota = Quota}); + +handle_call(Req, Channel) -> + ?LOG(error, "Unexpected call: ~p", [Req]), + reply(ignored, Channel). + +%%-------------------------------------------------------------------- +%% Handle Info +%%-------------------------------------------------------------------- + +-spec(handle_info(Info :: term(), channel()) + -> ok | {ok, channel()} | {shutdown, Reason :: term(), channel()}). + +%% XXX: Received from the emqx-management ??? +%handle_info({subscribe, TopicFilters}, Channel ) -> +% {_, NChannel} = lists:foldl( +% fun({TopicFilter, SubOpts}, {_, ChannelAcc}) -> +% do_subscribe(TopicFilter, SubOpts, ChannelAcc) +% end, {[], Channel}, parse_topic_filters(TopicFilters)), +% {ok, NChannel}; +% +%handle_info({unsubscribe, TopicFilters}, Channel) -> +% {_RC, NChannel} = process_unsubscribe(TopicFilters, #{}, Channel), +% {ok, NChannel}; + +handle_info({sock_closed, Reason}, + Channel = #channel{conn_state = idle}) -> + shutdown(Reason, Channel); + +handle_info({sock_closed, Reason}, + Channel = #channel{conn_state = connecting}) -> + shutdown(Reason, Channel); + +handle_info({sock_closed, Reason}, + Channel = #channel{conn_state = connected, + clientinfo = _ClientInfo}) -> + %% XXX: Flapping detect ??? + %% How to get the flapping detect policy ??? + %emqx_zone:enable_flapping_detect(Zone) + % andalso emqx_flapping:detect(ClientInfo), + NChannel = ensure_disconnected(Reason, Channel), + %% XXX: Session keepper detect here + shutdown(Reason, NChannel); + +handle_info({sock_closed, Reason}, + Channel = #channel{conn_state = disconnected}) -> + ?LOG(error, "Unexpected sock_closed: ~p", [Reason]), + {ok, Channel}; + +handle_info(clean_acl_cache, Channel) -> + ok = emqx_acl_cache:empty_acl_cache(), + {ok, Channel}; + +handle_info(Info, Channel) -> + ?LOG(error, "Unexpected info: ~p", [Info]), + {ok, Channel}. + +%%-------------------------------------------------------------------- +%% Ensure disconnected + +ensure_disconnected(Reason, Channel = #channel{ + ctx = Ctx, + conninfo = ConnInfo, + clientinfo = ClientInfo}) -> + NConnInfo = ConnInfo#{disconnected_at => erlang:system_time(millisecond)}, + ok = run_hooks(Ctx, 'client.disconnected', + [ClientInfo, Reason, NConnInfo]), + Channel#channel{conninfo = NConnInfo, conn_state = disconnected}. + +%%-------------------------------------------------------------------- +%% Handle Delivers from broker to client +%%-------------------------------------------------------------------- + +-spec(handle_deliver(list(emqx_types:deliver()), channel()) + -> {ok, channel()} + | {ok, replies(), channel()}). + +handle_deliver(Delivers, + Channel = #channel{ + ctx = Ctx, + clientinfo = ClientInfo, + subscriptions = Subs + }) -> + + %% TODO: Re-deliver ??? + %% Shared-subscription support ??? + + Frames0 = lists:foldl(fun({_, _, Message}, Acc) -> + Topic0 = emqx_message:topic(Message), + case lists:keyfind(Topic0, 2, Subs) of + {Id, Topic, Ack} -> + %% XXX: refactor later + metrics_inc('messages.delivered', Channel), + NMessage = run_hooks_without_metrics( + Ctx, + 'message.delivered', + [ClientInfo], + Message + ), + Topic = emqx_message:topic(NMessage), + Headers = emqx_message:get_headers(NMessage), + Payload = emqx_message:payload(NMessage), + Headers0 = [{<<"subscription">>, Id}, + {<<"message-id">>, next_msgid()}, + {<<"destination">>, Topic}, + {<<"content-type">>, <<"text/plain">>}], + Headers1 = case Ack of + _ when Ack =:= <<"client">>; + Ack =:= <<"client-individual">> -> + Headers0 ++ [{<<"ack">>, next_ackid()}]; + _ -> + Headers0 + end, + Frame = #stomp_frame{command = <<"MESSAGE">>, + headers = Headers1 ++ maps:get(stomp_headers, Headers, []), + body = Payload + }, + [Frame|Acc]; + false -> + ?LOG(error, "Dropped message ~0p due to not found " + "subscription id for ~s", + [Message, emqx_message:topic(Message)]), + metrics_inc('delivery.dropped', Channel), + metrics_inc('delivery.dropped.no_subid', Channel), + Acc + end + end, [], Delivers), + {ok, [{outgoing, lists:reverse(Frames0)}], Channel}. + +%%-------------------------------------------------------------------- +%% Handle timeout +%%-------------------------------------------------------------------- + +-spec(handle_timeout(reference(), Msg :: term(), channel()) + -> {ok, channel()} + | {ok, replies(), channel()} + | {shutdown, Reason :: term(), channel()}). + +handle_timeout(_TRef, {incoming, NewVal}, + Channel = #channel{heartbeat = HrtBt}) -> + case emqx_stomp_heartbeat:check(incoming, NewVal, HrtBt) of + {error, timeout} -> + shutdown(heartbeat_timeout, Channel); + {ok, NHrtBt} -> + {ok, reset_timer(incoming_timer, + Channel#channel{heartbeat = NHrtBt} + )} + end; + +handle_timeout(_TRef, {outgoing, NewVal}, + Channel = #channel{heartbeat = HrtBt}) -> + case emqx_stomp_heartbeat:check(outgoing, NewVal, HrtBt) of + {error, timeout} -> + NHrtBt = emqx_stomp_heartbeat:reset(outgoing, NewVal, HrtBt), + NChannel = Channel#channel{heartbeat = NHrtBt}, + {ok, emqx_stomp_frame:make(heartbeat), + reset_timer(outgoing_timer, NChannel)}; + {ok, NHrtBt} -> + {ok, reset_timer(outgoing_timer, + Channel#channel{heartbeat = NHrtBt} + )} + end; + +handle_timeout(_TRef, clean_trans, Channel = #channel{transaction = Trans}) -> + Now = erlang:system_time(millisecond), + NTrans = maps:filter(fun(_, {StartedAt, _}) -> + StartedAt + ?TRANS_TIMEOUT < Now + end, Trans), + {ok, ensure_clean_trans_timer(Channel#channel{transaction = NTrans})}. + +%%-------------------------------------------------------------------- +%% Terminate +%%-------------------------------------------------------------------- + +terminate(_Reason, _Channel) -> + ok. + +reply(Reply, Channel) -> + {reply, Reply, Channel}. + +shutdown(Reason, Channel) -> + {shutdown, Reason, Channel}. + +shutdown_with_recepit(Reason, ReceiptId, Channel) -> + case ReceiptId of + undefined -> + {shutdown, Reason, Channel}; + _ -> + {shutdown, Reason, receipt_frame(ReceiptId), Channel} + end. + +shutdown(Reason, AckFrame, Channel) -> + {shutdown, Reason, AckFrame, Channel}. + +shutdown_and_reply(Reason, Reply, Channel) -> + {shutdown, Reason, Reply, Channel}. + +do_negotiate_version(undefined) -> + {ok, <<"1.0">>}; + +do_negotiate_version(Accepts) -> + do_negotiate_version( + ?STOMP_VER, + lists:reverse(lists:sort(binary:split(Accepts, <<",">>, [global]))) + ). + +do_negotiate_version(Ver, []) -> + {error, <<"Supported protocol versions < ", Ver/binary>>}; +do_negotiate_version(Ver, [AcceptVer|_]) when Ver >= AcceptVer -> + {ok, AcceptVer}; +do_negotiate_version(Ver, [_|T]) -> + do_negotiate_version(Ver, T). + +header(Name, Headers) -> + get_value(Name, Headers). +header(Name, Headers, Val) -> + get_value(Name, Headers, Val). + +connected_frame(Headers) -> + emqx_stomp_frame:make(<<"CONNECTED">>, Headers). + +receipt_frame(ReceiptId) -> + emqx_stomp_frame:make(<<"RECEIPT">>, [{<<"receipt-id">>, ReceiptId}]). + +error_frame(ReceiptId, Msg) -> + error_frame([{<<"content-type">>, <<"text/plain">>}], ReceiptId, Msg). + +error_frame(Headers, undefined, Msg) -> + emqx_stomp_frame:make(<<"ERROR">>, Headers, Msg); +error_frame(Headers, ReceiptId, Msg) -> + emqx_stomp_frame:make(<<"ERROR">>, [{<<"receipt-id">>, ReceiptId} | Headers], Msg). + +next_msgid() -> + MsgId = case get(msgid) of + undefined -> 1; + I -> I + end, + put(msgid, MsgId + 1), + MsgId. + +next_ackid() -> + AckId = case get(ackid) of + undefined -> 1; + I -> I + end, + put(ackid, AckId + 1), + AckId. + +frame2message(?PACKET(?CMD_SEND, Headers, Body), + #channel{ + conninfo = #{proto_ver := ProtoVer}, + clientinfo = #{ + protocol := Protocol, + clientid := ClientId, + username := Username, + peerhost := PeerHost, + mountpoint := Mountpoint + }}) -> + Topic = header(<<"destination">>, Headers), + Msg = emqx_message:make(ClientId, Topic, Body), + StompHeaders = lists:foldl( + fun(Key, Headers0) -> + proplists:delete(Key, Headers0) + end, Headers, + [<<"destination">>, + <<"content-length">>, + <<"content-type">>, + <<"transaction">>, + <<"receipt">> + ]), + %% Pass-through of custom headers on the sending side + NMsg = emqx_message:set_headers(#{proto_ver => ProtoVer, + protocol => Protocol, + username => Username, + peerhost => PeerHost, + stomp_headers => StompHeaders + }, Msg), + emqx_mountpoint:mount(Mountpoint, NMsg). + +receipt_id(Headers) -> + header(<<"receipt">>, Headers). + +%%-------------------------------------------------------------------- +%% Trans + +add_action(TxId, Action, ReceiptId, Channel = #channel{transaction = Trans}) -> + case maps:get(TxId, Trans, undefined) of + {_StartedAt, Actions} -> + NTrans = Trans#{TxId => {_StartedAt, [Action|Actions]}}, + {ok, Channel#channel{transaction = NTrans}}; + _ -> + {ok, error_frame(ReceiptId, ["Transaction ", TxId, " not found"]), Channel} + end. + +%%-------------------------------------------------------------------- +%% Transaction Handle + +handle_recv_send_frame(Frame = ?PACKET(?CMD_SEND, Headers), Channel) -> + Msg = frame2message(Frame, Channel), + _ = emqx_broker:publish(Msg), + maybe_outgoing_receipt(receipt_id(Headers), Channel). + +handle_recv_ack_frame(?PACKET(?CMD_ACK, Headers), Channel) -> + maybe_outgoing_receipt(receipt_id(Headers), Channel). + +handle_recv_nack_frame(?PACKET(?CMD_NACK, Headers), Channel) -> + maybe_outgoing_receipt(receipt_id(Headers), Channel). + +maybe_outgoing_receipt(undefined, Channel) -> + {ok, [], Channel}; +maybe_outgoing_receipt(ReceiptId, Channel) -> + {ok, [{outgoing, receipt_frame(ReceiptId)}], Channel}. + +maybe_outgoing_receipt(undefined, Outgoings, Channel) -> + {ok, Outgoings, Channel}; +maybe_outgoing_receipt(ReceiptId, Outgoings, Channel) -> + {ok, lists:reverse([receipt_frame(ReceiptId)|Outgoings]), Channel}. + +ensure_clean_trans_timer(Channel = #channel{transaction = Trans}) -> + case maps:size(Trans) of + 0 -> Channel; + _ -> ensure_timer(clean_trans_timer, Channel) + end. + +%%-------------------------------------------------------------------- +%% Heartbeat + +reverse_heartbeats({Cx, Cy}) -> + iolist_to_binary(io_lib:format("~w,~w", [Cy, Cx])). + +ensure_heartbeart_timer(Channel = #channel{clientinfo = ClientInfo}) -> + Heartbeat = maps:get(heartbeat, ClientInfo), + ensure_timer( + [incoming_timer, outgoing_timer], + Channel#channel{heartbeat = emqx_stomp_heartbeat:init(Heartbeat)}). + +%%-------------------------------------------------------------------- +%% Timer + +ensure_timer([Name], Channel) -> + ensure_timer(Name, Channel); +ensure_timer([Name | Rest], Channel) -> + ensure_timer(Rest, ensure_timer(Name, Channel)); + +ensure_timer(Name, Channel = #channel{timers = Timers}) -> + TRef = maps:get(Name, Timers, undefined), + Time = interval(Name, Channel), + case TRef == undefined andalso is_integer(Time) andalso Time > 0 of + true -> ensure_timer(Name, Time, Channel); + false -> Channel %% Timer disabled or exists + end. + +ensure_timer(Name, Time, Channel = #channel{timers = Timers}) -> + Msg = maps:get(Name, ?TIMER_TABLE), + TRef = emqx_misc:start_timer(Time, Msg), + Channel#channel{timers = Timers#{Name => TRef}}. + +reset_timer(Name, Channel) -> + ensure_timer(Name, clean_timer(Name, Channel)). + +clean_timer(Name, Channel = #channel{timers = Timers}) -> + Channel#channel{timers = maps:remove(Name, Timers)}. + +interval(incoming_timer, #channel{heartbeat = HrtBt}) -> + emqx_stomp_heartbeat:interval(incoming, HrtBt); +interval(outgoing_timer, #channel{heartbeat = HrtBt}) -> + emqx_stomp_heartbeat:interval(outgoing, HrtBt); +interval(clean_trans_timer, _) -> + ?TRANS_TIMEOUT. + +%%-------------------------------------------------------------------- +%% Helper functions +%%-------------------------------------------------------------------- + +run_hooks(Ctx, Name, Args) -> + emqx_gateway_ctx:metrics_inc(Ctx, Name), + emqx_hooks:run(Name, Args). + +run_hooks(Ctx, Name, Args, Acc) -> + emqx_gateway_ctx:metrics_inc(Ctx, Name), + emqx_hooks:run_fold(Name, Args, Acc). + +run_hooks_without_metrics(_Ctx, Name, Args, Acc) -> + emqx_hooks:run_fold(Name, Args, Acc). + +metrics_inc(Name, #channel{ctx = Ctx}) -> + emqx_gateway_ctx:metrics_inc(Ctx, Name). diff --git a/apps/emqx_gateway/src/stomp/emqx_stomp_connection.erl b/apps/emqx_gateway/src/stomp/emqx_stomp_connection.erl new file mode 100644 index 000000000..a4b87fcd4 --- /dev/null +++ b/apps/emqx_gateway/src/stomp/emqx_stomp_connection.erl @@ -0,0 +1,908 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_stomp_connection). + +-include("src/stomp/include/emqx_stomp.hrl"). +-include_lib("emqx/include/logger.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +-logger_header("[Stomp-Conn]"). + +%% API +-export([ start_link/3 + , stop/1 + ]). + +-export([ info/1 + , stats/1 + ]). + +-export([ async_set_keepalive/3 + , async_set_keepalive/4 + , async_set_socket_options/2 + ]). + +-export([ call/2 + , call/3 + , cast/2 + ]). + +%% Callback +-export([init/4]). + +%% Sys callbacks +-export([ system_continue/3 + , system_terminate/4 + , system_code_change/4 + , system_get_state/1 + ]). + +%% Internal callback +-export([wakeup_from_hib/2, recvloop/2, get_state/1]). + +%% Export for CT +-export([set_field/3]). + +-import(emqx_misc, + [ maybe_apply/2 + ]). + +-record(state, { + %% TCP/TLS Transport + transport :: esockd:transport(), + %% TCP/TLS Socket + socket :: esockd:socket(), + %% Peername of the connection + peername :: emqx_types:peername(), + %% Sockname of the connection + sockname :: emqx_types:peername(), + %% Sock State + sockstate :: emqx_types:sockstate(), + %% The {active, N} option + active_n :: pos_integer(), + %% Limiter + limiter :: emqx_limiter:limiter() | undefined, + %% Limit Timer + limit_timer :: reference() | undefined, + %% Parse State + parse_state :: emqx_stomp_frame:parse_state(), + %% Serialize options + serialize :: emqx_stomp_frame:serialize_opts(), + %% Channel State + channel :: emqx_stomp_channel:channel(), + %% GC State + gc_state :: emqx_gc:gc_state() | undefined, + %% Stats Timer + stats_timer :: disabled | reference(), + %% Idle Timeout + idle_timeout :: integer(), + %% Idle Timer + idle_timer :: reference() | undefined + }). + +-type(state() :: #state{}). + +-define(ACTIVE_N, 100). +-define(INFO_KEYS, [socktype, peername, sockname, sockstate, active_n]). +-define(CONN_STATS, [recv_pkt, recv_msg, send_pkt, send_msg]). +-define(SOCK_STATS, [recv_oct, recv_cnt, send_oct, send_cnt, send_pend]). + +-define(ENABLED(X), (X =/= undefined)). + +%-define(ALARM_TCP_CONGEST(Channel), +% list_to_binary(io_lib:format("mqtt_conn/congested/~s/~s", +% [emqx_stomp_channel:info(clientid, Channel), +% emqx_stomp_channel:info(username, Channel)]))). +%-define(ALARM_CONN_INFO_KEYS, [ +% socktype, sockname, peername, +% clientid, username, proto_name, proto_ver, connected_at +%]). +%-define(ALARM_SOCK_STATS_KEYS, [send_pend, recv_cnt, recv_oct, send_cnt, send_oct]). +%-define(ALARM_SOCK_OPTS_KEYS, [high_watermark, high_msgq_watermark, sndbuf, recbuf, buffer]). + +-dialyzer({no_match, [info/2]}). +-dialyzer({nowarn_function, [ init/4 + , init_state/3 + , run_loop/2 + , system_terminate/4 + , system_code_change/4 + ]}). + +-dialyzer({nowarn_function, [ensure_stats_timer/2,cancel_stats_timer/1, + terminate/2,handle_call/3,handle_timeout/3, + parse_incoming/3,serialize_and_inc_stats_fun/1, + check_oom/1,inc_incoming_stats/1, + inc_outgoing_stats/1]}). + +-spec(start_link(esockd:transport(), esockd:socket(), proplists:proplist()) + -> {ok, pid()}). +start_link(Transport, Socket, Options) -> + Args = [self(), Transport, Socket, Options], + CPid = proc_lib:spawn_link(?MODULE, init, Args), + {ok, CPid}. + +%%-------------------------------------------------------------------- +%% API +%%-------------------------------------------------------------------- + +%% @doc Get infos of the connection/channel. +-spec(info(pid()|state()) -> emqx_types:infos()). +info(CPid) when is_pid(CPid) -> + call(CPid, info); +info(State = #state{channel = Channel}) -> + ChanInfo = emqx_stomp_channel:info(Channel), + SockInfo = maps:from_list( + info(?INFO_KEYS, State)), + ChanInfo#{sockinfo => SockInfo}. + +info(Keys, State) when is_list(Keys) -> + [{Key, info(Key, State)} || Key <- Keys]; +info(socktype, #state{transport = Transport, socket = Socket}) -> + Transport:type(Socket); +info(peername, #state{peername = Peername}) -> + Peername; +info(sockname, #state{sockname = Sockname}) -> + Sockname; +info(sockstate, #state{sockstate = SockSt}) -> + SockSt; +info(active_n, #state{active_n = ActiveN}) -> + ActiveN; +info(stats_timer, #state{stats_timer = StatsTimer}) -> + StatsTimer; +info(limit_timer, #state{limit_timer = LimitTimer}) -> + LimitTimer; +info(limiter, #state{limiter = Limiter}) -> + maybe_apply(fun emqx_limiter:info/1, Limiter). + +%% @doc Get stats of the connection/channel. +-spec(stats(pid()|state()) -> emqx_types:stats()). +stats(CPid) when is_pid(CPid) -> + call(CPid, stats); +stats(#state{transport = Transport, + socket = Socket, + channel = Channel}) -> + SockStats = case Transport:getstat(Socket, ?SOCK_STATS) of + {ok, Ss} -> Ss; + {error, _} -> [] + end, + ConnStats = emqx_pd:get_counters(?CONN_STATS), + ChanStats = emqx_stomp_channel:stats(Channel), + ProcStats = emqx_misc:proc_stats(), + lists:append([SockStats, ConnStats, ChanStats, ProcStats]). + +%% @doc Set TCP keepalive socket options to override system defaults. +%% Idle: The number of seconds a connection needs to be idle before +%% TCP begins sending out keep-alive probes (Linux default 7200). +%% Interval: The number of seconds between TCP keep-alive probes +%% (Linux default 75). +%% Probes: The maximum number of TCP keep-alive probes to send before +%% giving up and killing the connection if no response is +%% obtained from the other end (Linux default 9). +%% +%% NOTE: This API sets TCP socket options, which has nothing to do with +%% the MQTT layer's keepalive (PINGREQ and PINGRESP). +async_set_keepalive(Idle, Interval, Probes) -> + async_set_keepalive(self(), Idle, Interval, Probes). + +async_set_keepalive(Pid, Idle, Interval, Probes) -> + Options = [ {keepalive, true} + , {raw, 6, 4, <>} + , {raw, 6, 5, <>} + , {raw, 6, 6, <>} + ], + async_set_socket_options(Pid, Options). + +%% @doc Set custom socket options. +%% This API is made async because the call might be originated from +%% a hookpoint callback (otherwise deadlock). +%% If failed to set, the error message is logged. +async_set_socket_options(Pid, Options) -> + cast(Pid, {async_set_socket_options, Options}). + +cast(Pid, Req) -> + gen_server:cast(Pid, Req). + +call(Pid, Req) -> + call(Pid, Req, infinity). +call(Pid, Req, Timeout) -> + gen_server:call(Pid, Req, Timeout). + +stop(Pid) -> + gen_server:stop(Pid). + +%%-------------------------------------------------------------------- +%% callbacks +%%-------------------------------------------------------------------- + +init(Parent, Transport, RawSocket, Options) -> + case Transport:wait(RawSocket) of + {ok, Socket} -> + run_loop(Parent, init_state(Transport, Socket, Options)); + {error, Reason} -> + ok = Transport:fast_close(RawSocket), + exit_on_sock_error(Reason) + end. + +init_state(Transport, Socket, Options) -> + {ok, Peername} = Transport:ensure_ok_or_exit(peername, [Socket]), + {ok, Sockname} = Transport:ensure_ok_or_exit(sockname, [Socket]), + Peercert = Transport:ensure_ok_or_exit(peercert, [Socket]), + ConnInfo = #{socktype => Transport:type(Socket), + peername => Peername, + sockname => Sockname, + peercert => Peercert, + conn_mod => ?MODULE + }, + ActiveN = emqx_gateway_utils:active_n(Options), + %% TODO: RateLimit ? How ? + Limiter = undefined, + %RateLimit = emqx_gateway_utils:ratelimit(Options), + %%Limiter = emqx_limiter:init(Zone, RateLimit), + FrameOpts = emqx_gateway_utils:frame_options(Options), + ParseState = emqx_stomp_frame:initial_parse_state(FrameOpts), + Serialize = emqx_stomp_frame:serialize_opts(), + Channel = emqx_stomp_channel:init(ConnInfo, Options), + GcState = emqx_gateway_utils:init_gc_state(Options), + StatsTimer = emqx_gateway_utils:stats_timer(Options), + IdleTimeout = emqx_gateway_utils:idle_timeout(Options), + IdleTimer = emqx_misc:start_timer(IdleTimeout, idle_timeout), + #state{transport = Transport, + socket = Socket, + peername = Peername, + sockname = Sockname, + sockstate = idle, + active_n = ActiveN, + limiter = Limiter, + parse_state = ParseState, + serialize = Serialize, + channel = Channel, + gc_state = GcState, + stats_timer = StatsTimer, + idle_timeout = IdleTimeout, + idle_timer = IdleTimer + }. + +run_loop(Parent, State = #state{transport = Transport, + socket = Socket, + peername = Peername, + channel = _Channel}) -> + emqx_logger:set_metadata_peername(esockd:format(Peername)), + % TODO: How yo get oom_policy ??? + %emqx_misc:tune_heap_size(emqx_gateway_utils:oom_policy( + % emqx_stomp_channel:info(zone, Channel))), + case activate_socket(State) of + {ok, NState} -> hibernate(Parent, NState); + {error, Reason} -> + ok = Transport:fast_close(Socket), + exit_on_sock_error(Reason) + end. + +-spec exit_on_sock_error(any()) -> no_return(). +exit_on_sock_error(Reason) when Reason =:= einval; + Reason =:= enotconn; + Reason =:= closed -> + erlang:exit(normal); +exit_on_sock_error(timeout) -> + erlang:exit({shutdown, ssl_upgrade_timeout}); +exit_on_sock_error(Reason) -> + erlang:exit({shutdown, Reason}). + +%%-------------------------------------------------------------------- +%% Recv Loop + +recvloop(Parent, State = #state{idle_timeout = IdleTimeout}) -> + receive + Msg -> + handle_recv(Msg, Parent, State) + after + IdleTimeout + 100 -> + hibernate(Parent, cancel_stats_timer(State)) + end. + +handle_recv({system, From, Request}, Parent, State) -> + sys:handle_system_msg(Request, From, Parent, ?MODULE, [], State); +handle_recv({'EXIT', Parent, Reason}, Parent, State) -> + %% FIXME: it's not trapping exit, should never receive an EXIT + terminate(Reason, State); +handle_recv(Msg, Parent, State = #state{idle_timeout = IdleTimeout}) -> + case process_msg([Msg], ensure_stats_timer(IdleTimeout, State)) of + {ok, NewState} -> + ?MODULE:recvloop(Parent, NewState); + {stop, Reason, NewSate} -> + terminate(Reason, NewSate) + end. + +hibernate(Parent, State) -> + proc_lib:hibernate(?MODULE, wakeup_from_hib, [Parent, State]). + +%% Maybe do something here later. +wakeup_from_hib(Parent, State) -> + ?MODULE:recvloop(Parent, State). + +%%-------------------------------------------------------------------- +%% Ensure/cancel stats timer + +ensure_stats_timer(Timeout, State = #state{stats_timer = undefined}) -> + State#state{stats_timer = emqx_misc:start_timer(Timeout, emit_stats)}; +ensure_stats_timer(_Timeout, State) -> State. + +cancel_stats_timer(State = #state{stats_timer = TRef}) + when is_reference(TRef) -> + ?tp(debug, cancel_stats_timer, #{}), + ok = emqx_misc:cancel_timer(TRef), + State#state{stats_timer = undefined}; +cancel_stats_timer(State) -> State. + +%%-------------------------------------------------------------------- +%% Process next Msg + +process_msg([], State) -> + {ok, State}; +process_msg([Msg|More], State) -> + try + case handle_msg(Msg, State) of + ok -> + process_msg(More, State); + {ok, NState} -> + process_msg(More, NState); + {ok, Msgs, NState} -> + process_msg(append_msg(More, Msgs), NState); + {stop, Reason, NState} -> + {stop, Reason, NState} + end + catch + exit : normal -> + {stop, normal, State}; + exit : shutdown -> + {stop, shutdown, State}; + exit : {shutdown, _} = Shutdown -> + {stop, Shutdown, State}; + Exception : Context : Stack -> + {stop, #{exception => Exception, + context => Context, + stacktrace => Stack}, State} + end. + +append_msg([], Msgs) when is_list(Msgs) -> + Msgs; +append_msg([], Msg) -> [Msg]; +append_msg(Q, Msgs) when is_list(Msgs) -> + lists:append(Q, Msgs); +append_msg(Q, Msg) -> + lists:append(Q, [Msg]). + +%%-------------------------------------------------------------------- +%% Handle a Msg + +handle_msg({'$gen_call', From, Req}, State) -> + case handle_call(From, Req, State) of + {reply, Reply, NState} -> + gen_server:reply(From, Reply), + {ok, NState}; + {stop, Reason, Reply, NState} -> + gen_server:reply(From, Reply), + stop(Reason, NState) + end; +handle_msg({'$gen_cast', Req}, State) -> + NewState = handle_cast(Req, State), + {ok, NewState}; + +handle_msg({Inet, _Sock, Data}, State = #state{channel = Channel}) + when Inet == tcp; + Inet == ssl -> + ?LOG(debug, "RECV ~0p", [Data]), + Oct = iolist_size(Data), + inc_counter(incoming_bytes, Oct), + Ctx = emqx_stomp_channel:info(ctx, Channel), + ok = emqx_gateway_ctx:metrics_inc(Ctx, 'bytes.received', Oct), + parse_incoming(Data, State); + +handle_msg({incoming, Packet}, State = #state{idle_timer = undefined}) -> + handle_incoming(Packet, State); + +handle_msg({incoming, Packet}, + State = #state{idle_timer = IdleTimer}) -> + ok = emqx_misc:cancel_timer(IdleTimer), + %% XXX: Serialize with inpunt packets + %%Serialize = emqx_stomp_frame:serialize_opts(), + NState = State#state{idle_timer = undefined}, + handle_incoming(Packet, NState); + +handle_msg({outgoing, Packets}, State) -> + handle_outgoing(Packets, State); + +handle_msg({Error, _Sock, Reason}, State) + when Error == tcp_error; Error == ssl_error -> + handle_info({sock_error, Reason}, State); + +handle_msg({Closed, _Sock}, State) + when Closed == tcp_closed; Closed == ssl_closed -> + handle_info({sock_closed, Closed}, close_socket(State)); + +handle_msg({Passive, _Sock}, State) + when Passive == tcp_passive; Passive == ssl_passive -> + %% In Stats + Pubs = emqx_pd:reset_counter(incoming_pubs), + Bytes = emqx_pd:reset_counter(incoming_bytes), + InStats = #{cnt => Pubs, oct => Bytes}, + %% Ensure Rate Limit + NState = ensure_rate_limit(InStats, State), + %% Run GC and Check OOM + NState1 = check_oom(run_gc(InStats, NState)), + handle_info(activate_socket, NState1); + +handle_msg(Deliver = {deliver, _Topic, _Msg}, + #state{active_n = ActiveN} = State) -> + Delivers = [Deliver|emqx_misc:drain_deliver(ActiveN)], + with_channel(handle_deliver, [Delivers], State); + +%% Something sent +handle_msg({inet_reply, _Sock, ok}, State = #state{active_n = ActiveN}) -> + case emqx_pd:get_counter(outgoing_pubs) > ActiveN of + true -> + Pubs = emqx_pd:reset_counter(outgoing_pubs), + Bytes = emqx_pd:reset_counter(outgoing_bytes), + OutStats = #{cnt => Pubs, oct => Bytes}, + {ok, run_gc(OutStats, State)}; + %% FIXME: check oom ??? + %%{ok, check_oom(run_gc(OutStats, State))}; + false -> ok + end; + +handle_msg({inet_reply, _Sock, {error, Reason}}, State) -> + handle_info({sock_error, Reason}, State); + +handle_msg({connack, ConnAck}, State) -> + handle_outgoing(ConnAck, State); + +handle_msg({close, Reason}, State) -> + ?LOG(debug, "Force to close the socket due to ~p", [Reason]), + handle_info({sock_closed, Reason}, close_socket(State)); + +handle_msg({event, connected}, State = #state{channel = Channel}) -> + Ctx = emqx_stomp_channel:info(ctx, Channel), + ClientId = emqx_stomp_channel:info(clientid, Channel), + emqx_gateway_ctx:insert_channel_info( + Ctx, + ClientId, + info(State), + stats(State) + ); + +handle_msg({event, disconnected}, State = #state{channel = Channel}) -> + Ctx = emqx_stomp_channel:info(ctx, Channel), + ClientId = emqx_stomp_channel:info(clientid, Channel), + emqx_gateway_ctx:set_chan_info(Ctx, ClientId, info(State)), + emqx_gateway_ctx:connection_closed(Ctx, ClientId), + {ok, State}; + +handle_msg({event, _Other}, State = #state{channel = Channel}) -> + Ctx = emqx_stomp_channel:info(ctx, Channel), + ClientId = emqx_stomp_channel:info(clientid, Channel), + emqx_gateway_ctx:set_chan_info(Ctx, ClientId, info(State)), + emqx_gateway_ctx:set_chan_stats(Ctx, ClientId, stats(State)), + {ok, State}; + +handle_msg({timeout, TRef, TMsg}, State) -> + handle_timeout(TRef, TMsg, State); + +handle_msg(Shutdown = {shutdown, _Reason}, State) -> + stop(Shutdown, State); + +handle_msg(Msg, State) -> + handle_info(Msg, State). + +%%-------------------------------------------------------------------- +%% Terminate + +-spec terminate(any(), state()) -> no_return(). +terminate(Reason, State = #state{channel = Channel, transport = _Transport, + socket = _Socket}) -> + try + Channel1 = emqx_stomp_channel:set_conn_state(disconnected, Channel), + %emqx_congestion:cancel_alarms(Socket, Transport, Channel1), + emqx_stomp_channel:terminate(Reason, Channel1), + close_socket_ok(State) + catch + E : C : S -> + ?tp(warning, unclean_terminate, #{exception => E, context => C, stacktrace => S}) + end, + ?tp(info, terminate, #{reason => Reason}), + maybe_raise_excption(Reason). + +%% close socket, discard new state, always return ok. +close_socket_ok(State) -> + _ = close_socket(State), + ok. + +%% tell truth about the original exception +maybe_raise_excption(#{exception := Exception, + context := Context, + stacktrace := Stacktrace + }) -> + erlang:raise(Exception, Context, Stacktrace); +maybe_raise_excption(Reason) -> + exit(Reason). + +%%-------------------------------------------------------------------- +%% Sys callbacks + +system_continue(Parent, _Debug, State) -> + ?MODULE:recvloop(Parent, State). + +system_terminate(Reason, _Parent, _Debug, State) -> + terminate(Reason, State). + +system_code_change(State, _Mod, _OldVsn, _Extra) -> + {ok, State}. + +system_get_state(State) -> {ok, State}. + +%%-------------------------------------------------------------------- +%% Handle call + +handle_call(_From, info, State) -> + {reply, info(State), State}; + +handle_call(_From, stats, State) -> + {reply, stats(State), State}; + +%% TODO: How to set ratelimit ??? +%%handle_call(_From, {ratelimit, Policy}, State = #state{channel = Channel}) -> +%% Zone = emqx_stomp_channel:info(zone, Channel), +%% Limiter = emqx_limiter:init(Zone, Policy), +%% {reply, ok, State#state{limiter = Limiter}}; + +handle_call(_From, Req, State = #state{channel = Channel}) -> + case emqx_stomp_channel:handle_call(Req, Channel) of + {reply, Reply, NChannel} -> + {reply, Reply, State#state{channel = NChannel}}; + {shutdown, Reason, Reply, NChannel} -> + shutdown(Reason, Reply, State#state{channel = NChannel}); + {shutdown, Reason, Reply, OutPacket, NChannel} -> + NState = State#state{channel = NChannel}, + ok = handle_outgoing(OutPacket, NState), + shutdown(Reason, Reply, NState) + end. + +%%-------------------------------------------------------------------- +%% Handle timeout + +handle_timeout(_TRef, idle_timeout, State) -> + shutdown(idle_timeout, State); + +handle_timeout(_TRef, limit_timeout, State) -> + NState = State#state{sockstate = idle, + limit_timer = undefined + }, + handle_info(activate_socket, NState); + +handle_timeout(_TRef, emit_stats, State = #state{channel = Channel, + transport = _Transport, + socket = _Socket}) -> + %emqx_congestion:maybe_alarm_conn_congestion(Socket, Transport, Channel), + Ctx = emqx_stomp_channel:info(ctx, Channel), + ClientId = emqx_stomp_channel:info(clientid, Channel), + emqx_gateway_ctx:set_chan_stats(Ctx, ClientId, stats(State)), + {ok, State#state{stats_timer = undefined}}; + +%% Abstraction ??? +%handle_timeout(TRef, keepalive, State = #state{transport = Transport, +% socket = Socket, +% channel = Channel})-> +% case emqx_stomp_channel:info(conn_state, Channel) of +% disconnected -> {ok, State}; +% _ -> +% case Transport:getstat(Socket, [recv_oct]) of +% {ok, [{recv_oct, RecvOct}]} -> +% handle_timeout(TRef, {keepalive, RecvOct}, State); +% {error, Reason} -> +% handle_info({sock_error, Reason}, State) +% end +% end; + +handle_timeout(TRef, TMsg, State = #state{transport = Transport, + socket = Socket, + channel = Channel + }) + when TMsg =:= incoming; + TMsg =:= outgoing -> + Stat = case TMsg of incoming -> recv_oct; _ -> send_oct end, + case emqx_stomp_channel:info(conn_state, Channel) of + disconnected -> {ok, State}; + _ -> + case Transport:getstat(Socket, [Stat]) of + {ok, [{recv_oct, RecvOct}]} -> + handle_timeout(TRef, {incoming, RecvOct}, State); + {ok, [{send_oct, SendOct}]} -> + handle_timeout(TRef, {outgoing, SendOct}, State); + {error, Reason} -> + handle_info({sock_error, Reason}, State) + end + end; + +handle_timeout(TRef, Msg, State) -> + with_channel(handle_timeout, [TRef, Msg], State). + +%%-------------------------------------------------------------------- +%% Parse incoming data + +parse_incoming(Data, State) -> + {Packets, NState} = parse_incoming(Data, [], State), + {ok, next_incoming_msgs(Packets), NState}. + +parse_incoming(<<>>, Packets, State) -> + {Packets, State}; + +parse_incoming(Data, Packets, State = #state{parse_state = ParseState}) -> + try emqx_stomp_frame:parse(Data, ParseState) of + {more, NParseState} -> + {Packets, State#state{parse_state = NParseState}}; + {ok, Packet, Rest, NParseState} -> + NState = State#state{parse_state = NParseState}, + parse_incoming(Rest, [Packet|Packets], NState) + catch + error:Reason:Stk -> + ?LOG(error, "~nParse failed for ~0p~n~0p~nFrame data:~0p", + [Reason, Stk, Data]), + {[{frame_error, Reason}|Packets], State} + end. + +next_incoming_msgs([Packet]) -> + {incoming, Packet}; +next_incoming_msgs(Packets) -> + [{incoming, Packet} || Packet <- lists:reverse(Packets)]. + +%%-------------------------------------------------------------------- +%% Handle incoming packet + +handle_incoming(Packet, State) when is_record(Packet, stomp_frame) -> + ok = inc_incoming_stats(Packet), + ?LOG(debug, "RECV ~s", [emqx_stomp_frame:format(Packet)]), + with_channel(handle_in, [Packet], State); + +handle_incoming(FrameError, State) -> + with_channel(handle_in, [FrameError], State). + +%%-------------------------------------------------------------------- +%% With Channel + +with_channel(Fun, Args, State = #state{channel = Channel}) -> + case erlang:apply(emqx_stomp_channel, Fun, Args ++ [Channel]) of + ok -> {ok, State}; + {ok, NChannel} -> + {ok, State#state{channel = NChannel}}; + {ok, Replies, NChannel} -> + {ok, next_msgs(Replies), State#state{channel = NChannel}}; + {shutdown, Reason, NChannel} -> + shutdown(Reason, State#state{channel = NChannel}); + {shutdown, Reason, Packet, NChannel} -> + NState = State#state{channel = NChannel}, + ok = handle_outgoing(Packet, NState), + shutdown(Reason, NState) + end. + +%%-------------------------------------------------------------------- +%% Handle outgoing packets + +handle_outgoing(Packets, State) when is_list(Packets) -> + send(lists:map(serialize_and_inc_stats_fun(State), Packets), State); + +handle_outgoing(Packet, State) -> + send((serialize_and_inc_stats_fun(State))(Packet), State). + +serialize_and_inc_stats_fun(#state{serialize = Serialize, channel = Channel}) -> + Ctx = emqx_stomp_channel:info(ctx, Channel), + fun(Packet) -> + case emqx_stomp_frame:serialize_pkt(Packet, Serialize) of + <<>> -> ?LOG(warning, "~s is discarded due to the frame is too large!", + [emqx_stomp_frame:format(Packet)]), + ok = emqx_gateway_ctx:metrics_inc(Ctx, 'delivery.dropped.too_large'), + ok = emqx_gateway_ctx:metrics_inc(Ctx, 'delivery.dropped'), + <<>>; + Data -> ?LOG(debug, "SEND ~s", [emqx_stomp_frame:format(Packet)]), + ok = inc_outgoing_stats(Packet), + Data + end + end. + +%%-------------------------------------------------------------------- +%% Send data + +-spec(send(iodata(), state()) -> ok). +send(IoData, #state{transport = Transport, socket = Socket, channel = Channel}) -> + Ctx = emqx_stomp_channel:info(ctx, Channel), + Oct = iolist_size(IoData), + ok = emqx_gateway_ctx:metrics_inc(Ctx, 'bytes.sent', Oct), + inc_counter(outgoing_bytes, Oct), + %emqx_congestion:maybe_alarm_conn_congestion(Socket, Transport, Channel), + case Transport:async_send(Socket, IoData, [nosuspend]) of + ok -> ok; + Error = {error, _Reason} -> + %% Send an inet_reply to postpone handling the error + self() ! {inet_reply, Socket, Error}, + ok + end. + +%%-------------------------------------------------------------------- +%% Handle Info + +handle_info(activate_socket, State = #state{sockstate = OldSst}) -> + case activate_socket(State) of + {ok, NState = #state{sockstate = NewSst}} -> + case OldSst =/= NewSst of + true -> {ok, {event, NewSst}, NState}; + false -> {ok, NState} + end; + {error, Reason} -> + handle_info({sock_error, Reason}, State) + end; + +handle_info({sock_error, Reason}, State) -> + case Reason =/= closed andalso Reason =/= einval of + true -> ?LOG(warning, "socket_error: ~p", [Reason]); + false -> ok + end, + handle_info({sock_closed, Reason}, close_socket(State)); + +handle_info(Info, State) -> + with_channel(handle_info, [Info], State). + +%%-------------------------------------------------------------------- +%% Handle Info + +handle_cast({async_set_socket_options, Opts}, + State = #state{transport = Transport, + socket = Socket + }) -> + case Transport:setopts(Socket, Opts) of + ok -> ?tp(info, "custom_socket_options_successfully", #{opts => Opts}); + Err -> ?tp(error, "failed_to_set_custom_socket_optionn", #{reason => Err}) + end, + State; +handle_cast(Req, State) -> + ?tp(error, "received_unknown_cast", #{cast => Req}), + State. + +%%-------------------------------------------------------------------- +%% Ensure rate limit + +ensure_rate_limit(Stats, State = #state{limiter = Limiter}) -> + case ?ENABLED(Limiter) andalso emqx_limiter:check(Stats, Limiter) of + false -> State; + {ok, Limiter1} -> + State#state{limiter = Limiter1}; + {pause, Time, Limiter1} -> + ?LOG(warning, "Pause ~pms due to rate limit", [Time]), + TRef = emqx_misc:start_timer(Time, limit_timeout), + State#state{sockstate = blocked, + limiter = Limiter1, + limit_timer = TRef + } + end. + +%%-------------------------------------------------------------------- +%% Run GC and Check OOM + +run_gc(Stats, State = #state{gc_state = GcSt}) -> + case ?ENABLED(GcSt) andalso emqx_gc:run(Stats, GcSt) of + false -> State; + {_IsGC, GcSt1} -> + State#state{gc_state = GcSt1} + end. + +check_oom(State = #state{channel = Channel}) -> + Zone = emqx_stomp_channel:info(zone, Channel), + OomPolicy = emqx_gateway_utils:oom_policy(Zone), + ?tp(debug, check_oom, #{policy => OomPolicy}), + case ?ENABLED(OomPolicy) andalso emqx_misc:check_oom(OomPolicy) of + {shutdown, Reason} -> + %% triggers terminate/2 callback immediately + erlang:exit({shutdown, Reason}); + _Other -> + ok + end, + State. + +%%-------------------------------------------------------------------- +%% Activate Socket + +-compile({inline, [activate_socket/1]}). +activate_socket(State = #state{sockstate = closed}) -> + {ok, State}; +activate_socket(State = #state{sockstate = blocked}) -> + {ok, State}; +activate_socket(State = #state{transport = Transport, + socket = Socket, + active_n = N}) -> + case Transport:setopts(Socket, [{active, N}]) of + ok -> {ok, State#state{sockstate = running}}; + Error -> Error + end. + +%%-------------------------------------------------------------------- +%% Close Socket + +close_socket(State = #state{sockstate = closed}) -> State; +close_socket(State = #state{transport = Transport, socket = Socket}) -> + ok = Transport:fast_close(Socket), + State#state{sockstate = closed}. + +%%-------------------------------------------------------------------- +%% Inc incoming/outgoing stats + +%% XXX: Other packet type? +inc_incoming_stats(Packet = ?PACKET(Type)) -> + inc_counter(recv_pkt, 1), + case Type =:= ?CMD_SEND of + true -> + inc_counter(recv_msg, 1), + inc_counter(incoming_pubs, 1); + false -> + ok + end, + emqx_metrics:inc_recv(Packet). + +inc_outgoing_stats(Packet = ?PACKET(Type)) -> + inc_counter(send_pkt, 1), + case Type =:= ?CMD_MESSAGE of + true -> + inc_counter(send_msg, 1), + inc_counter(outgoing_pubs, 1); + false -> + ok + end, + emqx_metrics:inc_sent(Packet). + +%%-------------------------------------------------------------------- +%% Helper functions + +next_msgs(Packet) when is_record(Packet, stomp_frame) -> + {outgoing, Packet}; +next_msgs(Event) when is_tuple(Event) -> + Event; +next_msgs(More) when is_list(More) -> + More. + +shutdown(Reason, State) -> + stop({shutdown, Reason}, State). + +shutdown(Reason, Reply, State) -> + stop({shutdown, Reason}, Reply, State). + +stop(Reason, State) -> + {stop, Reason, State}. + +stop(Reason, Reply, State) -> + {stop, Reason, Reply, State}. + +inc_counter(Key, Inc) -> + _ = emqx_pd:inc_counter(Key, Inc), + ok. + +%%-------------------------------------------------------------------- +%% For CT tests +%%-------------------------------------------------------------------- + +set_field(Name, Value, State) -> + Pos = emqx_misc:index_of(Name, record_info(fields, state)), + setelement(Pos+1, State, Value). + +get_state(Pid) -> + State = sys:get_state(Pid), + maps:from_list(lists:zip(record_info(fields, state), + tl(tuple_to_list(State)))). diff --git a/apps/emqx_stomp/src/emqx_stomp_frame.erl b/apps/emqx_gateway/src/stomp/emqx_stomp_frame.erl similarity index 70% rename from apps/emqx_stomp/src/emqx_stomp_frame.erl rename to apps/emqx_gateway/src/stomp/emqx_stomp_frame.erl index fa9cb63a8..4db8a1f5f 100644 --- a/apps/emqx_stomp/src/emqx_stomp_frame.erl +++ b/apps/emqx_gateway/src/stomp/emqx_stomp_frame.erl @@ -68,14 +68,16 @@ -module(emqx_stomp_frame). --include("emqx_stomp.hrl"). +-include("src/stomp/include/emqx_stomp.hrl"). --export([ init_parer_state/1 +-export([ initial_parse_state/1 , parse/2 - , serialize/1 + , serialize_opts/0 + , serialize_pkt/2 ]). --export([ make/2 +-export([ make/1 + , make/2 , make/3 , format/1 ]). @@ -96,28 +98,33 @@ -record(frame_limit, {max_header_num, max_header_length, max_body_length}). --type(result() :: {ok, stomp_frame(), binary()} - | {more, parser()} - | {error, any()}). +-type(parse_result() :: {ok, stomp_frame(), binary()} + | {more, parse_state()}). --type(parser() :: #{phase := none | command | headers | hdname | hdvalue | body, - pre => binary(), - state := #parser_state{}}). +-type(parse_state() :: + #{phase := none | command | headers | hdname | hdvalue | body, + pre => binary(), + state := #parser_state{} + }). + +-dialyzer({nowarn_function, [serialize_pkt/2,make/1]}). %% @doc Initialize a parser --spec init_parer_state([proplists:property()]) -> parser(). -init_parer_state(Opts) -> +-spec initial_parse_state(map()) -> parse_state(). +initial_parse_state(Opts) -> #{phase => none, state => #parser_state{limit = limit(Opts)}}. limit(Opts) -> - #frame_limit{max_header_num = g(max_header_num, Opts, ?MAX_HEADER_NUM), - max_header_length = g(max_header_length, Opts, ?MAX_BODY_LENGTH), - max_body_length = g(max_body_length, Opts, ?MAX_BODY_LENGTH)}. + #frame_limit{ + max_header_num = g(max_header_num, Opts, ?MAX_HEADER_NUM), + max_header_length = g(max_header_length, Opts, ?MAX_BODY_LENGTH), + max_body_length = g(max_body_length, Opts, ?MAX_BODY_LENGTH) + }. g(Key, Opts, Val) -> - proplists:get_value(Key, Opts, Val). + maps:get(Key, Opts, Val). --spec parse(binary(), parser()) -> result(). +-spec parse(binary(), parse_state()) -> parse_result(). parse(<<>>, Parser) -> {more, Parser}; @@ -131,11 +138,14 @@ parse(<>, #{phase := Phase, state := State}) -> parse(<>, Parser) -> {more, Parser#{pre => <>}}; parse(<>, _Parser) -> - {error, linefeed_expected}; + error(linefeed_expected); -parse(<>, Parser = #{phase := Phase}) when Phase =:= hdname; Phase =:= hdvalue -> +parse(<>, Parser = #{phase := Phase}) when Phase =:= hdname; + Phase =:= hdvalue -> {more, Parser#{pre => <>}}; -parse(<>, #{phase := Phase, state := State}) when Phase =:= hdname; Phase =:= hdvalue -> +parse(<>, + #{phase := Phase, state := State}) when Phase =:= hdname; + Phase =:= hdvalue -> parse(Phase, Rest, acc(unescape(Ch), State)); parse(Bytes, #{phase := none, state := State}) -> @@ -153,14 +163,19 @@ parse(headers, Bin, State) -> parse(hdname, Bin, State); parse(hdname, <>, _State) -> - {error, unexpected_linefeed}; + error(unexpected_linefeed); parse(hdname, <>, State = #parser_state{acc = Acc}) -> parse(hdvalue, Rest, State#parser_state{hdname = Acc, acc = <<>>}); parse(hdname, <>, State) -> parse(hdname, Rest, acc(Ch, State)); -parse(hdvalue, <>, State = #parser_state{headers = Headers, hdname = Name, acc = Acc}) -> - parse(headers, Rest, State#parser_state{headers = add_header(Name, Acc, Headers), hdname = undefined, acc = <<>>}); +parse(hdvalue, <>, + State = #parser_state{headers = Headers, hdname = Name, acc = Acc}) -> + NState = State#parser_state{headers = add_header(Name, Acc, Headers), + hdname = undefined, + acc = <<>> + }, + parse(headers, Rest, NState); parse(hdvalue, <>, State) -> parse(hdvalue, Rest, acc(Ch, State)). @@ -170,15 +185,19 @@ parse(body, <<>>, State, Length) -> parse(body, Bin, State, none) -> case binary:split(Bin, <>) of [Chunk, Rest] -> - {ok, new_frame(acc(Chunk, State)), Rest}; + {ok, new_frame(acc(Chunk, State)), Rest, new_state(State)}; [Chunk] -> - {more, #{phase => body, length => none, state => acc(Chunk, State)}} + {more, #{phase => body, + length => none, + state => acc(Chunk, State)}} end; parse(body, Bin, State, Len) when byte_size(Bin) >= (Len+1) -> <> = Bin, - {ok, new_frame(acc(Chunk, State)), Rest}; + {ok, new_frame(acc(Chunk, State)), Rest, new_state(State)}; parse(body, Bin, State, Len) -> - {more, #{phase => body, length => Len - byte_size(Bin), state => acc(Bin, State)}}. + {more, #{phase => body, + length => Len - byte_size(Bin), + state => acc(Bin, State)}}. add_header(Name, Value, Headers) -> case lists:keyfind(Name, 1, Headers) of @@ -208,20 +227,33 @@ unescape($r) -> ?CR; unescape($n) -> ?LF; unescape($c) -> ?COLON; unescape($\\) -> ?BSL; -unescape(_Ch) -> {error, cannnot_unescape}. +unescape(_Ch) -> error(cannnot_unescape). -serialize(#stomp_frame{command = Cmd, headers = Headers, body = Body}) -> +%%-------------------------------------------------------------------- +%% Serialize funcs +%%-------------------------------------------------------------------- + +serialize_opts() -> + #{}. + +serialize_pkt(#stomp_frame{command = heartbeat}, _SerializeOpts) -> + <<$\n>>; + +serialize_pkt(#stomp_frame{command = Cmd, headers = Headers, body = Body}, + _SerializeOpts) -> Headers1 = lists:keydelete(<<"content-length">>, 1, Headers), Headers2 = case iolist_size(Body) of 0 -> Headers1; Len -> Headers1 ++ [{<<"content-length">>, Len}] end, - [Cmd, ?LF, [serialize(header, Header) || Header <- Headers2], ?LF, Body, 0]. + [Cmd, + ?LF, [serialize_pkt(header, Header) || Header <- Headers2], + ?LF, Body, 0]; -serialize(header, {Name, Val}) when is_integer(Val) -> +serialize_pkt(header, {Name, Val}) when is_integer(Val) -> [escape(Name), ?COLON, integer_to_list(Val), ?LF]; -serialize(header, {Name, Val}) -> +serialize_pkt(header, {Name, Val}) -> [escape(Name), ?COLON, escape(Val), ?LF]. escape(Bin) when is_binary(Bin) -> @@ -232,8 +264,18 @@ escape(?BSL) -> <>; escape(?COLON) -> <>; escape(Ch) -> <>. +new_state(#parser_state{limit = Limit}) -> + #{phase => none, state => #parser_state{limit = Limit}}. + +%%-------------------------------------------------------------------- +%% ??? +%%-------------------------------------------------------------------- %% @doc Make a frame + +make(heartbeat) -> + #stomp_frame{command = heartbeat}. + make(<<"CONNECTED">>, Headers) -> #stomp_frame{command = <<"CONNECTED">>, headers = [{<<"server">>, ?STOMP_SERVER} | Headers]}; @@ -245,5 +287,4 @@ make(Command, Headers, Body) -> #stomp_frame{command = Command, headers = Headers, body = Body}. %% @doc Format a frame -format(Frame) -> serialize(Frame). - +format(Frame) -> serialize_pkt(Frame, #{}). diff --git a/apps/emqx_stomp/src/emqx_stomp_heartbeat.erl b/apps/emqx_gateway/src/stomp/emqx_stomp_heartbeat.erl similarity index 89% rename from apps/emqx_stomp/src/emqx_stomp_heartbeat.erl rename to apps/emqx_gateway/src/stomp/emqx_stomp_heartbeat.erl index 145359e53..99a1508e1 100644 --- a/apps/emqx_stomp/src/emqx_stomp_heartbeat.erl +++ b/apps/emqx_gateway/src/stomp/emqx_stomp_heartbeat.erl @@ -17,10 +17,11 @@ %% @doc Stomp heartbeat. -module(emqx_stomp_heartbeat). --include("emqx_stomp.hrl"). +-include("src/stomp/include/emqx_stomp.hrl"). -export([ init/1 , check/3 + , reset/3 , info/1 , interval/2 ]). @@ -33,7 +34,6 @@ outgoing => #heartbeater{} }. - %%-------------------------------------------------------------------- %% APIs %%-------------------------------------------------------------------- @@ -77,6 +77,15 @@ check(NewVal, HrtBter = #heartbeater{statval = OldVal, true -> {error, timeout} end. +-spec reset(name(), pos_integer(), heartbeat()) + -> heartbeat(). +reset(Name, NewVal, HrtBt) -> + HrtBter = maps:get(Name, HrtBt), + HrtBt#{Name => reset(NewVal, HrtBter)}. + +reset(NewVal, HrtBter) -> + HrtBter#heartbeater{statval = NewVal, repeat = 1}. + -spec info(heartbeat()) -> map(). info(HrtBt) -> maps:map(fun(_, #heartbeater{interval = Intv, diff --git a/apps/emqx_gateway/src/stomp/emqx_stomp_impl.erl b/apps/emqx_gateway/src/stomp/emqx_stomp_impl.erl new file mode 100644 index 000000000..e6e62565a --- /dev/null +++ b/apps/emqx_gateway/src/stomp/emqx_stomp_impl.erl @@ -0,0 +1,153 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2017-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_stomp_impl). + +-include_lib("emqx_gateway/include/emqx_gateway.hrl"). + +-behavior(emqx_gateway_impl). + +%% APIs +-export([ load/0 + , unload/0 + ]). + +-export([ init/1 + , on_insta_create/3 + , on_insta_update/4 + , on_insta_destroy/3 + ]). + +-define(TCP_OPTS, [binary, {packet, raw}, {reuseaddr, true}, {nodelay, true}]). + +-dialyzer({nowarn_function, [load/0]}). + +%%-------------------------------------------------------------------- +%% APIs +%%-------------------------------------------------------------------- + +load() -> + RegistryOptions = [ {cbkmod, ?MODULE} + , {schema, emqx_stomp_schema} + ], + + YourOptions = [param1, param2], + emqx_gateway_registry:load(stomp, RegistryOptions, YourOptions). + +unload() -> + emqx_gateway_registry:unload(stomp). + +init([param1, param2]) -> + GwState = #{}, + {ok, GwState}. + +%%-------------------------------------------------------------------- +%% emqx_gateway_registry callbacks +%%-------------------------------------------------------------------- + +on_insta_create(_Insta = #{ id := InstaId, + rawconf := RawConf + }, Ctx, _GwState) -> + %% Step1. Fold the rawconfs to listeners + Listeners = emqx_gateway_utils:normalize_rawconf(RawConf), + %% Step2. Start listeners or escokd:specs + ListenerPids = lists:map(fun(Lis) -> + start_listener(InstaId, Ctx, Lis) + end, Listeners), + %% FIXME: How to throw an exception to interrupt the restart logic ? + %% FIXME: Assign ctx to InstaState + {ok, ListenerPids, _InstaState = #{ctx => Ctx}}. + +%% @private +on_insta_update(NewInsta, OldInstace, GwInstaState = #{ctx := Ctx}, GwState) -> + InstaId = maps:get(id, NewInsta), + try + %% XXX: 1. How hot-upgrade the changes ??? + %% XXX: 2. Check the New confs first before destroy old instance ??? + on_insta_destroy(OldInstace, GwInstaState, GwState), + on_insta_create(NewInsta, Ctx, GwState) + catch + Class : Reason : Stk -> + logger:error("Failed to update stomp instance ~s; " + "reason: {~0p, ~0p} stacktrace: ~0p", + [InstaId, Class, Reason, Stk]), + {error, {Class, Reason}} + end. + +on_insta_destroy(_Insta = #{ id := InstaId, + rawconf := RawConf + }, _GwInstaState, _GwState) -> + Listeners = emqx_gateway_utils:normalize_rawconf(RawConf), + lists:foreach(fun(Lis) -> + stop_listener(InstaId, Lis) + end, Listeners). + +%%-------------------------------------------------------------------- +%% Internal funcs +%%-------------------------------------------------------------------- + +start_listener(InstaId, Ctx, {Type, ListenOn, SocketOpts, Cfg}) -> + case start_listener(InstaId, Ctx, Type, ListenOn, SocketOpts, Cfg) of + {ok, Pid} -> + io:format("Start stomp ~s:~s listener on ~s successfully.~n", + [InstaId, Type, format(ListenOn)]), + Pid; + {error, Reason} -> + io:format(standard_error, + "Failed to start stomp ~s:~s listener on ~s: ~0p~n", + [InstaId, Type, format(ListenOn), Reason]), + throw({badconf, Reason}) + end. + +start_listener(InstaId, Ctx, Type, ListenOn, SocketOpts, Cfg) -> + Name = name(InstaId, Type), + esockd:open(Name, ListenOn, merge_default(SocketOpts), + {emqx_stomp_connection, start_link, [Cfg#{ctx => Ctx}]}). + +name(InstaId, Type) -> + list_to_atom(lists:concat([InstaId, ":", Type])). + +merge_default(Options) -> + case lists:keytake(tcp_options, 1, Options) of + {value, {tcp_options, TcpOpts}, Options1} -> + [{tcp_options, emqx_misc:merge_opts(?TCP_OPTS, TcpOpts)} | Options1]; + false -> + [{tcp_options, ?TCP_OPTS} | Options] + end. + +format(Port) when is_integer(Port) -> + io_lib:format("0.0.0.0:~w", [Port]); +format({Addr, Port}) when is_list(Addr) -> + io_lib:format("~s:~w", [Addr, Port]); +format({Addr, Port}) when is_tuple(Addr) -> + io_lib:format("~s:~w", [inet:ntoa(Addr), Port]). + +stop_listener(InstaId, {Type, ListenOn, SocketOpts, Cfg}) -> + StopRet = stop_listener(InstaId, Type, ListenOn, SocketOpts, Cfg), + case StopRet of + ok -> io:format("Stop stomp ~s:~s listener on ~s successfully.~n", + [InstaId, Type, format(ListenOn)]); + {error, Reason} -> + io:format(standard_error, + "Failed to stop stomp ~s:~s listener on ~s: ~0p~n", + [InstaId, Type, format(ListenOn), Reason] + ) + end, + StopRet. + +stop_listener(InstaId, Type, ListenOn, _SocketOpts, _Cfg) -> + Name = name(InstaId, Type), + esockd:close(Name, ListenOn). diff --git a/apps/emqx_gateway/src/stomp/include/emqx_stomp.hrl b/apps/emqx_gateway/src/stomp/include/emqx_stomp.hrl new file mode 100644 index 000000000..cffcb1bdf --- /dev/null +++ b/apps/emqx_gateway/src/stomp/include/emqx_stomp.hrl @@ -0,0 +1,92 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-ifndef(EMQX_STOMP_HRL). +-define(EMQX_STOMP_HRL, true). + +-define(STOMP_VER, <<"1.2">>). + +-define(STOMP_SERVER, <<"emqx-stomp/1.2">>). + +%%-------------------------------------------------------------------- +%% STOMP Frame +%%-------------------------------------------------------------------- + +%% client command +-define(CMD_STOMP, <<"STOMP">>). +-define(CMD_CONNECT, <<"CONNECT">>). +-define(CMD_SEND, <<"SEND">>). +-define(CMD_SUBSCRIBE, <<"SUBSCRIBE">>). +-define(CMD_UNSUBSCRIBE, <<"UNSUBSCRIBE">>). +-define(CMD_BEGIN, <<"BEGIN">>). +-define(CMD_COMMIT, <<"COMMIT">>). +-define(CMD_ABORT, <<"ABORT">>). +-define(CMD_ACK, <<"ACK">>). +-define(CMD_NACK, <<"NACK">>). +-define(CMD_DISCONNECT, <<"DISCONNECT">>). + +%% server command +-define(CMD_CONNECTED, <<"CONNECTED">>). +-define(CMD_MESSAGE, <<"MESSAGE">>). +-define(CMD_RECEIPT, <<"RECEIPT">>). +-define(CMD_ERROR, <<"ERROR">>). + +-type client_command() :: binary(). +%-type client_command() :: ?CMD_SEND | ?CMD_SUBSCRIBE | ?CMD_UNSUBSCRIBE +% | ?CMD_BEGIN | ?CMD_COMMIT | ?CMD_ABORT | ?CMD_ACK +% | ?CMD_NACK | ?CMD_DISCONNECT | ?CMD_CONNECT +% | ?CMD_STOMP. +% +-type server_command() :: binary(). +%-type server_command() :: ?CMD_CONNECTED | ?CMD_MESSAGE | ?CMD_RECEIPT +% | ?CMD_ERROR. + +-record(stomp_frame, { + command :: client_command() | server_command(), + headers = [], + body = <<>> :: iodata()} + ). + +-type stomp_frame() :: #stomp_frame{}. + +-define(PACKET(CMD), #stomp_frame{command = CMD}). + +-define(PACKET(CMD, Headers), #stomp_frame{command = CMD, headers = Headers}). + +-define(PACKET(CMD, Headers, Body), #stomp_frame{command = CMD, + headers = Headers, + body = Body + }). + +%%-------------------------------------------------------------------- +%% Frame Size Limits +%% +%% To prevent malicious clients from exploiting memory allocation in a server, +%% servers MAY place maximum limits on: +%% +%% the number of frame headers allowed in a single frame +%% the maximum length of header lines +%% the maximum size of a frame body +%% +%% If these limits are exceeded the server SHOULD send the client an ERROR frame +%% and then close the connection. +%%-------------------------------------------------------------------- + +-define(MAX_HEADER_NUM, 10). +-define(MAX_HEADER_LENGTH, 1024). +-define(MAX_BODY_LENGTH, 65536). + +-endif. diff --git a/apps/emqx_gateway/test/emqx_gateway_registry_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_registry_SUITE.erl new file mode 100644 index 000000000..cfc9399bb --- /dev/null +++ b/apps/emqx_gateway/test/emqx_gateway_registry_SUITE.erl @@ -0,0 +1,87 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_gateway_registry_SUITE). + +-include_lib("eunit/include/eunit.hrl"). + +-compile(export_all). +-compile(nowarn_export_all). + +all() -> emqx_ct:all(?MODULE). + +%%-------------------------------------------------------------------- +%% Setups +%%-------------------------------------------------------------------- + +init_per_suite(Cfg) -> + emqx_ct_helpers:start_apps([emqx_gateway], fun set_special_configs/1), + Cfg. + +end_per_suite(_Cfg) -> + emqx_ct_helpers:stop_apps([emqx_gateway]), + ok. + +set_special_configs(emqx_gateway) -> + emqx_config:put( + [emqx_gateway], + #{stomp => + #{'1' => + #{authenticator => allow_anonymous, + clientinfo_override => + #{password => "${Packet.headers.passcode}", + username => "${Packet.headers.login}"}, + frame => + #{max_body_length => 8192, + max_headers => 10, + max_headers_length => 1024}, + listener => + #{tcp => + #{'1' => + #{acceptors => 16,active_n => 100,backlog => 1024, + bind => 61613,high_watermark => 1048576, + max_conn_rate => 1000,max_connections => 1024000, + send_timeout => 15000,send_timeout_close => true}}}}}}), + ok; +set_special_configs(_) -> + ok. + +%%-------------------------------------------------------------------- +%% Test cases +%%-------------------------------------------------------------------- + +t_load_unload(_) -> + OldCnt = length(emqx_gateway_registry:list()), + RgOpts = [{cbkmod, ?MODULE}], + GwOpts = [paramsin], + ok = emqx_gateway_registry:load(test, RgOpts, GwOpts), + ?assertEqual(OldCnt+1, length(emqx_gateway_registry:list())), + + #{cbkmod := ?MODULE, + rgopts := RgOpts, + gwopts := GwOpts, + state := #{gwstate := 1}} = emqx_gateway_registry:lookup(test), + + {error, already_existed} = emqx_gateway_registry:load(test, [{cbkmod, ?MODULE}], GwOpts), + + ok = emqx_gateway_registry:unload(test), + undefined = emqx_gateway_registry:lookup(test), + OldCnt = length(emqx_gateway_registry:list()), + ok. + +init([paramsin]) -> + {ok, _GwState = #{gwstate => 1}}. + diff --git a/apps/emqx_stomp/test/emqx_stomp_SUITE.erl b/apps/emqx_gateway/test/emqx_stomp_SUITE.erl similarity index 82% rename from apps/emqx_stomp/test/emqx_stomp_SUITE.erl rename to apps/emqx_gateway/test/emqx_stomp_SUITE.erl index 9a5d9698e..cc2b0db54 100644 --- a/apps/emqx_stomp/test/emqx_stomp_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_stomp_SUITE.erl @@ -16,7 +16,7 @@ -module(emqx_stomp_SUITE). --include_lib("emqx_stomp/include/emqx_stomp.hrl"). +-include_lib("emqx_gateway/src/stomp/include/emqx_stomp.hrl"). -compile(export_all). -compile(nowarn_export_all). @@ -29,12 +29,37 @@ all() -> emqx_ct:all(?MODULE). %% Setups %%-------------------------------------------------------------------- -init_per_suite(Config) -> - emqx_ct_helpers:start_apps([emqx_stomp]), - Config. +init_per_suite(Cfg) -> + emqx_ct_helpers:start_apps([emqx_gateway], fun set_special_configs/1), + Cfg. -end_per_suite(_Config) -> - emqx_ct_helpers:stop_apps([emqx_stomp]). +end_per_suite(_Cfg) -> + emqx_ct_helpers:stop_apps([emqx_gateway]), + ok. + +set_special_configs(emqx_gateway) -> + emqx_config:put( + [emqx_gateway], + #{stomp => + #{'1' => + #{authenticator => allow_anonymous, + clientinfo_override => + #{password => "${Packet.headers.passcode}", + username => "${Packet.headers.login}"}, + frame => + #{max_body_length => 8192, + max_headers => 10, + max_headers_length => 1024}, + listener => + #{tcp => + #{'1' => + #{acceptors => 16,active_n => 100,backlog => 1024, + bind => 61613,high_watermark => 1048576, + max_conn_rate => 1000,max_connections => 1024000, + send_timeout => 15000,send_timeout_close => true}}}}}}), + ok; +set_special_configs(_) -> + ok. %%-------------------------------------------------------------------- %% Test Cases @@ -52,7 +77,7 @@ t_connect(_) -> {ok, Data} = gen_tcp:recv(Sock, 0), {ok, Frame = #stomp_frame{command = <<"CONNECTED">>, headers = _, - body = _}, _} = parse(Data), + body = _}, _, _} = parse(Data), <<"2000,1000">> = proplists:get_value(<<"heart-beat">>, Frame#stomp_frame.headers), gen_tcp:send(Sock, serialize(<<"DISCONNECT">>, @@ -61,22 +86,23 @@ t_connect(_) -> {ok, Data1} = gen_tcp:recv(Sock, 0), {ok, #stomp_frame{command = <<"RECEIPT">>, headers = [{<<"receipt-id">>, <<"12345">>}], - body = _}, _} = parse(Data1) + body = _}, _, _} = parse(Data1) end), %% Connect will be failed, because of bad login or passcode - with_connection(fun(Sock) -> - gen_tcp:send(Sock, serialize(<<"CONNECT">>, - [{<<"accept-version">>, ?STOMP_VER}, - {<<"host">>, <<"127.0.0.1:61613">>}, - {<<"login">>, <<"admin">>}, - {<<"passcode">>, <<"admin">>}, - {<<"heart-beat">>, <<"1000,2000">>}])), - {ok, Data} = gen_tcp:recv(Sock, 0), - {ok, #stomp_frame{command = <<"ERROR">>, - headers = _, - body = <<"Login or passcode error!">>}, _} = parse(Data) - end), + %% FIXME: Waiting for authentication works + %with_connection(fun(Sock) -> + % gen_tcp:send(Sock, serialize(<<"CONNECT">>, + % [{<<"accept-version">>, ?STOMP_VER}, + % {<<"host">>, <<"127.0.0.1:61613">>}, + % {<<"login">>, <<"admin">>}, + % {<<"passcode">>, <<"admin">>}, + % {<<"heart-beat">>, <<"1000,2000">>}])), + % {ok, Data} = gen_tcp:recv(Sock, 0), + % {ok, #stomp_frame{command = <<"ERROR">>, + % headers = _, + % body = <<"Login or passcode error!">>}, _, _} = parse(Data) + % end), %% Connect will be failed, because of bad version with_connection(fun(Sock) -> @@ -89,7 +115,7 @@ t_connect(_) -> {ok, Data} = gen_tcp:recv(Sock, 0), {ok, #stomp_frame{command = <<"ERROR">>, headers = _, - body = <<"Supported protocol versions < 1.2">>}, _} = parse(Data) + body = <<"Login Failed: Supported protocol versions < 1.2">>}, _, _} = parse(Data) end). t_heartbeat(_) -> @@ -104,7 +130,7 @@ t_heartbeat(_) -> {ok, Data} = gen_tcp:recv(Sock, 0), {ok, #stomp_frame{command = <<"CONNECTED">>, headers = _, - body = _}, _} = parse(Data), + body = _}, _, _} = parse(Data), {ok, ?HEARTBEAT} = gen_tcp:recv(Sock, 0), %% Server will close the connection because never receive the heart beat from client @@ -122,7 +148,7 @@ t_subscribe(_) -> {ok, Data} = gen_tcp:recv(Sock, 0), {ok, #stomp_frame{command = <<"CONNECTED">>, headers = _, - body = _}, _} = parse(Data), + body = _}, _, _} = parse(Data), %% Subscribe gen_tcp:send(Sock, serialize(<<"SUBSCRIBE">>, @@ -139,7 +165,7 @@ t_subscribe(_) -> {ok, Data1} = gen_tcp:recv(Sock, 0, 1000), {ok, Frame = #stomp_frame{command = <<"MESSAGE">>, headers = _, - body = <<"hello">>}, _} = parse(Data1), + body = <<"hello">>}, _, _} = parse(Data1), lists:foreach(fun({Key, Val}) -> Val = proplists:get_value(Key, Frame#stomp_frame.headers) end, [{<<"destination">>, <<"/queue/foo">>}, @@ -155,7 +181,7 @@ t_subscribe(_) -> {ok, #stomp_frame{command = <<"RECEIPT">>, headers = [{<<"receipt-id">>, <<"12345">>}], - body = _}, _} = parse(Data2), + body = _}, _, _} = parse(Data2), gen_tcp:send(Sock, serialize(<<"SEND">>, [{<<"destination">>, <<"/queue/foo">>}], @@ -175,7 +201,7 @@ t_transaction(_) -> {ok, Data} = gen_tcp:recv(Sock, 0), {ok, #stomp_frame{command = <<"CONNECTED">>, headers = _, - body = _}, _} = parse(Data), + body = _}, _, _} = parse(Data), %% Subscribe gen_tcp:send(Sock, serialize(<<"SUBSCRIBE">>, @@ -208,12 +234,12 @@ t_transaction(_) -> {ok, #stomp_frame{command = <<"MESSAGE">>, headers = _, - body = <<"hello">>}, Rest1} = parse(Data1), + body = <<"hello">>}, Rest1, _} = parse(Data1), %{ok, Data2} = gen_tcp:recv(Sock, 0, 500), {ok, #stomp_frame{command = <<"MESSAGE">>, headers = _, - body = <<"hello again">>}, _Rest2} = parse(Rest1), + body = <<"hello again">>}, _Rest2, _} = parse(Rest1), %% Transaction: tx2 gen_tcp:send(Sock, serialize(<<"BEGIN">>, @@ -236,7 +262,7 @@ t_transaction(_) -> {ok, Data3} = gen_tcp:recv(Sock, 0), {ok, #stomp_frame{command = <<"RECEIPT">>, headers = [{<<"receipt-id">>, <<"12345">>}], - body = _}, _} = parse(Data3) + body = _}, _, _} = parse(Data3) end). t_receipt_in_error(_) -> @@ -250,7 +276,7 @@ t_receipt_in_error(_) -> {ok, Data} = gen_tcp:recv(Sock, 0), {ok, #stomp_frame{command = <<"CONNECTED">>, headers = _, - body = _}, _} = parse(Data), + body = _}, _, _} = parse(Data), gen_tcp:send(Sock, serialize(<<"ABORT">>, [{<<"transaction">>, <<"tx1">>}, @@ -259,7 +285,7 @@ t_receipt_in_error(_) -> {ok, Data1} = gen_tcp:recv(Sock, 0), {ok, Frame = #stomp_frame{command = <<"ERROR">>, headers = _, - body = <<"Transaction tx1 not found">>}, _} = parse(Data1), + body = <<"Transaction tx1 not found">>}, _, _} = parse(Data1), <<"12345">> = proplists:get_value(<<"receipt-id">>, Frame#stomp_frame.headers) end). @@ -275,7 +301,7 @@ t_ack(_) -> {ok, Data} = gen_tcp:recv(Sock, 0), {ok, #stomp_frame{command = <<"CONNECTED">>, headers = _, - body = _}, _} = parse(Data), + body = _}, _, _} = parse(Data), %% Subscribe gen_tcp:send(Sock, serialize(<<"SUBSCRIBE">>, @@ -290,7 +316,7 @@ t_ack(_) -> {ok, Data1} = gen_tcp:recv(Sock, 0), {ok, Frame = #stomp_frame{command = <<"MESSAGE">>, headers = _, - body = <<"ack test">>}, _} = parse(Data1), + body = <<"ack test">>}, _, _} = parse(Data1), AckId = proplists:get_value(<<"ack">>, Frame#stomp_frame.headers), @@ -301,7 +327,7 @@ t_ack(_) -> {ok, Data2} = gen_tcp:recv(Sock, 0), {ok, #stomp_frame{command = <<"RECEIPT">>, headers = [{<<"receipt-id">>, <<"12345">>}], - body = _}, _} = parse(Data2), + body = _}, _, _} = parse(Data2), gen_tcp:send(Sock, serialize(<<"SEND">>, [{<<"destination">>, <<"/queue/foo">>}], @@ -310,7 +336,7 @@ t_ack(_) -> {ok, Data3} = gen_tcp:recv(Sock, 0), {ok, Frame1 = #stomp_frame{command = <<"MESSAGE">>, headers = _, - body = <<"nack test">>}, _} = parse(Data3), + body = <<"nack test">>}, _, _} = parse(Data3), AckId1 = proplists:get_value(<<"ack">>, Frame1#stomp_frame.headers), @@ -321,9 +347,16 @@ t_ack(_) -> {ok, Data4} = gen_tcp:recv(Sock, 0), {ok, #stomp_frame{command = <<"RECEIPT">>, headers = [{<<"receipt-id">>, <<"12345">>}], - body = _}, _} = parse(Data4) + body = _}, _, _} = parse(Data4) end). +%% TODO: Mountpoint, AuthChain, ACL + Mountpoint, ClientInfoOverride, +%% Listeners, Metrics, Stats, ClientInfo +%% +%% TODO: Start/Stop, List Instace +%% +%% TODO: RateLimit, OOM, + with_connection(DoFun) -> {ok, Sock} = gen_tcp:connect({127, 0, 0, 1}, 61613, @@ -336,14 +369,15 @@ with_connection(DoFun) -> end. serialize(Command, Headers) -> - emqx_stomp_frame:serialize(emqx_stomp_frame:make(Command, Headers)). + emqx_stomp_frame:serialize_pkt(emqx_stomp_frame:make(Command, Headers), #{}). serialize(Command, Headers, Body) -> - emqx_stomp_frame:serialize(emqx_stomp_frame:make(Command, Headers, Body)). + emqx_stomp_frame:serialize_pkt(emqx_stomp_frame:make(Command, Headers, Body), #{}). parse(Data) -> - ProtoEnv = [{max_headers, 10}, - {max_header_length, 1024}, - {max_body_length, 8192}], - Parser = emqx_stomp_frame:init_parer_state(ProtoEnv), + ProtoEnv = #{max_headers => 10, + max_header_length => 1024, + max_body_length => 8192 + }, + Parser = emqx_stomp_frame:initial_parse_state(ProtoEnv), emqx_stomp_frame:parse(Data, Parser). diff --git a/apps/emqx_stomp/test/emqx_stomp_heartbeat_SUITE.erl b/apps/emqx_gateway/test/emqx_stomp_heartbeat_SUITE.erl similarity index 100% rename from apps/emqx_stomp/test/emqx_stomp_heartbeat_SUITE.erl rename to apps/emqx_gateway/test/emqx_stomp_heartbeat_SUITE.erl diff --git a/apps/emqx_stomp/.gitignore b/apps/emqx_stomp/.gitignore deleted file mode 100644 index 95654d437..000000000 --- a/apps/emqx_stomp/.gitignore +++ /dev/null @@ -1,25 +0,0 @@ -.eunit -deps -*.o -*.beam -*.plt -erl_crash.dump -ebin -rel/example_project -.concrete/DEV_MODE -.rebar -.erlang.mk/ -emq_stomp.d -ct.coverdata -logs/ -test/ct.cover.spec -data/ -.DS_Store -emqx_stomp.d -_build/ -rebar.lock -erlang.mk -rebar3.crashdump -etc/emqx_stomp.conf.rendered -.rebar3/ -*.swp diff --git a/apps/emqx_stomp/etc/emqx_stomp.conf b/apps/emqx_stomp/etc/emqx_stomp.conf deleted file mode 100644 index 832b50655..000000000 --- a/apps/emqx_stomp/etc/emqx_stomp.conf +++ /dev/null @@ -1,124 +0,0 @@ -##-------------------------------------------------------------------- -## Stomp Plugin -##-------------------------------------------------------------------- - -##-------------------------------------------------------------------- -## Stomp listener - -## The Port that stomp listener will bind. -## -## Value: Port -stomp.listener.port = 61613 - -## The acceptor pool for stomp listener. -## -## Value: Number -stomp.listener.acceptors = 4 - -## Maximum number of concurrent stomp connections. -## -## Value: Number -stomp.listener.max_connections = 512 - -## Whether to enable SSL. -## -## Value: on | off -## stomp.listener.ssl = off - -## Path to the file containing the user's private PEM-encoded key. -## -## Value: File -## stomp.listener.keyfile = "etc/certs/key.pem" - -## Path to a file containing the user certificate. -## -## Value: File -## stomp.listener.certfile = "etc/certs/cert.pem" - -## Path to the file containing PEM-encoded CA certificates. -## -## Value: File -## stomp.listener.cacertfile = "etc/certs/cacert.pem" - -## See: 'listener.ssl..dhfile' in emq.conf -## -## Value: File -## stomp.listener.dhfile = "etc/certs/dh-params.pem" - -## See: 'listener.ssl..verify' in emq.conf -## -## Value: verify_peer | verify_none -## stomp.listener.verify = verify_peer - -## See: 'listener.ssl..fail_if_no_peer_cert' in emq.conf -## -## Value: false | true -## stomp.listener.fail_if_no_peer_cert = true - -## TLS versions only to protect from POODLE attack. -## -## Value: String, seperated by ',' -## NOTE: Do not use tlsv1.3 if emqx is running on OTP-22 or earlier -## stomp.listener.tls_versions = "tlsv1.3,tlsv1.2,tlsv1.1,tlsv1" - -## SSL Handshake timeout. -## -## Value: Duration -## stomp.listener.handshake_timeout = 15s - -## See: 'listener.ssl..ciphers' in emq.conf -## -## Value: Ciphers -## stomp.listener.ciphers = "TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256,TLS_CHACHA20_POLY1305_SHA256,TLS_AES_128_CCM_SHA256,TLS_AES_128_CCM_8_SHA256,ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA" - -## See: 'listener.ssl..secure_renegotiate' in emq.conf -## -## Value: on | off -## stomp.listener.secure_renegotiate = off - -## See: 'listener.ssl..reuse_sessions' in emq.conf -## -## Value: on | off -## stomp.listener.reuse_sessions = on - -## See: 'listener.ssl..honor_cipher_order' in emq.conf -## -## Value: on | off -## stomp.listener.honor_cipher_order = on - -##-------------------------------------------------------------------- -## Stomp login user and password - -## Default login user -## -## Value: String -stomp.default_user.login = guest - -## Default login password -## -## Value: String -stomp.default_user.passcode = guest - -## Allow anonymous authentication. -## -## Value: true | false -stomp.allow_anonymous = true - -##-------------------------------------------------------------------- -## Stomp frame - -## Maximum numbers of frame headers. -## -## Value: Number -stomp.frame.max_headers = 10 - -## Maximum length of frame header. -## -## Value: Number -stomp.frame.max_header_length = 1024 - -## Maximum body length of frame. -## -## Value: Number -stomp.frame.max_body_length = 8192 - diff --git a/apps/emqx_stomp/include/emqx_stomp.hrl b/apps/emqx_stomp/include/emqx_stomp.hrl deleted file mode 100644 index a9cf2cf48..000000000 --- a/apps/emqx_stomp/include/emqx_stomp.hrl +++ /dev/null @@ -1,48 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - -%% @doc Stomp Frame Header. - --define(STOMP_VER, <<"1.2">>). - --define(STOMP_SERVER, <<"emqx-stomp/1.2">>). - -%%-------------------------------------------------------------------- -%% STOMP Frame -%%-------------------------------------------------------------------- - --record(stomp_frame, {command, headers = [], body = <<>> :: iodata()}). - --type(stomp_frame() :: #stomp_frame{}). - -%%-------------------------------------------------------------------- -%% Frame Size Limits -%% -%% To prevent malicious clients from exploiting memory allocation in a server, -%% servers MAY place maximum limits on: -%% -%% the number of frame headers allowed in a single frame -%% the maximum length of header lines -%% the maximum size of a frame body -%% -%% If these limits are exceeded the server SHOULD send the client an ERROR frame -%% and then close the connection. -%%-------------------------------------------------------------------- - --define(MAX_HEADER_NUM, 10). --define(MAX_HEADER_LENGTH, 1024). --define(MAX_BODY_LENGTH, 65536). - diff --git a/apps/emqx_stomp/priv/emqx_stomp.schema b/apps/emqx_stomp/priv/emqx_stomp.schema deleted file mode 100644 index 32a3c272b..000000000 --- a/apps/emqx_stomp/priv/emqx_stomp.schema +++ /dev/null @@ -1,149 +0,0 @@ -%%-*- mode: erlang -*- -%% emqx_stomp config mapping - -{mapping, "stomp.listener.port", "emqx_stomp.listener", [ - {default, 61613}, - {datatype, [integer, ip]} -]}. - -{mapping, "stomp.listener.acceptors", "emqx_stomp.listener", [ - {default, 4}, - {datatype, integer} -]}. - -{mapping, "stomp.listener.max_connections", "emqx_stomp.listener", [ - {default, 512}, - {datatype, integer} -]}. - -{mapping, "stomp.listener.ssl", "emqx_stomp.listener", [ - {datatype, flag}, - {default, off} -]}. - -{mapping, "stomp.listener.tls_versions", "emqx_stomp.listener", [ - {datatype, string} -]}. - -{mapping, "stomp.listener.handshake_timeout", "emqx_stomp.listener", [ - {default, "15s"}, - {datatype, {duration, ms}} -]}. - -{mapping, "stomp.listener.dhfile", "emqx_stomp.listener", [ - {datatype, string} -]}. - -{mapping, "stomp.listener.keyfile", "emqx_stomp.listener", [ - {datatype, string} -]}. - -{mapping, "stomp.listener.certfile", "emqx_stomp.listener", [ - {datatype, string} -]}. - -{mapping, "stomp.listener.cacertfile", "emqx_stomp.listener", [ - {datatype, string} -]}. - -{mapping, "stomp.listener.verify", "emqx_stomp.listener", [ - {datatype, string} -]}. - -{mapping, "stomp.listener.fail_if_no_peer_cert", "emqx_stomp.listener", [ - {datatype, {enum, [true, false]}} -]}. - -{mapping, "stomp.listener.ciphers", "emqx_stomp.listener", [ - {datatype, string} -]}. - -{mapping, "stomp.listener.secure_renegotiate", "emqx_stomp.listener", [ - {datatype, flag} -]}. - -{mapping, "stomp.listener.reuse_sessions", "emqx_stomp.listener", [ - {default, on}, - {datatype, flag} -]}. - -{mapping, "stomp.listener.honor_cipher_order", "emqx_stomp.listener", [ - {datatype, flag} -]}. - -{translation, "emqx_stomp.listener", fun(Conf) -> - Port = cuttlefish:conf_get("stomp.listener.port", Conf), - Acceptors = cuttlefish:conf_get("stomp.listener.acceptors", Conf), - MaxConnections = cuttlefish:conf_get("stomp.listener.max_connections", Conf), - Filter = fun(Opts) -> [{K, V} || {K, V} <- Opts, V =/= undefined] end, - SplitFun = fun(undefined) -> undefined; (S) -> string:tokens(S, ",") end, - SslOpts = fun(Prefix) -> - Versions = case SplitFun(cuttlefish:conf_get(Prefix ++ ".tls_versions", Conf, undefined)) of - undefined -> undefined; - L -> [list_to_atom(V) || V <- L] - end, - Filter([{versions, Versions}, - {ciphers, SplitFun(cuttlefish:conf_get(Prefix ++ ".ciphers", Conf, undefined))}, - {handshake_timeout, cuttlefish:conf_get(Prefix ++ ".handshake_timeout", Conf, undefined)}, - {dhfile, cuttlefish:conf_get(Prefix ++ ".dhfile", Conf, undefined)}, - {keyfile, cuttlefish:conf_get(Prefix ++ ".keyfile", Conf, undefined)}, - {certfile, cuttlefish:conf_get(Prefix ++ ".certfile", Conf, undefined)}, - {cacertfile, cuttlefish:conf_get(Prefix ++ ".cacertfile", Conf, undefined)}, - {verify, cuttlefish:conf_get(Prefix ++ ".verify", Conf, undefined)}, - {fail_if_no_peer_cert, cuttlefish:conf_get(Prefix ++ ".fail_if_no_peer_cert", Conf, undefined)}, - {secure_renegotiate, cuttlefish:conf_get(Prefix ++ ".secure_renegotiate", Conf, undefined)}, - {reuse_sessions, cuttlefish:conf_get(Prefix ++ ".reuse_sessions", Conf, undefined)}, - {honor_cipher_order, cuttlefish:conf_get(Prefix ++ ".honor_cipher_order", Conf, undefined)}]) - end, - Opts = [{acceptors, Acceptors}, {max_connections, MaxConnections}], - {Port, case cuttlefish:conf_get("stomp.listener.ssl", Conf) of - true -> - [{sslopts, SslOpts("stomp.listener")} | Opts]; - false -> - Opts - end} -end}. - -{mapping, "stomp.default_user.login", "emqx_stomp.default_user", [ - {default, "guest"}, - {datatype, string} -]}. - -{mapping, "stomp.default_user.passcode", "emqx_stomp.default_user", [ - {default, "guest"}, - {datatype, string} -]}. - -{translation, "emqx_stomp.default_user", fun(Conf) -> - Login = cuttlefish:conf_get("stomp.default_user.login", Conf), - Passcode = cuttlefish:conf_get("stomp.default_user.passcode", Conf), - [{login, Login}, {passcode, Passcode}] -end}. - -{mapping, "stomp.allow_anonymous", "emqx_stomp.allow_anonymous", [ - {default, true}, - {datatype, {enum, [true, false]}} -]}. - -{mapping, "stomp.frame.max_headers", "emqx_stomp.frame", [ - {default, 10}, - {datatype, integer} -]}. - -{mapping, "stomp.frame.max_header_length", "emqx_stomp.frame", [ - {default, 1024}, - {datatype, integer} -]}. - -{mapping, "stomp.frame.max_body_length", "emqx_stomp.frame", [ - {default, 8192}, - {datatype, integer} -]}. - -{translation, "emqx_stomp.frame", fun(Conf) -> - MaxHeaders = cuttlefish:conf_get("stomp.frame.max_headers", Conf), - MaxHeaderLength = cuttlefish:conf_get("stomp.frame.max_header_length", Conf), - MaxBodyLength = cuttlefish:conf_get("stomp.frame.max_body_length", Conf), - [{max_headers, MaxHeaders}, {max_header_length, MaxHeaderLength}, {max_body_length, MaxBodyLength}] -end}. - diff --git a/apps/emqx_stomp/rebar.config b/apps/emqx_stomp/rebar.config deleted file mode 100644 index 7ac3b98c8..000000000 --- a/apps/emqx_stomp/rebar.config +++ /dev/null @@ -1,16 +0,0 @@ -{deps, []}. - -{edoc_opts, [{preprocess, true}]}. -{erl_opts, [warn_unused_vars, - warn_shadow_vars, - warn_unused_import, - warn_obsolete_guard, - debug_info, - {parse_transform}]}. - -{xref_checks, [undefined_function_calls, undefined_functions, - locals_not_used, deprecated_function_calls, - warnings_as_errors, deprecated_functions]}. -{cover_enabled, true}. -{cover_opts, [verbose]}. -{cover_export_enabled, true}. diff --git a/apps/emqx_stomp/src/emqx_stomp.app.src b/apps/emqx_stomp/src/emqx_stomp.app.src deleted file mode 100644 index 2e66734ec..000000000 --- a/apps/emqx_stomp/src/emqx_stomp.app.src +++ /dev/null @@ -1,14 +0,0 @@ -{application, emqx_stomp, - [{description, "EMQ X Stomp Protocol Plugin"}, - {vsn, "4.4.0"}, % strict semver, bump manually! - {modules, []}, - {registered, [emqx_stomp_sup]}, - {applications, [kernel,stdlib]}, - {mod, {emqx_stomp,[]}}, - {env, []}, - {licenses, ["Apache-2.0"]}, - {maintainers, ["EMQ X Team "]}, - {links, [{"Homepage", "https://emqx.io/"}, - {"Github", "https://github.com/emqx/emqx-stomp"} - ]} - ]}. diff --git a/apps/emqx_stomp/src/emqx_stomp.erl b/apps/emqx_stomp/src/emqx_stomp.erl deleted file mode 100644 index 9eafe3cf7..000000000 --- a/apps/emqx_stomp/src/emqx_stomp.erl +++ /dev/null @@ -1,142 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_stomp). - --behaviour(application). --behaviour(supervisor). - --emqx_plugin(protocol). - --export([ start/2 - , stop/1 - ]). - --export([ start_listeners/0 - , start_listener/1 - , start_listener/3 - , stop_listeners/0 - , stop_listener/1 - , stop_listener/3 - ]). - --export([init/1]). - --define(APP, ?MODULE). --define(TCP_OPTS, [binary, {packet, raw}, {reuseaddr, true}, {nodelay, true}]). - --type(listener() :: {esockd:proto(), esockd:listen_on(), [esockd:option()]}). - -%%-------------------------------------------------------------------- -%% Application callbacks -%%-------------------------------------------------------------------- - -start(_StartType, _StartArgs) -> - {ok, Sup} = supervisor:start_link({local, emqx_stomp_sup}, ?MODULE, []), - start_listeners(), - {ok, Sup}. - -stop(_State) -> - stop_listeners(). - -%%-------------------------------------------------------------------- -%% Supervisor callbacks -%%-------------------------------------------------------------------- - -init([]) -> - {ok, {{one_for_all, 10, 100}, []}}. - -%%-------------------------------------------------------------------- -%% Start/Stop listeners -%%-------------------------------------------------------------------- - --spec(start_listeners() -> ok). -start_listeners() -> - lists:foreach(fun start_listener/1, listeners_confs()). - --spec(start_listener(listener()) -> ok). -start_listener({Proto, ListenOn, Options}) -> - case start_listener(Proto, ListenOn, Options) of - {ok, _} -> io:format("Start stomp:~s listener on ~s successfully.~n", - [Proto, format(ListenOn)]); - {error, Reason} -> - io:format(standard_error, "Failed to start stomp:~s listener on ~s: ~0p~n", - [Proto, format(ListenOn), Reason]), - error(Reason) - end. - --spec(start_listener(esockd:proto(), esockd:listen_on(), [esockd:option()]) - -> {ok, pid()} | {error, term()}). -start_listener(tcp, ListenOn, Options) -> - start_stomp_listener('stomp:tcp', ListenOn, Options); -start_listener(ssl, ListenOn, Options) -> - start_stomp_listener('stomp:ssl', ListenOn, Options). - -%% @private -start_stomp_listener(Name, ListenOn, Options) -> - SockOpts = esockd:parse_opt(Options), - esockd:open(Name, ListenOn, merge_default(SockOpts), - {emqx_stomp_connection, start_link, [Options -- SockOpts]}). - --spec(stop_listeners() -> ok). -stop_listeners() -> - lists:foreach(fun stop_listener/1, listeners_confs()). - --spec(stop_listener(listener()) -> ok | {error, term()}). -stop_listener({Proto, ListenOn, Opts}) -> - StopRet = stop_listener(Proto, ListenOn, Opts), - case StopRet of - ok -> io:format("Stop stomp:~s listener on ~s successfully.~n", - [Proto, format(ListenOn)]); - {error, Reason} -> - io:format(standard_error, "Failed to stop stomp:~s listener on ~s: ~0p~n", - [Proto, format(ListenOn), Reason]) - end, - StopRet. - --spec(stop_listener(esockd:proto(), esockd:listen_on(), [esockd:option()]) - -> ok | {error, term()}). -stop_listener(tcp, ListenOn, _Opts) -> - esockd:close('stomp:tcp', ListenOn); -stop_listener(ssl, ListenOn, _Opts) -> - esockd:close('stomp:ssl', ListenOn). - -%%-------------------------------------------------------------------- -%% Internal funcs -%%-------------------------------------------------------------------- - -listeners_confs() -> - {ok, {Port, Opts}} = application:get_env(?APP, listener), - Options = application:get_env(?APP, frame, []), - Anonymous = application:get_env(emqx_stomp, allow_anonymous, false), - {ok, DefaultUser} = application:get_env(emqx_stomp, default_user), - [{tcp, Port, [{allow_anonymous, Anonymous}, - {default_user, DefaultUser} | Options ++ Opts]}]. - -merge_default(Options) -> - case lists:keytake(tcp_options, 1, Options) of - {value, {tcp_options, TcpOpts}, Options1} -> - [{tcp_options, emqx_misc:merge_opts(?TCP_OPTS, TcpOpts)} | Options1]; - false -> - [{tcp_options, ?TCP_OPTS} | Options] - end. - -format(Port) when is_integer(Port) -> - io_lib:format("0.0.0.0:~w", [Port]); -format({Addr, Port}) when is_list(Addr) -> - io_lib:format("~s:~w", [Addr, Port]); -format({Addr, Port}) when is_tuple(Addr) -> - io_lib:format("~s:~w", [inet:ntoa(Addr), Port]). diff --git a/apps/emqx_stomp/src/emqx_stomp_connection.erl b/apps/emqx_stomp/src/emqx_stomp_connection.erl deleted file mode 100644 index d4e7f6475..000000000 --- a/apps/emqx_stomp/src/emqx_stomp_connection.erl +++ /dev/null @@ -1,274 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_stomp_connection). - --behaviour(gen_server). - --include("emqx_stomp.hrl"). --include_lib("emqx/include/logger.hrl"). - --logger_header("[Stomp-Conn]"). - --export([ start_link/3 - , info/1 - ]). - -%% gen_server Function Exports --export([ init/1 - , handle_call/3 - , handle_cast/2 - , handle_info/2 - , code_change/3 - , terminate/2 - ]). - -%% for protocol --export([send/4, heartbeat/2]). - --record(state, {transport, socket, peername, conn_name, conn_state, - await_recv, rate_limit, parser, pstate, - proto_env, heartbeat}). - --define(INFO_KEYS, [peername, await_recv, conn_state]). --define(SOCK_STATS, [recv_oct, recv_cnt, send_oct, send_cnt]). - -start_link(Transport, Sock, ProtoEnv) -> - {ok, proc_lib:spawn_link(?MODULE, init, [[Transport, Sock, ProtoEnv]])}. - -info(CPid) -> - gen_server:call(CPid, info, infinity). - -init([Transport, Sock, ProtoEnv]) -> - process_flag(trap_exit, true), - case Transport:wait(Sock) of - {ok, NewSock} -> - {ok, Peername} = Transport:ensure_ok_or_exit(peername, [NewSock]), - ConnName = esockd:format(Peername), - SendFun = {fun ?MODULE:send/4, [Transport, Sock, self()]}, - HrtBtFun = {fun ?MODULE:heartbeat/2, [Transport, Sock]}, - Parser = emqx_stomp_frame:init_parer_state(ProtoEnv), - PState = emqx_stomp_protocol:init(#{peername => Peername, - sendfun => SendFun, - heartfun => HrtBtFun}, ProtoEnv), - RateLimit = init_rate_limit(proplists:get_value(rate_limit, ProtoEnv)), - State = run_socket(#state{transport = Transport, - socket = NewSock, - peername = Peername, - conn_name = ConnName, - conn_state = running, - await_recv = false, - rate_limit = RateLimit, - parser = Parser, - proto_env = ProtoEnv, - pstate = PState}), - emqx_logger:set_metadata_peername(esockd:format(Peername)), - gen_server:enter_loop(?MODULE, [{hibernate_after, 5000}], State, 20000); - {error, Reason} -> - {stop, Reason} - end. - -init_rate_limit(undefined) -> - undefined; -init_rate_limit({Rate, Burst}) -> - esockd_rate_limit:new(Rate, Burst). - -send(Data, Transport, Sock, ConnPid) -> - try Transport:async_send(Sock, Data) of - ok -> ok; - {error, Reason} -> ConnPid ! {shutdown, Reason} - catch - error:Error -> ConnPid ! {shutdown, Error} - end. - -heartbeat(Transport, Sock) -> - Transport:send(Sock, <<$\n>>). - -handle_call(info, _From, State = #state{transport = Transport, - socket = Sock, - peername = Peername, - await_recv = AwaitRecv, - conn_state = ConnState, - pstate = PState}) -> - ClientInfo = [{peername, Peername}, {await_recv, AwaitRecv}, - {conn_state, ConnState}], - ProtoInfo = emqx_stomp_protocol:info(PState), - case Transport:getstat(Sock, ?SOCK_STATS) of - {ok, SockStats} -> - {reply, lists:append([ClientInfo, ProtoInfo, SockStats]), State}; - {error, Reason} -> - {stop, Reason, lists:append([ClientInfo, ProtoInfo]), State} - end; - -handle_call(Req, _From, State) -> - ?LOG(error, "unexpected request: ~p", [Req]), - {reply, ignored, State}. - -handle_cast(Msg, State) -> - ?LOG(error, "unexpected msg: ~p", [Msg]), - noreply(State). - -handle_info(timeout, State) -> - shutdown(idle_timeout, State); - -handle_info({shutdown, Reason}, State) -> - shutdown(Reason, State); - -handle_info({timeout, TRef, TMsg}, State) when TMsg =:= incoming; - TMsg =:= outgoing -> - - Stat = case TMsg of - incoming -> recv_oct; - _ -> send_oct - end, - case getstat(Stat, State) of - {ok, Val} -> - with_proto(timeout, [TRef, {TMsg, Val}], State); - {error, Reason} -> - shutdown({sock_error, Reason}, State) - end; - -handle_info({timeout, TRef, TMsg}, State) -> - with_proto(timeout, [TRef, TMsg], State); - -handle_info({'EXIT', HbProc, Error}, State = #state{heartbeat = HbProc}) -> - stop(Error, State); - -handle_info(activate_sock, State) -> - noreply(run_socket(State#state{conn_state = running})); - -handle_info({inet_async, _Sock, _Ref, {ok, Bytes}}, State) -> - ?LOG(debug, "RECV ~p", [Bytes]), - received(Bytes, rate_limit(size(Bytes), State#state{await_recv = false})); - -handle_info({inet_async, _Sock, _Ref, {error, Reason}}, State) -> - shutdown(Reason, State); - -handle_info({inet_reply, _Ref, ok}, State) -> - noreply(State); - -handle_info({inet_reply, _Sock, {error, Reason}}, State) -> - shutdown(Reason, State); - -handle_info({deliver, _Topic, Msg}, State = #state{pstate = PState}) -> - noreply(State#state{pstate = case emqx_stomp_protocol:send(Msg, PState) of - {ok, PState1} -> - PState1; - {error, dropped, PState1} -> - PState1 - end}); - -handle_info(Info, State) -> - ?LOG(error, "Unexpected info: ~p", [Info]), - noreply(State). - -terminate(Reason, #state{transport = Transport, - socket = Sock, - pstate = PState}) -> - ?LOG(info, "terminated for ~p", [Reason]), - Transport:fast_close(Sock), - case {PState, Reason} of - {undefined, _} -> ok; - {_, {shutdown, Error}} -> - emqx_stomp_protocol:shutdown(Error, PState); - {_, Reason} -> - emqx_stomp_protocol:shutdown(Reason, PState) - end. - -code_change(_OldVsn, State, _Extra) -> - {ok, State}. - -%%-------------------------------------------------------------------- -%% Receive and Parse data -%%-------------------------------------------------------------------- - -with_proto(Fun, Args, State = #state{pstate = PState}) -> - case erlang:apply(emqx_stomp_protocol, Fun, Args ++ [PState]) of - {ok, NPState} -> - noreply(State#state{pstate = NPState}); - {F, Reason, NPState} when F == stop; - F == error; - F == shutdown -> - shutdown(Reason, State#state{pstate = NPState}) - end. - -received(<<>>, State) -> - noreply(State); - -received(Bytes, State = #state{parser = Parser, - pstate = PState}) -> - try emqx_stomp_frame:parse(Bytes, Parser) of - {more, NewParser} -> - noreply(State#state{parser = NewParser}); - {ok, Frame, Rest} -> - ?LOG(info, "RECV Frame: ~s", [emqx_stomp_frame:format(Frame)]), - case emqx_stomp_protocol:received(Frame, PState) of - {ok, PState1} -> - received(Rest, reset_parser(State#state{pstate = PState1})); - {error, Error, PState1} -> - shutdown(Error, State#state{pstate = PState1}); - {stop, Reason, PState1} -> - stop(Reason, State#state{pstate = PState1}) - end; - {error, Error} -> - ?LOG(error, "Framing error - ~s", [Error]), - ?LOG(error, "Bytes: ~p", [Bytes]), - shutdown(frame_error, State) - catch - _Error:Reason -> - ?LOG(error, "Parser failed for ~p", [Reason]), - ?LOG(error, "Error data: ~p", [Bytes]), - shutdown(parse_error, State) - end. - -reset_parser(State = #state{proto_env = ProtoEnv}) -> - State#state{parser = emqx_stomp_frame:init_parer_state(ProtoEnv)}. - -rate_limit(_Size, State = #state{rate_limit = undefined}) -> - run_socket(State); -rate_limit(Size, State = #state{rate_limit = Rl}) -> - case esockd_rate_limit:check(Size, Rl) of - {0, Rl1} -> - run_socket(State#state{conn_state = running, rate_limit = Rl1}); - {Pause, Rl1} -> - ?LOG(error, "Rate limiter pause for ~p", [Pause]), - erlang:send_after(Pause, self(), activate_sock), - State#state{conn_state = blocked, rate_limit = Rl1} - end. - -run_socket(State = #state{conn_state = blocked}) -> - State; -run_socket(State = #state{await_recv = true}) -> - State; -run_socket(State = #state{transport = Transport, socket = Sock}) -> - Transport:async_recv(Sock, 0, infinity), - State#state{await_recv = true}. - -getstat(Stat, #state{transport = Transport, socket = Sock}) -> - case Transport:getstat(Sock, [Stat]) of - {ok, [{Stat, Val}]} -> {ok, Val}; - {error, Error} -> {error, Error} - end. - -noreply(State) -> - {noreply, State}. - -stop(Reason, State) -> - {stop, Reason, State}. - -shutdown(Reason, State) -> - stop({shutdown, Reason}, State). - diff --git a/apps/emqx_stomp/src/emqx_stomp_protocol.erl b/apps/emqx_stomp/src/emqx_stomp_protocol.erl deleted file mode 100644 index cc5c28ce9..000000000 --- a/apps/emqx_stomp/src/emqx_stomp_protocol.erl +++ /dev/null @@ -1,468 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - -%% @doc Stomp Protocol Processor. --module(emqx_stomp_protocol). - --include("emqx_stomp.hrl"). - --include_lib("emqx/include/emqx.hrl"). --include_lib("emqx/include/logger.hrl"). --include_lib("emqx/include/emqx_mqtt.hrl"). - --logger_header("[Stomp-Proto]"). - --import(proplists, [get_value/2, get_value/3]). - -%% API --export([ init/2 - , info/1 - ]). - --export([ received/2 - , send/2 - , shutdown/2 - , timeout/3 - ]). - -%% for trans callback --export([ handle_recv_send_frame/2 - , handle_recv_ack_frame/2 - , handle_recv_nack_frame/2 - ]). - --record(pstate, { - peername, - heartfun, - sendfun, - connected = false, - proto_ver, - proto_name, - heart_beats, - login, - allow_anonymous, - default_user, - subscriptions = [], - timers :: #{atom() => disable | undefined | reference()}, - transaction :: #{binary() => list()} - }). - --define(TIMER_TABLE, #{ - incoming_timer => incoming, - outgoing_timer => outgoing, - clean_trans_timer => clean_trans - }). - --define(TRANS_TIMEOUT, 60000). - --type(pstate() :: #pstate{}). - -%% @doc Init protocol -init(#{peername := Peername, - sendfun := SendFun, - heartfun := HeartFun}, Env) -> - AllowAnonymous = get_value(allow_anonymous, Env, false), - DefaultUser = get_value(default_user, Env), - #pstate{peername = Peername, - heartfun = HeartFun, - sendfun = SendFun, - timers = #{}, - transaction = #{}, - allow_anonymous = AllowAnonymous, - default_user = DefaultUser}. - -info(#pstate{connected = Connected, - proto_ver = ProtoVer, - proto_name = ProtoName, - heart_beats = Heartbeats, - login = Login, - subscriptions = Subscriptions}) -> - [{connected, Connected}, - {proto_ver, ProtoVer}, - {proto_name, ProtoName}, - {heart_beats, Heartbeats}, - {login, Login}, - {subscriptions, Subscriptions}]. - --spec(received(stomp_frame(), pstate()) - -> {ok, pstate()} - | {error, any(), pstate()} - | {stop, any(), pstate()}). -received(Frame = #stomp_frame{command = <<"STOMP">>}, State) -> - received(Frame#stomp_frame{command = <<"CONNECT">>}, State); - -received(#stomp_frame{command = <<"CONNECT">>, headers = Headers}, - State = #pstate{connected = false, allow_anonymous = AllowAnonymous, default_user = DefaultUser}) -> - case negotiate_version(header(<<"accept-version">>, Headers)) of - {ok, Version} -> - Login = header(<<"login">>, Headers), - Passc = header(<<"passcode">>, Headers), - case check_login(Login, Passc, AllowAnonymous, DefaultUser) of - true -> - emqx_logger:set_metadata_clientid(Login), - - Heartbeats = parse_heartbeats(header(<<"heart-beat">>, Headers, <<"0,0">>)), - NState = start_heartbeart_timer(Heartbeats, State#pstate{connected = true, - proto_ver = Version, login = Login}), - send(connected_frame([{<<"version">>, Version}, - {<<"heart-beat">>, reverse_heartbeats(Heartbeats)}]), NState); - false -> - _ = send(error_frame(undefined, <<"Login or passcode error!">>), State), - {error, login_or_passcode_error, State} - end; - {error, Msg} -> - _ = send(error_frame([{<<"version">>, <<"1.0,1.1,1.2">>}, - {<<"content-type">>, <<"text/plain">>}], undefined, Msg), State), - {error, unsupported_version, State} - end; - -received(#stomp_frame{command = <<"CONNECT">>}, State = #pstate{connected = true}) -> - {error, unexpected_connect, State}; - -received(Frame = #stomp_frame{command = <<"SEND">>, headers = Headers}, State) -> - case header(<<"transaction">>, Headers) of - undefined -> {ok, handle_recv_send_frame(Frame, State)}; - TransactionId -> add_action(TransactionId, {fun ?MODULE:handle_recv_send_frame/2, [Frame]}, receipt_id(Headers), State) - end; - -received(#stomp_frame{command = <<"SUBSCRIBE">>, headers = Headers}, - State = #pstate{subscriptions = Subscriptions}) -> - Id = header(<<"id">>, Headers), - Topic = header(<<"destination">>, Headers), - Ack = header(<<"ack">>, Headers, <<"auto">>), - {ok, State1} = case lists:keyfind(Id, 1, Subscriptions) of - {Id, Topic, Ack} -> - {ok, State}; - false -> - emqx_broker:subscribe(Topic), - {ok, State#pstate{subscriptions = [{Id, Topic, Ack}|Subscriptions]}} - end, - maybe_send_receipt(receipt_id(Headers), State1); - -received(#stomp_frame{command = <<"UNSUBSCRIBE">>, headers = Headers}, - State = #pstate{subscriptions = Subscriptions}) -> - Id = header(<<"id">>, Headers), - - {ok, State1} = case lists:keyfind(Id, 1, Subscriptions) of - {Id, Topic, _Ack} -> - ok = emqx_broker:unsubscribe(Topic), - {ok, State#pstate{subscriptions = lists:keydelete(Id, 1, Subscriptions)}}; - false -> - {ok, State} - end, - maybe_send_receipt(receipt_id(Headers), State1); - -%% ACK -%% id:12345 -%% transaction:tx1 -%% -%% ^@ -received(Frame = #stomp_frame{command = <<"ACK">>, headers = Headers}, State) -> - case header(<<"transaction">>, Headers) of - undefined -> {ok, handle_recv_ack_frame(Frame, State)}; - TransactionId -> add_action(TransactionId, {fun ?MODULE:handle_recv_ack_frame/2, [Frame]}, receipt_id(Headers), State) - end; - -%% NACK -%% id:12345 -%% transaction:tx1 -%% -%% ^@ -received(Frame = #stomp_frame{command = <<"NACK">>, headers = Headers}, State) -> - case header(<<"transaction">>, Headers) of - undefined -> {ok, handle_recv_nack_frame(Frame, State)}; - TransactionId -> add_action(TransactionId, {fun ?MODULE:handle_recv_nack_frame/2, [Frame]}, receipt_id(Headers), State) - end; - -%% BEGIN -%% transaction:tx1 -%% -%% ^@ -received(#stomp_frame{command = <<"BEGIN">>, headers = Headers}, - State = #pstate{transaction = Trans}) -> - Id = header(<<"transaction">>, Headers), - case maps:get(Id, Trans, undefined) of - undefined -> - Ts = erlang:system_time(millisecond), - NState = ensure_clean_trans_timer(State#pstate{transaction = Trans#{Id => {Ts, []}}}), - maybe_send_receipt(receipt_id(Headers), NState); - _ -> - send(error_frame(receipt_id(Headers), ["Transaction ", Id, " already started"]), State) - end; - -%% COMMIT -%% transaction:tx1 -%% -%% ^@ -received(#stomp_frame{command = <<"COMMIT">>, headers = Headers}, - State = #pstate{transaction = Trans}) -> - Id = header(<<"transaction">>, Headers), - case maps:get(Id, Trans, undefined) of - {_, Actions} -> - NState = lists:foldr(fun({Func, Args}, S) -> - erlang:apply(Func, Args ++ [S]) - end, State#pstate{transaction = maps:remove(Id, Trans)}, Actions), - maybe_send_receipt(receipt_id(Headers), NState); - _ -> - send(error_frame(receipt_id(Headers), ["Transaction ", Id, " not found"]), State) - end; - -%% ABORT -%% transaction:tx1 -%% -%% ^@ -received(#stomp_frame{command = <<"ABORT">>, headers = Headers}, - State = #pstate{transaction = Trans}) -> - Id = header(<<"transaction">>, Headers), - case maps:get(Id, Trans, undefined) of - {_, _Actions} -> - NState = State#pstate{transaction = maps:remove(Id, Trans)}, - maybe_send_receipt(receipt_id(Headers), NState); - _ -> - send(error_frame(receipt_id(Headers), ["Transaction ", Id, " not found"]), State) - end; - -received(#stomp_frame{command = <<"DISCONNECT">>, headers = Headers}, State) -> - _ = maybe_send_receipt(receipt_id(Headers), State), - {stop, normal, State}. - -send(Msg = #message{topic = Topic, headers = Headers, payload = Payload}, - State = #pstate{subscriptions = Subscriptions}) -> - case lists:keyfind(Topic, 2, Subscriptions) of - {Id, Topic, Ack} -> - Headers0 = [{<<"subscription">>, Id}, - {<<"message-id">>, next_msgid()}, - {<<"destination">>, Topic}, - {<<"content-type">>, <<"text/plain">>}], - Headers1 = case Ack of - _ when Ack =:= <<"client">> orelse Ack =:= <<"client-individual">> -> - Headers0 ++ [{<<"ack">>, next_ackid()}]; - _ -> - Headers0 - end, - Frame = #stomp_frame{command = <<"MESSAGE">>, - headers = Headers1 ++ maps:get(stomp_headers, Headers, []), - body = Payload}, - send(Frame, State); - false -> - ?LOG(error, "Stomp dropped: ~p", [Msg]), - {error, dropped, State} - end; - -send(Frame, State = #pstate{sendfun = {Fun, Args}}) -> - ?LOG(info, "SEND Frame: ~s", [emqx_stomp_frame:format(Frame)]), - Data = emqx_stomp_frame:serialize(Frame), - ?LOG(debug, "SEND ~p", [Data]), - erlang:apply(Fun, [Data] ++ Args), - {ok, State}. - -shutdown(_Reason, _State) -> - ok. - -timeout(_TRef, {incoming, NewVal}, - State = #pstate{heart_beats = HrtBt}) -> - case emqx_stomp_heartbeat:check(incoming, NewVal, HrtBt) of - {error, timeout} -> - {shutdown, heartbeat_timeout, State}; - {ok, NHrtBt} -> - {ok, reset_timer(incoming_timer, State#pstate{heart_beats = NHrtBt})} - end; - -timeout(_TRef, {outgoing, NewVal}, - State = #pstate{heart_beats = HrtBt, - heartfun = {Fun, Args}}) -> - case emqx_stomp_heartbeat:check(outgoing, NewVal, HrtBt) of - {error, timeout} -> - _ = erlang:apply(Fun, Args), - {ok, State}; - {ok, NHrtBt} -> - {ok, reset_timer(outgoing_timer, State#pstate{heart_beats = NHrtBt})} - end; - -timeout(_TRef, clean_trans, State = #pstate{transaction = Trans}) -> - Now = erlang:system_time(millisecond), - NTrans = maps:filter(fun(_, {Ts, _}) -> Ts + ?TRANS_TIMEOUT < Now end, Trans), - {ok, ensure_clean_trans_timer(State#pstate{transaction = NTrans})}. - -negotiate_version(undefined) -> - {ok, <<"1.0">>}; -negotiate_version(Accepts) -> - negotiate_version(?STOMP_VER, - lists:reverse( - lists:sort( - binary:split(Accepts, <<",">>, [global])))). - -negotiate_version(Ver, []) -> - {error, <<"Supported protocol versions < ", Ver/binary>>}; -negotiate_version(Ver, [AcceptVer|_]) when Ver >= AcceptVer -> - {ok, AcceptVer}; -negotiate_version(Ver, [_|T]) -> - negotiate_version(Ver, T). - -check_login(undefined, _, AllowAnonymous, _) -> - AllowAnonymous; -check_login(_, _, _, undefined) -> - false; -check_login(Login, Passcode, _, DefaultUser) -> - case {list_to_binary(get_value(login, DefaultUser)), - list_to_binary(get_value(passcode, DefaultUser))} of - {Login, Passcode} -> true; - {_, _ } -> false - end. - -add_action(Id, Action, ReceiptId, State = #pstate{transaction = Trans}) -> - case maps:get(Id, Trans, undefined) of - {Ts, Actions} -> - NTrans = Trans#{Id => {Ts, [Action|Actions]}}, - {ok, State#pstate{transaction = NTrans}}; - _ -> - send(error_frame(ReceiptId, ["Transaction ", Id, " not found"]), State) - end. - -maybe_send_receipt(undefined, State) -> - {ok, State}; -maybe_send_receipt(ReceiptId, State) -> - send(receipt_frame(ReceiptId), State). - -ack(_Id, State) -> - State. - -nack(_Id, State) -> State. - -header(Name, Headers) -> - get_value(Name, Headers). -header(Name, Headers, Val) -> - get_value(Name, Headers, Val). - -connected_frame(Headers) -> - emqx_stomp_frame:make(<<"CONNECTED">>, Headers). - -receipt_frame(ReceiptId) -> - emqx_stomp_frame:make(<<"RECEIPT">>, [{<<"receipt-id">>, ReceiptId}]). - -error_frame(ReceiptId, Msg) -> - error_frame([{<<"content-type">>, <<"text/plain">>}], ReceiptId, Msg). - -error_frame(Headers, undefined, Msg) -> - emqx_stomp_frame:make(<<"ERROR">>, Headers, Msg); -error_frame(Headers, ReceiptId, Msg) -> - emqx_stomp_frame:make(<<"ERROR">>, [{<<"receipt-id">>, ReceiptId} | Headers], Msg). - -next_msgid() -> - MsgId = case get(msgid) of - undefined -> 1; - I -> I - end, - put(msgid, MsgId + 1), - MsgId. - -next_ackid() -> - AckId = case get(ackid) of - undefined -> 1; - I -> I - end, - put(ackid, AckId + 1), - AckId. - -make_mqtt_message(Topic, Headers, Body) -> - Msg = emqx_message:make(stomp, Topic, Body), - Headers1 = lists:foldl(fun(Key, Headers0) -> - proplists:delete(Key, Headers0) - end, Headers, [<<"destination">>, - <<"content-length">>, - <<"content-type">>, - <<"transaction">>, - <<"receipt">>]), - emqx_message:set_headers(#{stomp_headers => Headers1}, Msg). - -receipt_id(Headers) -> - header(<<"receipt">>, Headers). - -%%-------------------------------------------------------------------- -%% Transaction Handle - -handle_recv_send_frame(#stomp_frame{command = <<"SEND">>, headers = Headers, body = Body}, State) -> - Topic = header(<<"destination">>, Headers), - _ = maybe_send_receipt(receipt_id(Headers), State), - _ = emqx_broker:publish( - make_mqtt_message(Topic, Headers, iolist_to_binary(Body)) - ), - State. - -handle_recv_ack_frame(#stomp_frame{command = <<"ACK">>, headers = Headers}, State) -> - Id = header(<<"id">>, Headers), - _ = maybe_send_receipt(receipt_id(Headers), State), - ack(Id, State). - -handle_recv_nack_frame(#stomp_frame{command = <<"NACK">>, headers = Headers}, State) -> - Id = header(<<"id">>, Headers), - _ = maybe_send_receipt(receipt_id(Headers), State), - nack(Id, State). - -ensure_clean_trans_timer(State = #pstate{transaction = Trans}) -> - case maps:size(Trans) of - 0 -> State; - _ -> ensure_timer(clean_trans_timer, State) - end. - -%%-------------------------------------------------------------------- -%% Heartbeat - -parse_heartbeats(Heartbeats) -> - CxCy = re:split(Heartbeats, <<",">>, [{return, list}]), - list_to_tuple([list_to_integer(S) || S <- CxCy]). - -reverse_heartbeats({Cx, Cy}) -> - iolist_to_binary(io_lib:format("~w,~w", [Cy, Cx])). - -start_heartbeart_timer(Heartbeats, State) -> - ensure_timer( - [incoming_timer, outgoing_timer], - State#pstate{heart_beats = emqx_stomp_heartbeat:init(Heartbeats)}). - -%%-------------------------------------------------------------------- -%% Timer - -ensure_timer([Name], State) -> - ensure_timer(Name, State); -ensure_timer([Name | Rest], State) -> - ensure_timer(Rest, ensure_timer(Name, State)); - -ensure_timer(Name, State = #pstate{timers = Timers}) -> - TRef = maps:get(Name, Timers, undefined), - Time = interval(Name, State), - case TRef == undefined andalso is_integer(Time) andalso Time > 0 of - true -> ensure_timer(Name, Time, State); - false -> State %% Timer disabled or exists - end. - -ensure_timer(Name, Time, State = #pstate{timers = Timers}) -> - Msg = maps:get(Name, ?TIMER_TABLE), - TRef = emqx_misc:start_timer(Time, Msg), - State#pstate{timers = Timers#{Name => TRef}}. - -reset_timer(Name, State) -> - ensure_timer(Name, clean_timer(Name, State)). - -clean_timer(Name, State = #pstate{timers = Timers}) -> - State#pstate{timers = maps:remove(Name, Timers)}. - -interval(incoming_timer, #pstate{heart_beats = HrtBt}) -> - emqx_stomp_heartbeat:interval(incoming, HrtBt); -interval(outgoing_timer, #pstate{heart_beats = HrtBt}) -> - emqx_stomp_heartbeat:interval(outgoing, HrtBt); -interval(clean_trans_timer, _) -> - ?TRANS_TIMEOUT. diff --git a/apps/emqx_stomp/test/client.py b/apps/emqx_stomp/test/client.py deleted file mode 100644 index f9f9e6577..000000000 --- a/apps/emqx_stomp/test/client.py +++ /dev/null @@ -1,19 +0,0 @@ -from stompest.config import StompConfig -from stompest.protocol import StompSpec -from stompest.sync import Stomp - -CONFIG = StompConfig('tcp://localhost:61613', version=StompSpec.VERSION_1_1) -QUEUE = '/queue/test' - -if __name__ == '__main__': - client = Stomp(CONFIG) - client.connect(heartBeats=(0, 10000)) - client.subscribe(QUEUE, {StompSpec.ID_HEADER: 1, StompSpec.ACK_HEADER: StompSpec.ACK_CLIENT_INDIVIDUAL}) - client.send(QUEUE, 'test message 1') - client.send(QUEUE, 'test message 2') - while True: - frame = client.receiveFrame() - print 'Got %s' % frame.info() - client.ack(frame) - client.disconnect() - diff --git a/rebar.config.erl b/rebar.config.erl index 56599dd54..de42601af 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -253,6 +253,7 @@ relx_apps(ReleaseType) -> , emqx_connector , emqx_authn , emqx_authz + , emqx_gateway , emqx_data_bridge , emqx_rule_engine , emqx_rule_actions