diff --git a/.ci/build_packages/tests.sh b/.ci/build_packages/tests.sh index 87c19621a..53ab9ac57 100755 --- a/.ci/build_packages/tests.sh +++ b/.ci/build_packages/tests.sh @@ -91,6 +91,12 @@ emqx_test(){ ;; "rpm") packagename=$(basename "${PACKAGE_PATH}/${EMQX_NAME}"-*.rpm) + + if [[ "${ARCH}" == "amd64" && $(rpm -E '%{rhel}') == 7 ]] ; + then + # EMQX OTP requires openssl11 to have TLS1.3 support + yum install -y openssl11; + fi rpm -ivh "${PACKAGE_PATH}/${packagename}" if ! rpm -q emqx | grep -q emqx; then echo "package install error" @@ -126,7 +132,7 @@ export EMQX_LOG__FILE_HANDLERS__DEFAULT__LEVEL=debug EOF ## for ARM, due to CI env issue, skip start of quic listener for the moment [[ $(arch) == *arm* || $(arch) == aarch64 ]] && tee -a "$emqx_env_vars" < $HOME/.git-credentials + git config --global credential.helper store + make emqx-ee-zip + - uses: actions/upload-artifact@v2 + with: + name: emqx-broker + path: _packages/**/*.zip + api-test: + needs: build + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + script_name: + - api_metrics + - api_subscriptions + - api_clients + - api_routes + - api_publish + steps: + - uses: actions/checkout@v2 + with: + repository: emqx/emqx-fvt + path: . + - uses: actions/setup-java@v1 + with: + java-version: '8.0.282' # The JDK version to make available on the path. + java-package: jdk # (jre, jdk, or jdk+fx) - defaults to jdk + architecture: x64 # (x64 or x86) - defaults to x64 + - uses: actions/download-artifact@v2 + with: + name: emqx-broker + path: . + - name: start emqx-broker + env: + EMQX_LISTENERS__WSS__DEFAULT__BIND: "0.0.0.0:8085" + run: | + unzip ./emqx/*.zip + ./emqx/bin/emqx start + - name: install jmeter + timeout-minutes: 10 + env: + JMETER_VERSION: 5.3 + run: | + wget --no-verbose --no-check-certificate -O /tmp/apache-jmeter.tgz https://downloads.apache.org/jmeter/binaries/apache-jmeter-$JMETER_VERSION.tgz + cd /tmp && tar -xvf apache-jmeter.tgz + echo "jmeter.save.saveservice.output_format=xml" >> /tmp/apache-jmeter-$JMETER_VERSION/user.properties + echo "jmeter.save.saveservice.response_data.on_error=true" >> /tmp/apache-jmeter-$JMETER_VERSION/user.properties + wget --no-verbose -O /tmp/apache-jmeter-$JMETER_VERSION/lib/ext/mqtt-xmeter-2.0.2-jar-with-dependencies.jar https://raw.githubusercontent.com/xmeter-net/mqtt-jmeter/master/Download/v2.0.2/mqtt-xmeter-2.0.2-jar-with-dependencies.jar + ln -s /tmp/apache-jmeter-$JMETER_VERSION /opt/jmeter + - name: run ${{ matrix.script_name }} + run: | + /opt/jmeter/bin/jmeter.sh \ + -Jjmeter.save.saveservice.output_format=xml -n \ + -t .ci/api-test-suite/${{ matrix.script_name }}.jmx \ + -Demqx_ip="127.0.0.1" \ + -l jmeter_logs/${{ matrix.script_name }}.jtl \ + -j jmeter_logs/logs/${{ matrix.script_name }}.log + - name: check test logs + run: | + if cat jmeter_logs/${{ matrix.script_name }}.jtl | grep -e 'true' > /dev/null 2>&1; then + grep -A 5 -B 3 'true' jmeter_logs/${{ matrix.script_name }}.jtl > jmeter_logs/${{ matrix.script_name }}_err_api.txt + echo "check logs failed" + exit 1 + fi + - uses: actions/upload-artifact@v1 + if: failure() + with: + name: jmeter_logs + path: ./jmeter_logs + - uses: actions/upload-artifact@v1 + if: failure() + with: + name: jmeter_logs + path: emqx/log + delete-package: + runs-on: ubuntu-20.04 + needs: api-test + if: always() + steps: + - uses: geekyeggo/delete-artifact@v1 + with: + name: emqx-broker diff --git a/Makefile b/Makefile index 52b0ea209..2ef0e1806 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ BUILD = $(CURDIR)/build SCRIPTS = $(CURDIR)/scripts export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh) export EMQX_DESC ?= EMQ X -export EMQX_DASHBOARD_VERSION ?= v5.0.0-beta.9 +export EMQX_DASHBOARD_VERSION ?= v5.0.0-beta.13 ifeq ($(OS),Windows_NT) export REBAR_COLOR=none endif diff --git a/README-CN.md b/README-CN.md index b430d4b5f..80e926199 100644 --- a/README-CN.md +++ b/README-CN.md @@ -4,7 +4,7 @@ [![Build Status](https://travis-ci.org/emqx/emqx.svg)](https://travis-ci.org/emqx/emqx) [![Coverage Status](https://coveralls.io/repos/github/emqx/emqx/badge.svg)](https://coveralls.io/github/emqx/emqx) [![Docker Pulls](https://img.shields.io/docker/pulls/emqx/emqx)](https://hub.docker.com/r/emqx/emqx) -[![Slack Invite]()](https://slack-invite.emqx.io) +[![Slack](https://img.shields.io/badge/Slack-EMQ%20X-39AE85?logo=slack)](https://slack-invite.emqx.io/) [![Twitter](https://img.shields.io/badge/Twitter-EMQ-1DA1F2?logo=twitter)](https://twitter.com/EMQTech) [![Community](https://img.shields.io/badge/Community-EMQ%20X-yellow)](https://askemq.com) [![YouTube](https://img.shields.io/badge/Subscribe-EMQ%20中文-FF0000?logo=youtube)](https://www.youtube.com/channel/UCir_r04HIsLjf2qqyZ4A8Cg) @@ -90,7 +90,7 @@ make eunit ct ### 执行部分应用的 common tests ```bash -make apps/emqx_bridge_mqtt-ct +make apps/emqx_retainer-ct ``` ### 静态分析(Dialyzer) diff --git a/README-JP.md b/README-JP.md index 6e1c62f2f..57c9a1809 100644 --- a/README-JP.md +++ b/README-JP.md @@ -4,7 +4,7 @@ [![Build Status](https://travis-ci.org/emqx/emqx.svg)](https://travis-ci.org/emqx/emqx) [![Coverage Status](https://coveralls.io/repos/github/emqx/emqx/badge.svg)](https://coveralls.io/github/emqx/emqx) [![Docker Pulls](https://img.shields.io/docker/pulls/emqx/emqx)](https://hub.docker.com/r/emqx/emqx) -[![Slack Invite]()](https://slack-invite.emqx.io) +[![Slack](https://img.shields.io/badge/Slack-EMQ%20X-39AE85?logo=slack)](https://slack-invite.emqx.io/) [![Twitter](https://img.shields.io/badge/Twitter-EMQ-1DA1F2?logo=twitter)](https://twitter.com/EMQTech) [![YouTube](https://img.shields.io/badge/Subscribe-EMQ-FF0000?logo=youtube)](https://www.youtube.com/channel/UC5FjR77ErAxvZENEWzQaO5Q) @@ -84,7 +84,7 @@ make eunit ct ### common test の一部を実行する ```bash -make apps/emqx_bridge_mqtt-ct +make apps/emqx_retainer-ct ``` ### Dialyzer diff --git a/README-RU.md b/README-RU.md index 2a06dac71..e02f47aa4 100644 --- a/README-RU.md +++ b/README-RU.md @@ -4,7 +4,7 @@ [![Build Status](https://travis-ci.org/emqx/emqx.svg)](https://travis-ci.org/emqx/emqx) [![Coverage Status](https://coveralls.io/repos/github/emqx/emqx/badge.svg?branch=master)](https://coveralls.io/github/emqx/emqx?branch=master) [![Docker Pulls](https://img.shields.io/docker/pulls/emqx/emqx)](https://hub.docker.com/r/emqx/emqx) -[![Slack Invite]()](https://slack-invite.emqx.io) +[![Slack](https://img.shields.io/badge/Slack-EMQ%20X-39AE85?logo=slack)](https://slack-invite.emqx.io/) [![Twitter](https://img.shields.io/badge/Follow-EMQ-1DA1F2?logo=twitter)](https://twitter.com/EMQTech) [![Community](https://img.shields.io/badge/Community-EMQ%20X-yellow?logo=github)](https://github.com/emqx/emqx/discussions) [![YouTube](https://img.shields.io/badge/Subscribe-EMQ-FF0000?logo=youtube)](https://www.youtube.com/channel/UC5FjR77ErAxvZENEWzQaO5Q) @@ -93,7 +93,7 @@ make eunit ct Пример: ```bash -make apps/emqx_bridge_mqtt-ct +make apps/emqx_retainer-ct ``` ### Dialyzer diff --git a/README.md b/README.md index 1726d426b..f60ed3cd9 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ [![Build Status](https://travis-ci.org/emqx/emqx.svg)](https://travis-ci.org/emqx/emqx) [![Coverage Status](https://coveralls.io/repos/github/emqx/emqx/badge.svg?branch=master)](https://coveralls.io/github/emqx/emqx?branch=master) [![Docker Pulls](https://img.shields.io/docker/pulls/emqx/emqx)](https://hub.docker.com/r/emqx/emqx) -[![Slack Invite]()](https://slack-invite.emqx.io) +[![Slack](https://img.shields.io/badge/Slack-EMQ%20X-39AE85?logo=slack)](https://slack-invite.emqx.io/) [![Twitter](https://img.shields.io/badge/Follow-EMQ-1DA1F2?logo=twitter)](https://twitter.com/EMQTech) [![YouTube](https://img.shields.io/badge/Subscribe-EMQ-FF0000?logo=youtube)](https://www.youtube.com/channel/UC5FjR77ErAxvZENEWzQaO5Q) @@ -92,7 +92,7 @@ make eunit ct Examples ```bash -make apps/emqx_bridge_mqtt-ct +make apps/emqx_retainer-ct ``` ### Dialyzer diff --git a/apps/emqx/etc/emqx.conf b/apps/emqx/etc/emqx.conf index 7c404ff2d..6834d2a6e 100644 --- a/apps/emqx/etc/emqx.conf +++ b/apps/emqx/etc/emqx.conf @@ -64,7 +64,7 @@ listeners.tcp.default { proxy_protocol = false ## Sets the timeout for proxy protocol. EMQ X will close the TCP connection - ## if no proxy protocol packet recevied within the timeout. + ## if no proxy protocol packet received within the timeout. ## ## @doc listeners.tcp..proxy_protocol_timeout ## ValueType: Duration @@ -163,7 +163,7 @@ listeners.ssl.default { proxy_protocol = false ## Sets the timeout for proxy protocol. EMQ X will close the TCP connection - ## if no proxy protocol packet recevied within the timeout. + ## if no proxy protocol packet received within the timeout. ## ## @doc listeners.ssl..proxy_protocol_timeout ## ValueType: Duration @@ -345,7 +345,7 @@ listeners.ws.default { proxy_protocol = false ## Sets the timeout for proxy protocol. EMQ X will close the TCP connection - ## if no proxy protocol packet recevied within the timeout. + ## if no proxy protocol packet received within the timeout. ## ## @doc listeners.ws..proxy_protocol_timeout ## ValueType: Duration @@ -448,7 +448,7 @@ listeners.wss.default { proxy_protocol = false ## Sets the timeout for proxy protocol. EMQ X will close the TCP connection - ## if no proxy protocol packet recevied within the timeout. + ## if no proxy protocol packet received within the timeout. ## ## @doc listeners.wss..proxy_protocol_timeout ## ValueType: Duration diff --git a/apps/emqx/include/emqx.hrl b/apps/emqx/include/emqx.hrl index 633527b57..550e650a2 100644 --- a/apps/emqx/include/emqx.hrl +++ b/apps/emqx/include/emqx.hrl @@ -134,3 +134,19 @@ }). -endif. + +%%-------------------------------------------------------------------- +%% Authentication +%%-------------------------------------------------------------------- + +-record(authenticator, + { id :: binary() + , provider :: module() + , enable :: boolean() + , state :: map() + }). + +-record(chain, + { name :: atom() + , authenticators :: [#authenticator{}] + }). \ No newline at end of file diff --git a/apps/emqx/include/emqx_release.hrl b/apps/emqx/include/emqx_release.hrl index 572d23155..7391b765a 100644 --- a/apps/emqx/include/emqx_release.hrl +++ b/apps/emqx/include/emqx_release.hrl @@ -29,7 +29,7 @@ -ifndef(EMQX_ENTERPRISE). --define(EMQX_RELEASE, {opensource, "5.0-alpha.5"}). +-define(EMQX_RELEASE, {opensource, "5.0-alpha.6"}). -else. diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index 271558f6d..a8462ad82 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -15,7 +15,7 @@ , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.2"}}} , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.10.8"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.1"}}} - , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.14.0"}}} + , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.17.0"}}} , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}} , {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.14.1"}}} diff --git a/apps/emqx/src/emqx_access_control.erl b/apps/emqx/src/emqx_access_control.erl index 7d5b009ba..914651535 100644 --- a/apps/emqx/src/emqx_access_control.erl +++ b/apps/emqx/src/emqx_access_control.erl @@ -29,9 +29,9 @@ -spec(authenticate(emqx_types:clientinfo()) -> {ok, map()} | {ok, map(), binary()} | {continue, map()} | {continue, binary(), map()} | {error, term()}). authenticate(Credential) -> - case run_hooks('client.authenticate', [Credential], {ok, #{superuser => false}}) of + case run_hooks('client.authenticate', [Credential], {ok, #{is_superuser => false}}) of ok -> - {ok, #{superuser => false}}; + {ok, #{is_superuser => false}}; Other -> Other end. diff --git a/apps/emqx/src/emqx_alarm.erl b/apps/emqx/src/emqx_alarm.erl index b43f5c52e..6bd2d5d49 100644 --- a/apps/emqx/src/emqx_alarm.erl +++ b/apps/emqx/src/emqx_alarm.erl @@ -408,6 +408,8 @@ normalize_message(high_cpu_usage, #{usage := Usage}) -> list_to_binary(io_lib:format("~s cpu usage", [Usage])); normalize_message(too_many_processes, #{usage := Usage}) -> list_to_binary(io_lib:format("~s process usage", [Usage])); +normalize_message(cluster_rpc_apply_failed, #{tnx_id := TnxId}) -> + list_to_binary(io_lib:format("cluster_rpc_apply_failed:~w", [TnxId])); normalize_message(partition, #{occurred := Node}) -> list_to_binary(io_lib:format("Partition occurs at node ~s", [Node])); normalize_message(<<"resource", _/binary>>, #{type := Type, id := ID}) -> diff --git a/apps/emqx/src/emqx_authentication.erl b/apps/emqx/src/emqx_authentication.erl new file mode 100644 index 000000000..2e53d85eb --- /dev/null +++ b/apps/emqx/src/emqx_authentication.erl @@ -0,0 +1,779 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_authentication). + +-behaviour(gen_server). +-behaviour(hocon_schema). +-behaviour(emqx_config_handler). + +-include("emqx.hrl"). +-include("logger.hrl"). + +-export([ roots/0 + , fields/1 + ]). + +-export([ pre_config_update/2 + , post_config_update/4 + ]). + +-export([ authenticate/2 + ]). + +-export([ initialize_authentication/2 ]). + +-export([ start_link/0 + , stop/0 + ]). + +-export([ add_provider/2 + , remove_provider/1 + , create_chain/1 + , delete_chain/1 + , lookup_chain/1 + , list_chains/0 + , create_authenticator/2 + , delete_authenticator/2 + , update_authenticator/3 + , lookup_authenticator/2 + , list_authenticators/1 + , move_authenticator/3 + ]). + +-export([ import_users/3 + , add_user/3 + , delete_user/3 + , update_user/4 + , lookup_user/3 + , list_users/2 + ]). + +-export([ generate_id/1 ]). + +%% gen_server callbacks +-export([ init/1 + , handle_call/3 + , handle_cast/2 + , handle_info/2 + , terminate/2 + , code_change/3 + ]). + +-define(CHAINS_TAB, emqx_authn_chains). + +-define(VER_1, <<"1">>). +-define(VER_2, <<"2">>). + +-type chain_name() :: atom(). +-type authenticator_id() :: binary(). +-type position() :: top | bottom | {before, authenticator_id()}. +-type update_request() :: {create_authenticator, chain_name(), map()} + | {delete_authenticator, chain_name(), authenticator_id()} + | {update_authenticator, chain_name(), authenticator_id(), map()} + | {move_authenticator, chain_name(), authenticator_id(), position()}. +-type authn_type() :: atom() | {atom(), atom()}. +-type provider() :: module(). + +-type chain() :: #{name := chain_name(), + authenticators := [authenticator()]}. + +-type authenticator() :: #{id := authenticator_id(), + provider := provider(), + enable := boolean(), + state := map()}. + + +-type config() :: #{atom() => term()}. +-type state() :: #{atom() => term()}. +-type extra() :: #{is_superuser := boolean(), + atom() => term()}. +-type user_info() :: #{user_id := binary(), + atom() => term()}. + +-callback refs() -> [{ref, Module, Name}] when Module::module(), Name::atom(). + +-callback create(Config) + -> {ok, State} + | {error, term()} + when Config::config(), State::state(). + +-callback update(Config, State) + -> {ok, NewState} + | {error, term()} + when Config::config(), State::state(), NewState::state(). + +-callback authenticate(Credential, State) + -> ignore + | {ok, Extra} + | {ok, Extra, AuthData} + | {continue, AuthCache} + | {continue, AuthData, AuthCache} + | {error, term()} + when Credential::map(), State::state(), Extra::extra(), AuthData::binary(), AuthCache::map(). + +-callback destroy(State) + -> ok + when State::state(). + +-callback import_users(Filename, State) + -> ok + | {error, term()} + when Filename::binary(), State::state(). + +-callback add_user(UserInfo, State) + -> {ok, User} + | {error, term()} + when UserInfo::user_info(), State::state(), User::user_info(). + +-callback delete_user(UserID, State) + -> ok + | {error, term()} + when UserID::binary(), State::state(). + +-callback update_user(UserID, UserInfo, State) + -> {ok, User} + | {error, term()} + when UserID::binary(), UserInfo::map(), State::state(), User::user_info(). + +-callback lookup_user(UserID, UserInfo, State) + -> {ok, User} + | {error, term()} + when UserID::binary(), UserInfo::map(), State::state(), User::user_info(). + +-callback list_users(State) + -> {ok, Users} + when State::state(), Users::[user_info()]. + +-optional_callbacks([ import_users/2 + , add_user/2 + , delete_user/2 + , update_user/3 + , lookup_user/3 + , list_users/1 + ]). + +%%------------------------------------------------------------------------------ +%% Hocon Schema +%%------------------------------------------------------------------------------ + +roots() -> [{authentication, fun authentication/1}]. + +fields(_) -> []. + +authentication(type) -> + {ok, Refs} = get_refs(), + hoconsc:union([hoconsc:array(hoconsc:union(Refs)) | Refs]); +authentication(default) -> []; +authentication(_) -> undefined. + +%%------------------------------------------------------------------------------ +%% Callbacks of config handler +%%------------------------------------------------------------------------------ + +-spec pre_config_update(update_request(), emqx_config:raw_config()) + -> {ok, map() | list()} | {error, term()}. +pre_config_update(UpdateReq, OldConfig) -> + case do_pre_config_update(UpdateReq, to_list(OldConfig)) of + {error, Reason} -> {error, Reason}; + {ok, NewConfig} -> {ok, may_to_map(NewConfig)} + end. + +do_pre_config_update({create_authenticator, _ChainName, Config}, OldConfig) -> + {ok, OldConfig ++ [Config]}; +do_pre_config_update({delete_authenticator, _ChainName, AuthenticatorID}, OldConfig) -> + NewConfig = lists:filter(fun(OldConfig0) -> + AuthenticatorID =/= generate_id(OldConfig0) + end, OldConfig), + {ok, NewConfig}; +do_pre_config_update({update_authenticator, _ChainName, AuthenticatorID, Config}, OldConfig) -> + NewConfig = lists:map(fun(OldConfig0) -> + case AuthenticatorID =:= generate_id(OldConfig0) of + true -> maps:merge(OldConfig0, Config); + false -> OldConfig0 + end + end, OldConfig), + {ok, NewConfig}; +do_pre_config_update({move_authenticator, _ChainName, AuthenticatorID, Position}, OldConfig) -> + case split_by_id(AuthenticatorID, OldConfig) of + {error, Reason} -> {error, Reason}; + {ok, Part1, [Found | Part2]} -> + case Position of + top -> + {ok, [Found | Part1] ++ Part2}; + bottom -> + {ok, Part1 ++ Part2 ++ [Found]}; + {before, Before} -> + case split_by_id(Before, Part1 ++ Part2) of + {error, Reason} -> + {error, Reason}; + {ok, NPart1, [NFound | NPart2]} -> + {ok, NPart1 ++ [Found, NFound | NPart2]} + end + end + end. + +-spec post_config_update(update_request(), map() | list(), emqx_config:raw_config(), emqx_config:app_envs()) + -> ok | {ok, map()} | {error, term()}. +post_config_update(UpdateReq, NewConfig, OldConfig, AppEnvs) -> + do_post_config_update(UpdateReq, check_config(to_list(NewConfig)), OldConfig, AppEnvs). + +do_post_config_update({create_authenticator, ChainName, Config}, _NewConfig, _OldConfig, _AppEnvs) -> + NConfig = check_config(Config), + _ = create_chain(ChainName), + create_authenticator(ChainName, NConfig); + +do_post_config_update({delete_authenticator, ChainName, AuthenticatorID}, _NewConfig, _OldConfig, _AppEnvs) -> + delete_authenticator(ChainName, AuthenticatorID); + +do_post_config_update({update_authenticator, ChainName, AuthenticatorID, _Config}, NewConfig, _OldConfig, _AppEnvs) -> + [Config] = lists:filter(fun(NewConfig0) -> + AuthenticatorID =:= generate_id(NewConfig0) + end, NewConfig), + NConfig = check_config(Config), + update_authenticator(ChainName, AuthenticatorID, NConfig); + +do_post_config_update({move_authenticator, ChainName, AuthenticatorID, Position}, _NewConfig, _OldConfig, _AppEnvs) -> + move_authenticator(ChainName, AuthenticatorID, Position). + +check_config(Config) -> + #{authentication := CheckedConfig} = hocon_schema:check_plain(emqx_authentication, + #{<<"authentication">> => Config}, #{nullable => true, atom_key => true}), + CheckedConfig. + +%%------------------------------------------------------------------------------ +%% Authenticate +%%------------------------------------------------------------------------------ + +authenticate(#{listener := Listener, protocol := Protocol} = Credential, _AuthResult) -> + case ets:lookup(?CHAINS_TAB, Listener) of + [#chain{authenticators = Authenticators}] when Authenticators =/= [] -> + do_authenticate(Authenticators, Credential); + _ -> + case ets:lookup(?CHAINS_TAB, global_chain(Protocol)) of + [#chain{authenticators = Authenticators}] when Authenticators =/= [] -> + do_authenticate(Authenticators, Credential); + _ -> + ignore + end + end. + +do_authenticate([], _) -> + {stop, {error, not_authorized}}; +do_authenticate([#authenticator{provider = Provider, state = State} | More], Credential) -> + case Provider:authenticate(Credential, State) of + ignore -> + do_authenticate(More, Credential); + Result -> + %% {ok, Extra} + %% {ok, Extra, AuthData} + %% {continue, AuthCache} + %% {continue, AuthData, AuthCache} + %% {error, Reason} + {stop, Result} + end. + +%%------------------------------------------------------------------------------ +%% APIs +%%------------------------------------------------------------------------------ + +-spec initialize_authentication(chain_name(), [#{binary() => term()}]) -> ok. +initialize_authentication(_, []) -> + ok; +initialize_authentication(ChainName, AuthenticatorsConfig) -> + _ = create_chain(ChainName), + CheckedConfig = check_config(to_list(AuthenticatorsConfig)), + lists:foreach(fun(AuthenticatorConfig) -> + case create_authenticator(ChainName, AuthenticatorConfig) of + {ok, _} -> + ok; + {error, Reason} -> + ?LOG(error, "Failed to create authenticator '~s': ~p", [generate_id(AuthenticatorConfig), Reason]) + end + end, CheckedConfig). + +-spec start_link() -> {ok, pid()} | ignore | {error, term()}. +start_link() -> + gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). + +-spec stop() -> ok. +stop() -> + gen_server:stop(?MODULE). + +-spec get_refs() -> {ok, Refs} when Refs :: [{authn_type(), module()}]. +get_refs() -> + gen_server:call(?MODULE, get_refs). + +-spec add_provider(authn_type(), module()) -> ok. +add_provider(AuthNType, Provider) -> + gen_server:call(?MODULE, {add_provider, AuthNType, Provider}). + +-spec remove_provider(authn_type()) -> ok. +remove_provider(AuthNType) -> + gen_server:call(?MODULE, {remove_provider, AuthNType}). + +-spec create_chain(chain_name()) -> {ok, chain()} | {error, term()}. +create_chain(Name) -> + gen_server:call(?MODULE, {create_chain, Name}). + +-spec delete_chain(chain_name()) -> ok | {error, term()}. +delete_chain(Name) -> + gen_server:call(?MODULE, {delete_chain, Name}). + +-spec lookup_chain(chain_name()) -> {ok, chain()} | {error, term()}. +lookup_chain(Name) -> + gen_server:call(?MODULE, {lookup_chain, Name}). + +-spec list_chains() -> {ok, [chain()]}. +list_chains() -> + Chains = ets:tab2list(?CHAINS_TAB), + {ok, [serialize_chain(Chain) || Chain <- Chains]}. + +-spec create_authenticator(chain_name(), config()) -> {ok, authenticator()} | {error, term()}. +create_authenticator(ChainName, Config) -> + gen_server:call(?MODULE, {create_authenticator, ChainName, Config}). + +-spec delete_authenticator(chain_name(), authenticator_id()) -> ok | {error, term()}. +delete_authenticator(ChainName, AuthenticatorID) -> + gen_server:call(?MODULE, {delete_authenticator, ChainName, AuthenticatorID}). + +-spec update_authenticator(chain_name(), authenticator_id(), config()) -> {ok, authenticator()} | {error, term()}. +update_authenticator(ChainName, AuthenticatorID, Config) -> + gen_server:call(?MODULE, {update_authenticator, ChainName, AuthenticatorID, Config}). + +-spec lookup_authenticator(chain_name(), authenticator_id()) -> {ok, authenticator()} | {error, term()}. +lookup_authenticator(ChainName, AuthenticatorID) -> + case ets:lookup(?CHAINS_TAB, ChainName) of + [] -> + {error, {not_found, {chain, ChainName}}}; + [#chain{authenticators = Authenticators}] -> + case lists:keyfind(AuthenticatorID, #authenticator.id, Authenticators) of + false -> + {error, {not_found, {authenticator, AuthenticatorID}}}; + Authenticator -> + {ok, serialize_authenticator(Authenticator)} + end + end. + +-spec list_authenticators(chain_name()) -> {ok, [authenticator()]} | {error, term()}. +list_authenticators(ChainName) -> + case ets:lookup(?CHAINS_TAB, ChainName) of + [] -> + {error, {not_found, {chain, ChainName}}}; + [#chain{authenticators = Authenticators}] -> + {ok, serialize_authenticators(Authenticators)} + end. + +-spec move_authenticator(chain_name(), authenticator_id(), position()) -> ok | {error, term()}. +move_authenticator(ChainName, AuthenticatorID, Position) -> + gen_server:call(?MODULE, {move_authenticator, ChainName, AuthenticatorID, Position}). + +-spec import_users(chain_name(), authenticator_id(), binary()) -> ok | {error, term()}. +import_users(ChainName, AuthenticatorID, Filename) -> + gen_server:call(?MODULE, {import_users, ChainName, AuthenticatorID, Filename}). + +-spec add_user(chain_name(), authenticator_id(), user_info()) -> {ok, user_info()} | {error, term()}. +add_user(ChainName, AuthenticatorID, UserInfo) -> + gen_server:call(?MODULE, {add_user, ChainName, AuthenticatorID, UserInfo}). + +-spec delete_user(chain_name(), authenticator_id(), binary()) -> ok | {error, term()}. +delete_user(ChainName, AuthenticatorID, UserID) -> + gen_server:call(?MODULE, {delete_user, ChainName, AuthenticatorID, UserID}). + +-spec update_user(chain_name(), authenticator_id(), binary(), map()) -> {ok, user_info()} | {error, term()}. +update_user(ChainName, AuthenticatorID, UserID, NewUserInfo) -> + gen_server:call(?MODULE, {update_user, ChainName, AuthenticatorID, UserID, NewUserInfo}). + +-spec lookup_user(chain_name(), authenticator_id(), binary()) -> {ok, user_info()} | {error, term()}. +lookup_user(ChainName, AuthenticatorID, UserID) -> + gen_server:call(?MODULE, {lookup_user, ChainName, AuthenticatorID, UserID}). + +%% TODO: Support pagination +-spec list_users(chain_name(), authenticator_id()) -> {ok, [user_info()]} | {error, term()}. +list_users(ChainName, AuthenticatorID) -> + gen_server:call(?MODULE, {list_users, ChainName, AuthenticatorID}). + +-spec generate_id(config()) -> authenticator_id(). +generate_id(#{mechanism := Mechanism0, backend := Backend0}) -> + Mechanism = atom_to_binary(Mechanism0), + Backend = atom_to_binary(Backend0), + <>; +generate_id(#{mechanism := Mechanism}) -> + atom_to_binary(Mechanism); +generate_id(#{<<"mechanism">> := Mechanism, <<"backend">> := Backend}) -> + <>; +generate_id(#{<<"mechanism">> := Mechanism}) -> + Mechanism. + +%%-------------------------------------------------------------------- +%% gen_server callbacks +%%-------------------------------------------------------------------- + +init(_Opts) -> + _ = ets:new(?CHAINS_TAB, [ named_table, set, public + , {keypos, #chain.name} + , {read_concurrency, true}]), + ok = emqx_config_handler:add_handler([authentication], ?MODULE), + ok = emqx_config_handler:add_handler([listeners, '?', '?', authentication], ?MODULE), + {ok, #{hooked => false, providers => #{}}}. + +handle_call({add_provider, AuthNType, Provider}, _From, #{providers := Providers} = State) -> + reply(ok, State#{providers := Providers#{AuthNType => Provider}}); + +handle_call({remove_provider, AuthNType}, _From, #{providers := Providers} = State) -> + reply(ok, State#{providers := maps:remove(AuthNType, Providers)}); + +handle_call(get_refs, _From, #{providers := Providers} = State) -> + Refs = lists:foldl(fun({_, Provider}, Acc) -> + Acc ++ Provider:refs() + end, [], maps:to_list(Providers)), + reply({ok, Refs}, State); + +handle_call({create_chain, Name}, _From, State) -> + case ets:member(?CHAINS_TAB, Name) of + true -> + reply({error, {already_exists, {chain, Name}}}, State); + false -> + Chain = #chain{name = Name, + authenticators = []}, + true = ets:insert(?CHAINS_TAB, Chain), + reply({ok, serialize_chain(Chain)}, State) + end; + +handle_call({delete_chain, Name}, _From, State) -> + case ets:lookup(?CHAINS_TAB, Name) of + [] -> + reply({error, {not_found, {chain, Name}}}, State); + [#chain{authenticators = Authenticators}] -> + _ = [do_delete_authenticator(Authenticator) || Authenticator <- Authenticators], + true = ets:delete(?CHAINS_TAB, Name), + reply(ok, may_unhook(State)) + end; + +handle_call({lookup_chain, Name}, _From, State) -> + case ets:lookup(?CHAINS_TAB, Name) of + [] -> + reply({error, {not_found, {chain, Name}}}, State); + [Chain] -> + reply({ok, serialize_chain(Chain)}, State) + end; + +handle_call({create_authenticator, ChainName, Config}, _From, #{providers := Providers} = State) -> + UpdateFun = + fun(#chain{authenticators = Authenticators} = Chain) -> + AuthenticatorID = generate_id(Config), + case lists:keymember(AuthenticatorID, #authenticator.id, Authenticators) of + true -> + {error, {already_exists, {authenticator, AuthenticatorID}}}; + false -> + case do_create_authenticator(ChainName, AuthenticatorID, Config, Providers) of + {ok, Authenticator} -> + NAuthenticators = Authenticators ++ [Authenticator], + true = ets:insert(?CHAINS_TAB, Chain#chain{authenticators = NAuthenticators}), + {ok, serialize_authenticator(Authenticator)}; + {error, Reason} -> + {error, Reason} + end + end + end, + Reply = update_chain(ChainName, UpdateFun), + reply(Reply, may_hook(State)); + +handle_call({delete_authenticator, ChainName, AuthenticatorID}, _From, State) -> + UpdateFun = + fun(#chain{authenticators = Authenticators} = Chain) -> + case lists:keytake(AuthenticatorID, #authenticator.id, Authenticators) of + false -> + {error, {not_found, {authenticator, AuthenticatorID}}}; + {value, Authenticator, NAuthenticators} -> + _ = do_delete_authenticator(Authenticator), + true = ets:insert(?CHAINS_TAB, Chain#chain{authenticators = NAuthenticators}), + ok + end + end, + Reply = update_chain(ChainName, UpdateFun), + reply(Reply, may_unhook(State)); + +handle_call({update_authenticator, ChainName, AuthenticatorID, Config}, _From, State) -> + UpdateFun = + fun(#chain{authenticators = Authenticators} = Chain) -> + case lists:keyfind(AuthenticatorID, #authenticator.id, Authenticators) of + false -> + {error, {not_found, {authenticator, AuthenticatorID}}}; + #authenticator{provider = Provider, + state = #{version := Version} = ST} = Authenticator -> + case AuthenticatorID =:= generate_id(Config) of + true -> + Unique = unique(ChainName, AuthenticatorID, Version), + case Provider:update(Config#{'_unique' => Unique}, ST) of + {ok, NewST} -> + NewAuthenticator = Authenticator#authenticator{state = switch_version(NewST)}, + NewAuthenticators = replace_authenticator(AuthenticatorID, NewAuthenticator, Authenticators), + true = ets:insert(?CHAINS_TAB, Chain#chain{authenticators = NewAuthenticators}), + {ok, serialize_authenticator(NewAuthenticator)}; + {error, Reason} -> + {error, Reason} + end; + false -> + {error, change_of_authentication_type_is_not_allowed} + end + end + end, + Reply = update_chain(ChainName, UpdateFun), + reply(Reply, State); + +handle_call({move_authenticator, ChainName, AuthenticatorID, Position}, _From, State) -> + UpdateFun = + fun(#chain{authenticators = Authenticators} = Chain) -> + case do_move_authenticator(AuthenticatorID, Authenticators, Position) of + {ok, NAuthenticators} -> + true = ets:insert(?CHAINS_TAB, Chain#chain{authenticators = NAuthenticators}), + ok; + {error, Reason} -> + {error, Reason} + end + end, + Reply = update_chain(ChainName, UpdateFun), + reply(Reply, State); + +handle_call({import_users, ChainName, AuthenticatorID, Filename}, _From, State) -> + Reply = call_authenticator(ChainName, AuthenticatorID, import_users, [Filename]), + reply(Reply, State); + +handle_call({add_user, ChainName, AuthenticatorID, UserInfo}, _From, State) -> + Reply = call_authenticator(ChainName, AuthenticatorID, add_user, [UserInfo]), + reply(Reply, State); + +handle_call({delete_user, ChainName, AuthenticatorID, UserID}, _From, State) -> + Reply = call_authenticator(ChainName, AuthenticatorID, delete_user, [UserID]), + reply(Reply, State); + +handle_call({update_user, ChainName, AuthenticatorID, UserID, NewUserInfo}, _From, State) -> + Reply = call_authenticator(ChainName, AuthenticatorID, update_user, [UserID, NewUserInfo]), + reply(Reply, State); + +handle_call({lookup_user, ChainName, AuthenticatorID, UserID}, _From, State) -> + Reply = call_authenticator(ChainName, AuthenticatorID, lookup_user, [UserID]), + reply(Reply, State); + +handle_call({list_users, ChainName, AuthenticatorID}, _From, State) -> + Reply = call_authenticator(ChainName, AuthenticatorID, list_users, []), + reply(Reply, State); + +handle_call(Req, _From, State) -> + ?LOG(error, "Unexpected call: ~p", [Req]), + {reply, ignored, State}. + +handle_cast(Req, State) -> + ?LOG(error, "Unexpected case: ~p", [Req]), + {noreply, State}. + +handle_info(Info, State) -> + ?LOG(error, "Unexpected info: ~p", [Info]), + {noreply, State}. + +terminate(_Reason, _State) -> + emqx_config_handler:remove_handler([authentication]), + emqx_config_handler:remove_handler([listeners, '?', '?', authentication]), + ok. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +reply(Reply, State) -> + {reply, Reply, State}. + +%%------------------------------------------------------------------------------ +%% Internal functions +%%------------------------------------------------------------------------------ + +split_by_id(ID, AuthenticatorsConfig) -> + case lists:foldl( + fun(C, {P1, P2, F0}) -> + F = case ID =:= generate_id(C) of + true -> true; + false -> F0 + end, + case F of + false -> {[C | P1], P2, F}; + true -> {P1, [C | P2], F} + end + end, {[], [], false}, AuthenticatorsConfig) of + {_, _, false} -> + {error, {not_found, {authenticator, ID}}}; + {Part1, Part2, true} -> + {ok, lists:reverse(Part1), lists:reverse(Part2)} + end. + +global_chain(mqtt) -> + 'mqtt:global'; +global_chain('mqtt-sn') -> + 'mqtt-sn:global'; +global_chain(coap) -> + 'coap:global'; +global_chain(lwm2m) -> + 'lwm2m:global'; +global_chain(stomp) -> + 'stomp:global'; +global_chain(_) -> + 'unknown:global'. + +may_hook(#{hooked := false} = State) -> + case lists:any(fun(#chain{authenticators = []}) -> false; + (_) -> true + end, ets:tab2list(?CHAINS_TAB)) of + true -> + _ = emqx:hook('client.authenticate', {emqx_authentication, authenticate, []}), + State#{hooked => true}; + false -> + State + end; +may_hook(State) -> + State. + +may_unhook(#{hooked := true} = State) -> + case lists:all(fun(#chain{authenticators = []}) -> true; + (_) -> false + end, ets:tab2list(?CHAINS_TAB)) of + true -> + _ = emqx:unhook('client.authenticate', {emqx_authentication, authenticate, []}), + State#{hooked => false}; + false -> + State + end; +may_unhook(State) -> + State. + +do_create_authenticator(ChainName, AuthenticatorID, #{enable := Enable} = Config, Providers) -> + case maps:get(authn_type(Config), Providers, undefined) of + undefined -> + {error, no_available_provider}; + Provider -> + Unique = unique(ChainName, AuthenticatorID, ?VER_1), + case Provider:create(Config#{'_unique' => Unique}) of + {ok, State} -> + Authenticator = #authenticator{id = AuthenticatorID, + provider = Provider, + enable = Enable, + state = switch_version(State)}, + {ok, Authenticator}; + {error, Reason} -> + {error, Reason} + end + end. + +do_delete_authenticator(#authenticator{provider = Provider, state = State}) -> + _ = Provider:destroy(State), + ok. + +replace_authenticator(ID, Authenticator, Authenticators) -> + lists:keyreplace(ID, #authenticator.id, Authenticators, Authenticator). + +do_move_authenticator(ID, Authenticators, Position) -> + case lists:keytake(ID, #authenticator.id, Authenticators) of + false -> + {error, {not_found, {authenticator, ID}}}; + {value, Authenticator, NAuthenticators} -> + case Position of + top -> + {ok, [Authenticator | NAuthenticators]}; + bottom -> + {ok, NAuthenticators ++ [Authenticator]}; + {before, ID0} -> + insert(Authenticator, NAuthenticators, ID0, []) + end + end. + +insert(_, [], ID, _) -> + {error, {not_found, {authenticator, ID}}}; +insert(Authenticator, [#authenticator{id = ID} | _] = Authenticators, ID, Acc) -> + {ok, lists:reverse(Acc) ++ [Authenticator | Authenticators]}; +insert(Authenticator, [Authenticator0 | More], ID, Acc) -> + insert(Authenticator, More, ID, [Authenticator0 | Acc]). + +update_chain(ChainName, UpdateFun) -> + case ets:lookup(?CHAINS_TAB, ChainName) of + [] -> + {error, {not_found, {chain, ChainName}}}; + [Chain] -> + UpdateFun(Chain) + end. + +call_authenticator(ChainName, AuthenticatorID, Func, Args) -> + UpdateFun = + fun(#chain{authenticators = Authenticators}) -> + case lists:keyfind(AuthenticatorID, #authenticator.id, Authenticators) of + false -> + {error, {not_found, {authenticator, AuthenticatorID}}}; + #authenticator{provider = Provider, state = State} -> + case erlang:function_exported(Provider, Func, length(Args) + 1) of + true -> + erlang:apply(Provider, Func, Args ++ [State]); + false -> + {error, unsupported_operation} + end + end + end, + update_chain(ChainName, UpdateFun). + +serialize_chain(#chain{name = Name, + authenticators = Authenticators}) -> + #{ name => Name + , authenticators => serialize_authenticators(Authenticators) + }. + +serialize_authenticators(Authenticators) -> + [serialize_authenticator(Authenticator) || Authenticator <- Authenticators]. + +serialize_authenticator(#authenticator{id = ID, + provider = Provider, + enable = Enable, + state = State}) -> + #{ id => ID + , provider => Provider + , enable => Enable + , state => State + }. + +unique(ChainName, AuthenticatorID, Version) -> + NChainName = atom_to_binary(ChainName), + <>. + +switch_version(State = #{version := ?VER_1}) -> + State#{version := ?VER_2}; +switch_version(State = #{version := ?VER_2}) -> + State#{version := ?VER_1}; +switch_version(State) -> + State#{version => ?VER_1}. + +authn_type(#{mechanism := Mechanism, backend := Backend}) -> + {Mechanism, Backend}; +authn_type(#{mechanism := Mechanism}) -> + Mechanism. + +may_to_map([L]) -> + L; +may_to_map(L) -> + L. + +to_list(undefined) -> + []; +to_list(M) when M =:= #{} -> + []; +to_list(M) when is_map(M) -> + [M]; +to_list(L) when is_list(L) -> + L. diff --git a/apps/emqx/src/emqx_banned.erl b/apps/emqx/src/emqx_banned.erl index 715548d41..608734363 100644 --- a/apps/emqx/src/emqx_banned.erl +++ b/apps/emqx/src/emqx_banned.erl @@ -37,6 +37,7 @@ , delete/1 , info/1 , format/1 + , parse/1 ]). %% gen_server callbacks @@ -107,6 +108,33 @@ format(#banned{who = Who0, until => to_rfc3339(Until) }. +parse(Params) -> + Who = pares_who(Params), + By = maps:get(<<"by">>, Params, <<"mgmt_api">>), + Reason = maps:get(<<"reason">>, Params, <<"">>), + At = pares_time(maps:get(<<"at">>, Params, undefined), erlang:system_time(second)), + Until = pares_time(maps:get(<<"until">>, Params, undefined), At + 5 * 60), + #banned{ + who = Who, + by = By, + reason = Reason, + at = At, + until = Until + }. + +pares_who(#{as := As, who := Who}) -> + pares_who(#{<<"as">> => As, <<"who">> => Who}); +pares_who(#{<<"as">> := <<"peerhost">>, <<"who">> := Peerhost0}) -> + {ok, Peerhost} = inet:parse_address(binary_to_list(Peerhost0)), + {peerhost, Peerhost}; +pares_who(#{<<"as">> := As, <<"who">> := Who}) -> + {binary_to_atom(As, utf8), Who}. + +pares_time(undefined, Default) -> + Default; +pares_time(Rfc3339, _Default) -> + to_timestamp(Rfc3339). + maybe_format_host({peerhost, Host}) -> AddrBinary = list_to_binary(inet:ntoa(Host)), {peerhost, AddrBinary}; @@ -116,6 +144,11 @@ maybe_format_host({As, Who}) -> to_rfc3339(Timestamp) -> list_to_binary(calendar:system_time_to_rfc3339(Timestamp, [{unit, second}])). +to_timestamp(Rfc3339) when is_binary(Rfc3339) -> + to_timestamp(binary_to_list(Rfc3339)); +to_timestamp(Rfc3339) -> + calendar:rfc3339_to_system_time(Rfc3339, [{unit, second}]). + -spec(create(emqx_types:banned() | map()) -> ok). create(#{who := Who, by := By, @@ -130,12 +163,16 @@ create(#{who := Who, create(Banned) when is_record(Banned, banned) -> ekka_mnesia:dirty_write(?BANNED_TAB, Banned). +look_up(Who) when is_map(Who) -> + look_up(pares_who(Who)); look_up(Who) -> mnesia:dirty_read(?BANNED_TAB, Who). -spec(delete({clientid, emqx_types:clientid()} | {username, emqx_types:username()} | {peerhost, emqx_types:peerhost()}) -> ok). +delete(Who) when is_map(Who)-> + delete(pares_who(Who)); delete(Who) -> ekka_mnesia:dirty_delete(?BANNED_TAB, Who). diff --git a/apps/emqx/src/emqx_broker_sup.erl b/apps/emqx/src/emqx_broker_sup.erl index 69df72408..a479e9ff1 100644 --- a/apps/emqx/src/emqx_broker_sup.erl +++ b/apps/emqx/src/emqx_broker_sup.erl @@ -43,6 +43,14 @@ init([]) -> type => worker, modules => [emqx_shared_sub]}, + %% Authentication + AuthN = #{id => authn, + start => {emqx_authentication, start_link, []}, + restart => permanent, + shutdown => 2000, + type => worker, + modules => [emqx_authentication]}, + %% Broker helper Helper = #{id => helper, start => {emqx_broker_helper, start_link, []}, @@ -51,5 +59,5 @@ init([]) -> type => worker, modules => [emqx_broker_helper]}, - {ok, {{one_for_all, 0, 1}, [BrokerPool, SharedSub, Helper]}}. + {ok, {{one_for_all, 0, 1}, [BrokerPool, SharedSub, AuthN, Helper]}}. diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl index e25a9c8d6..0b1ff7e25 100644 --- a/apps/emqx/src/emqx_channel.erl +++ b/apps/emqx/src/emqx_channel.erl @@ -214,7 +214,7 @@ init(ConnInfo = #{peername := {PeerHost, _Port}, ClientInfo = set_peercert_infos( Peercert, #{zone => Zone, - listener => Listener, + listener => emqx_listeners:listener_id(Type, Listener), protocol => Protocol, peerhost => PeerHost, sockport => SockPort, @@ -223,7 +223,7 @@ init(ConnInfo = #{peername := {PeerHost, _Port}, mountpoint => MountPoint, is_bridge => false, is_superuser => false - }, Zone, Listener), + }, Zone), {NClientInfo, NConnInfo} = take_ws_cookie(ClientInfo, ConnInfo), #channel{conninfo = NConnInfo, clientinfo = NClientInfo, @@ -244,12 +244,12 @@ quota_policy(RawPolicy) -> erlang:trunc(hocon_postprocess:duration(StrWind) / 1000)}} || {Name, [StrCount, StrWind]} <- maps:to_list(RawPolicy)]. -set_peercert_infos(NoSSL, ClientInfo, _, _) +set_peercert_infos(NoSSL, ClientInfo, _) when NoSSL =:= nossl; NoSSL =:= undefined -> ClientInfo#{username => undefined}; -set_peercert_infos(Peercert, ClientInfo, Zone, _Listener) -> +set_peercert_infos(Peercert, ClientInfo, Zone) -> {DN, CN} = {esockd_peercert:subject(Peercert), esockd_peercert:common_name(Peercert)}, PeercetAs = fun(Key) -> @@ -1303,11 +1303,11 @@ do_authenticate(#{auth_method := AuthMethod} = Credential, #channel{clientinfo = case emqx_access_control:authenticate(Credential) of {ok, Result} -> {ok, Properties, - Channel#channel{clientinfo = ClientInfo#{is_superuser => maps:get(superuser, Result, false)}, + Channel#channel{clientinfo = ClientInfo#{is_superuser => maps:get(is_superuser, Result, false)}, auth_cache = #{}}}; {ok, Result, AuthData} -> {ok, Properties#{'Authentication-Data' => AuthData}, - Channel#channel{clientinfo = ClientInfo#{is_superuser => maps:get(superuser, Result, false)}, + Channel#channel{clientinfo = ClientInfo#{is_superuser => maps:get(is_superuser, Result, false)}, auth_cache = #{}}}; {continue, AuthCache} -> {continue, Properties, Channel#channel{auth_cache = AuthCache}}; @@ -1320,8 +1320,8 @@ do_authenticate(#{auth_method := AuthMethod} = Credential, #channel{clientinfo = do_authenticate(Credential, #channel{clientinfo = ClientInfo} = Channel) -> case emqx_access_control:authenticate(Credential) of - {ok, #{superuser := Superuser}} -> - {ok, #{}, Channel#channel{clientinfo = ClientInfo#{is_superuser => Superuser}}}; + {ok, #{is_superuser := IsSuperuser}} -> + {ok, #{}, Channel#channel{clientinfo = ClientInfo#{is_superuser => IsSuperuser}}}; {error, Reason} -> {error, emqx_reason_codes:connack_error(Reason)} end. diff --git a/apps/emqx/src/emqx_cm_sup.erl b/apps/emqx/src/emqx_cm_sup.erl index f332a0868..cddd8aa5e 100644 --- a/apps/emqx/src/emqx_cm_sup.erl +++ b/apps/emqx/src/emqx_cm_sup.erl @@ -22,49 +22,38 @@ -export([init/1]). +%%-------------------------------------------------------------------- +%% API +%%-------------------------------------------------------------------- + start_link() -> supervisor:start_link({local, ?MODULE}, ?MODULE, []). +%%-------------------------------------------------------------------- +%% Supervisor callbacks +%%-------------------------------------------------------------------- + init([]) -> - Banned = #{id => banned, - start => {emqx_banned, start_link, []}, - restart => permanent, - shutdown => 1000, - type => worker, - modules => [emqx_banned]}, - Flapping = #{id => flapping, - start => {emqx_flapping, start_link, []}, - restart => permanent, - shutdown => 1000, - type => worker, - modules => [emqx_flapping]}, - %% Channel locker - Locker = #{id => locker, - start => {emqx_cm_locker, start_link, []}, - restart => permanent, - shutdown => 5000, - type => worker, - modules => [emqx_cm_locker] - }, - %% Channel registry - Registry = #{id => registry, - start => {emqx_cm_registry, start_link, []}, - restart => permanent, - shutdown => 5000, - type => worker, - modules => [emqx_cm_registry] - }, - %% Channel Manager - Manager = #{id => manager, - start => {emqx_cm, start_link, []}, - restart => permanent, - shutdown => 5000, - type => worker, - modules => [emqx_cm] - }, SupFlags = #{strategy => one_for_one, intensity => 100, period => 10 }, + Banned = child_spec(emqx_banned, 1000, worker), + Flapping = child_spec(emqx_flapping, 1000, worker), + Locker = child_spec(emqx_cm_locker, 5000, worker), + Registry = child_spec(emqx_cm_registry, 5000, worker), + Manager = child_spec(emqx_cm, 5000, worker), {ok, {SupFlags, [Banned, Flapping, Locker, Registry, Manager]}}. +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- + +child_spec(Mod, Shutdown, Type) -> + #{id => Mod, + start => {Mod, start_link, []}, + restart => permanent, + shutdown => Shutdown, + type => Type, + modules => [Mod] + }. diff --git a/apps/emqx/src/emqx_config.erl b/apps/emqx/src/emqx_config.erl index 317aba401..bd6e14e8e 100644 --- a/apps/emqx/src/emqx_config.erl +++ b/apps/emqx/src/emqx_config.erl @@ -87,8 +87,14 @@ -type update_request() :: term(). -type update_cmd() :: {update, update_request()} | remove. -type update_opts() :: #{ - %% fill the default values into the rawconf map - rawconf_with_defaults => boolean() + %% rawconf_with_defaults: + %% fill the default values into the `raw_config` field of the return value + %% defaults to `false` + rawconf_with_defaults => boolean(), + %% persistent: + %% save the updated config to the emqx_override.conf file + %% defaults to `true` + persistent => boolean() }. -type update_args() :: {update_cmd(), Opts :: update_opts()}. -type update_stage() :: pre_config_update | post_config_update. @@ -235,7 +241,7 @@ put_raw(KeyPath, Config) -> do_put(?RAW_CONF, KeyPath, Config). %% in the rear of the list overrides prior values. -spec init_load(module(), [string()] | binary() | hocon:config()) -> ok. init_load(SchemaMod, Conf) when is_list(Conf) orelse is_binary(Conf) -> - ParseOptions = #{format => richmap}, + ParseOptions = #{format => map}, Parser = case is_binary(Conf) of true -> fun hocon:binary/2; false -> fun hocon:files/2 @@ -249,19 +255,17 @@ init_load(SchemaMod, Conf) when is_list(Conf) orelse is_binary(Conf) -> }), error(failed_to_load_hocon_conf) end; -init_load(SchemaMod, RawRichConf) when is_map(RawRichConf) -> - %% check with richmap for line numbers in error reports (future enhancement) - Opts = #{return_plain => true, - nullable => true - }, - %% this call throws exception in case of check failure - {_AppEnvs, CheckedConf} = hocon_schema:map_translate(SchemaMod, RawRichConf, Opts), +init_load(SchemaMod, RawConf0) when is_map(RawConf0) -> ok = save_schema_mod_and_names(SchemaMod), - ok = save_to_config_map(emqx_map_lib:unsafe_atom_key_map(normalize_conf(CheckedConf)), - normalize_conf(hocon_schema:richmap_to_map(RawRichConf))). + %% override part of the input conf using emqx_override.conf + RawConf = merge_with_override_conf(RawConf0), + %% check and save configs + {_AppEnvs, CheckedConf} = check_config(SchemaMod, RawConf), + ok = save_to_config_map(maps:with(get_atom_root_names(), CheckedConf), + maps:with(get_root_names(), RawConf)). -normalize_conf(Conf) -> - maps:with(get_root_names(), Conf). +merge_with_override_conf(RawConf) -> + maps:merge(RawConf, maps:with(maps:keys(RawConf), read_override_conf())). -spec check_config(module(), raw_config()) -> {AppEnvs, CheckedConf} when AppEnvs :: app_envs(), CheckedConf :: config(). @@ -320,6 +324,9 @@ get_schema_mod(RootName) -> get_root_names() -> maps:get(names, persistent_term:get(?PERSIS_SCHEMA_MODS, #{names => []})). +get_atom_root_names() -> + [atom(N) || N <- get_root_names()]. + -spec save_configs(app_envs(), config(), raw_config(), raw_config()) -> ok | {error, term()}. save_configs(_AppEnvs, Conf, RawConf, OverrideConf) -> %% We may need also support hot config update for the apps that use application envs. @@ -341,14 +348,19 @@ save_to_config_map(Conf, RawConf) -> ?MODULE:put_raw(RawConf). -spec save_to_override_conf(raw_config()) -> ok | {error, term()}. +save_to_override_conf(undefined) -> + ok; save_to_override_conf(RawConf) -> - FileName = emqx_override_conf_name(), - ok = filelib:ensure_dir(FileName), - case file:write_file(FileName, jsx:prettify(jsx:encode(RawConf))) of - ok -> ok; - {error, Reason} -> - logger:error("write to ~s failed, ~p", [FileName, Reason]), - {error, Reason} + case emqx_override_conf_name() of + undefined -> ok; + FileName -> + ok = filelib:ensure_dir(FileName), + case file:write_file(FileName, jsx:prettify(jsx:encode(RawConf))) of + ok -> ok; + {error, Reason} -> + logger:error("write to ~s failed, ~p", [FileName, Reason]), + {error, Reason} + end end. load_hocon_file(FileName, LoadType) -> @@ -360,7 +372,7 @@ load_hocon_file(FileName, LoadType) -> end. emqx_override_conf_name() -> - application:get_env(emqx, override_conf_file, "emqx_override.conf"). + application:get_env(emqx, override_conf_file, undefined). do_get(Type, KeyPath) -> Ref = make_ref(), diff --git a/apps/emqx/src/emqx_config_handler.erl b/apps/emqx/src/emqx_config_handler.erl index f16f8a97a..d92f1d35a 100644 --- a/apps/emqx/src/emqx_config_handler.erl +++ b/apps/emqx/src/emqx_config_handler.erl @@ -39,6 +39,7 @@ code_change/3]). -define(MOD, {mod}). +-define(WKEY, '?'). -define(ATOM_CONF_PATH(PATH, EXP, EXP_ON_FAIL), try [safe_atom(Key) || Key <- PATH] of @@ -80,11 +81,11 @@ update_config(SchemaModule, ConfKeyPath, UpdateArgs) -> -spec add_handler(emqx_config:config_key_path(), handler_name()) -> ok. add_handler(ConfKeyPath, HandlerName) -> - gen_server:call(?MODULE, {add_child, ConfKeyPath, HandlerName}). + gen_server:call(?MODULE, {add_handler, ConfKeyPath, HandlerName}). -spec remove_handler(emqx_config:config_key_path()) -> ok. remove_handler(ConfKeyPath) -> - gen_server:call(?MODULE, {remove_child, ConfKeyPath}). + gen_server:call(?MODULE, {remove_handler, ConfKeyPath}). %%============================================================================ @@ -92,15 +93,18 @@ remove_handler(ConfKeyPath) -> init(_) -> {ok, #{handlers => #{?MOD => ?MODULE}}}. -handle_call({add_child, ConfKeyPath, HandlerName}, _From, - State = #{handlers := Handlers}) -> - {reply, ok, State#{handlers => - emqx_map_lib:deep_put(ConfKeyPath, Handlers, #{?MOD => HandlerName})}}; +handle_call({add_handler, ConfKeyPath, HandlerName}, _From, State = #{handlers := Handlers}) -> + case deep_put_handler(ConfKeyPath, Handlers, HandlerName) of + {ok, NewHandlers} -> + {reply, ok, State#{handlers => NewHandlers}}; + Error -> + {reply, Error, State} + end; -handle_call({remove_child, ConfKeyPath}, _From, +handle_call({remove_handler, ConfKeyPath}, _From, State = #{handlers := Handlers}) -> {reply, ok, State#{handlers => - emqx_map_lib:deep_remove(ConfKeyPath, Handlers)}}; + emqx_map_lib:deep_remove(ConfKeyPath ++ [?MOD], Handlers)}}; handle_call({change_config, SchemaModule, ConfKeyPath, UpdateArgs}, _From, #{handlers := Handlers} = State) -> @@ -134,17 +138,40 @@ terminate(_Reason, _State) -> code_change(_OldVsn, State, _Extra) -> {ok, State}. -process_update_request(ConfKeyPath, _Handlers, {remove, _Opts}) -> +deep_put_handler([], Handlers, Mod) when is_map(Handlers) -> + {ok, Handlers#{?MOD => Mod}}; +deep_put_handler([], _Handlers, Mod) -> + {ok, #{?MOD => Mod}}; +deep_put_handler([?WKEY | KeyPath], Handlers, Mod) -> + deep_put_handler2(?WKEY, KeyPath, Handlers, Mod); +deep_put_handler([Key | KeyPath], Handlers, Mod) -> + case maps:find(?WKEY, Handlers) of + error -> + deep_put_handler2(Key, KeyPath, Handlers, Mod); + {ok, _SubHandlers} -> + {error, {cannot_override_a_wildcard_path, [?WKEY | KeyPath]}} + end. + +deep_put_handler2(Key, KeyPath, Handlers, Mod) -> + SubHandlers = maps:get(Key, Handlers, #{}), + case deep_put_handler(KeyPath, SubHandlers, Mod) of + {ok, SubHandlers1} -> + {ok, Handlers#{Key => SubHandlers1}}; + Error -> + Error + end. + +process_update_request(ConfKeyPath, _Handlers, {remove, Opts}) -> OldRawConf = emqx_config:get_root_raw(ConfKeyPath), BinKeyPath = bin_path(ConfKeyPath), NewRawConf = emqx_map_lib:deep_remove(BinKeyPath, OldRawConf), - OverrideConf = emqx_map_lib:deep_remove(BinKeyPath, emqx_config:read_override_conf()), + OverrideConf = remove_from_override_config(BinKeyPath, Opts), {ok, NewRawConf, OverrideConf}; -process_update_request(ConfKeyPath, Handlers, {{update, UpdateReq}, _Opts}) -> +process_update_request(ConfKeyPath, Handlers, {{update, UpdateReq}, Opts}) -> OldRawConf = emqx_config:get_root_raw(ConfKeyPath), case do_update_config(ConfKeyPath, Handlers, OldRawConf, UpdateReq) of {ok, NewRawConf} -> - OverrideConf = update_override_config(NewRawConf), + OverrideConf = update_override_config(NewRawConf, Opts), {ok, NewRawConf, OverrideConf}; Error -> Error end. @@ -153,7 +180,7 @@ do_update_config([], Handlers, OldRawConf, UpdateReq) -> call_pre_config_update(Handlers, OldRawConf, UpdateReq); do_update_config([ConfKey | ConfKeyPath], Handlers, OldRawConf, UpdateReq) -> SubOldRawConf = get_sub_config(bin(ConfKey), OldRawConf), - SubHandlers = maps:get(ConfKey, Handlers, #{}), + SubHandlers = get_sub_handlers(ConfKey, Handlers), case do_update_config(ConfKeyPath, SubHandlers, SubOldRawConf, UpdateReq) of {ok, NewUpdateReq} -> call_pre_config_update(Handlers, OldRawConf, #{bin(ConfKey) => NewUpdateReq}); @@ -184,7 +211,7 @@ do_post_config_update([ConfKey | ConfKeyPath], Handlers, OldConf, NewConf, AppEn Result) -> SubOldConf = get_sub_config(ConfKey, OldConf), SubNewConf = get_sub_config(ConfKey, NewConf), - SubHandlers = maps:get(ConfKey, Handlers, #{}), + SubHandlers = get_sub_handlers(ConfKey, Handlers), case do_post_config_update(ConfKeyPath, SubHandlers, SubOldConf, SubNewConf, AppEnvs, UpdateArgs, Result) of {ok, Result1} -> @@ -193,6 +220,12 @@ do_post_config_update([ConfKey | ConfKeyPath], Handlers, OldConf, NewConf, AppEn Error -> Error end. +get_sub_handlers(ConfKey, Handlers) -> + case maps:find(ConfKey, Handlers) of + error -> maps:get(?WKEY, Handlers, #{}); + {ok, SubHandlers} -> SubHandlers + end. + get_sub_config(ConfKey, Conf) when is_map(Conf) -> maps:get(ConfKey, Conf, undefined); get_sub_config(_, _Conf) -> %% the Conf is a primitive @@ -237,7 +270,15 @@ merge_to_old_config(UpdateReq, RawConf) when is_map(UpdateReq), is_map(RawConf) merge_to_old_config(UpdateReq, _RawConf) -> {ok, UpdateReq}. -update_override_config(RawConf) -> +remove_from_override_config(_BinKeyPath, #{persistent := false}) -> + undefined; +remove_from_override_config(BinKeyPath, _Opts) -> + OldConf = emqx_config:read_override_conf(), + emqx_map_lib:deep_remove(BinKeyPath, OldConf). + +update_override_config(_RawConf, #{persistent := false}) -> + undefined; +update_override_config(RawConf, _Opts) -> OldConf = emqx_config:read_override_conf(), maps:merge(OldConf, RawConf). diff --git a/apps/emqx/src/emqx_frame.erl b/apps/emqx/src/emqx_frame.erl index 082801bad..79a740bed 100644 --- a/apps/emqx/src/emqx_frame.erl +++ b/apps/emqx/src/emqx_frame.erl @@ -100,14 +100,10 @@ parse(<>, StrictMode andalso validate_header(Type, Dup, QoS, Retain), Header = #mqtt_packet_header{type = Type, dup = bool(Dup), - qos = QoS, + qos = fixqos(Type, QoS), retain = bool(Retain) }, - Header1 = case fixqos(Type, QoS) of - QoS -> Header; - FixedQoS -> Header#mqtt_packet_header{qos = FixedQoS} - end, - parse_remaining_len(Rest, Header1, Options); + parse_remaining_len(Rest, Header, Options); parse(Bin, {{len, #{hdr := Header, len := {Multiplier, Length}} diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl index 399bd3d08..06d89c86d 100644 --- a/apps/emqx/src/emqx_listeners.erl +++ b/apps/emqx/src/emqx_listeners.erl @@ -26,6 +26,8 @@ , restart/0 , stop/0 , is_running/1 + , current_conns/2 + , max_conns/2 ]). -export([ start_listener/1 @@ -48,7 +50,7 @@ %% @doc List configured listeners. -spec(list() -> [{ListenerId :: atom(), ListenerConf :: map()}]). list() -> - [{listener_id(ZoneName, LName), LConf} || {ZoneName, LName, LConf} <- do_list()]. + [{listener_id(Type, LName), LConf} || {Type, LName, LConf} <- do_list()]. do_list() -> Listeners = maps:to_list(emqx:get_config([listeners], #{})), @@ -62,7 +64,7 @@ list(Type, Conf) -> -spec is_running(ListenerId :: atom()) -> boolean() | {error, no_found}. is_running(ListenerId) -> - case lists:filtermap(fun({_Zone, Id, #{running := IsRunning}}) -> + case lists:filtermap(fun({_Type, Id, #{running := IsRunning}}) -> Id =:= ListenerId andalso {true, IsRunning} end, do_list()) of [IsRunning] -> IsRunning; @@ -89,6 +91,28 @@ is_running(quic, _ListenerId, _Conf)-> %% TODO: quic support {error, no_found}. +current_conns(ID, ListenOn) -> + {Type, Name} = parse_listener_id(ID), + current_conns(Type, Name, ListenOn). + +current_conns(Type, Name, ListenOn) when Type == tcl; Type == ssl -> + esockd:get_current_connections({listener_id(Type, Name), ListenOn}); +current_conns(Type, Name, _ListenOn) when Type =:= ws; Type =:= wss -> + proplists:get_value(all_connections, ranch:info(listener_id(Type, Name))); +current_conns(_, _, _) -> + {error, not_support}. + +max_conns(ID, ListenOn) -> + {Type, Name} = parse_listener_id(ID), + max_conns(Type, Name, ListenOn). + +max_conns(Type, Name, ListenOn) when Type == tcl; Type == ssl -> + esockd:get_max_connections({listener_id(Type, Name), ListenOn}); +max_conns(Type, Name, _ListenOn) when Type =:= ws; Type =:= wss -> + proplists:get_value(max_connections, ranch:info(listener_id(Type, Name))); +max_conns(_, _, _) -> + {error, not_support}. + %% @doc Start all listeners. -spec(start() -> ok). start() -> @@ -228,11 +252,15 @@ do_start_listener(quic, ListenerName, #{bind := ListenOn} = Opts) -> {ok, {skipped, quic_app_missing}} end. +delete_authentication(Type, ListenerName, _Conf) -> + emqx_authentication:delete_chain(listener_id(Type, ListenerName)). + %% Update the listeners at runtime post_config_update(_Req, NewListeners, OldListeners, _AppEnvs) -> #{added := Added, removed := Removed, changed := Updated} = diff_listeners(NewListeners, OldListeners), perform_listener_changes(fun stop_listener/3, Removed), + perform_listener_changes(fun delete_authentication/3, Removed), perform_listener_changes(fun start_listener/3, Added), perform_listener_changes(fun restart_listener/3, Updated). @@ -252,7 +280,7 @@ flatten_listeners(Conf0) -> || {Type, Conf} <- maps:to_list(Conf0)])). do_flatten_listeners(Type, Conf0) -> - [{listener_id(Type, Name), Conf} || {Name, Conf} <- maps:to_list(Conf0)]. + [{listener_id(Type, Name), maps:remove(authentication, Conf)} || {Name, Conf} <- maps:to_list(Conf0)]. esockd_opts(Type, Opts0) -> Opts1 = maps:with([acceptors, max_connections, proxy_protocol, proxy_protocol_timeout], Opts0), diff --git a/apps/emqx/src/emqx_map_lib.erl b/apps/emqx/src/emqx_map_lib.erl index d5e851971..6aa6606c0 100644 --- a/apps/emqx/src/emqx_map_lib.erl +++ b/apps/emqx/src/emqx_map_lib.erl @@ -65,13 +65,11 @@ deep_find(_KeyPath, Data) -> {not_found, _KeyPath, Data}. -spec deep_put(config_key_path(), map(), term()) -> map(). -deep_put([], Map, Data) when is_map(Map) -> - Data; -deep_put([], _Map, Data) -> %% not map, replace it +deep_put([], _Map, Data) -> Data; deep_put([Key | KeyPath], Map, Data) -> - SubMap = deep_put(KeyPath, maps:get(Key, Map, #{}), Data), - Map#{Key => SubMap}. + SubMap = maps:get(Key, Map, #{}), + Map#{Key => deep_put(KeyPath, SubMap, Data)}. -spec deep_remove(config_key_path(), map()) -> map(). deep_remove([], Map) -> diff --git a/apps/emqx/src/emqx_metrics.erl b/apps/emqx/src/emqx_metrics.erl index 736bb05b0..282b8b5f3 100644 --- a/apps/emqx/src/emqx_metrics.erl +++ b/apps/emqx/src/emqx_metrics.erl @@ -22,8 +22,6 @@ -include("logger.hrl"). -include("types.hrl"). -include("emqx_mqtt.hrl"). --include("emqx.hrl"). - -export([ start_link/0 , stop/0 diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 812e52f9b..ed3f64d0e 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -49,6 +49,10 @@ -typerefl_from_string({cipher/0, emqx_schema, to_erl_cipher_suite}). -typerefl_from_string({comma_separated_atoms/0, emqx_schema, to_comma_separated_atoms}). +-export([ validate_heap_size/1 + , parse_user_lookup_fun/1 + ]). + % workaround: prevent being recognized as unused functions -export([to_duration/1, to_duration_s/1, to_duration_ms/1, to_bytesize/1, to_wordsize/1, @@ -65,204 +69,571 @@ cipher/0, comma_separated_atoms/0]). --export([roots/0, fields/1]). --export([t/1, t/3, t/4, ref/1]). +-export([namespace/0, roots/0, roots/1, fields/1]). -export([conf_get/2, conf_get/3, keys/2, filter/1]). -export([ssl/1]). +namespace() -> undefined. + roots() -> - ["zones", "mqtt", "flapping_detect", "force_shutdown", "force_gc", - "conn_congestion", "rate_limit", "quota", "listeners", "broker", "plugins", - "stats", "sysmon", "alarm", "authorization"]. + %% TODO change config importance to a field metadata + roots(high) ++ roots(medium) ++ roots(low). + +roots(high) -> + [ {"listeners", + sc(ref("listeners"), + #{ desc => "MQTT listeners identified by their protocol type and assigned names" + }) + } + , {"zones", + sc(map("name", ref("zone")), + #{ desc => "A zone is a set of configs grouped by the zone name.
" + "For flexible configuration mapping, the name " + "can be set to a listener's zone config.
" + "NOTE: A builtin zone named default is auto created " + "and can not be deleted." + })} + , {"mqtt", + sc(ref("mqtt"), + #{ desc => "Global MQTT configuration.
" + "The configs here work as default values which can be overriden " + "in zone configs" + })} + , {"authentication", + sc(hoconsc:lazy(hoconsc:array(map())), + #{ desc => "Default authentication configs for all MQTT listeners.
" + "For per-listener overrides see authentication " + "in listener configs" + })} + , {"authorization", + sc(ref("authorization"), + #{})} + ]; +roots(medium) -> + [ {"broker", + sc(ref("broker"), + #{})} + , {"rate_limit", + sc(ref("rate_limit"), + #{})} + , {"force_shutdown", + sc(ref("force_shutdown"), + #{})} + ]; +roots(low) -> + [ {"force_gc", + sc(ref("force_gc"), + #{})} + , {"conn_congestion", + sc(ref("conn_congestion"), + #{})} + , {"quota", + sc(ref("quota"), + #{})} + , {"plugins", %% TODO: move to emqx_machine_schema + sc(ref("plugins"), + #{})} + , {"stats", + sc(ref("stats"), + #{})} + , {"sysmon", + sc(ref("sysmon"), + #{})} + , {"alarm", + sc(ref("alarm"), + #{})} + , {"flapping_detect", + sc(ref("flapping_detect"), + #{})} + ]. fields("stats") -> - [ {"enable", t(boolean(), undefined, true)} + [ {"enable", + sc(boolean(), + #{ default => true + })} ]; fields("authorization") -> - [ {"no_match", t(union(allow, deny), undefined, allow)} - , {"deny_action", t(union(ignore, disconnect), undefined, ignore)} - , {"cache", ref("authorization_cache")} + [ {"no_match", + sc(union(allow, deny), + #{ default => allow + })} + , {"deny_action", + sc(union(ignore, disconnect), + #{ default => ignore + })} + , {"cache", + sc(ref(?MODULE, "cache"), + #{ + })} ]; -fields("authorization_cache") -> - [ {"enable", t(boolean(), undefined, true)} - , {"max_size", t(range(1, 1048576), undefined, 32)} - , {"ttl", t(duration(), undefined, "1m")} +fields("cache") -> + [ {"enable", + sc(boolean(), + #{ default => true + }) + } + , {"max_size", + sc(range(1, 1048576), + #{ default => 32 + }) + } + , {"ttl", + sc(duration(), + #{ default => "1m" + }) + } ]; fields("mqtt") -> - [ {"idle_timeout", maybe_infinity(duration(), "15s")} - , {"max_packet_size", t(bytesize(), undefined, "1MB")} - , {"max_clientid_len", t(range(23, 65535), undefined, 65535)} - , {"max_topic_levels", t(range(1, 65535), undefined, 65535)} - , {"max_qos_allowed", t(range(0, 2), undefined, 2)} - , {"max_topic_alias", t(range(0, 65535), undefined, 65535)} - , {"retain_available", t(boolean(), undefined, true)} - , {"wildcard_subscription", t(boolean(), undefined, true)} - , {"shared_subscription", t(boolean(), undefined, true)} - , {"ignore_loop_deliver", t(boolean(), undefined, false)} - , {"strict_mode", t(boolean(), undefined, false)} - , {"response_information", t(string(), undefined, "")} - , {"server_keepalive", maybe_disabled(integer())} - , {"keepalive_backoff", t(float(), undefined, 0.75)} - , {"max_subscriptions", maybe_infinity(range(1, inf))} - , {"upgrade_qos", t(boolean(), undefined, false)} - , {"max_inflight", t(range(1, 65535), undefined, 32)} - , {"retry_interval", t(duration(), undefined, "30s")} - , {"max_awaiting_rel", maybe_infinity(integer(), 100)} - , {"await_rel_timeout", t(duration(), undefined, "300s")} - , {"session_expiry_interval", t(duration(), undefined, "2h")} - , {"max_mqueue_len", maybe_infinity(range(0, inf), 1000)} - , {"mqueue_priorities", maybe_disabled(map())} - , {"mqueue_default_priority", t(union(highest, lowest), undefined, lowest)} - , {"mqueue_store_qos0", t(boolean(), undefined, true)} - , {"use_username_as_clientid", t(boolean(), undefined, false)} - , {"peer_cert_as_username", maybe_disabled(union([cn, dn, crt, pem, md5]))} - , {"peer_cert_as_clientid", maybe_disabled(union([cn, dn, crt, pem, md5]))} + [ {"idle_timeout", + sc(hoconsc:union([infinity, duration()]), + #{ default => "15s" + })} + , {"max_packet_size", + sc(bytesize(), + #{ default => "1MB" + })} + , {"max_clientid_len", + sc(range(23, 65535), + #{ default => 65535 + })} + , {"max_topic_levels", + sc(range(1, 65535), + #{ default => 65535 + })} + , {"max_qos_allowed", + sc(range(0, 2), + #{ default => 2 + })} + , {"max_topic_alias", + sc(range(0, 65535), + #{ default => 65535 + })} + , {"retain_available", + sc(boolean(), + #{ default => true + })} + , {"wildcard_subscription", + sc(boolean(), + #{ default => true + })} + , {"shared_subscription", + sc(boolean(), + #{ default => true + })} + , {"ignore_loop_deliver", + sc(boolean(), + #{ default => false + })} + , {"strict_mode", + sc(boolean(), + #{default => false + }) + } + , {"response_information", + sc(string(), + #{default => "" + }) + } + , {"server_keepalive", + sc(hoconsc:union([integer(), disabled]), + #{ default => disabled + }) + } + , {"keepalive_backoff", + sc(float(), + #{default => 0.75 + }) + } + , {"max_subscriptions", + sc(hoconsc:union([range(1, inf), infinity]), + #{ default => infinity + }) + } + , {"upgrade_qos", + sc(boolean(), + #{ default => false + }) + } + , {"max_inflight", + sc(range(1, 65535), + #{ default => 32 + }) + } + , {"retry_interval", + sc(duration(), + #{default => "30s" + }) + } + , {"max_awaiting_rel", + sc(hoconsc:union([integer(), infinity]), + #{ default => 100 + }) + } + , {"await_rel_timeout", + sc(duration(), + #{ default => "300s" + }) + } + , {"session_expiry_interval", + sc(duration(), + #{ default => "2h" + }) + } + , {"max_mqueue_len", + sc(hoconsc:union([range(0, inf), infinity]), + #{ default => 1000 + }) + } + , {"mqueue_priorities", + sc(hoconsc:union([map(), disabled]), + #{ default => disabled + }) + } + , {"mqueue_default_priority", + sc(union(highest, lowest), + #{ default => lowest + }) + } + , {"mqueue_store_qos0", + sc(boolean(), + #{ default => true + }) + } + , {"use_username_as_clientid", + sc(boolean(), + #{ default => false + }) + } + , {"peer_cert_as_username", + sc(hoconsc:union([disabled, cn, dn, crt, pem, md5]), + #{ default => disabled + })} + , {"peer_cert_as_clientid", + sc(hoconsc:union([disabled, cn, dn, crt, pem, md5]), + #{ default => disabled + })} ]; -fields("zones") -> - [ {"$name", ref("zone_settings")}]; - -fields("zone_settings") -> - Fields = ["mqtt", "stats", "authorization", "flapping_detect", "force_shutdown", +fields("zone") -> + Fields = ["mqtt", "stats", "flapping_detect", "force_shutdown", "conn_congestion", "rate_limit", "quota", "force_gc"], - [{F, ref("strip_default:" ++ F)} || F <- Fields]; + [{F, ref(emqx_zone_schema, F)} || F <- Fields]; fields("rate_limit") -> - [ {"max_conn_rate", maybe_infinity(integer(), 1000)} - , {"conn_messages_in", maybe_infinity(comma_separated_list())} - , {"conn_bytes_in", maybe_infinity(comma_separated_list())} + [ {"max_conn_rate", + sc(hoconsc:union([infinity, integer()]), + #{ default => 1000 + }) + } + , {"conn_messages_in", + sc(hoconsc:union([infinity, comma_separated_list()]), + #{ default => infinity + }) + } + , {"conn_bytes_in", + sc(hoconsc:union([infinity, comma_separated_list()]), + #{ default => infinity + }) + } ]; fields("quota") -> - [ {"conn_messages_routing", maybe_infinity(comma_separated_list())} - , {"overall_messages_routing", maybe_infinity(comma_separated_list())} + [ {"conn_messages_routing", + sc(hoconsc:union([infinity, comma_separated_list()]), + #{ default => infinity + }) + } + , {"overall_messages_routing", + sc(hoconsc:union([infinity, comma_separated_list()]), + #{ default => infinity + }) + } ]; fields("flapping_detect") -> - [ {"enable", t(boolean(), undefined, false)} - , {"max_count", t(integer(), undefined, 15)} - , {"window_time", t(duration(), undefined, "1m")} - , {"ban_time", t(duration(), undefined, "5m")} + [ {"enable", + sc(boolean(), + #{ default => false + })} + , {"max_count", + sc(integer(), + #{ default => 15 + })} + , {"window_time", + sc(duration(), + #{ default => "1m" + })} + , {"ban_time", + sc(duration(), + #{ default => "5m" + })} ]; fields("force_shutdown") -> - [ {"enable", t(boolean(), undefined, true)} - , {"max_message_queue_len", t(range(0, inf), undefined, 1000)} - , {"max_heap_size", t(wordsize(), undefined, "32MB", undefined, - fun(Siz) -> - MaxSiz = case erlang:system_info(wordsize) of - 8 -> % arch_64 - (1 bsl 59) - 1; - 4 -> % arch_32 - (1 bsl 27) - 1 - end, - case Siz > MaxSiz of - true -> - error(io_lib:format("force_shutdown_policy: heap-size ~s is too large", [Siz])); - false -> - ok - end - end)} + [ {"enable", + sc(boolean(), + #{ default => true})} + , {"max_message_queue_len", + sc(range(0, inf), + #{ default => 1000 + })} + , {"max_heap_size", + sc(wordsize(), + #{ default => "32MB", + validator => fun ?MODULE:validate_heap_size/1 + })} ]; fields("conn_congestion") -> - [ {"enable_alarm", t(boolean(), undefined, false)} - , {"min_alarm_sustain_duration", t(duration(), undefined, "1m")} + [ {"enable_alarm", + sc(boolean(), + #{ default => false + })} + , {"min_alarm_sustain_duration", + sc(duration(), + #{ default => "1m" + })} ]; fields("force_gc") -> - [ {"enable", t(boolean(), undefined, true)} - , {"count", t(range(0, inf), undefined, 16000)} - , {"bytes", t(bytesize(), undefined, "16MB")} + [ {"enable", + sc(boolean(), + #{ default => true + })} + , {"count", + sc(range(0, inf), + #{ default => 16000 + })} + , {"bytes", + sc(bytesize(), + #{ default => "16MB" + })} ]; fields("listeners") -> - [ {"tcp", ref("t_tcp_listeners")} - , {"ssl", ref("t_ssl_listeners")} - , {"ws", ref("t_ws_listeners")} - , {"wss", ref("t_wss_listeners")} - , {"quic", ref("t_quic_listeners")} - ]; - -fields("t_tcp_listeners") -> - [ {"$name", ref("mqtt_tcp_listener")} - ]; -fields("t_ssl_listeners") -> - [ {"$name", ref("mqtt_ssl_listener")} - ]; -fields("t_ws_listeners") -> - [ {"$name", ref("mqtt_ws_listener")} - ]; -fields("t_wss_listeners") -> - [ {"$name", ref("mqtt_wss_listener")} - ]; -fields("t_quic_listeners") -> - [ {"$name", ref("mqtt_quic_listener")} + [ {"tcp", + sc(map(name, ref("mqtt_tcp_listener")), + #{ desc => "TCP listeners" + , nullable => {true, recursive} + }) + } + , {"ssl", + sc(map(name, ref("mqtt_ssl_listener")), + #{ desc => "SSL listeners" + , nullable => {true, recursive} + }) + } + , {"ws", + sc(map(name, ref("mqtt_ws_listener")), + #{ desc => "HTTP websocket listeners" + , nullable => {true, recursive} + }) + } + , {"wss", + sc(map(name, ref("mqtt_wss_listener")), + #{ desc => "HTTPS websocket listeners" + , nullable => {true, recursive} + }) + } + , {"quic", + sc(map(name, ref("mqtt_quic_listener")), + #{ desc => "QUIC listeners" + , nullable => {true, recursive} + }) + } ]; fields("mqtt_tcp_listener") -> - [ {"tcp", ref("tcp_opts")} + [ {"tcp", + sc(ref("tcp_opts"), + #{ desc => "TCP listener options" + }) + } ] ++ mqtt_listener(); fields("mqtt_ssl_listener") -> - [ {"tcp", ref("tcp_opts")} - , {"ssl", ref("ssl_opts")} + [ {"tcp", + sc(ref("tcp_opts"), + #{}) + } + , {"ssl", + sc(ref("listener_ssl_opts"), + #{}) + } ] ++ mqtt_listener(); fields("mqtt_ws_listener") -> - [ {"tcp", ref("tcp_opts")} - , {"websocket", ref("ws_opts")} + [ {"tcp", + sc(ref("tcp_opts"), + #{}) + } + , {"websocket", + sc(ref("ws_opts"), + #{}) + } ] ++ mqtt_listener(); fields("mqtt_wss_listener") -> - [ {"tcp", ref("tcp_opts")} - , {"ssl", ref("ssl_opts")} - , {"websocket", ref("ws_opts")} + [ {"tcp", + sc(ref("tcp_opts"), + #{}) + } + , {"ssl", + sc(ref("listener_ssl_opts"), + #{}) + } + , {"websocket", + sc(ref("ws_opts"), + #{}) + } ] ++ mqtt_listener(); fields("mqtt_quic_listener") -> - [ {"enabled", t(boolean(), undefined, true)} - , {"certfile", t(string(), undefined, undefined)} - , {"keyfile", t(string(), undefined, undefined)} - , {"ciphers", t(comma_separated_list(), undefined, "TLS_AES_256_GCM_SHA384," - "TLS_AES_128_GCM_SHA256,TLS_CHACHA20_POLY1305_SHA256")} - , {"idle_timeout", t(duration(), undefined, "15s")} + [ {"enabled", + sc(boolean(), + #{ default => true + }) + } + , {"certfile", + sc(string(), + #{}) + } + , {"keyfile", + sc(string(), + #{}) + } + , {"ciphers", + sc(comma_separated_list(), + #{ default => "TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256," + "TLS_CHACHA20_POLY1305_SHA256" + })} + , {"idle_timeout", + sc(duration(), + #{ default => "15s" + }) + } ] ++ base_listener(); fields("ws_opts") -> - [ {"mqtt_path", t(string(), undefined, "/mqtt")} - , {"mqtt_piggyback", t(union(single, multiple), undefined, multiple)} - , {"compress", t(boolean(), undefined, false)} - , {"idle_timeout", t(duration(), undefined, "15s")} - , {"max_frame_size", maybe_infinity(integer())} - , {"fail_if_no_subprotocol", t(boolean(), undefined, true)} - , {"supported_subprotocols", t(comma_separated_list(), undefined, - "mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5")} - , {"check_origin_enable", t(boolean(), undefined, false)} - , {"allow_origin_absence", t(boolean(), undefined, true)} - , {"check_origins", t(hoconsc:array(binary()), undefined, [])} - , {"proxy_address_header", t(string(), undefined, "x-forwarded-for")} - , {"proxy_port_header", t(string(), undefined, "x-forwarded-port")} - , {"deflate_opts", ref("deflate_opts")} + [ {"mqtt_path", + sc(string(), + #{ default => "/mqtt" + }) + } + , {"mqtt_piggyback", + sc(hoconsc:union([single, multiple]), + #{ default => multiple + }) + } + , {"compress", + sc(boolean(), + #{ default => false + }) + } + , {"idle_timeout", + sc(duration(), + #{ default => "15s" + }) + } + , {"max_frame_size", + sc(hoconsc:union([infinity, integer()]), + #{ default => infinity + }) + } + , {"fail_if_no_subprotocol", + sc(boolean(), + #{ default => true + }) + } + , {"supported_subprotocols", + sc(comma_separated_list(), + #{ default => "mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5" + }) + } + , {"check_origin_enable", + sc(boolean(), + #{ default => false + }) + } + , {"allow_origin_absence", + sc(boolean(), + #{ default => true + }) + } + , {"check_origins", + sc(hoconsc:array(binary()), + #{ default => [] + }) + } + , {"proxy_address_header", + sc(string(), + #{ default => "x-forwarded-for" + }) + } + , {"proxy_port_header", + sc(string(), + #{ default => "x-forwarded-port" + }) + } + , {"deflate_opts", + sc(ref("deflate_opts"), + #{}) + } ]; fields("tcp_opts") -> - [ {"active_n", t(integer(), undefined, 100)} - , {"backlog", t(integer(), undefined, 1024)} - , {"send_timeout", t(duration(), undefined, "15s")} - , {"send_timeout_close", t(boolean(), undefined, true)} - , {"recbuf", t(bytesize())} - , {"sndbuf", t(bytesize())} - , {"buffer", t(bytesize())} - , {"high_watermark", t(bytesize(), undefined, "1MB")} - , {"nodelay", t(boolean(), undefined, false)} - , {"reuseaddr", t(boolean(), undefined, true)} + [ {"active_n", + sc(integer(), + #{ default => 100 + }) + } + , {"backlog", + sc(integer(), + #{ default => 1024 + }) + } + , {"send_timeout", + sc(duration(), + #{ default => "15s" + }) + } + , {"send_timeout_close", + sc(boolean(), + #{ default => true + }) + } + , {"recbuf", + sc(bytesize(), + #{}) + } + , {"sndbuf", + sc(bytesize(), + #{}) + } + , {"buffer", + sc(bytesize(), + #{}) + } + , {"high_watermark", + sc(bytesize(), + #{ default => "1MB"}) + } + , {"nodelay", + sc(boolean(), + #{ default => false}) + } + , {"reuseaddr", + sc(boolean(), + #{ default => true + }) + } ]; -fields("ssl_opts") -> +fields("listener_ssl_opts") -> ssl(#{handshake_timeout => "15s" , depth => 10 , reuse_sessions => true @@ -271,82 +642,241 @@ fields("ssl_opts") -> }); fields("deflate_opts") -> - [ {"level", t(union([none, default, best_compression, best_speed]))} - , {"mem_level", t(range(1, 9), undefined, 8)} - , {"strategy", t(union([default, filtered, huffman_only, rle]))} - , {"server_context_takeover", t(union(takeover, no_takeover))} - , {"client_context_takeover", t(union(takeover, no_takeover))} - , {"server_max_window_bits", t(range(8, 15), undefined, 15)} - , {"client_max_window_bits", t(range(8, 15), undefined, 15)} + [ {"level", + sc(hoconsc:union([none, default, best_compression, best_speed]), + #{}) + } + , {"mem_level", + sc(range(1, 9), + #{ default => 8 + }) + } + , {"strategy", + sc(hoconsc:union([default, filtered, huffman_only, rle]), + #{}) + } + , {"server_context_takeover", + sc(hoconsc:union([takeover, no_takeover]), + #{}) + } + , {"client_context_takeover", + sc(hoconsc:union([takeover, no_takeover]), + #{}) + } + , {"server_max_window_bits", + sc(range(8, 15), + #{ default => 15 + }) + } + , {"client_max_window_bits", + sc(range(8, 15), + #{ default => 15 + }) + } ]; fields("plugins") -> - [ {"expand_plugins_dir", t(string())} + [ {"expand_plugins_dir", + sc(string(), + #{}) + } ]; fields("broker") -> - [ {"sys_msg_interval", maybe_disabled(duration(), "1m")} - , {"sys_heartbeat_interval", maybe_disabled(duration(), "30s")} - , {"enable_session_registry", t(boolean(), undefined, true)} - , {"session_locking_strategy", t(union([local, leader, quorum, all]), undefined, quorum)} - , {"shared_subscription_strategy", t(union(random, round_robin), undefined, round_robin)} - , {"shared_dispatch_ack_enabled", t(boolean(), undefined, false)} - , {"route_batch_clean", t(boolean(), undefined, true)} - , {"perf", ref("perf")} + [ {"sys_msg_interval", + sc(hoconsc:union([disabled, duration()]), + #{ default => "1m" + }) + } + , {"sys_heartbeat_interval", + sc(hoconsc:union([disabled, duration()]), + #{ default => "30s" + }) + } + , {"enable_session_registry", + sc(boolean(), + #{ default => true + }) + } + , {"session_locking_strategy", + sc(hoconsc:union([local, leader, quorum, all]), + #{ default => quorum + }) + } + , {"shared_subscription_strategy", + sc(hoconsc:union([random, round_robin]), + #{ default => round_robin + }) + } + , {"shared_dispatch_ack_enabled", + sc(boolean(), + #{ default => false + }) + } + , {"route_batch_clean", + sc(boolean(), + #{ default => true + })} + , {"perf", + sc(ref("broker_perf"), + #{ desc => "Broker performance tuning pamaters" + }) + } ]; -fields("perf") -> - [ {"route_lock_type", t(union([key, tab, global]), undefined, key)} - , {"trie_compaction", t(boolean(), undefined, true)} +fields("broker_perf") -> + [ {"route_lock_type", + sc(hoconsc:union([key, tab, global]), + #{ default => key + })} + , {"trie_compaction", + sc(boolean(), + #{ default => true + })} ]; fields("sysmon") -> - [ {"vm", ref("sysmon_vm")} - , {"os", ref("sysmon_os")} + [ {"vm", + sc(ref("sysmon_vm"), + #{}) + } + , {"os", + sc(ref("sysmon_os"), + #{}) + } ]; fields("sysmon_vm") -> - [ {"process_check_interval", t(duration(), undefined, "30s")} - , {"process_high_watermark", t(percent(), undefined, "80%")} - , {"process_low_watermark", t(percent(), undefined, "60%")} - , {"long_gc", maybe_disabled(duration())} - , {"long_schedule", maybe_disabled(duration(), "240ms")} - , {"large_heap", maybe_disabled(bytesize(), "32MB")} - , {"busy_dist_port", t(boolean(), undefined, true)} - , {"busy_port", t(boolean(), undefined, true)} + [ {"process_check_interval", + sc(duration(), + #{ default => "30s" + }) + } + , {"process_high_watermark", + sc(percent(), + #{ default => "80%" + }) + } + , {"process_low_watermark", + sc(percent(), + #{ default => "60%" + }) + } + , {"long_gc", + sc(hoconsc:union([disabled, duration()]), + #{}) + } + , {"long_schedule", + sc(hoconsc:union([disabled, duration()]), + #{ default => "240ms" + }) + } + , {"large_heap", + sc(hoconsc:union([disabled, bytesize()]), + #{default => "32MB"}) + } + , {"busy_dist_port", + sc(boolean(), + #{ default => true + }) + } + , {"busy_port", + sc(boolean(), + #{ default => true + })} ]; fields("sysmon_os") -> - [ {"cpu_check_interval", t(duration(), undefined, "60s")} - , {"cpu_high_watermark", t(percent(), undefined, "80%")} - , {"cpu_low_watermark", t(percent(), undefined, "60%")} - , {"mem_check_interval", maybe_disabled(duration(), "60s")} - , {"sysmem_high_watermark", t(percent(), undefined, "70%")} - , {"procmem_high_watermark", t(percent(), undefined, "5%")} + [ {"cpu_check_interval", + sc(duration(), + #{ default => "60s"}) + } + , {"cpu_high_watermark", + sc(percent(), + #{ default => "80%" + }) + } + , {"cpu_low_watermark", + sc(percent(), + #{ default => "60%" + }) + } + , {"mem_check_interval", + sc(hoconsc:union([disabled, duration()]), + #{ default => "60s" + })} + , {"sysmem_high_watermark", + sc(percent(), + #{ default => "70%" + }) + } + , {"procmem_high_watermark", + sc(percent(), + #{ default => "5%" + }) + } ]; fields("alarm") -> - [ {"actions", t(hoconsc:array(atom()), undefined, [log, publish])} - , {"size_limit", t(integer(), undefined, 1000)} - , {"validity_period", t(duration(), undefined, "24h")} - ]; - -fields("strip_default:" ++ Name) -> - strip_default(fields(Name)). + [ {"actions", + sc(hoconsc:array(atom()), + #{ default => [log, publish] + }) + } + , {"size_limit", + sc(integer(), + #{ default => 1000 + }) + } + , {"validity_period", + sc(duration(), + #{ default => "24h" + }) + } + ]. mqtt_listener() -> base_listener() ++ - [ {"access_rules", t(hoconsc:array(string()))} - , {"proxy_protocol", t(boolean(), undefined, false)} - , {"proxy_protocol_timeout", t(duration())} + [ {"access_rules", + sc(hoconsc:array(string()), + #{}) + } + , {"proxy_protocol", + sc(boolean(), + #{ default => false + }) + } + , {"proxy_protocol_timeout", + sc(duration(), + #{}) + } + , {"authentication", + sc(hoconsc:lazy(hoconsc:array(map())), + #{}) + } ]. base_listener() -> - [ {"bind", hoconsc:t(union(ip_port(), integer()), #{nullable => false})} - , {"acceptors", t(integer(), undefined, 16)} - , {"max_connections", maybe_infinity(integer(), infinity)} - , {"mountpoint", t(binary(), undefined, <<>>)} - , {"zone", t(atom(), undefined, default)} + [ {"bind", + sc(hoconsc:union([ip_port(), integer()]), + #{ nullable => false + })} + , {"acceptors", + sc(integer(), + #{ default => 16 + })} + , {"max_connections", + sc(hoconsc:union([infinity, integer()]), + #{ default => infinity + })} + , {"mountpoint", + sc(binary(), + #{ default => <<>> + })} + , {"zone", + sc(atom(), + #{ default => 'default' + })} ]. %% utils @@ -372,43 +902,101 @@ conf_get(Key, Conf, Default) -> filter(Opts) -> [{K, V} || {K, V} <- Opts, V =/= undefined]. -%% generate a ssl field. -%% ssl(#{"verify" => verify_peer}) will return: -%% [ {"cacertfile", t(string(), undefined, undefined)} -%% , {"certfile", t(string(), undefined, undefined)} -%% , {"keyfile", t(string(), undefined, undefined)} -%% , {"verify", t(union(verify_peer, verify_none), undefined, verify_peer)} -%% , {"server_name_indication", undefined, undefined)} -%% ...] ssl(Defaults) -> D = fun (Field) -> maps:get(to_atom(Field), Defaults, undefined) end, - [ {"enable", t(boolean(), undefined, D("enable"))} - , {"cacertfile", t(string(), undefined, D("cacertfile"))} - , {"certfile", t(string(), undefined, D("certfile"))} - , {"keyfile", t(string(), undefined, D("keyfile"))} - , {"verify", t(union(verify_peer, verify_none), undefined, D("verify"))} - , {"fail_if_no_peer_cert", t(boolean(), undefined, D("fail_if_no_peer_cert"))} - , {"secure_renegotiate", t(boolean(), undefined, D("secure_renegotiate"))} - , {"reuse_sessions", t(boolean(), undefined, D("reuse_sessions"))} - , {"honor_cipher_order", t(boolean(), undefined, D("honor_cipher_order"))} - , {"handshake_timeout", t(duration(), undefined, D("handshake_timeout"))} - , {"depth", t(integer(), undefined, D("depth"))} - , {"password", hoconsc:t(string(), #{default => D("key_password"), - sensitive => true - })} - , {"dhfile", t(string(), undefined, D("dhfile"))} - , {"server_name_indication", t(union(disable, string()), undefined, - D("server_name_indication"))} - , {"versions", #{ type => list(atom()) - , default => maps:get(versions, Defaults, default_tls_vsns()) - , converter => fun (Vsns) -> [tls_vsn(V) || V <- Vsns] end - }} - , {"ciphers", t(hoconsc:array(string()), undefined, D("ciphers"))} - , {"user_lookup_fun", t(any(), undefined, {fun emqx_psk:lookup/3, <<>>})} + [ {"enable", + sc(boolean(), + #{ default => D("enable") + }) + } + , {"cacertfile", + sc(string(), + #{ default => D("cacertfile") + }) + } + , {"certfile", + sc(string(), + #{ default => D("certfile") + }) + } + , {"keyfile", + sc(string(), + #{ default => D("keyfile") + }) + } + , {"verify", + sc(hoconsc:union([verify_peer, verify_none]), + #{ default => D("verify") + }) + } + , {"fail_if_no_peer_cert", + sc(boolean(), + #{ default => D("fail_if_no_peer_cert") + }) + } + , {"secure_renegotiate", + sc(boolean(), + #{ default => D("secure_renegotiate") + }) + } + , {"reuse_sessions", + sc(boolean(), + #{ default => D("reuse_sessions") + }) + } + , {"honor_cipher_order", + sc(boolean(), + #{ default => D("honor_cipher_order") + }) + } + , {"handshake_timeout", + sc(duration(), + #{ default => D("handshake_timeout") + }) + } + , {"depth", + sc(integer(), + #{default => D("depth") + }) + } + , {"password", + sc(string(), + #{ default => D("key_password") + , sensitive => true + }) + } + , {"dhfile", + sc(string(), + #{ default => D("dhfile") + }) + } + , {"server_name_indication", + sc(hoconsc:union([disable, string()]), + #{ default => D("server_name_indication") + }) + } + , {"versions", + sc(typerefl:alias("string", list(atom())), + #{ default => maps:get(versions, Defaults, default_tls_vsns()) + , converter => fun (Vsns) -> [tls_vsn(iolist_to_binary(V)) || V <- Vsns] end + }) + } + , {"ciphers", + sc(hoconsc:array(string()), + #{ default => D("ciphers") + }) + } + , {"user_lookup_fun", + sc(typerefl:alias("string", any()), + #{ default => "emqx_psk:lookup" + , converter => fun ?MODULE:parse_user_lookup_fun/1 + }) + } ]. %% on erl23.2.7.2-emqx-2, sufficient_crypto_support('tlsv1.3') -> false default_tls_vsns() -> [<<"tlsv1.2">>, <<"tlsv1.1">>, <<"tlsv1">>]. + tls_vsn(<<"tlsv1.3">>) -> 'tlsv1.3'; tls_vsn(<<"tlsv1.2">>) -> 'tlsv1.2'; tls_vsn(<<"tlsv1.1">>) -> 'tlsv1.1'; @@ -451,40 +1039,13 @@ ceiling(X) -> %% types -t(Type) -> hoconsc:t(Type). +sc(Type, Meta) -> hoconsc:mk(Type, Meta). -t(Type, Mapping, Default) -> - hoconsc:t(Type, #{mapping => Mapping, default => Default}). +map(Name, Type) -> hoconsc:map(Name, Type). -t(Type, Mapping, Default, OverrideEnv) -> - hoconsc:t(Type, #{ mapping => Mapping - , default => Default - , override_env => OverrideEnv - }). +ref(Field) -> hoconsc:ref(?MODULE, Field). -t(Type, Mapping, Default, OverrideEnv, Validator) -> - hoconsc:t(Type, #{ mapping => Mapping - , default => Default - , override_env => OverrideEnv - , validator => Validator - }). - -ref(Field) -> hoconsc:t(hoconsc:ref(?MODULE, Field)). - -maybe_disabled(T) -> - maybe_sth(disabled, T, disabled). - -maybe_disabled(T, Default) -> - maybe_sth(disabled, T, Default). - -maybe_infinity(T) -> - maybe_sth(infinity, T, infinity). - -maybe_infinity(T, Default) -> - maybe_sth(infinity, T, Default). - -maybe_sth(What, Type, Default) -> - t(union([What, Type]), undefined, Default). +ref(Module, Field) -> hoconsc:ref(Module, Field). to_duration(Str) -> case hocon_postprocess:duration(Str) of @@ -545,22 +1106,26 @@ to_erl_cipher_suite(Str) -> Cipher -> Cipher end. -strip_default(Fields) -> - [do_strip_default(F) || F <- Fields]. - -do_strip_default({Name, #{type := {ref, Ref}}}) -> - {Name, nullable_no_def(ref("strip_default:" ++ Ref))}; -do_strip_default({Name, #{type := {ref, _Mod, Ref}}}) -> - {Name, nullable_no_def(ref("strip_default:" ++ Ref))}; -do_strip_default({Name, Type}) -> - {Name, nullable_no_def(Type)}. - -nullable_no_def(Type) when is_map(Type) -> - Type#{default => undefined, nullable => true}. - to_atom(Atom) when is_atom(Atom) -> Atom; to_atom(Str) when is_list(Str) -> list_to_atom(Str); to_atom(Bin) when is_binary(Bin) -> binary_to_atom(Bin, utf8). + +validate_heap_size(Siz) -> + MaxSiz = case erlang:system_info(wordsize) of + 8 -> % arch_64 + (1 bsl 59) - 1; + 4 -> % arch_32 + (1 bsl 27) - 1 + end, + case Siz > MaxSiz of + true -> error(io_lib:format("force_shutdown_policy: heap-size ~s is too large", [Siz])); + false -> ok + end. +parse_user_lookup_fun(StrConf) -> + [ModStr, FunStr] = string:tokens(StrConf, ":"), + Mod = list_to_atom(ModStr), + Fun = list_to_atom(FunStr), + {fun Mod:Fun/3, <<>>}. diff --git a/apps/emqx_gateway/src/emqx_gateway_api_client.erl b/apps/emqx/src/emqx_zone_schema.erl similarity index 63% rename from apps/emqx_gateway/src/emqx_gateway_api_client.erl rename to apps/emqx/src/emqx_zone_schema.erl index 03fb056ad..013ffb22f 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api_client.erl +++ b/apps/emqx/src/emqx_zone_schema.erl @@ -13,13 +13,22 @@ %% See the License for the specific language governing permissions and %% limitations under the License. %%-------------------------------------------------------------------- -%% --module(emqx_gateway_api_client). --behaviour(minirest_api). +-module(emqx_zone_schema). -%% minirest behaviour callbacks --export([api_spec/0]). +-export([namespace/0, roots/0, fields/1]). -api_spec() -> - {[], []}. +namespace() -> zone. + +roots() -> []. + +%% zone schemas are clones from the same name from root level +%% only not allowed to have default values. +fields(Name) -> + [{N, no_default(Sc)} || {N, Sc} <- emqx_schema:fields(Name)]. + +%% no default values for zone settings +no_default(Sc) -> + fun(default) -> undefined; + (Other) -> hocon_schema:field_schema(Sc, Other) + end. diff --git a/apps/emqx/test/emqx_authentication_SUITE.erl b/apps/emqx/test/emqx_authentication_SUITE.erl new file mode 100644 index 000000000..001a4b40e --- /dev/null +++ b/apps/emqx/test/emqx_authentication_SUITE.erl @@ -0,0 +1,238 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_authentication_SUITE). + +-behaviour(hocon_schema). +-behaviour(emqx_authentication). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("typerefl/include/types.hrl"). + +-export([ fields/1 ]). + +-export([ refs/0 + , create/1 + , update/2 + , authenticate/2 + , destroy/1 + ]). + +-define(AUTHN, emqx_authentication). + +%%------------------------------------------------------------------------------ +%% Hocon Schema +%%------------------------------------------------------------------------------ + +fields(type1) -> + [ {mechanism, {enum, ['password-based']}} + , {backend, {enum, ['built-in-database']}} + , {enable, fun enable/1} + ]; + +fields(type2) -> + [ {mechanism, {enum, ['password-based']}} + , {backend, {enum, ['mysql']}} + , {enable, fun enable/1} + ]. + +enable(type) -> boolean(); +enable(default) -> true; +enable(_) -> undefined. + +%%------------------------------------------------------------------------------ +%% Callbacks +%%------------------------------------------------------------------------------ + +refs() -> + [ hoconsc:ref(?MODULE, type1) + , hoconsc:ref(?MODULE, type2) + ]. + +create(_Config) -> + {ok, #{mark => 1}}. + +update(_Config, _State) -> + {ok, #{mark => 2}}. + +authenticate(#{username := <<"good">>}, _State) -> + {ok, #{is_superuser => true}}; +authenticate(#{username := _}, _State) -> + {error, bad_username_or_password}. + +destroy(_State) -> + ok. + +all() -> + emqx_ct:all(?MODULE). + +init_per_suite(Config) -> + application:set_env(ekka, strict_mode, true), + emqx_ct_helpers:start_apps([]), + Config. + +end_per_suite(_) -> + emqx_ct_helpers:stop_apps([]), + ok. + +t_chain(_) -> + % CRUD of authentication chain + ChainName = 'test', + ?assertMatch({ok, []}, ?AUTHN:list_chains()), + ?assertMatch({ok, #{name := ChainName, authenticators := []}}, ?AUTHN:create_chain(ChainName)), + ?assertEqual({error, {already_exists, {chain, ChainName}}}, ?AUTHN:create_chain(ChainName)), + ?assertMatch({ok, #{name := ChainName, authenticators := []}}, ?AUTHN:lookup_chain(ChainName)), + ?assertMatch({ok, [#{name := ChainName}]}, ?AUTHN:list_chains()), + ?assertEqual(ok, ?AUTHN:delete_chain(ChainName)), + ?assertMatch({error, {not_found, {chain, ChainName}}}, ?AUTHN:lookup_chain(ChainName)), + ok. + +t_authenticator(_) -> + ChainName = 'test', + AuthenticatorConfig1 = #{mechanism => 'password-based', + backend => 'built-in-database', + enable => true}, + + % Create an authenticator when the authentication chain does not exist + ?assertEqual({error, {not_found, {chain, ChainName}}}, ?AUTHN:create_authenticator(ChainName, AuthenticatorConfig1)), + ?AUTHN:create_chain(ChainName), + % Create an authenticator when the provider does not exist + ?assertEqual({error, no_available_provider}, ?AUTHN:create_authenticator(ChainName, AuthenticatorConfig1)), + + AuthNType1 = {'password-based', 'built-in-database'}, + ?AUTHN:add_provider(AuthNType1, ?MODULE), + ID1 = <<"password-based:built-in-database">>, + + % CRUD of authencaticator + ?assertMatch({ok, #{id := ID1, state := #{mark := 1}}}, ?AUTHN:create_authenticator(ChainName, AuthenticatorConfig1)), + ?assertMatch({ok, #{id := ID1}}, ?AUTHN:lookup_authenticator(ChainName, ID1)), + ?assertMatch({ok, [#{id := ID1}]}, ?AUTHN:list_authenticators(ChainName)), + ?assertEqual({error, {already_exists, {authenticator, ID1}}}, ?AUTHN:create_authenticator(ChainName, AuthenticatorConfig1)), + ?assertMatch({ok, #{id := ID1, state := #{mark := 2}}}, ?AUTHN:update_authenticator(ChainName, ID1, AuthenticatorConfig1)), + ?assertEqual(ok, ?AUTHN:delete_authenticator(ChainName, ID1)), + ?assertEqual({error, {not_found, {authenticator, ID1}}}, ?AUTHN:update_authenticator(ChainName, ID1, AuthenticatorConfig1)), + ?assertMatch({ok, []}, ?AUTHN:list_authenticators(ChainName)), + + % Multiple authenticators exist at the same time + AuthNType2 = {'password-based', mysql}, + ?AUTHN:add_provider(AuthNType2, ?MODULE), + ID2 = <<"password-based:mysql">>, + AuthenticatorConfig2 = #{mechanism => 'password-based', + backend => mysql, + enable => true}, + ?assertMatch({ok, #{id := ID1}}, ?AUTHN:create_authenticator(ChainName, AuthenticatorConfig1)), + ?assertMatch({ok, #{id := ID2}}, ?AUTHN:create_authenticator(ChainName, AuthenticatorConfig2)), + + % Move authenticator + ?assertMatch({ok, [#{id := ID1}, #{id := ID2}]}, ?AUTHN:list_authenticators(ChainName)), + ?assertEqual(ok, ?AUTHN:move_authenticator(ChainName, ID2, top)), + ?assertMatch({ok, [#{id := ID2}, #{id := ID1}]}, ?AUTHN:list_authenticators(ChainName)), + ?assertEqual(ok, ?AUTHN:move_authenticator(ChainName, ID2, bottom)), + ?assertMatch({ok, [#{id := ID1}, #{id := ID2}]}, ?AUTHN:list_authenticators(ChainName)), + ?assertEqual(ok, ?AUTHN:move_authenticator(ChainName, ID2, {before, ID1})), + ?assertMatch({ok, [#{id := ID2}, #{id := ID1}]}, ?AUTHN:list_authenticators(ChainName)), + + ?AUTHN:delete_chain(ChainName), + ?AUTHN:remove_provider(AuthNType1), + ?AUTHN:remove_provider(AuthNType2), + ok. + +t_authenticate(_) -> + ListenerID = 'tcp:default', + ClientInfo = #{zone => default, + listener => ListenerID, + protocol => mqtt, + username => <<"good">>, + password => <<"any">>}, + ?assertEqual({ok, #{is_superuser => false}}, emqx_access_control:authenticate(ClientInfo)), + + AuthNType = {'password-based', 'built-in-database'}, + ?AUTHN:add_provider(AuthNType, ?MODULE), + + AuthenticatorConfig = #{mechanism => 'password-based', + backend => 'built-in-database', + enable => true}, + ?AUTHN:create_chain(ListenerID), + ?assertMatch({ok, _}, ?AUTHN:create_authenticator(ListenerID, AuthenticatorConfig)), + ?assertEqual({ok, #{is_superuser => true}}, emqx_access_control:authenticate(ClientInfo)), + ?assertEqual({error, bad_username_or_password}, emqx_access_control:authenticate(ClientInfo#{username => <<"bad">>})), + + ?AUTHN:delete_chain(ListenerID), + ?AUTHN:remove_provider(AuthNType), + ok. + +t_update_config(_) -> + emqx_config_handler:add_handler([authentication], emqx_authentication), + + AuthNType1 = {'password-based', 'built-in-database'}, + AuthNType2 = {'password-based', mysql}, + ?AUTHN:add_provider(AuthNType1, ?MODULE), + ?AUTHN:add_provider(AuthNType2, ?MODULE), + + Global = 'mqtt:global', + AuthenticatorConfig1 = #{mechanism => 'password-based', + backend => 'built-in-database', + enable => true}, + AuthenticatorConfig2 = #{mechanism => 'password-based', + backend => mysql, + enable => true}, + ID1 = <<"password-based:built-in-database">>, + ID2 = <<"password-based:mysql">>, + + ?assertMatch({ok, []}, ?AUTHN:list_chains()), + ?assertMatch({ok, _}, update_config([authentication], {create_authenticator, Global, AuthenticatorConfig1})), + ?assertMatch({ok, #{id := ID1, state := #{mark := 1}}}, ?AUTHN:lookup_authenticator(Global, ID1)), + + ?assertMatch({ok, _}, update_config([authentication], {create_authenticator, Global, AuthenticatorConfig2})), + ?assertMatch({ok, #{id := ID2, state := #{mark := 1}}}, ?AUTHN:lookup_authenticator(Global, ID2)), + + ?assertMatch({ok, _}, update_config([authentication], {update_authenticator, Global, ID1, #{}})), + ?assertMatch({ok, #{id := ID1, state := #{mark := 2}}}, ?AUTHN:lookup_authenticator(Global, ID1)), + + ?assertMatch({ok, _}, update_config([authentication], {move_authenticator, Global, ID2, top})), + ?assertMatch({ok, [#{id := ID2}, #{id := ID1}]}, ?AUTHN:list_authenticators(Global)), + + ?assertMatch({ok, _}, update_config([authentication], {delete_authenticator, Global, ID1})), + ?assertEqual({error, {not_found, {authenticator, ID1}}}, ?AUTHN:lookup_authenticator(Global, ID1)), + + ListenerID = 'tcp:default', + ConfKeyPath = [listeners, tcp, default, authentication], + ?assertMatch({ok, _}, update_config(ConfKeyPath, {create_authenticator, ListenerID, AuthenticatorConfig1})), + ?assertMatch({ok, #{id := ID1, state := #{mark := 1}}}, ?AUTHN:lookup_authenticator(ListenerID, ID1)), + + ?assertMatch({ok, _}, update_config(ConfKeyPath, {create_authenticator, ListenerID, AuthenticatorConfig2})), + ?assertMatch({ok, #{id := ID2, state := #{mark := 1}}}, ?AUTHN:lookup_authenticator(ListenerID, ID2)), + + ?assertMatch({ok, _}, update_config(ConfKeyPath, {update_authenticator, ListenerID, ID1, #{}})), + ?assertMatch({ok, #{id := ID1, state := #{mark := 2}}}, ?AUTHN:lookup_authenticator(ListenerID, ID1)), + + ?assertMatch({ok, _}, update_config(ConfKeyPath, {move_authenticator, ListenerID, ID2, top})), + ?assertMatch({ok, [#{id := ID2}, #{id := ID1}]}, ?AUTHN:list_authenticators(ListenerID)), + + ?assertMatch({ok, _}, update_config(ConfKeyPath, {delete_authenticator, ListenerID, ID1})), + ?assertEqual({error, {not_found, {authenticator, ID1}}}, ?AUTHN:lookup_authenticator(ListenerID, ID1)), + + ?AUTHN:delete_chain(Global), + ?AUTHN:remove_provider(AuthNType1), + ?AUTHN:remove_provider(AuthNType2), + ok. + +update_config(Path, ConfigRequest) -> + emqx:update_config(Path, ConfigRequest, #{rawconf_with_defaults => true}). diff --git a/apps/emqx/test/emqx_channel_SUITE.erl b/apps/emqx/test/emqx_channel_SUITE.erl index 031f89612..775b40ee8 100644 --- a/apps/emqx/test/emqx_channel_SUITE.erl +++ b/apps/emqx/test/emqx_channel_SUITE.erl @@ -144,7 +144,7 @@ init_per_suite(Config) -> %% Access Control Meck ok = meck:new(emqx_access_control, [passthrough, no_history, no_link]), ok = meck:expect(emqx_access_control, authenticate, - fun(_) -> {ok, #{superuser => false}} end), + fun(_) -> {ok, #{is_superuser => false}} end), ok = meck:expect(emqx_access_control, authorize, fun(_, _, _) -> allow end), %% Broker Meck ok = meck:new(emqx_broker, [passthrough, no_history, no_link]), diff --git a/apps/emqx/test/emqx_client_SUITE.erl b/apps/emqx/test/emqx_client_SUITE.erl index 117a0f5b9..0a3a050ac 100644 --- a/apps/emqx/test/emqx_client_SUITE.erl +++ b/apps/emqx/test/emqx_client_SUITE.erl @@ -114,8 +114,8 @@ t_cm(_) -> emqx_config:put_zone_conf(default, [mqtt, idle_timeout], 15000). t_cm_registry(_) -> - Info = supervisor:which_children(emqx_cm_sup), - {_, Pid, _, _} = lists:keyfind(registry, 1, Info), + Children = supervisor:which_children(emqx_cm_sup), + {_, Pid, _, _} = lists:keyfind(emqx_cm_registry, 1, Children), ignored = gen_server:call(Pid, <<"Unexpected call">>), gen_server:cast(Pid, <<"Unexpected cast">>), Pid ! <<"Unexpected info">>. diff --git a/apps/emqx/test/emqx_flapping_SUITE.erl b/apps/emqx/test/emqx_flapping_SUITE.erl index 5ac6b9cdf..a8e783c49 100644 --- a/apps/emqx/test/emqx_flapping_SUITE.erl +++ b/apps/emqx/test/emqx_flapping_SUITE.erl @@ -55,8 +55,8 @@ t_detect_check(_) -> true = emqx_banned:check(ClientInfo), timer:sleep(3000), false = emqx_banned:check(ClientInfo), - Childrens = supervisor:which_children(emqx_cm_sup), - {flapping, Pid, _, _} = lists:keyfind(flapping, 1, Childrens), + Children = supervisor:which_children(emqx_cm_sup), + {emqx_flapping, Pid, _, _} = lists:keyfind(emqx_flapping, 1, Children), gen_server:call(Pid, unexpected_msg), gen_server:cast(Pid, unexpected_msg), Pid ! test, @@ -72,4 +72,4 @@ t_expired_detecting(_) -> (_) -> false end, ets:tab2list(emqx_flapping))), timer:sleep(200), ?assertEqual(true, lists:all(fun({flapping, <<"client008">>, _, _, _}) -> false; - (_) -> true end, ets:tab2list(emqx_flapping))). \ No newline at end of file + (_) -> true end, ets:tab2list(emqx_flapping))). diff --git a/apps/emqx_authn/data/user-credentials.csv b/apps/emqx_authn/data/user-credentials.csv index 0548308b7..cbadaefbc 100644 --- a/apps/emqx_authn/data/user-credentials.csv +++ b/apps/emqx_authn/data/user-credentials.csv @@ -1,3 +1,3 @@ -user_id,password_hash,salt,superuser +user_id,password_hash,salt,is_superuser myuser3,b6c743545a7817ae8c8f624371d5f5f0373234bb0ff36b8ffbf19bce0e06ab75,de1024f462fb83910fd13151bd4bd235,true myuser4,ee68c985a69208b6eda8c6c9b4c7c2d2b15ee2352cdd64a903171710a99182e8,ad773b5be9dd0613fe6c2f4d8c403139,false diff --git a/apps/emqx_authn/data/user-credentials.json b/apps/emqx_authn/data/user-credentials.json index e54501233..94375df22 100644 --- a/apps/emqx_authn/data/user-credentials.json +++ b/apps/emqx_authn/data/user-credentials.json @@ -3,12 +3,12 @@ "user_id":"myuser1", "password_hash":"c5e46903df45e5dc096dc74657610dbee8deaacae656df88a1788f1847390242", "salt": "e378187547bf2d6f0545a3f441aa4d8a", - "superuser": true + "is_superuser": true }, { "user_id":"myuser2", "password_hash":"f4d17f300b11e522fd33f497c11b126ef1ea5149c74d2220f9a16dc876d4567b", "salt": "6d3f9bd5b54d94b98adbcfe10b6d181f", - "superuser": false + "is_superuser": false } ] diff --git a/apps/emqx_authn/etc/emqx_authn.conf b/apps/emqx_authn/etc/emqx_authn.conf index 59f4aa9ee..d1d3d16f8 100644 --- a/apps/emqx_authn/etc/emqx_authn.conf +++ b/apps/emqx_authn/etc/emqx_authn.conf @@ -1,37 +1,6 @@ -authentication { - enable = false - authenticators = [ - # { - # name: "authenticator1" - # mechanism: password-based - # server_type: built-in-database - # user_id_type: clientid - # }, - # { - # name: "authenticator2" - # mechanism: password-based - # server_type: mongodb - # server: "127.0.0.1:27017" - # database: mqtt - # collection: users - # selector: { - # username: "${mqtt-username}" - # } - # password_hash_field: password_hash - # salt_field: salt - # password_hash_algorithm: sha256 - # salt_position: prefix - # }, - # { - # name: "authenticator 3" - # mechanism: password-based - # server_type: redis - # server: "127.0.0.1:6379" - # password: "public" - # database: 0 - # query: "HMGET ${mqtt-username} password_hash salt" - # password_hash_algorithm: sha256 - # salt_position: prefix - # } - ] -} +# authentication: { +# mechanism: password-based +# backend: built-in-database +# user_id_type: clientid +# } + diff --git a/apps/emqx_authn/include/emqx_authn.hrl b/apps/emqx_authn/include/emqx_authn.hrl index c5a392fd0..5eef08012 100644 --- a/apps/emqx_authn/include/emqx_authn.hrl +++ b/apps/emqx_authn/include/emqx_authn.hrl @@ -15,24 +15,11 @@ %%-------------------------------------------------------------------- -define(APP, emqx_authn). --define(CHAIN, <<"mqtt">>). --define(VER_1, <<"1">>). --define(VER_2, <<"2">>). +-define(AUTHN, emqx_authentication). + +-define(GLOBAL, 'mqtt:global'). -define(RE_PLACEHOLDER, "\\$\\{[a-z0-9\\-]+\\}"). --record(authenticator, - { id :: binary() - , name :: binary() - , provider :: module() - , state :: map() - }). - --record(chain, - { id :: binary() - , authenticators :: [{binary(), binary(), #authenticator{}}] - , created_at :: integer() - }). - -define(AUTH_SHARD, emqx_authn_shard). diff --git a/apps/emqx_authn/rebar.config b/apps/emqx_authn/rebar.config index 32b5a43e0..73696b033 100644 --- a/apps/emqx_authn/rebar.config +++ b/apps/emqx_authn/rebar.config @@ -1,6 +1,4 @@ -{deps, [ - {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.1"}}} -]}. +{deps, []}. {edoc_opts, [{preprocess, true}]}. {erl_opts, [warn_unused_vars, diff --git a/apps/emqx_authn/src/emqx_authn.erl b/apps/emqx_authn/src/emqx_authn.erl index 1034682e5..3ab05e6b0 100644 --- a/apps/emqx_authn/src/emqx_authn.erl +++ b/apps/emqx_authn/src/emqx_authn.erl @@ -15,640 +15,3 @@ %%-------------------------------------------------------------------- -module(emqx_authn). - --behaviour(gen_server). - --behaviour(emqx_config_handler). - --include("emqx_authn.hrl"). --include_lib("emqx/include/logger.hrl"). - --export([ pre_config_update/2 - , post_config_update/4 - , update_config/2 - ]). - --export([ enable/0 - , disable/0 - , is_enabled/0 - ]). - --export([authenticate/2]). - --export([ start_link/0 - , stop/0 - ]). - --export([ create_chain/1 - , delete_chain/1 - , lookup_chain/1 - , list_chains/0 - , create_authenticator/2 - , delete_authenticator/2 - , update_authenticator/3 - , update_or_create_authenticator/3 - , lookup_authenticator/2 - , list_authenticators/1 - , move_authenticator/3 - ]). - --export([ import_users/3 - , add_user/3 - , delete_user/3 - , update_user/4 - , lookup_user/3 - , list_users/2 - ]). - -%% gen_server callbacks --export([ init/1 - , handle_call/3 - , handle_cast/2 - , handle_info/2 - , terminate/2 - , code_change/3 - ]). - --define(CHAIN_TAB, emqx_authn_chain). - -%%------------------------------------------------------------------------------ -%% APIs -%%------------------------------------------------------------------------------ - -pre_config_update({enable, Enable}, _OldConfig) -> - {ok, Enable}; -pre_config_update({create_authenticator, Config}, OldConfig) -> - {ok, OldConfig ++ [Config]}; -pre_config_update({delete_authenticator, ID}, OldConfig) -> - case lookup_authenticator(?CHAIN, ID) of - {error, Reason} -> {error, Reason}; - {ok, #{name := Name}} -> - NewConfig = lists:filter(fun(#{<<"name">> := N}) -> - N =/= Name - end, OldConfig), - {ok, NewConfig} - end; -pre_config_update({update_authenticator, ID, Config}, OldConfig) -> - case lookup_authenticator(?CHAIN, ID) of - {error, Reason} -> {error, Reason}; - {ok, #{name := Name}} -> - NewConfig = lists:map(fun(#{<<"name">> := N} = C) -> - case N =:= Name of - true -> Config; - false -> C - end - end, OldConfig), - {ok, NewConfig} - end; -pre_config_update({update_or_create_authenticator, ID, Config}, OldConfig) -> - case lookup_authenticator(?CHAIN, ID) of - {error, _Reason} -> OldConfig ++ [Config]; - {ok, #{name := Name}} -> - NewConfig = lists:map(fun(#{<<"name">> := N} = C) -> - case N =:= Name of - true -> Config; - false -> C - end - end, OldConfig), - {ok, NewConfig} - end; -pre_config_update({move_authenticator, ID, Position}, OldConfig) -> - case lookup_authenticator(?CHAIN, ID) of - {error, Reason} -> {error, Reason}; - {ok, #{name := Name}} -> - {ok, Found, Part1, Part2} = split_by_name(Name, OldConfig), - case Position of - <<"top">> -> - {ok, [Found | Part1] ++ Part2}; - <<"bottom">> -> - {ok, Part1 ++ Part2 ++ [Found]}; - Before -> - case binary:split(Before, <<":">>, [global]) of - [<<"before">>, ID0] -> - case lookup_authenticator(?CHAIN, ID0) of - {error, Reason} -> {error, Reason}; - {ok, #{name := Name1}} -> - {ok, NFound, NPart1, NPart2} = split_by_name(Name1, Part1 ++ Part2), - {ok, NPart1 ++ [Found, NFound | NPart2]} - end; - _ -> - {error, {invalid_parameter, position}} - end - end - end. - -post_config_update({enable, true}, _NewConfig, _OldConfig, _AppEnvs) -> - emqx_authn:enable(); -post_config_update({enable, false}, _NewConfig, _OldConfig, _AppEnvs) -> - emqx_authn:disable(); -post_config_update({create_authenticator, #{<<"name">> := Name}}, NewConfig, _OldConfig, _AppEnvs) -> - case lists:filter( - fun(#{name := N}) -> - N =:= Name - end, NewConfig) of - [Config] -> - create_authenticator(?CHAIN, Config); - [_Config | _] -> - {error, name_has_be_used} - end; -post_config_update({delete_authenticator, ID}, _NewConfig, _OldConfig, _AppEnvs) -> - case delete_authenticator(?CHAIN, ID) of - ok -> ok; - {error, Reason} -> throw(Reason) - end; -post_config_update({update_authenticator, ID, #{<<"name">> := Name}}, NewConfig, _OldConfig, _AppEnvs) -> - case lists:filter( - fun(#{name := N}) -> - N =:= Name - end, NewConfig) of - [Config] -> - update_authenticator(?CHAIN, ID, Config); - [_Config | _] -> - {error, name_has_be_used} - end; -post_config_update({update_or_create_authenticator, ID, #{<<"name">> := Name}}, NewConfig, _OldConfig, _AppEnvs) -> - case lists:filter( - fun(#{name := N}) -> - N =:= Name - end, NewConfig) of - [Config] -> - update_or_create_authenticator(?CHAIN, ID, Config); - [_Config | _] -> - {error, name_has_be_used} - end; -post_config_update({move_authenticator, ID, Position}, _NewConfig, _OldConfig, _AppEnvs) -> - NPosition = case Position of - <<"top">> -> top; - <<"bottom">> -> bottom; - Before -> - case binary:split(Before, <<":">>, [global]) of - [<<"before">>, ID0] -> - {before, ID0}; - _ -> - {error, {invalid_parameter, position}} - end - end, - move_authenticator(?CHAIN, ID, NPosition). - -update_config(Path, ConfigRequest) -> - emqx:update_config(Path, ConfigRequest, #{rawconf_with_defaults => true}). - -enable() -> - case emqx:hook('client.authenticate', {?MODULE, authenticate, []}) of - ok -> ok; - {error, already_exists} -> ok - end. - -disable() -> - emqx:unhook('client.authenticate', {?MODULE, authenticate, []}), - ok. - -is_enabled() -> - Callbacks = emqx_hooks:lookup('client.authenticate'), - lists:any(fun({callback, {?MODULE, authenticate, []}, _, _}) -> - true; - (_) -> - false - end, Callbacks). - -authenticate(Credential, _AuthResult) -> - case ets:lookup(?CHAIN_TAB, ?CHAIN) of - [#chain{authenticators = Authenticators}] -> - do_authenticate(Authenticators, Credential); - [] -> - {stop, {error, not_authorized}} - end. - -do_authenticate([], _) -> - {stop, {error, not_authorized}}; -do_authenticate([{_, _, #authenticator{provider = Provider, state = State}} | More], Credential) -> - case Provider:authenticate(Credential, State) of - ignore -> - do_authenticate(More, Credential); - Result -> - %% {ok, Extra} - %% {ok, Extra, AuthData} - %% {ok, MetaData} - %% {continue, AuthCache} - %% {continue, AuthData, AuthCache} - %% {error, Reason} - {stop, Result} - end. - -start_link() -> - gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). - -stop() -> - gen_server:stop(?MODULE). - -create_chain(#{id := ID}) -> - gen_server:call(?MODULE, {create_chain, ID}). - -delete_chain(ID) -> - gen_server:call(?MODULE, {delete_chain, ID}). - -lookup_chain(ID) -> - gen_server:call(?MODULE, {lookup_chain, ID}). - -list_chains() -> - Chains = ets:tab2list(?CHAIN_TAB), - {ok, [serialize_chain(Chain) || Chain <- Chains]}. - -create_authenticator(ChainID, Config) -> - gen_server:call(?MODULE, {create_authenticator, ChainID, Config}). - -delete_authenticator(ChainID, AuthenticatorID) -> - gen_server:call(?MODULE, {delete_authenticator, ChainID, AuthenticatorID}). - -update_authenticator(ChainID, AuthenticatorID, Config) -> - gen_server:call(?MODULE, {update_authenticator, ChainID, AuthenticatorID, Config}). - -update_or_create_authenticator(ChainID, AuthenticatorID, Config) -> - gen_server:call(?MODULE, {update_or_create_authenticator, ChainID, AuthenticatorID, Config}). - -lookup_authenticator(ChainID, AuthenticatorID) -> - case ets:lookup(?CHAIN_TAB, ChainID) of - [] -> - {error, {not_found, {chain, ChainID}}}; - [#chain{authenticators = Authenticators}] -> - case lists:keyfind(AuthenticatorID, 1, Authenticators) of - false -> - {error, {not_found, {authenticator, AuthenticatorID}}}; - {_, _, Authenticator} -> - {ok, serialize_authenticator(Authenticator)} - end - end. - -list_authenticators(ChainID) -> - case ets:lookup(?CHAIN_TAB, ChainID) of - [] -> - {error, {not_found, {chain, ChainID}}}; - [#chain{authenticators = Authenticators}] -> - {ok, serialize_authenticators(Authenticators)} - end. - -move_authenticator(ChainID, AuthenticatorID, Position) -> - gen_server:call(?MODULE, {move_authenticator, ChainID, AuthenticatorID, Position}). - -import_users(ChainID, AuthenticatorID, Filename) -> - gen_server:call(?MODULE, {import_users, ChainID, AuthenticatorID, Filename}). - -add_user(ChainID, AuthenticatorID, UserInfo) -> - gen_server:call(?MODULE, {add_user, ChainID, AuthenticatorID, UserInfo}). - -delete_user(ChainID, AuthenticatorID, UserID) -> - gen_server:call(?MODULE, {delete_user, ChainID, AuthenticatorID, UserID}). - -update_user(ChainID, AuthenticatorID, UserID, NewUserInfo) -> - gen_server:call(?MODULE, {update_user, ChainID, AuthenticatorID, UserID, NewUserInfo}). - -lookup_user(ChainID, AuthenticatorID, UserID) -> - gen_server:call(?MODULE, {lookup_user, ChainID, AuthenticatorID, UserID}). - -%% TODO: Support pagination -list_users(ChainID, AuthenticatorID) -> - gen_server:call(?MODULE, {list_users, ChainID, AuthenticatorID}). - -%%-------------------------------------------------------------------- -%% gen_server callbacks -%%-------------------------------------------------------------------- - -init(_Opts) -> - _ = ets:new(?CHAIN_TAB, [ named_table, set, public - , {keypos, #chain.id} - , {read_concurrency, true}]), - {ok, #{}}. - -handle_call({create_chain, ID}, _From, State) -> - case ets:member(?CHAIN_TAB, ID) of - true -> - reply({error, {already_exists, {chain, ID}}}, State); - false -> - Chain = #chain{id = ID, - authenticators = [], - created_at = erlang:system_time(millisecond)}, - true = ets:insert(?CHAIN_TAB, Chain), - reply({ok, serialize_chain(Chain)}, State) - end; - -handle_call({delete_chain, ID}, _From, State) -> - case ets:lookup(?CHAIN_TAB, ID) of - [] -> - reply({error, {not_found, {chain, ID}}}, State); - [#chain{authenticators = Authenticators}] -> - _ = [do_delete_authenticator(Authenticator) || {_, _, Authenticator} <- Authenticators], - true = ets:delete(?CHAIN_TAB, ID), - reply(ok, State) - end; - -handle_call({lookup_chain, ID}, _From, State) -> - case ets:lookup(?CHAIN_TAB, ID) of - [] -> - reply({error, {not_found, {chain, ID}}}, State); - [Chain] -> - reply({ok, serialize_chain(Chain)}, State) - end; - -handle_call({create_authenticator, ChainID, #{name := Name} = Config}, _From, State) -> - UpdateFun = - fun(#chain{authenticators = Authenticators} = Chain) -> - case lists:keymember(Name, 2, Authenticators) of - true -> - {error, name_has_be_used}; - false -> - AlreadyExist = fun(ID) -> - lists:keymember(ID, 1, Authenticators) - end, - AuthenticatorID = gen_id(AlreadyExist), - case do_create_authenticator(ChainID, AuthenticatorID, Config) of - {ok, Authenticator} -> - NAuthenticators = Authenticators ++ [{AuthenticatorID, Name, Authenticator}], - true = ets:insert(?CHAIN_TAB, Chain#chain{authenticators = NAuthenticators}), - {ok, serialize_authenticator(Authenticator)}; - {error, Reason} -> - {error, Reason} - end - end - end, - Reply = update_chain(ChainID, UpdateFun), - reply(Reply, State); - -handle_call({delete_authenticator, ChainID, AuthenticatorID}, _From, State) -> - UpdateFun = - fun(#chain{authenticators = Authenticators} = Chain) -> - case lists:keytake(AuthenticatorID, 1, Authenticators) of - false -> - {error, {not_found, {authenticator, AuthenticatorID}}}; - {value, {_, _, Authenticator}, NAuthenticators} -> - _ = do_delete_authenticator(Authenticator), - true = ets:insert(?CHAIN_TAB, Chain#chain{authenticators = NAuthenticators}), - ok - end - end, - Reply = update_chain(ChainID, UpdateFun), - reply(Reply, State); - -handle_call({update_authenticator, ChainID, AuthenticatorID, Config}, _From, State) -> - Reply = update_or_create_authenticator(ChainID, AuthenticatorID, Config, false), - reply(Reply, State); - -handle_call({update_or_create_authenticator, ChainID, AuthenticatorID, Config}, _From, State) -> - Reply = update_or_create_authenticator(ChainID, AuthenticatorID, Config, true), - reply(Reply, State); - -handle_call({move_authenticator, ChainID, AuthenticatorID, Position}, _From, State) -> - UpdateFun = - fun(#chain{authenticators = Authenticators} = Chain) -> - case do_move_authenticator(AuthenticatorID, Authenticators, Position) of - {ok, NAuthenticators} -> - true = ets:insert(?CHAIN_TAB, Chain#chain{authenticators = NAuthenticators}), - ok; - {error, Reason} -> - {error, Reason} - end - end, - Reply = update_chain(ChainID, UpdateFun), - reply(Reply, State); - -handle_call({import_users, ChainID, AuthenticatorID, Filename}, _From, State) -> - Reply = call_authenticator(ChainID, AuthenticatorID, import_users, [Filename]), - reply(Reply, State); - -handle_call({add_user, ChainID, AuthenticatorID, UserInfo}, _From, State) -> - Reply = call_authenticator(ChainID, AuthenticatorID, add_user, [UserInfo]), - reply(Reply, State); - -handle_call({delete_user, ChainID, AuthenticatorID, UserID}, _From, State) -> - Reply = call_authenticator(ChainID, AuthenticatorID, delete_user, [UserID]), - reply(Reply, State); - -handle_call({update_user, ChainID, AuthenticatorID, UserID, NewUserInfo}, _From, State) -> - Reply = call_authenticator(ChainID, AuthenticatorID, update_user, [UserID, NewUserInfo]), - reply(Reply, State); - -handle_call({lookup_user, ChainID, AuthenticatorID, UserID}, _From, State) -> - Reply = call_authenticator(ChainID, AuthenticatorID, lookup_user, [UserID]), - reply(Reply, State); - -handle_call({list_users, ChainID, AuthenticatorID}, _From, State) -> - Reply = call_authenticator(ChainID, AuthenticatorID, list_users, []), - reply(Reply, State); - -handle_call(Req, _From, State) -> - ?LOG(error, "Unexpected call: ~p", [Req]), - {reply, ignored, State}. - -handle_cast(Req, State) -> - ?LOG(error, "Unexpected case: ~p", [Req]), - {noreply, State}. - -handle_info(Info, State) -> - ?LOG(error, "Unexpected info: ~p", [Info]), - {noreply, State}. - -terminate(_Reason, _State) -> - ok. - -code_change(_OldVsn, State, _Extra) -> - {ok, State}. - -reply(Reply, State) -> - {reply, Reply, State}. - -%%------------------------------------------------------------------------------ -%% Internal functions -%%------------------------------------------------------------------------------ - -authenticator_provider(#{mechanism := 'password-based', server_type := 'built-in-database'}) -> - emqx_authn_mnesia; -authenticator_provider(#{mechanism := 'password-based', server_type := 'mysql'}) -> - emqx_authn_mysql; -authenticator_provider(#{mechanism := 'password-based', server_type := 'pgsql'}) -> - emqx_authn_pgsql; -authenticator_provider(#{mechanism := 'password-based', server_type := 'mongodb'}) -> - emqx_authn_mongodb; -authenticator_provider(#{mechanism := 'password-based', server_type := 'redis'}) -> - emqx_authn_redis; -authenticator_provider(#{mechanism := 'password-based', server_type := 'http-server'}) -> - emqx_authn_http; -authenticator_provider(#{mechanism := jwt}) -> - emqx_authn_jwt; -authenticator_provider(#{mechanism := scram, server_type := 'built-in-database'}) -> - emqx_enhanced_authn_scram_mnesia. - -gen_id(AlreadyExist) -> - ID = list_to_binary(emqx_rule_id:gen()), - case AlreadyExist(ID) of - true -> gen_id(AlreadyExist); - false -> ID - end. - -switch_version(State = #{version := ?VER_1}) -> - State#{version := ?VER_2}; -switch_version(State = #{version := ?VER_2}) -> - State#{version := ?VER_1}; -switch_version(State) -> - State#{version => ?VER_1}. - -split_by_name(Name, Config) -> - {Part1, Part2, true} = lists:foldl( - fun(#{<<"name">> := N} = C, {P1, P2, F0}) -> - F = case N =:= Name of - true -> true; - false -> F0 - end, - case F of - false -> {[C | P1], P2, F}; - true -> {P1, [C | P2], F} - end - end, {[], [], false}, Config), - [Found | NPart2] = lists:reverse(Part2), - {ok, Found, lists:reverse(Part1), NPart2}. - -do_create_authenticator(ChainID, AuthenticatorID, #{name := Name} = Config) -> - Provider = authenticator_provider(Config), - Unique = <>, - case Provider:create(Config#{'_unique' => Unique}) of - {ok, State} -> - Authenticator = #authenticator{id = AuthenticatorID, - name = Name, - provider = Provider, - state = switch_version(State)}, - {ok, Authenticator}; - {error, Reason} -> - {error, Reason} - end. - -do_delete_authenticator(#authenticator{provider = Provider, state = State}) -> - _ = Provider:destroy(State), - ok. - -update_or_create_authenticator(ChainID, AuthenticatorID, #{name := NewName} = Config, CreateWhenNotFound) -> - UpdateFun = - fun(#chain{authenticators = Authenticators} = Chain) -> - case lists:keytake(AuthenticatorID, 1, Authenticators) of - false -> - case CreateWhenNotFound of - true -> - case lists:keymember(NewName, 2, Authenticators) of - true -> - {error, name_has_be_used}; - false -> - case do_create_authenticator(ChainID, AuthenticatorID, Config) of - {ok, Authenticator} -> - NAuthenticators = Authenticators ++ [{AuthenticatorID, NewName, Authenticator}], - true = ets:insert(?CHAIN_TAB, Chain#chain{authenticators = NAuthenticators}), - {ok, serialize_authenticator(Authenticator)}; - {error, Reason} -> - {error, Reason} - end - end; - false -> - {error, {not_found, {authenticator, AuthenticatorID}}} - end; - {value, - {_, _, #authenticator{provider = Provider, - state = #{version := Version} = State} = Authenticator}, - Others} -> - case lists:keymember(NewName, 2, Others) of - true -> - {error, name_has_be_used}; - false -> - case (NewProvider = authenticator_provider(Config)) =:= Provider of - true -> - Unique = <>, - case Provider:update(Config#{'_unique' => Unique}, State) of - {ok, NewState} -> - NewAuthenticator = Authenticator#authenticator{name = NewName, - state = switch_version(NewState)}, - NewAuthenticators = replace_authenticator(AuthenticatorID, NewAuthenticator, Authenticators), - true = ets:insert(?CHAIN_TAB, Chain#chain{authenticators = NewAuthenticators}), - {ok, serialize_authenticator(NewAuthenticator)}; - {error, Reason} -> - {error, Reason} - end; - false -> - Unique = <>, - case NewProvider:create(Config#{'_unique' => Unique}) of - {ok, NewState} -> - NewAuthenticator = Authenticator#authenticator{name = NewName, - provider = NewProvider, - state = switch_version(NewState)}, - NewAuthenticators = replace_authenticator(AuthenticatorID, NewAuthenticator, Authenticators), - true = ets:insert(?CHAIN_TAB, Chain#chain{authenticators = NewAuthenticators}), - _ = Provider:destroy(State), - {ok, serialize_authenticator(NewAuthenticator)}; - {error, Reason} -> - {error, Reason} - end - end - end - end - end, - update_chain(ChainID, UpdateFun). - -replace_authenticator(ID, #authenticator{name = Name} = Authenticator, Authenticators) -> - lists:keyreplace(ID, 1, Authenticators, {ID, Name, Authenticator}). - -do_move_authenticator(AuthenticatorID, Authenticators, Position) when is_binary(AuthenticatorID) -> - case lists:keytake(AuthenticatorID, 1, Authenticators) of - false -> - {error, {not_found, {authenticator, AuthenticatorID}}}; - {value, Authenticator, NAuthenticators} -> - do_move_authenticator(Authenticator, NAuthenticators, Position) - end; - -do_move_authenticator(Authenticator, Authenticators, top) -> - {ok, [Authenticator | Authenticators]}; -do_move_authenticator(Authenticator, Authenticators, bottom) -> - {ok, Authenticators ++ [Authenticator]}; -do_move_authenticator(Authenticator, Authenticators, {before, ID}) -> - insert(Authenticator, Authenticators, ID, []). - -insert(_, [], ID, _) -> - {error, {not_found, {authenticator, ID}}}; -insert(Authenticator, [{ID, _, _} | _] = Authenticators, ID, Acc) -> - {ok, lists:reverse(Acc) ++ [Authenticator | Authenticators]}; -insert(Authenticator, [{_, _, _} = Authenticator0 | More], ID, Acc) -> - insert(Authenticator, More, ID, [Authenticator0 | Acc]). - -update_chain(ChainID, UpdateFun) -> - case ets:lookup(?CHAIN_TAB, ChainID) of - [] -> - {error, {not_found, {chain, ChainID}}}; - [Chain] -> - UpdateFun(Chain) - end. - -call_authenticator(ChainID, AuthenticatorID, Func, Args) -> - UpdateFun = - fun(#chain{authenticators = Authenticators}) -> - case lists:keyfind(AuthenticatorID, 1, Authenticators) of - false -> - {error, {not_found, {authenticator, AuthenticatorID}}}; - {_, _, #authenticator{provider = Provider, state = State}} -> - case erlang:function_exported(Provider, Func, length(Args) + 1) of - true -> - erlang:apply(Provider, Func, Args ++ [State]); - false -> - {error, unsupported_feature} - end - end - end, - update_chain(ChainID, UpdateFun). - -serialize_chain(#chain{id = ID, - authenticators = Authenticators, - created_at = CreatedAt}) -> - #{id => ID, - authenticators => serialize_authenticators(Authenticators), - created_at => CreatedAt}. - -serialize_authenticators(Authenticators) -> - [serialize_authenticator(Authenticator) || {_, _, Authenticator} <- Authenticators]. - -serialize_authenticator(#authenticator{id = ID, - name = Name, - provider = Provider, - state = State}) -> - #{id => ID, name => Name, provider => Provider, state => State}. diff --git a/apps/emqx_authn/src/emqx_authn_api.erl b/apps/emqx_authn/src/emqx_authn_api.erl index 5f2b96b57..e492100ee 100644 --- a/apps/emqx_authn/src/emqx_authn_api.erl +++ b/apps/emqx_authn/src/emqx_authn_api.erl @@ -22,37 +22,39 @@ -export([ api_spec/0 , authentication/2 - , authenticators/2 - , authenticators2/2 + , authentication2/2 + , authentication3/2 + , authentication4/2 , move/2 + , move2/2 , import_users/2 + , import_users2/2 , users/2 , users2/2 + , users3/2 + , users4/2 ]). --define(EXAMPLE_1, #{name => <<"example 1">>, - mechanism => <<"password-based">>, - server_type => <<"built-in-database">>, +-define(EXAMPLE_1, #{mechanism => <<"password-based">>, + backend => <<"built-in-database">>, user_id_type => <<"username">>, password_hash_algorithm => #{ name => <<"sha256">> }}). --define(EXAMPLE_2, #{name => <<"example 2">>, - mechanism => <<"password-based">>, - server_type => <<"http-server">>, +-define(EXAMPLE_2, #{mechanism => <<"password-based">>, + backend => <<"http-server">>, method => <<"post">>, url => <<"http://localhost:80/login">>, headers => #{ <<"content-type">> => <<"application/json">> }, - form_data => #{ + body => #{ <<"username">> => <<"${mqtt-username}">>, <<"password">> => <<"${mqtt-password}">> }}). --define(EXAMPLE_3, #{name => <<"example 3">>, - mechanism => <<"jwt">>, +-define(EXAMPLE_3, #{mechanism => <<"jwt">>, use_jwks => false, algorithm => <<"hmac-based">>, secret => <<"mysecret">>, @@ -61,9 +63,8 @@ <<"username">> => <<"${mqtt-username}">> }}). --define(EXAMPLE_4, #{name => <<"example 4">>, - mechanism => <<"password-based">>, - server_type => <<"mongodb">>, +-define(EXAMPLE_4, #{mechanism => <<"password-based">>, + backend => <<"mongodb">>, server => <<"127.0.0.1:27017">>, database => example, collection => users, @@ -72,13 +73,13 @@ }, password_hash_field => <<"password_hash">>, salt_field => <<"salt">>, + is_superuser_field => <<"is_superuser">>, password_hash_algorithm => <<"sha256">>, salt_position => <<"prefix">> }). --define(EXAMPLE_5, #{name => <<"example 5">>, - mechanism => <<"password-based">>, - server_type => <<"redis">>, +-define(EXAMPLE_5, #{mechanism => <<"password-based">>, + backend => <<"redis">>, server => <<"127.0.0.1:6379">>, database => 0, query => <<"HMGET ${mqtt-username} password_hash salt">>, @@ -86,10 +87,53 @@ salt_position => <<"prefix">> }). +-define(INSTANCE_EXAMPLE_1, maps:merge(?EXAMPLE_1, #{id => <<"password-based:built-in-database">>, + enable => true})). + +-define(INSTANCE_EXAMPLE_2, maps:merge(?EXAMPLE_2, #{id => <<"password-based:http-server">>, + connect_timeout => 5000, + enable_pipelining => true, + headers => #{ + <<"accept">> => <<"application/json">>, + <<"cache-control">> => <<"no-cache">>, + <<"connection">> => <<"keepalive">>, + <<"content-type">> => <<"application/json">>, + <<"keep-alive">> => <<"timeout=5">> + }, + max_retries => 5, + pool_size => 8, + request_timeout => 5000, + retry_interval => 1000, + enable => true})). + +-define(INSTANCE_EXAMPLE_3, maps:merge(?EXAMPLE_3, #{id => <<"jwt">>, + enable => true})). + +-define(INSTANCE_EXAMPLE_4, maps:merge(?EXAMPLE_4, #{id => <<"password-based:mongodb">>, + mongo_type => <<"single">>, + pool_size => 8, + ssl => #{ + enable => false + }, + topology => #{ + max_overflow => 8, + pool_size => 8 + }, + enable => true})). + +-define(INSTANCE_EXAMPLE_5, maps:merge(?EXAMPLE_5, #{id => <<"password-based:redis">>, + auto_reconnect => true, + redis_type => single, + pool_size => 8, + ssl => #{ + enable => false + }, + enable => true})). + -define(ERR_RESPONSE(Desc), #{description => Desc, content => #{ 'application/json' => #{ - schema => minirest:ref(<<"error">>), + schema => minirest:ref(<<"Error">>), examples => #{ example1 => #{ summary => <<"Not Found">>, @@ -107,532 +151,642 @@ api_spec() -> {[ authentication_api() - , authenticators_api() - , authenticators_api2() + , authentication_api2() , move_api() + , authentication_api3() + , authentication_api4() + , move_api2() , import_users_api() + , import_users_api2() , users_api() , users2_api() + , users3_api() + , users4_api() ], definitions()}. authentication_api() -> Metadata = #{ - post => #{ - description => "Enable or disbale authentication", - requestBody => #{ - content => #{ - 'application/json' => #{ - schema => #{ - type => object, - required => [enable], - properties => #{ - enable => #{ - type => boolean, - example => true - } - } - } - } - } - }, - responses => #{ - <<"204">> => #{ - description => <<"No Content">> - }, - <<"400">> => ?ERR_RESPONSE(<<"Bad Request">>) - } - }, - get => #{ - description => "Get status of authentication", - responses => #{ - <<"200">> => #{ - description => <<"OK">>, - content => #{ - 'application/json' => #{ - schema => #{ - type => object, - properties => #{ - enabled => #{ - type => boolean, - example => true - } - } - } - } - } - } - } - } + post => create_authenticator_api_spec(), + get => list_authenticators_api_spec() }, {"/authentication", Metadata, authentication}. -authenticators_api() -> +authentication_api2() -> Metadata = #{ - post => #{ - description => "Create authenticator", - requestBody => #{ - content => #{ - 'application/json' => #{ - schema => minirest:ref(<<"authenticator">>), - examples => #{ - default => #{ - summary => <<"Default">>, - value => emqx_json:encode(?EXAMPLE_1) - }, - http => #{ - summary => <<"Authentication provided by HTTP Server">>, - value => emqx_json:encode(?EXAMPLE_2) - }, - jwt => #{ - summary => <<"JWT Authentication">>, - value => emqx_json:encode(?EXAMPLE_3) - }, - mongodb => #{ - summary => <<"Authentication with MongoDB">>, - value => emqx_json:encode(?EXAMPLE_4) - }, - redis => #{ - summary => <<"Authentication with Redis">>, - value => emqx_json:encode(?EXAMPLE_5) - } - } - } - } - }, - responses => #{ - <<"201">> => #{ - description => <<"Created">>, - content => #{ - 'application/json' => #{ - schema => minirest:ref(<<"returned_authenticator">>), - examples => #{ - %% TODO: return full content - example1 => #{ - summary => <<"Example 1">>, - value => emqx_json:encode(maps:put(id, <<"example 1">>, ?EXAMPLE_1)) - }, - example2 => #{ - summary => <<"Example 2">>, - value => emqx_json:encode(maps:put(id, <<"example 2">>, ?EXAMPLE_2)) - }, - example3 => #{ - summary => <<"Example 3">>, - value => emqx_json:encode(maps:put(id, <<"example 3">>, ?EXAMPLE_3)) - }, - example4 => #{ - summary => <<"Example 4">>, - value => emqx_json:encode(maps:put(id, <<"example 4">>, ?EXAMPLE_4)) - }, - example5 => #{ - summary => <<"Example 4">>, - value => emqx_json:encode(maps:put(id, <<"example 5">>, ?EXAMPLE_5)) - } - } - } - } - }, - <<"400">> => ?ERR_RESPONSE(<<"Bad Request">>), - <<"409">> => ?ERR_RESPONSE(<<"Conflict">>) - } - }, - get => #{ - description => "List authenticators", - responses => #{ - <<"200">> => #{ - description => <<"OK">>, - content => #{ - 'application/json' => #{ - schema => #{ - type => array, - items => minirest:ref(<<"returned_authenticator">>) - }, - examples => #{ - example1 => #{ - summary => <<"Example 1">>, - value => emqx_json:encode([ maps:put(id, <<"example 1">>, ?EXAMPLE_1) - , maps:put(id, <<"example 2">>, ?EXAMPLE_2) - , maps:put(id, <<"example 3">>, ?EXAMPLE_3) - , maps:put(id, <<"example 4">>, ?EXAMPLE_4) - , maps:put(id, <<"example 5">>, ?EXAMPLE_5) - ]) - } - } - } - } - } - } - } + get => find_authenticator_api_spec(), + put => update_authenticator_api_spec(), + delete => delete_authenticator_api_spec() }, - {"/authentication/authenticators", Metadata, authenticators}. + {"/authentication/:id", Metadata, authentication2}. -authenticators_api2() -> +authentication_api3() -> Metadata = #{ - get => #{ - description => "Get authenicator by id", - parameters => [ - #{ - name => id, - in => path, - schema => #{ - type => string - }, - required => true - } - ], - responses => #{ - <<"200">> => #{ - description => <<"OK">>, - content => #{ - 'application/json' => #{ - schema => minirest:ref(<<"returned_authenticator">>), - examples => #{ - example1 => #{ - summary => <<"Example 1">>, - value => emqx_json:encode(maps:put(id, <<"example 1">>, ?EXAMPLE_1)) - }, - example2 => #{ - summary => <<"Example 2">>, - value => emqx_json:encode(maps:put(id, <<"example 2">>, ?EXAMPLE_2)) - }, - example3 => #{ - summary => <<"Example 3">>, - value => emqx_json:encode(maps:put(id, <<"example 3">>, ?EXAMPLE_3)) - }, - example4 => #{ - summary => <<"Example 4">>, - value => emqx_json:encode(maps:put(id, <<"example 4">>, ?EXAMPLE_4)) - }, - example5 => #{ - summary => <<"Example 5">>, - value => emqx_json:encode(maps:put(id, <<"example 5">>, ?EXAMPLE_5)) - } - } - } - } - }, - <<"404">> => ?ERR_RESPONSE(<<"Not Found">>) - } - }, - put => #{ - description => "Update authenticator", - parameters => [ - #{ - name => id, - in => path, - schema => #{ - type => string - }, - required => true - } - ], - requestBody => #{ - content => #{ - 'application/json' => #{ - schema => #{ - oneOf => [ minirest:ref(<<"password_based">>) - , minirest:ref(<<"jwt">>) - , minirest:ref(<<"scram">>) - ] - }, - examples => #{ - example1 => #{ - summary => <<"Example 1">>, - value => emqx_json:encode(?EXAMPLE_1) - }, - example2 => #{ - summary => <<"Example 2">>, - value => emqx_json:encode(?EXAMPLE_2) - } - } - } - } - }, - responses => #{ - <<"200">> => #{ - description => <<"OK">>, - content => #{ - 'application/json' => #{ - schema => minirest:ref(<<"returned_authenticator">>), - examples => #{ - example1 => #{ - summary => <<"Example 1">>, - value => emqx_json:encode(maps:put(id, <<"example 1">>, ?EXAMPLE_1)) - }, - example2 => #{ - summary => <<"Example 2">>, - value => emqx_json:encode(maps:put(id, <<"example 2">>, ?EXAMPLE_2)) - }, - example3 => #{ - summary => <<"Example 3">>, - value => emqx_json:encode(maps:put(id, <<"example 3">>, ?EXAMPLE_3)) - }, - example4 => #{ - summary => <<"Example 4">>, - value => emqx_json:encode(maps:put(id, <<"example 4">>, ?EXAMPLE_4)) - }, - example5 => #{ - summary => <<"Example 5">>, - value => emqx_json:encode(maps:put(id, <<"example 5">>, ?EXAMPLE_5)) - } - } - } - } - }, - <<"400">> => ?ERR_RESPONSE(<<"Bad Request">>), - <<"404">> => ?ERR_RESPONSE(<<"Not Found">>), - <<"409">> => ?ERR_RESPONSE(<<"Conflict">>) - } - }, - delete => #{ - description => "Delete authenticator", - parameters => [ - #{ - name => id, - in => path, - schema => #{ - type => string - }, - required => true - } - ], - responses => #{ - <<"204">> => #{ - description => <<"No Content">> - }, - <<"404">> => ?ERR_RESPONSE(<<"Not Found">>) - } - } + post => create_authenticator_api_spec2(), + get => list_authenticators_api_spec2() }, - {"/authentication/authenticators/:id", Metadata, authenticators2}. + {"/listeners/:listener_id/authentication", Metadata, authentication3}. + +authentication_api4() -> + Metadata = #{ + get => find_authenticator_api_spec2(), + put => update_authenticator_api_spec2(), + delete => delete_authenticator_api_spec2() + }, + {"/listeners/:listener_id/authentication/:id", Metadata, authentication4}. move_api() -> Metadata = #{ - post => #{ - description => "Move authenticator", - parameters => [ - #{ - name => id, - in => path, - schema => #{ - type => string - }, - required => true - } - ], - requestBody => #{ - content => #{ - 'application/json' => #{ - schema => #{ - oneOf => [ - #{ - type => object, - required => [position], - properties => #{ - position => #{ - type => string, - enum => [<<"top">>, <<"bottom">>], - example => <<"top">> - } - } - }, - #{ - type => object, - required => [position], - properties => #{ - position => #{ - type => string, - description => <<"before:">>, - example => <<"before:67e4c9d3">> - } - } - } - ] - } - } - } - }, - responses => #{ - <<"204">> => #{ - description => <<"No Content">> - }, - <<"400">> => ?ERR_RESPONSE(<<"Bad Request">>), - <<"404">> => ?ERR_RESPONSE(<<"Not Found">>) - } - } + post => move_authenticator_api_spec() }, - {"/authentication/authenticators/:id/move", Metadata, move}. + {"/authentication/:id/move", Metadata, move}. + +move_api2() -> + Metadata = #{ + post => move_authenticator_api_spec2() + }, + {"/listeners/:listener_id/authentication/:id/move", Metadata, move2}. import_users_api() -> Metadata = #{ - post => #{ - description => "Import users from json/csv file", - parameters => [ - #{ - name => id, - in => path, - schema => #{ - type => string - }, - required => true + post => import_users_api_spec() + }, + {"/authentication/:id/import_users", Metadata, import_users}. + +import_users_api2() -> + Metadata = #{ + post => import_users_api_spec2() + }, + {"/listeners/:listener_id/authentication/:id/import_users", Metadata, import_users2}. + +users_api() -> + Metadata = #{ + post => create_user_api_spec(), + get => list_users_api_spec() + }, + {"/authentication/:id/users", Metadata, users}. + +users2_api() -> + Metadata = #{ + put => update_user_api_spec(), + get => find_user_api_spec(), + delete => delete_user_api_spec() + }, + {"/authentication/:id/users/:user_id", Metadata, users2}. + +users3_api() -> + Metadata = #{ + post => create_user_api_spec2(), + get => list_users_api_spec2() + }, + {"/listeners/:listener_id/authentication/:id/users", Metadata, users3}. + +users4_api() -> + Metadata = #{ + put => update_user_api_spec2(), + get => find_user_api_spec2(), + delete => delete_user_api_spec2() + }, + {"/listeners/:listener_id/authentication/:id/users/:user_id", Metadata, users4}. + +create_authenticator_api_spec() -> + #{ + description => "Create a authenticator for global authentication", + requestBody => #{ + content => #{ + 'application/json' => #{ + schema => minirest:ref(<<"AuthenticatorConfig">>), + examples => #{ + default => #{ + summary => <<"Default">>, + value => emqx_json:encode(?EXAMPLE_1) + }, + http => #{ + summary => <<"Authentication provided by HTTP Server">>, + value => emqx_json:encode(?EXAMPLE_2) + }, + jwt => #{ + summary => <<"JWT Authentication">>, + value => emqx_json:encode(?EXAMPLE_3) + }, + mongodb => #{ + summary => <<"Authentication with MongoDB">>, + value => emqx_json:encode(?EXAMPLE_4) + }, + redis => #{ + summary => <<"Authentication with Redis">>, + value => emqx_json:encode(?EXAMPLE_5) + } + } } - ], - requestBody => #{ + } + }, + responses => #{ + <<"201">> => #{ + description => <<"Created">>, content => #{ 'application/json' => #{ - schema => #{ - type => object, - required => [filename], - properties => #{ - filename => #{ - type => string - } + schema => minirest:ref(<<"AuthenticatorInstance">>), + examples => #{ + example1 => #{ + summary => <<"Example 1">>, + value => emqx_json:encode(?INSTANCE_EXAMPLE_1) + }, + example2 => #{ + summary => <<"Example 2">>, + value => emqx_json:encode(?INSTANCE_EXAMPLE_2) + }, + example3 => #{ + summary => <<"Example 3">>, + value => emqx_json:encode(?INSTANCE_EXAMPLE_3) + }, + example4 => #{ + summary => <<"Example 4">>, + value => emqx_json:encode(?INSTANCE_EXAMPLE_4) + }, + example5 => #{ + summary => <<"Example 5">>, + value => emqx_json:encode(?INSTANCE_EXAMPLE_5) } } } } }, - responses => #{ - <<"204">> => #{ - description => <<"No Content">> - }, - <<"400">> => ?ERR_RESPONSE(<<"Bad Request">>), - <<"404">> => ?ERR_RESPONSE(<<"Not Found">>) - } - } - }, - {"/authentication/authenticators/:id/import-users", Metadata, import_users}. + <<"400">> => ?ERR_RESPONSE(<<"Bad Request">>), + <<"409">> => ?ERR_RESPONSE(<<"Conflict">>) + } + }. -users_api() -> - Metadata = #{ - post => #{ - description => "Add user", - parameters => [ - #{ - name => id, - in => path, - schema => #{ - type => string - }, - required => true +create_authenticator_api_spec2() -> + Spec = create_authenticator_api_spec(), + Spec#{ + description => "Create a authenticator for listener", + parameters => [ + #{ + name => listener_id, + in => path, + schema => #{ + type => string + }, + required => true + } + ] + }. + +list_authenticators_api_spec() -> + #{ + description => "List authenticators for global authentication", + responses => #{ + <<"200">> => #{ + description => <<"OK">>, + content => #{ + 'application/json' => #{ + schema => #{ + type => array, + items => minirest:ref(<<"AuthenticatorInstance">>) + }, + examples => #{ + example => #{ + summary => <<"Example">>, + value => emqx_json:encode([ ?INSTANCE_EXAMPLE_1 + , ?INSTANCE_EXAMPLE_2 + , ?INSTANCE_EXAMPLE_3 + , ?INSTANCE_EXAMPLE_4 + , ?INSTANCE_EXAMPLE_5 + ])}}}}}}}. + +list_authenticators_api_spec2() -> + Spec = list_authenticators_api_spec(), + Spec#{ + description => "List authenticators for listener", + parameters => [ + #{ + name => listener_id, + in => path, + schema => #{ + type => string + }, + required => true + } + ] + }. + +find_authenticator_api_spec() -> + #{ + description => "Get authenticator by id", + parameters => [ + #{ + name => id, + in => path, + description => "Authenticator id", + schema => #{ + type => string + }, + required => true + } + ], + responses => #{ + <<"200">> => #{ + description => <<"OK">>, + content => #{ + 'application/json' => #{ + schema => minirest:ref(<<"AuthenticatorInstance">>), + examples => #{ + example1 => #{ + summary => <<"Example 1">>, + value => emqx_json:encode(?INSTANCE_EXAMPLE_1) + }, + example2 => #{ + summary => <<"Example 2">>, + value => emqx_json:encode(?INSTANCE_EXAMPLE_2) + }, + example3 => #{ + summary => <<"Example 3">>, + value => emqx_json:encode(?INSTANCE_EXAMPLE_3) + }, + example4 => #{ + summary => <<"Example 4">>, + value => emqx_json:encode(?INSTANCE_EXAMPLE_4) + }, + example5 => #{ + summary => <<"Example 5">>, + value => emqx_json:encode(?INSTANCE_EXAMPLE_5) + } + } + } } - ], - requestBody => #{ + }, + <<"404">> => ?ERR_RESPONSE(<<"Not Found">>) + } + }. + +find_authenticator_api_spec2() -> + Spec = find_authenticator_api_spec(), + Spec#{ + parameters => [ + #{ + name => listener_id, + in => path, + description => "Listener id", + schema => #{ + type => string + }, + required => true + }, + #{ + name => id, + in => path, + description => "Authenticator id", + schema => #{ + type => string + }, + required => true + } + ] + }. + +update_authenticator_api_spec() -> + #{ + description => "Update authenticator", + parameters => [ + #{ + name => id, + in => path, + description => "Authenticator id", + schema => #{ + type => string + }, + required => true + } + ], + requestBody => #{ + content => #{ + 'application/json' => #{ + schema => minirest:ref(<<"AuthenticatorConfig">>), + examples => #{ + example1 => #{ + summary => <<"Example 1">>, + value => emqx_json:encode(?EXAMPLE_1) + }, + example2 => #{ + summary => <<"Example 2">>, + value => emqx_json:encode(?EXAMPLE_2) + }, + example3 => #{ + summary => <<"Example 3">>, + value => emqx_json:encode(?EXAMPLE_3) + }, + example4 => #{ + summary => <<"Example 4">>, + value => emqx_json:encode(?EXAMPLE_4) + }, + example5 => #{ + summary => <<"Example 5">>, + value => emqx_json:encode(?EXAMPLE_5) + } + } + } + } + }, + responses => #{ + <<"200">> => #{ + description => <<"OK">>, + content => #{ + 'application/json' => #{ + schema => minirest:ref(<<"AuthenticatorInstance">>), + examples => #{ + example1 => #{ + summary => <<"Example 1">>, + value => emqx_json:encode(?INSTANCE_EXAMPLE_1) + }, + example2 => #{ + summary => <<"Example 2">>, + value => emqx_json:encode(?INSTANCE_EXAMPLE_2) + }, + example3 => #{ + summary => <<"Example 3">>, + value => emqx_json:encode(?INSTANCE_EXAMPLE_3) + }, + example4 => #{ + summary => <<"Example 4">>, + value => emqx_json:encode(?INSTANCE_EXAMPLE_4) + }, + example5 => #{ + summary => <<"Example 5">>, + value => emqx_json:encode(?INSTANCE_EXAMPLE_5) + } + } + } + } + }, + <<"400">> => ?ERR_RESPONSE(<<"Bad Request">>), + <<"404">> => ?ERR_RESPONSE(<<"Not Found">>), + <<"409">> => ?ERR_RESPONSE(<<"Conflict">>) + } + }. + +update_authenticator_api_spec2() -> + Spec = update_authenticator_api_spec(), + Spec#{ + parameters => [ + #{ + name => listener_id, + in => path, + description => "Listener id", + schema => #{ + type => string + }, + required => true + }, + #{ + name => id, + in => path, + description => "Authenticator id", + schema => #{ + type => string + }, + required => true + } + ] + }. + +delete_authenticator_api_spec() -> + #{ + description => "Delete authenticator", + parameters => [ + #{ + name => id, + in => path, + description => "Authenticator id", + schema => #{ + type => string + }, + required => true + } + ], + responses => #{ + <<"204">> => #{ + description => <<"No Content">> + }, + <<"404">> => ?ERR_RESPONSE(<<"Not Found">>) + } + }. + +delete_authenticator_api_spec2() -> + Spec = delete_authenticator_api_spec(), + Spec#{ + parameters => [ + #{ + name => listener_id, + in => path, + description => "Listener id", + schema => #{ + type => string + }, + required => true + }, + #{ + name => id, + in => path, + description => "Authenticator id", + schema => #{ + type => string + }, + required => true + } + ] + }. + +move_authenticator_api_spec() -> + #{ + description => "Move authenticator", + parameters => [ + #{ + name => id, + in => path, + description => "Authenticator id", + schema => #{ + type => string + }, + required => true + } + ], + requestBody => #{ + content => #{ + 'application/json' => #{ + schema => #{ + oneOf => [ + #{ + type => object, + required => [position], + properties => #{ + position => #{ + type => string, + enum => [<<"top">>, <<"bottom">>], + example => <<"top">> + } + } + }, + #{ + type => object, + required => [position], + properties => #{ + position => #{ + type => string, + description => <<"before:">>, + example => <<"before:password-based:mysql">> + } + } + } + ] + } + } + } + }, + responses => #{ + <<"204">> => #{ + description => <<"No Content">> + }, + <<"400">> => ?ERR_RESPONSE(<<"Bad Request">>), + <<"404">> => ?ERR_RESPONSE(<<"Not Found">>) + } + }. + +move_authenticator_api_spec2() -> + Spec = move_authenticator_api_spec(), + Spec#{ + parameters => [ + #{ + name => listener_id, + in => path, + description => "Listener id", + schema => #{ + type => string + }, + required => true + }, + #{ + name => id, + in => path, + description => "Authenticator id", + schema => #{ + type => string + }, + required => true + } + ] + }. + +import_users_api_spec() -> + #{ + description => "Import users from json/csv file", + parameters => [ + #{ + name => id, + in => path, + description => "Authenticator id", + schema => #{ + type => string + }, + required => true + } + ], + requestBody => #{ + content => #{ + 'application/json' => #{ + schema => #{ + type => object, + required => [filename], + properties => #{ + filename => #{ + type => string + } + } + } + } + } + }, + responses => #{ + <<"204">> => #{ + description => <<"No Content">> + }, + <<"400">> => ?ERR_RESPONSE(<<"Bad Request">>), + <<"404">> => ?ERR_RESPONSE(<<"Not Found">>) + } + }. + +import_users_api_spec2() -> + Spec = import_users_api_spec(), + Spec#{ + parameters => [ + #{ + name => listener_id, + in => path, + description => "Listener id", + schema => #{ + type => string + }, + required => true + }, + #{ + name => id, + in => path, + description => "Authenticator id", + schema => #{ + type => string + }, + required => true + } + ] + }. + +create_user_api_spec() -> + #{ + description => "Add user", + parameters => [ + #{ + name => id, + in => path, + description => "Authenticator id", + schema => #{ + type => string + }, + required => true + } + ], + requestBody => #{ + content => #{ + 'application/json' => #{ + schema => #{ + type => object, + required => [user_id, password], + properties => #{ + user_id => #{ + type => string + }, + password => #{ + type => string + }, + is_superuser => #{ + type => boolean, + default => false + } + } + } + } + } + }, + responses => #{ + <<"201">> => #{ + description => <<"Created">>, content => #{ 'application/json' => #{ schema => #{ type => object, - required => [user_id, password], properties => #{ user_id => #{ type => string }, - password => #{ - type => string - }, - superuser => #{ - type => boolean, - default => false - } - } - } - } - } - }, - responses => #{ - <<"201">> => #{ - description => <<"Created">>, - content => #{ - 'application/json' => #{ - schema => #{ - type => object, - properties => #{ - user_id => #{ - type => string - }, - superuser => #{ - type => boolean - } - } - } - } - } - }, - <<"400">> => ?ERR_RESPONSE(<<"Bad Request">>), - <<"404">> => ?ERR_RESPONSE(<<"Not Found">>) - } - }, - get => #{ - description => "List users", - parameters => [ - #{ - name => id, - in => path, - schema => #{ - type => string - }, - required => true - } - ], - responses => #{ - <<"200">> => #{ - description => <<"OK">>, - content => #{ - 'application/json' => #{ - schema => #{ - type => array, - items => #{ - type => object, - properties => #{ - user_id => #{ - type => string - }, - superuser => #{ - type => boolean - } - } - } - } - } - } - }, - <<"404">> => ?ERR_RESPONSE(<<"Not Found">>) - } - } - }, - {"/authentication/authenticators/:id/users", Metadata, users}. - -users2_api() -> - Metadata = #{ - patch => #{ - description => "Update user", - parameters => [ - #{ - name => id, - in => path, - schema => #{ - type => string - }, - required => true - }, - #{ - name => user_id, - in => path, - schema => #{ - type => string - }, - required => true - } - ], - requestBody => #{ - content => #{ - 'application/json' => #{ - schema => #{ - type => object, - properties => #{ - password => #{ - type => string - }, - superuser => #{ + is_superuser => #{ type => boolean } } @@ -640,116 +794,379 @@ users2_api() -> } } }, - responses => #{ - <<"200">> => #{ - description => <<"OK">>, - content => #{ - 'application/json' => #{ - schema => #{ - type => array, - items => #{ - type => object, - properties => #{ - user_id => #{ - type => string - }, - superuser => #{ - type => boolean - } - } - } - } - } - } - }, - <<"400">> => ?ERR_RESPONSE(<<"Bad Request">>), - <<"404">> => ?ERR_RESPONSE(<<"Not Found">>) - } - }, - get => #{ - description => "Get user info", - parameters => [ - #{ - name => id, - in => path, - schema => #{ - type => string - }, - required => true - }, - #{ - name => user_id, - in => path, - schema => #{ - type => string - }, - required => true - } - ], - responses => #{ - <<"200">> => #{ - description => <<"OK">>, - content => #{ - 'application/json' => #{ - schema => #{ - type => array, - items => #{ - type => object, - properties => #{ - user_id => #{ - type => string - }, - superuser => #{ - type => boolean - } - } - } - } - } - } - }, - <<"404">> => ?ERR_RESPONSE(<<"Not Found">>) - } - }, - delete => #{ - description => "Delete user", - parameters => [ - #{ - name => id, - in => path, - schema => #{ - type => string - }, - required => true - }, - #{ - name => user_id, - in => path, - schema => #{ - type => string - }, - required => true - } - ], - responses => #{ - <<"204">> => #{ - description => <<"No Content">> - }, - <<"404">> => ?ERR_RESPONSE(<<"Not Found">>) - } + <<"400">> => ?ERR_RESPONSE(<<"Bad Request">>), + <<"404">> => ?ERR_RESPONSE(<<"Not Found">>) } - }, - {"/authentication/authenticators/:id/users/:user_id", Metadata, users2}. + }. + +create_user_api_spec2() -> + Spec = create_user_api_spec(), + Spec#{ + parameters => [ + #{ + name => listener_id, + in => path, + description => "Listener id", + schema => #{ + type => string + }, + required => true + }, + #{ + name => id, + in => path, + description => "Authenticator id", + schema => #{ + type => string + }, + required => true + } + ] + }. + +list_users_api_spec() -> + #{ + description => "List users", + parameters => [ + #{ + name => id, + in => path, + description => "Authenticator id", + schema => #{ + type => string + }, + required => true + } + ], + responses => #{ + <<"200">> => #{ + description => <<"OK">>, + content => #{ + 'application/json' => #{ + schema => #{ + type => array, + items => #{ + type => object, + properties => #{ + user_id => #{ + type => string + }, + is_superuser => #{ + type => boolean + } + } + } + } + } + } + }, + <<"404">> => ?ERR_RESPONSE(<<"Not Found">>) + } + }. + +list_users_api_spec2() -> + Spec = list_users_api_spec(), + Spec#{ + parameters => [ + #{ + name => listener_id, + in => path, + description => "Listener id", + schema => #{ + type => string + }, + required => true + }, + #{ + name => id, + in => path, + description => "Authenticator id", + schema => #{ + type => string + }, + required => true + } + ] + }. + +update_user_api_spec() -> + #{ + description => "Update user", + parameters => [ + #{ + name => id, + in => path, + description => "Authenticator id", + schema => #{ + type => string + }, + required => true + }, + #{ + name => user_id, + in => path, + description => "User id", + schema => #{ + type => string + }, + required => true + } + ], + requestBody => #{ + content => #{ + 'application/json' => #{ + schema => #{ + type => object, + properties => #{ + password => #{ + type => string + }, + is_superuser => #{ + type => boolean + } + } + } + } + } + }, + responses => #{ + <<"200">> => #{ + description => <<"OK">>, + content => #{ + 'application/json' => #{ + schema => #{ + type => array, + items => #{ + type => object, + properties => #{ + user_id => #{ + type => string + }, + is_superuser => #{ + type => boolean + } + } + } + } + } + } + }, + <<"400">> => ?ERR_RESPONSE(<<"Bad Request">>), + <<"404">> => ?ERR_RESPONSE(<<"Not Found">>) + } + }. + +update_user_api_spec2() -> + Spec = update_user_api_spec(), + Spec#{ + parameters => [ + #{ + name => listener_id, + in => path, + description => "Listener id", + schema => #{ + type => string + }, + required => true + }, + #{ + name => id, + in => path, + description => "Authenticator id", + schema => #{ + type => string + }, + required => true + }, + #{ + name => user_id, + in => path, + description => "User id", + schema => #{ + type => string + }, + required => true + } + ] + }. + +find_user_api_spec() -> + #{ + description => "Get user info", + parameters => [ + #{ + name => id, + in => path, + description => "Authenticator id", + schema => #{ + type => string + }, + required => true + }, + #{ + name => user_id, + in => path, + description => "User id", + schema => #{ + type => string + }, + required => true + } + ], + responses => #{ + <<"200">> => #{ + description => <<"OK">>, + content => #{ + 'application/json' => #{ + schema => #{ + type => array, + items => #{ + type => object, + properties => #{ + user_id => #{ + type => string + }, + is_superuser => #{ + type => boolean + } + } + } + } + } + } + }, + <<"404">> => ?ERR_RESPONSE(<<"Not Found">>) + } + }. + +find_user_api_spec2() -> + Spec = find_user_api_spec(), + Spec#{ + parameters => [ + #{ + name => listener_id, + in => path, + description => "Listener id", + schema => #{ + type => string + }, + required => true + }, + #{ + name => id, + in => path, + description => "Authenticator id", + schema => #{ + type => string + }, + required => true + }, + #{ + name => user_id, + in => path, + description => "User id", + schema => #{ + type => string + }, + required => true + } + ] + }. + +delete_user_api_spec() -> + #{ + description => "Delete user", + parameters => [ + #{ + name => id, + in => path, + description => "Authenticator id", + schema => #{ + type => string + }, + required => true + }, + #{ + name => user_id, + in => path, + description => "User id", + schema => #{ + type => string + }, + required => true + } + ], + responses => #{ + <<"204">> => #{ + description => <<"No Content">> + }, + <<"404">> => ?ERR_RESPONSE(<<"Not Found">>) + } + }. + +delete_user_api_spec2() -> + Spec = delete_user_api_spec(), + Spec#{ + parameters => [ + #{ + name => listener_id, + in => path, + description => "Listener id", + schema => #{ + type => string + }, + required => true + }, + #{ + name => id, + in => path, + description => "Authenticator id", + schema => #{ + type => string + }, + required => true + }, + #{ + name => user_id, + in => path, + description => "User id", + schema => #{ + type => string + }, + required => true + } + ] + }. + definitions() -> - AuthenticatorDef = #{ - oneOf => [ minirest:ref(<<"password_based">>) - , minirest:ref(<<"jwt">>) - , minirest:ref(<<"scram">>) - ] + AuthenticatorConfigDef = #{ + allOf => [ + #{ + type => object, + properties => #{ + enable => #{ + type => boolean, + default => true, + example => true + } + } + }, + #{ + oneOf => [ minirest:ref(<<"PasswordBasedBuiltInDatabase">>) + , minirest:ref(<<"PasswordBasedMySQL">>) + , minirest:ref(<<"PasswordBasedPostgreSQL">>) + , minirest:ref(<<"PasswordBasedMongoDB">>) + , minirest:ref(<<"PasswordBasedRedis">>) + , minirest:ref(<<"PasswordBasedHTTPServer">>) + , minirest:ref(<<"JWT">>) + , minirest:ref(<<"SCRAMBuiltInDatabase">>) + ] + } + ] }, - ReturnedAuthenticatorDef = #{ + AuthenticatorInstanceDef = #{ allOf => [ #{ type => object, @@ -758,124 +1175,20 @@ definitions() -> type => string } } - }, - #{ - oneOf => [ minirest:ref(<<"password_based">>) - , minirest:ref(<<"jwt">>) - , minirest:ref(<<"scram">>) - ] } - ] - }, - - PasswordBasedDef = #{ - allOf => [ - #{ - type => object, - required => [name, mechanism], - properties => #{ - name => #{ - type => string, - example => "exmaple" - }, - mechanism => #{ - type => string, - enum => [<<"password-based">>], - example => <<"password-based">> - } - } - }, - #{ - oneOf => [ minirest:ref(<<"password_based_built_in_database">>) - , minirest:ref(<<"password_based_mysql">>) - , minirest:ref(<<"password_based_pgsql">>) - , minirest:ref(<<"password_based_mongodb">>) - , minirest:ref(<<"password_based_redis">>) - , minirest:ref(<<"password_based_http_server">>) - ] - } - ] - }, - - JWTDef = #{ - type => object, - required => [name, mechanism], - properties => #{ - name => #{ - type => string, - example => "exmaple" - }, - mechanism => #{ - type => string, - enum => [<<"jwt">>], - example => <<"jwt">> - }, - use_jwks => #{ - type => boolean, - default => false, - example => false - }, - algorithm => #{ - type => string, - enum => [<<"hmac-based">>, <<"public-key">>], - default => <<"hmac-based">>, - example => <<"hmac-based">> - }, - secret => #{ - type => string - }, - secret_base64_encoded => #{ - type => boolean, - default => false - }, - certificate => #{ - type => string - }, - verify_claims => #{ - type => object, - additionalProperties => #{ - type => string - } - }, - ssl => minirest:ref(<<"ssl">>) - } - }, - - SCRAMDef = #{ - type => object, - required => [name, mechanism, server_type], - properties => #{ - name => #{ - type => string, - example => "exmaple" - }, - mechanism => #{ - type => string, - enum => [<<"scram">>], - example => <<"scram">> - }, - server_type => #{ - type => string, - enum => [<<"built-in-database">>], - default => <<"built-in-database">> - }, - algorithm => #{ - type => string, - enum => [<<"sha256">>, <<"sha512">>], - default => <<"sha256">> - }, - iteration_count => #{ - type => integer, - default => 4096 - } - } + ] ++ maps:get(allOf, AuthenticatorConfigDef) }, PasswordBasedBuiltInDatabaseDef = #{ type => object, - required => [server_type], + required => [mechanism, backend], properties => #{ - server_type => #{ + mechanism => #{ + type => string, + enum => [<<"password-based">>], + example => <<"password-based">> + }, + backend => #{ type => string, enum => [<<"built-in-database">>], example => <<"built-in-database">> @@ -883,23 +1196,28 @@ definitions() -> user_id_type => #{ type => string, enum => [<<"username">>, <<"clientid">>], - default => <<"username">>, example => <<"username">> }, - password_hash_algorithm => minirest:ref(<<"password_hash_algorithm">>) + password_hash_algorithm => minirest:ref(<<"PasswordHashAlgorithm">>) } }, PasswordBasedMySQLDef = #{ type => object, - required => [ server_type + required => [ mechanism + , backend , server , database , username , password , query], properties => #{ - server_type => #{ + mechanism => #{ + type => string, + enum => [<<"password-based">>], + example => <<"password-based">> + }, + backend => #{ type => string, enum => [<<"mysql">>], example => <<"mysql">> @@ -925,7 +1243,7 @@ definitions() -> type => boolean, default => true }, - ssl => minirest:ref(<<"ssl">>), + ssl => minirest:ref(<<"SSL">>), password_hash_algorithm => #{ type => string, enum => [<<"plain">>, <<"md5">>, <<"sha">>, <<"sha256">>, <<"sha512">>, <<"bcrypt">>], @@ -948,19 +1266,25 @@ definitions() -> } }, - PasswordBasedPgSQLDef = #{ + PasswordBasedPostgreSQLDef = #{ type => object, - required => [ server_type + required => [ mechanism + , backend , server , database , username , password , query], properties => #{ - server_type => #{ + mechanism => #{ type => string, - enum => [<<"pgsql">>], - example => <<"pgsql">> + enum => [<<"password-based">>], + example => <<"password-based">> + }, + backend => #{ + type => string, + enum => [<<"postgresql">>], + example => <<"postgresql">> }, server => #{ type => string, @@ -1002,7 +1326,8 @@ definitions() -> PasswordBasedMongoDBDef = #{ type => object, - required => [ server_type + required => [ mechanism + , backend , server , servers , replica_set_name @@ -1014,10 +1339,15 @@ definitions() -> , password_hash_field ], properties => #{ - server_type => #{ + mechanism => #{ + type => string, + enum => [<<"password-based">>], + example => <<"password-based">> + }, + backend => #{ type => string, enum => [<<"mongodb">>], - example => [<<"mongodb">>] + example => <<"mongodb">> }, server => #{ description => <<"Mutually exclusive with the 'servers' field, only valid in standalone mode">>, @@ -1069,6 +1399,10 @@ definitions() -> type => string, example => <<"salt">> }, + is_superuser_field => #{ + type => string, + example => <<"is_superuser">> + }, password_hash_algorithm => #{ type => string, enum => [<<"plain">>, <<"md5">>, <<"sha">>, <<"sha256">>, <<"sha512">>, <<"bcrypt">>], @@ -1087,7 +1421,8 @@ definitions() -> PasswordBasedRedisDef = #{ type => object, - required => [ server_type + required => [ mechanism + , backend , server , servers , password @@ -1095,10 +1430,15 @@ definitions() -> , query ], properties => #{ - server_type => #{ + mechanism => #{ + type => string, + enum => [<<"password-based">>], + example => <<"password-based">> + }, + backend => #{ type => string, enum => [<<"redis">>], - example => [<<"redis">>] + example => <<"redis">> }, server => #{ description => <<"Mutually exclusive with the 'servers' field, only valid in standalone mode">>, @@ -1153,12 +1493,18 @@ definitions() -> PasswordBasedHTTPServerDef = #{ type => object, - required => [ server_type + required => [ mechanism + , backend , url - , form_data + , body ], properties => #{ - server_type => #{ + mechanism => #{ + type => string, + enum => [<<"password-based">>], + example => <<"password-based">> + }, + backend => #{ type => string, enum => [<<"http-server">>], example => <<"http-server">> @@ -1178,8 +1524,8 @@ definitions() -> type => string } }, - form_data => #{ - type => string + body => #{ + type => object }, connect_timeout => #{ type => integer, @@ -1204,6 +1550,77 @@ definitions() -> enable_pipelining => #{ type => boolean, default => true + }, + ssl => minirest:ref(<<"SSL">>) + } + }, + + JWTDef = #{ + type => object, + required => [mechanism], + properties => #{ + mechanism => #{ + type => string, + enum => [<<"jwt">>], + example => <<"jwt">> + }, + use_jwks => #{ + type => boolean, + default => false, + example => false + }, + algorithm => #{ + type => string, + enum => [<<"hmac-based">>, <<"public-key">>], + default => <<"hmac-based">>, + example => <<"hmac-based">> + }, + secret => #{ + type => string + }, + secret_base64_encoded => #{ + type => boolean, + default => false + }, + certificate => #{ + type => string + }, + endpoint => #{ + type => string, + example => <<"http://localhost:80">> + }, + verify_claims => #{ + type => object, + additionalProperties => #{ + type => string + } + }, + ssl => minirest:ref(<<"SSL">>) + } + }, + + SCRAMBuiltInDatabaseDef = #{ + type => object, + required => [mechanism, backend], + properties => #{ + mechanism => #{ + type => string, + enum => [<<"scram">>], + example => <<"scram">> + }, + backend => #{ + type => string, + enum => [<<"built-in-database">>], + example => <<"built-in-database">> + }, + algorithm => #{ + type => string, + enum => [<<"sha256">>, <<"sha512">>], + default => <<"sha256">> + }, + iteration_count => #{ + type => integer, + default => 4096 } } }, @@ -1219,6 +1636,7 @@ definitions() -> }, salt_rounds => #{ type => integer, + description => <<"Only valid when the name field is set to bcrypt">>, default => 10 } } @@ -1273,176 +1691,357 @@ definitions() -> } }, - [ #{<<"authenticator">> => AuthenticatorDef} - , #{<<"returned_authenticator">> => ReturnedAuthenticatorDef} - , #{<<"password_based">> => PasswordBasedDef} - , #{<<"jwt">> => JWTDef} - , #{<<"scram">> => SCRAMDef} - , #{<<"password_based_built_in_database">> => PasswordBasedBuiltInDatabaseDef} - , #{<<"password_based_mysql">> => PasswordBasedMySQLDef} - , #{<<"password_based_pgsql">> => PasswordBasedPgSQLDef} - , #{<<"password_based_mongodb">> => PasswordBasedMongoDBDef} - , #{<<"password_based_redis">> => PasswordBasedRedisDef} - , #{<<"password_based_http_server">> => PasswordBasedHTTPServerDef} - , #{<<"password_hash_algorithm">> => PasswordHashAlgorithmDef} - , #{<<"ssl">> => SSLDef} - , #{<<"error">> => ErrorDef} + [ #{<<"AuthenticatorConfig">> => AuthenticatorConfigDef} + , #{<<"AuthenticatorInstance">> => AuthenticatorInstanceDef} + , #{<<"PasswordBasedBuiltInDatabase">> => PasswordBasedBuiltInDatabaseDef} + , #{<<"PasswordBasedMySQL">> => PasswordBasedMySQLDef} + , #{<<"PasswordBasedPostgreSQL">> => PasswordBasedPostgreSQLDef} + , #{<<"PasswordBasedMongoDB">> => PasswordBasedMongoDBDef} + , #{<<"PasswordBasedRedis">> => PasswordBasedRedisDef} + , #{<<"PasswordBasedHTTPServer">> => PasswordBasedHTTPServerDef} + , #{<<"JWT">> => JWTDef} + , #{<<"SCRAMBuiltInDatabase">> => SCRAMBuiltInDatabaseDef} + , #{<<"PasswordHashAlgorithm">> => PasswordHashAlgorithmDef} + , #{<<"SSL">> => SSLDef} + , #{<<"Error">> => ErrorDef} ]. authentication(post, #{body := Config}) -> - case Config of - #{<<"enable">> := Enable} -> - {ok, _} = emqx_authn:update_config([authentication, enable], {enable, Enable}), - {204}; - _ -> - serialize_error({missing_parameter, enable}) - end; + create_authenticator([authentication], ?GLOBAL, Config); + authentication(get, _Params) -> - Enabled = emqx_authn:is_enabled(), - {200, #{enabled => Enabled}}. + list_authenticators([authentication]). -authenticators(post, #{body := Config}) -> - case emqx_authn:update_config([authentication, authenticators], {create_authenticator, Config}) of - {ok, #{post_config_update := #{emqx_authn := #{id := ID, name := Name}}, - raw_config := RawConfig}} -> - [RawConfig1] = [RC || #{<<"name">> := N} = RC <- RawConfig, N =:= Name], - {200, RawConfig1#{id => ID}}; - {error, {_, _, Reason}} -> - serialize_error(Reason) - end; -authenticators(get, _Params) -> - RawConfig = get_raw_config([authentication, authenticators]), - {ok, Authenticators} = emqx_authn:list_authenticators(?CHAIN), - NAuthenticators = lists:zipwith(fun(#{<<"name">> := Name} = Config, #{id := ID, name := Name}) -> - Config#{id => ID} - end, RawConfig, Authenticators), - {200, NAuthenticators}. +authentication2(get, #{bindings := #{id := AuthenticatorID}}) -> + list_authenticator([authentication], AuthenticatorID); -authenticators2(get, #{bindings := #{id := AuthenticatorID}}) -> - case emqx_authn:lookup_authenticator(?CHAIN, AuthenticatorID) of - {ok, #{id := ID, name := Name}} -> - RawConfig = get_raw_config([authentication, authenticators]), - [RawConfig1] = [RC || #{<<"name">> := N} = RC <- RawConfig, N =:= Name], - {200, RawConfig1#{id => ID}}; +authentication2(put, #{bindings := #{id := AuthenticatorID}, body := Config}) -> + update_authenticator([authentication], ?GLOBAL, AuthenticatorID, Config); + +authentication2(delete, #{bindings := #{id := AuthenticatorID}}) -> + delete_authenticator([authentication], ?GLOBAL, AuthenticatorID). + +authentication3(post, #{bindings := #{listener_id := ListenerID}, body := Config}) -> + case find_listener(ListenerID) of + {ok, {Type, Name}} -> + create_authenticator([listeners, Type, Name, authentication], ListenerID, Config); {error, Reason} -> serialize_error(Reason) end; -authenticators2(put, #{bindings := #{id := AuthenticatorID}, body := Config}) -> - case emqx_authn:update_config([authentication, authenticators], - {update_or_create_authenticator, AuthenticatorID, Config}) of - {ok, #{post_config_update := #{emqx_authn := #{id := ID, name := Name}}, - raw_config := RawConfig}} -> - [RawConfig0] = [RC || #{<<"name">> := N} = RC <- RawConfig, N =:= Name], - {200, RawConfig0#{id => ID}}; - {error, {_, _, Reason}} -> +authentication3(get, #{bindings := #{listener_id := ListenerID}}) -> + case find_listener(ListenerID) of + {ok, {Type, Name}} -> + list_authenticators([listeners, Type, Name, authentication]); + {error, Reason} -> + serialize_error(Reason) + end. + +authentication4(get, #{bindings := #{listener_id := ListenerID, id := AuthenticatorID}}) -> + case find_listener(ListenerID) of + {ok, {Type, Name}} -> + list_authenticator([listeners, Type, Name, authentication], AuthenticatorID); + {error, Reason} -> serialize_error(Reason) end; -authenticators2(delete, #{bindings := #{id := AuthenticatorID}}) -> - case emqx_authn:update_config([authentication, authenticators], {delete_authenticator, AuthenticatorID}) of +authentication4(put, #{bindings := #{listener_id := ListenerID, id := AuthenticatorID}, body := Config}) -> + case find_listener(ListenerID) of + {ok, {Type, Name}} -> + update_authenticator([listeners, Type, Name, authentication], ListenerID, AuthenticatorID, Config); + {error, Reason} -> + serialize_error(Reason) + end; +authentication4(delete, #{bindings := #{listener_id := ListenerID, id := AuthenticatorID}}) -> + case find_listener(ListenerID) of + {ok, {Type, Name}} -> + delete_authenticator([listeners, Type, Name, authentication], ListenerID, AuthenticatorID); + {error, Reason} -> + serialize_error(Reason) + end. + +move(post, #{bindings := #{id := AuthenticatorID}, body := #{<<"position">> := Position}}) -> + move_authenitcator([authentication], ?GLOBAL, AuthenticatorID, Position); +move(post, #{bindings := #{id := _}, body := _}) -> + serialize_error({missing_parameter, position}). + +move2(post, #{bindings := #{listener_id := ListenerID, id := AuthenticatorID}, body := #{<<"position">> := Position}}) -> + case find_listener(ListenerID) of + {ok, {Type, Name}} -> + move_authenitcator([listeners, Type, Name, authentication], ListenerID, AuthenticatorID, Position); + {error, Reason} -> + serialize_error(Reason) + end; +move2(post, #{bindings := #{listener_id := _, id := _}, body := _}) -> + serialize_error({missing_parameter, position}). + +import_users(post, #{bindings := #{id := AuthenticatorID}, body := #{<<"filename">> := Filename}}) -> + case ?AUTHN:import_users(?GLOBAL, AuthenticatorID, Filename) of + ok -> {204}; + {error, Reason} -> serialize_error(Reason) + end; +import_users(post, #{bindings := #{id := _}, body := _}) -> + serialize_error({missing_parameter, filename}). + +import_users2(post, #{bindings := #{listener_id := ListenerID, id := AuthenticatorID}, body := #{<<"filename">> := Filename}}) -> + case ?AUTHN:import_users(ListenerID, AuthenticatorID, Filename) of + ok -> {204}; + {error, Reason} -> serialize_error(Reason) + end; +import_users2(post, #{bindings := #{listener_id := _, id := _}, body := _}) -> + serialize_error({missing_parameter, filename}). + +users(post, #{bindings := #{id := AuthenticatorID}, body := UserInfo}) -> + add_user(?GLOBAL, AuthenticatorID, UserInfo); +users(get, #{bindings := #{id := AuthenticatorID}}) -> + list_users(?GLOBAL, AuthenticatorID). + +users2(put, #{bindings := #{id := AuthenticatorID, + user_id := UserID}, body := UserInfo}) -> + update_user(?GLOBAL, AuthenticatorID, UserID, UserInfo); +users2(get, #{bindings := #{id := AuthenticatorID, user_id := UserID}}) -> + find_user(?GLOBAL, AuthenticatorID, UserID); +users2(delete, #{bindings := #{id := AuthenticatorID, user_id := UserID}}) -> + delete_user(?GLOBAL, AuthenticatorID, UserID). + +users3(post, #{bindings := #{listener_id := ListenerID, + id := AuthenticatorID}, body := UserInfo}) -> + add_user(ListenerID, AuthenticatorID, UserInfo); +users3(get, #{bindings := #{listener_id := ListenerID, + id := AuthenticatorID}}) -> + list_users(ListenerID, AuthenticatorID). + +users4(put, #{bindings := #{listener_id := ListenerID, + id := AuthenticatorID, + user_id := UserID}, body := UserInfo}) -> + update_user(ListenerID, AuthenticatorID, UserID, UserInfo); +users4(get, #{bindings := #{listener_id := ListenerID, + id := AuthenticatorID, + user_id := UserID}}) -> + find_user(ListenerID, AuthenticatorID, UserID); +users4(delete, #{bindings := #{listener_id := ListenerID, + id := AuthenticatorID, + user_id := UserID}}) -> + delete_user(ListenerID, AuthenticatorID, UserID). + +%%------------------------------------------------------------------------------ +%% Internal functions +%%------------------------------------------------------------------------------ + +find_listener(ListenerID) -> + {Type, Name} = emqx_listeners:parse_listener_id(ListenerID), + case emqx_config:find([listeners, Type, Name]) of + {not_found, _, _} -> + {error, {not_found, {listener, ListenerID}}}; + {ok, _} -> + {ok, {Type, Name}} + end. + +create_authenticator(ConfKeyPath, ChainName0, Config) -> + ChainName = to_atom(ChainName0), + case update_config(ConfKeyPath, {create_authenticator, ChainName, Config}) of + {ok, #{post_config_update := #{?AUTHN := #{id := ID}}, + raw_config := AuthenticatorsConfig}} -> + {ok, AuthenticatorConfig} = find_config(ID, AuthenticatorsConfig), + {200, maps:put(id, ID, fill_defaults(AuthenticatorConfig))}; + {error, {_, _, Reason}} -> + serialize_error(Reason) + end. + +list_authenticators(ConfKeyPath) -> + AuthenticatorsConfig = get_raw_config_with_defaults(ConfKeyPath), + NAuthenticators = [maps:put(id, ?AUTHN:generate_id(AuthenticatorConfig), AuthenticatorConfig) + || AuthenticatorConfig <- AuthenticatorsConfig], + {200, NAuthenticators}. + +list_authenticator(ConfKeyPath, AuthenticatorID) -> + AuthenticatorsConfig = get_raw_config_with_defaults(ConfKeyPath), + case find_config(AuthenticatorID, AuthenticatorsConfig) of + {ok, AuthenticatorConfig} -> + {200, AuthenticatorConfig#{id => AuthenticatorID}}; + {error, Reason} -> + serialize_error(Reason) + end. + +update_authenticator(ConfKeyPath, ChainName0, AuthenticatorID, Config) -> + ChainName = to_atom(ChainName0), + case update_config(ConfKeyPath, {update_authenticator, ChainName, AuthenticatorID, Config}) of + {ok, #{post_config_update := #{?AUTHN := #{id := ID}}, + raw_config := AuthenticatorsConfig}} -> + {ok, AuthenticatorConfig} = find_config(ID, AuthenticatorsConfig), + {200, maps:put(id, ID, fill_defaults(AuthenticatorConfig))}; + {error, {_, _, Reason}} -> + serialize_error(Reason) + end. + +delete_authenticator(ConfKeyPath, ChainName0, AuthenticatorID) -> + ChainName = to_atom(ChainName0), + case update_config(ConfKeyPath, {delete_authenticator, ChainName, AuthenticatorID}) of {ok, _} -> {204}; {error, {_, _, Reason}} -> serialize_error(Reason) end. -move(post, #{bindings := #{id := AuthenticatorID}, body := Body}) -> - case Body of - #{<<"position">> := Position} -> - case emqx_authn:update_config([authentication, authenticators], {move_authenticator, AuthenticatorID, Position}) of - {ok, _} -> {204}; - {error, {_, _, Reason}} -> serialize_error(Reason) - end; - _ -> - serialize_error({missing_parameter, position}) - end. - -import_users(post, #{bindings := #{id := AuthenticatorID}, body := Body}) -> - case Body of - #{<<"filename">> := Filename} -> - case emqx_authn:import_users(?CHAIN, AuthenticatorID, Filename) of - ok -> {204}; - {error, Reason} -> serialize_error(Reason) - end; - _ -> - serialize_error({missing_parameter, filename}) - end. - -users(post, #{bindings := #{id := AuthenticatorID}, body := UserInfo}) -> - case UserInfo of - #{ <<"user_id">> := UserID, <<"password">> := Password} -> - Superuser = maps:get(<<"superuser">>, UserInfo, false), - case emqx_authn:add_user(?CHAIN, AuthenticatorID, #{ user_id => UserID - , password => Password - , superuser => Superuser}) of - {ok, User} -> - {201, User}; - {error, Reason} -> +move_authenitcator(ConfKeyPath, ChainName0, AuthenticatorID, Position) -> + ChainName = to_atom(ChainName0), + case parse_position(Position) of + {ok, NPosition} -> + case update_config(ConfKeyPath, {move_authenticator, ChainName, AuthenticatorID, NPosition}) of + {ok, _} -> + {204}; + {error, {_, _, Reason}} -> serialize_error(Reason) end; - #{<<"user_id">> := _} -> - serialize_error({missing_parameter, password}); - _ -> - serialize_error({missing_parameter, user_id}) - end; -users(get, #{bindings := #{id := AuthenticatorID}}) -> - case emqx_authn:list_users(?CHAIN, AuthenticatorID) of - {ok, Users} -> - {200, Users}; {error, Reason} -> serialize_error(Reason) end. -users2(patch, #{bindings := #{id := AuthenticatorID, - user_id := UserID}, - body := UserInfo}) -> - NUserInfo = maps:with([<<"password">>, <<"superuser">>], UserInfo), - case NUserInfo =:= #{} of +add_user(ChainName0, AuthenticatorID, #{<<"user_id">> := UserID, <<"password">> := Password} = UserInfo) -> + ChainName = to_atom(ChainName0), + IsSuperuser = maps:get(<<"is_superuser">>, UserInfo, false), + case ?AUTHN:add_user(ChainName, AuthenticatorID, #{ user_id => UserID + , password => Password + , is_superuser => IsSuperuser}) of + {ok, User} -> + {201, User}; + {error, Reason} -> + serialize_error(Reason) + end; +add_user(_, _, #{<<"user_id">> := _}) -> + serialize_error({missing_parameter, password}); +add_user(_, _, _) -> + serialize_error({missing_parameter, user_id}). + +update_user(ChainName0, AuthenticatorID, UserID, UserInfo) -> + ChainName = to_atom(ChainName0), + case maps:with([<<"password">>, <<"is_superuser">>], UserInfo) =:= #{} of true -> serialize_error({missing_parameter, password}); false -> - case emqx_authn:update_user(?CHAIN, AuthenticatorID, UserID, UserInfo) of + case ?AUTHN:update_user(ChainName, AuthenticatorID, UserID, UserInfo) of {ok, User} -> {200, User}; {error, Reason} -> serialize_error(Reason) end - end; -users2(get, #{bindings := #{id := AuthenticatorID, user_id := UserID}}) -> - case emqx_authn:lookup_user(?CHAIN, AuthenticatorID, UserID) of + end. + +find_user(ChainName0, AuthenticatorID, UserID) -> + ChainName = to_atom(ChainName0), + case ?AUTHN:lookup_user(ChainName, AuthenticatorID, UserID) of {ok, User} -> {200, User}; {error, Reason} -> serialize_error(Reason) - end; -users2(delete, #{bindings := #{id := AuthenticatorID, user_id := UserID}}) -> - case emqx_authn:delete_user(?CHAIN, AuthenticatorID, UserID) of + end. + +delete_user(ChainName0, AuthenticatorID, UserID) -> + ChainName = to_atom(ChainName0), + case ?AUTHN:delete_user(ChainName, AuthenticatorID, UserID) of ok -> {204}; {error, Reason} -> serialize_error(Reason) end. -get_raw_config(ConfKeyPath) -> - %% TODO: call emqx_config:get_raw(ConfKeyPath) directly +list_users(ChainName0, AuthenticatorID) -> + ChainName = to_atom(ChainName0), + case ?AUTHN:list_users(ChainName, AuthenticatorID) of + {ok, Users} -> + {200, Users}; + {error, Reason} -> + serialize_error(Reason) + end. + +update_config(Path, ConfigRequest) -> + emqx:update_config(Path, ConfigRequest, #{rawconf_with_defaults => true}). + +get_raw_config_with_defaults(ConfKeyPath) -> NConfKeyPath = [atom_to_binary(Key, utf8) || Key <- ConfKeyPath], - emqx_map_lib:deep_get(NConfKeyPath, emqx_config:fill_defaults(emqx_config:get_raw([]))). + RawConfig = emqx_map_lib:deep_get(NConfKeyPath, emqx_config:get_raw([]), []), + to_list(fill_defaults(RawConfig)). + +find_config(AuthenticatorID, AuthenticatorsConfig) -> + case [AC || AC <- to_list(AuthenticatorsConfig), AuthenticatorID =:= ?AUTHN:generate_id(AC)] of + [] -> {error, {not_found, {authenticator, AuthenticatorID}}}; + [AuthenticatorConfig] -> {ok, AuthenticatorConfig} + end. + +fill_defaults(Config) -> + #{<<"authentication">> := CheckedConfig} = hocon_schema:check_plain( + ?AUTHN, #{<<"authentication">> => Config}, #{nullable => true, no_conversion => true}), + CheckedConfig. serialize_error({not_found, {authenticator, ID}}) -> {404, #{code => <<"NOT_FOUND">>, - message => list_to_binary(io_lib:format("Authenticator '~s' does not exist", [ID]))}}; -serialize_error(name_has_be_used) -> + message => list_to_binary( + io_lib:format("Authenticator '~s' does not exist", [ID]) + )}}; + +serialize_error({not_found, {listener, ID}}) -> + {404, #{code => <<"NOT_FOUND">>, + message => list_to_binary( + io_lib:format("Listener '~s' does not exist", [ID]) + )}}; + +serialize_error({not_found, {chain, ?GLOBAL}}) -> + {500, #{code => <<"INTERNAL_SERVER_ERROR">>, + message => <<"Authentication status is abnormal">>}}; + +serialize_error({not_found, {chain, Name}}) -> + {400, #{code => <<"BAD_REQUEST">>, + message => list_to_binary( + io_lib:format("No authentication has been create for listener '~s'", [Name]) + )}}; + +serialize_error({already_exists, {authenticator, ID}}) -> {409, #{code => <<"ALREADY_EXISTS">>, - message => <<"Name has be used">>}}; + message => list_to_binary( + io_lib:format("Authenticator '~s' already exist", [ID]) + )}}; + +serialize_error(no_available_provider) -> + {400, #{code => <<"BAD_REQUEST">>, + message => <<"Unsupported authentication type">>}}; + +serialize_error(change_of_authentication_type_is_not_allowed) -> + {400, #{code => <<"BAD_REQUEST">>, + message => <<"Change of authentication type is not allowed">>}}; + +serialize_error(unsupported_operation) -> + {400, #{code => <<"BAD_REQUEST">>, + message => <<"Operation not supported in this authentication type">>}}; + serialize_error({missing_parameter, Name}) -> {400, #{code => <<"MISSING_PARAMETER">>, message => list_to_binary( io_lib:format("The input parameter '~p' that is mandatory for processing this request is not supplied", [Name]) )}}; + serialize_error({invalid_parameter, Name}) -> {400, #{code => <<"INVALID_PARAMETER">>, message => list_to_binary( io_lib:format("The value of input parameter '~p' is invalid", [Name]) )}}; + serialize_error(Reason) -> {400, #{code => <<"BAD_REQUEST">>, - message => list_to_binary(io_lib:format("Todo: ~p", [Reason]))}}. + message => list_to_binary(io_lib:format("~p", [Reason]))}}. + +parse_position(<<"top">>) -> + {ok, top}; +parse_position(<<"bottom">>) -> + {ok, bottom}; +parse_position(<<"before:", Before/binary>>) -> + {ok, {before, Before}}; +parse_position(_) -> + {error, {invalid_parameter, position}}. + +to_list(M) when is_map(M) -> + [M]; +to_list(L) when is_list(L) -> + L. + +to_atom(B) when is_binary(B) -> + binary_to_atom(B); +to_atom(A) when is_atom(A) -> + A. \ No newline at end of file diff --git a/apps/emqx_authn/src/emqx_authn_app.erl b/apps/emqx_authn/src/emqx_authn_app.erl index b7f409bc9..016decdd2 100644 --- a/apps/emqx_authn/src/emqx_authn_app.erl +++ b/apps/emqx_authn/src/emqx_authn_app.erl @@ -17,7 +17,6 @@ -module(emqx_authn_app). -include("emqx_authn.hrl"). --include_lib("emqx/include/logger.hrl"). -behaviour(application). @@ -26,33 +25,45 @@ , stop/1 ]). +%%------------------------------------------------------------------------------ +%% APIs +%%------------------------------------------------------------------------------ + start(_StartType, _StartArgs) -> ok = ekka_rlog:wait_for_shards([?AUTH_SHARD], infinity), {ok, Sup} = emqx_authn_sup:start_link(), - emqx_config_handler:add_handler([authentication, authenticators], emqx_authn), - initialize(), + ok = add_providers(), + ok = initialize(), {ok, Sup}. stop(_State) -> + ok = remove_providers(), ok. +%%------------------------------------------------------------------------------ +%% Internal functions +%%------------------------------------------------------------------------------ + +add_providers() -> + _ = [?AUTHN:add_provider(AuthNType, Provider) || {AuthNType, Provider} <- providers()], ok. + +remove_providers() -> + _ = [?AUTHN:remove_provider(AuthNType) || {AuthNType, _} <- providers()], ok. + initialize() -> - AuthNConfig = emqx:get_config([authentication], #{enable => false, - authenticators => []}), - initialize(AuthNConfig). - -initialize(#{enable := Enable, authenticators := AuthenticatorsConfig}) -> - {ok, _} = emqx_authn:create_chain(#{id => ?CHAIN}), - initialize_authenticators(AuthenticatorsConfig), - Enable =:= true andalso emqx_authn:enable(), + ?AUTHN:initialize_authentication(?GLOBAL, emqx:get_raw_config([authentication], [])), + lists:foreach(fun({ListenerID, ListenerConfig}) -> + ?AUTHN:initialize_authentication(ListenerID, maps:get(authentication, ListenerConfig, [])) + end, emqx_listeners:list()), ok. -initialize_authenticators([]) -> - ok; -initialize_authenticators([#{name := Name} = AuthenticatorConfig | More]) -> - case emqx_authn:create_authenticator(?CHAIN, AuthenticatorConfig) of - {ok, _} -> - initialize_authenticators(More); - {error, Reason} -> - ?LOG(error, "Failed to create authenticator '~s': ~p", [Name, Reason]) - end. +providers() -> + [ {{'password-based', 'built-in-database'}, emqx_authn_mnesia} + , {{'password-based', mysql}, emqx_authn_mysql} + , {{'password-based', posgresql}, emqx_authn_pgsql} + , {{'password-based', mongodb}, emqx_authn_mongodb} + , {{'password-based', redis}, emqx_authn_redis} + , {{'password-based', 'http-server'}, emqx_authn_http} + , {jwt, emqx_authn_jwt} + , {{scram, 'built-in-database'}, emqx_enhanced_authn_scram_mnesia} + ]. \ No newline at end of file diff --git a/apps/emqx_authn/src/emqx_authn_schema.erl b/apps/emqx_authn/src/emqx_authn_schema.erl index de0de9fcc..23e412088 100644 --- a/apps/emqx_authn/src/emqx_authn_schema.erl +++ b/apps/emqx_authn/src/emqx_authn_schema.erl @@ -16,53 +16,15 @@ -module(emqx_authn_schema). --include("emqx_authn.hrl"). -include_lib("typerefl/include/types.hrl"). --behaviour(hocon_schema). - --export([ roots/0 - , fields/1 +-export([ common_fields/0 ]). --export([ authenticator_name/1 - ]). - -%% Export it for emqx_gateway_schema module --export([ authenticators/1 - ]). - -roots() -> [ "authentication" ]. - -fields("authentication") -> - [ {enable, fun enable/1} - , {authenticators, fun authenticators/1} +common_fields() -> + [ {enable, fun enable/1} ]. -authenticator_name(type) -> binary(); -authenticator_name(nullable) -> false; -authenticator_name(_) -> undefined. - enable(type) -> boolean(); -enable(default) -> false; +enable(default) -> true; enable(_) -> undefined. - -authenticators(type) -> - hoconsc:array({union, [ hoconsc:ref(emqx_authn_mnesia, config) - , hoconsc:ref(emqx_authn_mysql, config) - , hoconsc:ref(emqx_authn_pgsql, config) - , hoconsc:ref(emqx_authn_mongodb, standalone) - , hoconsc:ref(emqx_authn_mongodb, 'replica-set') - , hoconsc:ref(emqx_authn_mongodb, 'sharded-cluster') - , hoconsc:ref(emqx_authn_redis, standalone) - , hoconsc:ref(emqx_authn_redis, cluster) - , hoconsc:ref(emqx_authn_redis, sentinel) - , hoconsc:ref(emqx_authn_http, get) - , hoconsc:ref(emqx_authn_http, post) - , hoconsc:ref(emqx_authn_jwt, 'hmac-based') - , hoconsc:ref(emqx_authn_jwt, 'public-key') - , hoconsc:ref(emqx_authn_jwt, 'jwks') - , hoconsc:ref(emqx_enhanced_authn_scram_mnesia, config) - ]}); -authenticators(default) -> []; -authenticators(_) -> undefined. diff --git a/apps/emqx_authn/src/emqx_authn_sup.erl b/apps/emqx_authn/src/emqx_authn_sup.erl index 56fcf299a..dd672a7c7 100644 --- a/apps/emqx_authn/src/emqx_authn_sup.erl +++ b/apps/emqx_authn/src/emqx_authn_sup.erl @@ -26,11 +26,5 @@ start_link() -> supervisor:start_link({local, ?MODULE}, ?MODULE, []). init([]) -> - ChildSpecs = [ - #{id => emqx_authn, - start => {emqx_authn, start_link, []}, - restart => permanent, - type => worker, - modules => [emqx_authn]} - ], + ChildSpecs = [], {ok, {{one_for_one, 10, 10}, ChildSpecs}}. diff --git a/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl b/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl index 0ca281aa0..e0f37a50d 100644 --- a/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl +++ b/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl @@ -20,12 +20,15 @@ -include_lib("typerefl/include/types.hrl"). -behaviour(hocon_schema). +-behaviour(emqx_authentication). --export([ roots/0 +-export([ namespace/0 + , roots/0 , fields/1 ]). --export([ create/1 +-export([ refs/0 + , create/1 , update/2 , authenticate/2 , destroy/1 @@ -50,7 +53,7 @@ , stored_key , server_key , salt - , superuser + , is_superuser }). %%------------------------------------------------------------------------------ @@ -74,19 +77,16 @@ mnesia(copy) -> %% Hocon Schema %%------------------------------------------------------------------------------ +namespace() -> "authn:scram:builtin-db". + roots() -> [config]. fields(config) -> - [ {name, fun emqx_authn_schema:authenticator_name/1} - , {mechanism, {enum, [scram]}} - , {server_type, fun server_type/1} + [ {mechanism, {enum, [scram]}} + , {backend, {enum, ['built-in-database']}} , {algorithm, fun algorithm/1} , {iteration_count, fun iteration_count/1} - ]. - -server_type(type) -> hoconsc:enum(['built-in-database']); -server_type(default) -> 'built-in-database'; -server_type(_) -> undefined. + ] ++ emqx_authn_schema:common_fields(). algorithm(type) -> hoconsc:enum([sha256, sha512]); algorithm(default) -> sha256; @@ -100,6 +100,9 @@ iteration_count(_) -> undefined. %% APIs %%------------------------------------------------------------------------------ +refs() -> + [hoconsc:ref(?MODULE, config)]. + create(#{ algorithm := Algorithm , iteration_count := IterationCount , '_unique' := Unique @@ -144,9 +147,9 @@ add_user(#{user_id := UserID, fun() -> case mnesia:read(?TAB, {UserGroup, UserID}, write) of [] -> - Superuser = maps:get(superuser, UserInfo, false), - add_user(UserID, Password, Superuser, State), - {ok, #{user_id => UserID, superuser => Superuser}}; + IsSuperuser = maps:get(is_superuser, UserInfo, false), + add_user(UserID, Password, IsSuperuser, State), + {ok, #{user_id => UserID, is_superuser => IsSuperuser}}; [_] -> {error, already_exist} end @@ -170,8 +173,8 @@ update_user(UserID, User, case mnesia:read(?TAB, {UserGroup, UserID}, write) of [] -> {error, not_found}; - [#user_info{superuser = Superuser} = UserInfo] -> - UserInfo1 = UserInfo#user_info{superuser = maps:get(superuser, User, Superuser)}, + [#user_info{is_superuser = IsSuperuser} = UserInfo] -> + UserInfo1 = UserInfo#user_info{is_superuser = maps:get(is_superuser, User, IsSuperuser)}, UserInfo2 = case maps:get(password, User, undefined) of undefined -> UserInfo1; @@ -226,36 +229,36 @@ check_client_first_message(Bin, _Cache, #{iteration_count := IterationCount} = S {error, not_authorized} end. -check_client_final_message(Bin, #{superuser := Superuser} = Cache, #{algorithm := Alg}) -> +check_client_final_message(Bin, #{is_superuser := IsSuperuser} = Cache, #{algorithm := Alg}) -> case esasl_scram:check_client_final_message( Bin, Cache#{algorithm => Alg} ) of {ok, ServerFinalMessage} -> - {ok, #{superuser => Superuser}, ServerFinalMessage}; + {ok, #{is_superuser => IsSuperuser}, ServerFinalMessage}; {error, _Reason} -> {error, not_authorized} end. -add_user(UserID, Password, Superuser, State) -> +add_user(UserID, Password, IsSuperuser, State) -> {StoredKey, ServerKey, Salt} = esasl_scram:generate_authentication_info(Password, State), - UserInfo = #user_info{user_id = UserID, - stored_key = StoredKey, - server_key = ServerKey, - salt = Salt, - superuser = Superuser}, + UserInfo = #user_info{user_id = UserID, + stored_key = StoredKey, + server_key = ServerKey, + salt = Salt, + is_superuser = IsSuperuser}, mnesia:write(?TAB, UserInfo, write). retrieve(UserID, #{user_group := UserGroup}) -> case mnesia:dirty_read(?TAB, {UserGroup, UserID}) of - [#user_info{stored_key = StoredKey, - server_key = ServerKey, - salt = Salt, - superuser = Superuser}] -> + [#user_info{stored_key = StoredKey, + server_key = ServerKey, + salt = Salt, + is_superuser = IsSuperuser}] -> {ok, #{stored_key => StoredKey, server_key => ServerKey, salt => Salt, - superuser => Superuser}}; + is_superuser => IsSuperuser}}; [] -> {error, not_found} end. @@ -270,5 +273,5 @@ trans(Fun, Args) -> {aborted, Reason} -> {error, Reason} end. -serialize_user_info(#user_info{user_id = {_, UserID}, superuser = Superuser}) -> - #{user_id => UserID, superuser => Superuser}. +serialize_user_info(#user_info{user_id = {_, UserID}, is_superuser = IsSuperuser}) -> + #{user_id => UserID, is_superuser => IsSuperuser}. diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl index c5cbc0f02..daa7f8073 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl @@ -21,13 +21,16 @@ -include_lib("typerefl/include/types.hrl"). -behaviour(hocon_schema). +-behaviour(emqx_authentication). --export([ roots/0 +-export([ namespace/0 + , roots/0 , fields/1 , validations/0 ]). --export([ create/1 +-export([ refs/0 + , create/1 , update/2 , authenticate/2 , destroy/1 @@ -37,9 +40,11 @@ %% Hocon Schema %%------------------------------------------------------------------------------ +namespace() -> "authn:password-based:http-server". + roots() -> - [ {config, {union, [ hoconsc:t(get) - , hoconsc:t(post) + [ {config, {union, [ hoconsc:ref(?MODULE, get) + , hoconsc:ref(?MODULE, post) ]}} ]. @@ -56,15 +61,15 @@ fields(post) -> ] ++ common_fields(). common_fields() -> - [ {name, fun emqx_authn_schema:authenticator_name/1} - , {mechanism, {enum, ['password-based']}} - , {server_type, {enum, ['http-server']}} + [ {mechanism, {enum, ['password-based']}} + , {backend, {enum, ['http-server']}} , {url, fun url/1} - , {form_data, fun form_data/1} + , {body, fun body/1} , {request_timeout, fun request_timeout/1} - ] ++ maps:to_list(maps:without([ base_url - , pool_type], - maps:from_list(emqx_connector_http:fields(config)))). + ] ++ emqx_authn_schema:common_fields() + ++ maps:to_list(maps:without([ base_url + , pool_type], + maps:from_list(emqx_connector_http:fields(config)))). validations() -> [ {check_ssl_opts, fun check_ssl_opts/1} @@ -92,11 +97,10 @@ headers_no_content_type(converter) -> headers_no_content_type(default) -> default_headers_no_content_type(); headers_no_content_type(_) -> undefined. -%% TODO: Using map() -form_data(type) -> map(); -form_data(nullable) -> false; -form_data(validate) -> [fun check_form_data/1]; -form_data(_) -> undefined. +body(type) -> map(); +body(nullable) -> false; +body(validate) -> [fun check_body/1]; +body(_) -> undefined. request_timeout(type) -> non_neg_integer(); request_timeout(default) -> 5000; @@ -106,10 +110,15 @@ request_timeout(_) -> undefined. %% APIs %%------------------------------------------------------------------------------ +refs() -> + [ hoconsc:ref(?MODULE, get) + , hoconsc:ref(?MODULE, post) + ]. + create(#{ method := Method , url := URL , headers := Headers - , form_data := FormData + , body := Body , request_timeout := RequestTimeout , '_unique' := Unique } = Config) -> @@ -118,8 +127,8 @@ create(#{ method := Method State = #{ method => Method , path => Path , base_query => cow_qs:parse_qs(list_to_binary(Query)) - , headers => normalize_headers(Headers) - , form_data => maps:to_list(FormData) + , headers => maps:to_list(Headers) + , body => maps:to_list(Body) , request_timeout => RequestTimeout , '_unique' => Unique }, @@ -152,16 +161,16 @@ authenticate(Credential, #{'_unique' := Unique, try Request = generate_request(Credential, State), case emqx_resource:query(Unique, {Method, Request, RequestTimeout}) of - {ok, 204, _Headers} -> {ok, #{superuser => false}}; + {ok, 204, _Headers} -> {ok, #{is_superuser => false}}; {ok, 200, Headers, Body} -> ContentType = proplists:get_value(<<"content-type">>, Headers, <<"application/json">>), case safely_parse_body(ContentType, Body) of {ok, NBody} -> %% TODO: Return by user property - {ok, #{superuser => maps:get(<<"superuser">>, NBody, false), + {ok, #{is_superuser => maps:get(<<"is_superuser">>, NBody, false), user_property => NBody}}; {error, _Reason} -> - {ok, #{superuser => false}} + {ok, #{is_superuser => false}} end; {error, _Reason} -> ignore @@ -186,10 +195,10 @@ check_url(URL) -> {error, _} -> false end. -check_form_data(FormData) -> +check_body(Body) -> lists:any(fun({_, V}) -> not is_binary(V) - end, maps:to_list(FormData)). + end, maps:to_list(Body)). default_headers() -> maps:put(<<"content-type">>, @@ -205,7 +214,7 @@ default_headers_no_content_type() -> transform_header_name(Headers) -> maps:fold(fun(K0, V, Acc) -> - K = list_to_binary(string:to_lower(binary_to_list(K0))), + K = list_to_binary(string:to_lower(to_list(K0))), maps:put(K, V, Acc) end, #{}, Headers). @@ -229,24 +238,21 @@ parse_url(URL) -> URIMap end. -normalize_headers(Headers) -> - [{atom_to_binary(K), V} || {K, V} <- maps:to_list(Headers)]. - generate_request(Credential, #{method := Method, path := Path, base_query := BaseQuery, headers := Headers, - form_data := FormData0}) -> - FormData = replace_placeholders(FormData0, Credential), + body := Body0}) -> + Body = replace_placeholders(Body0, Credential), case Method of get -> - NPath = append_query(Path, BaseQuery ++ FormData), + NPath = append_query(Path, BaseQuery ++ Body), {NPath, Headers}; post -> NPath = append_query(Path, BaseQuery), ContentType = proplists:get_value(<<"content-type">>, Headers), - Body = serialize_body(ContentType, FormData), - {NPath, Headers, Body} + NBody = serialize_body(ContentType, Body), + {NPath, Headers, NBody} end. replace_placeholders(KVs, Credential) -> @@ -276,10 +282,10 @@ qs([], Acc) -> qs([{K, V} | More], Acc) -> qs(More, [["&", emqx_http_lib:uri_encode(K), "=", emqx_http_lib:uri_encode(V)] | Acc]). -serialize_body(<<"application/json">>, FormData) -> - emqx_json:encode(FormData); -serialize_body(<<"application/x-www-form-urlencoded">>, FormData) -> - qs(FormData). +serialize_body(<<"application/json">>, Body) -> + emqx_json:encode(Body); +serialize_body(<<"application/x-www-form-urlencoded">>, Body) -> + qs(Body). safely_parse_body(ContentType, Body) -> try parse_body(ContentType, Body) of @@ -295,3 +301,8 @@ parse_body(<<"application/x-www-form-urlencoded">>, Body) -> {ok, maps:from_list(cow_qs:parse_qs(Body))}; parse_body(ContentType, _) -> {error, {unsupported_content_type, ContentType}}. + +to_list(A) when is_atom(A) -> + atom_to_list(A); +to_list(B) when is_binary(B) -> + binary_to_list(B). diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl index bc26bf70e..774d75157 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl @@ -19,12 +19,15 @@ -include_lib("typerefl/include/types.hrl"). -behaviour(hocon_schema). +-behaviour(emqx_authentication). --export([ roots/0 +-export([ namespace/0 + , roots/0 , fields/1 ]). --export([ create/1 +-export([ refs/0 + , create/1 , update/2 , authenticate/2 , destroy/1 @@ -34,10 +37,12 @@ %% Hocon Schema %%------------------------------------------------------------------------------ +namespace() -> "authn:jwt". + roots() -> - [ {config, {union, [ hoconsc:t('hmac-based') - , hoconsc:t('public-key') - , hoconsc:t('jwks') + [ {config, {union, [ hoconsc:mk('hmac-based') + , hoconsc:mk('public-key') + , hoconsc:mk('jwks') ]}} ]. @@ -78,12 +83,11 @@ fields(ssl_disable) -> [ {enable, #{type => false}} ]. common_fields() -> - [ {name, fun emqx_authn_schema:authenticator_name/1} - , {mechanism, {enum, [jwt]}} + [ {mechanism, {enum, [jwt]}} , {verify_claims, fun verify_claims/1} - ]. + ] ++ emqx_authn_schema:common_fields(). -secret(type) -> string(); +secret(type) -> binary(); secret(_) -> undefined. secret_base64_encoded(type) -> boolean(); @@ -130,6 +134,12 @@ verify_claims(_) -> undefined. %% APIs %%------------------------------------------------------------------------------ +refs() -> + [ hoconsc:ref(?MODULE, 'hmac-based') + , hoconsc:ref(?MODULE, 'public-key') + , hoconsc:ref(?MODULE, 'jwks') + ]. + create(#{verify_claims := VerifyClaims} = Config) -> create2(Config#{verify_claims => handle_verify_claims(VerifyClaims)}). @@ -239,7 +249,7 @@ verify(JWS, [JWK | More], VerifyClaims) -> Claims = emqx_json:decode(Payload, [return_maps]), case verify_claims(Claims, VerifyClaims) of ok -> - {ok, #{superuser => maps:get(<<"superuser">>, Claims, false)}}; + {ok, #{is_superuser => maps:get(<<"is_superuser">>, Claims, false)}}; {error, Reason} -> {error, Reason} end; diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl index c525efbf1..b69d613f8 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl @@ -20,10 +20,15 @@ -include_lib("typerefl/include/types.hrl"). -behaviour(hocon_schema). +-behaviour(emqx_authentication). --export([ roots/0, fields/1 ]). +-export([ namespace/0 + , roots/0 + , fields/1 + ]). --export([ create/1 +-export([ refs/0 + , create/1 , update/2 , authenticate/2 , destroy/1 @@ -46,7 +51,7 @@ { user_id :: {user_group(), user_id()} , password_hash :: binary() , salt :: binary() - , superuser :: boolean() + , is_superuser :: boolean() }). -reflect_type([ user_id_type/0 ]). @@ -79,15 +84,16 @@ mnesia(copy) -> %% Hocon Schema %%------------------------------------------------------------------------------ +namespace() -> "authn:password-based:builtin-db". + roots() -> [config]. fields(config) -> - [ {name, fun emqx_authn_schema:authenticator_name/1} - , {mechanism, {enum, ['password-based']}} - , {server_type, {enum, ['built-in-database']}} + [ {mechanism, {enum, ['password-based']}} + , {backend, {enum, ['built-in-database']}} , {user_id_type, fun user_id_type/1} , {password_hash_algorithm, fun password_hash_algorithm/1} - ]; + ] ++ emqx_authn_schema:common_fields(); fields(bcrypt) -> [ {name, {enum, [bcrypt]}} @@ -102,7 +108,8 @@ user_id_type(type) -> user_id_type(); user_id_type(default) -> username; user_id_type(_) -> undefined. -password_hash_algorithm(type) -> {union, [hoconsc:ref(bcrypt), hoconsc:ref(other_algorithms)]}; +password_hash_algorithm(type) -> hoconsc:union([hoconsc:ref(?MODULE, bcrypt), + hoconsc:ref(?MODULE, other_algorithms)]); password_hash_algorithm(default) -> #{<<"name">> => sha256}; password_hash_algorithm(_) -> undefined. @@ -114,6 +121,9 @@ salt_rounds(_) -> undefined. %% APIs %%------------------------------------------------------------------------------ +refs() -> + [hoconsc:ref(?MODULE, config)]. + create(#{ user_id_type := Type , password_hash_algorithm := #{name := bcrypt, salt_rounds := SaltRounds} @@ -148,13 +158,13 @@ authenticate(#{password := Password} = Credential, case mnesia:dirty_read(?TAB, {UserGroup, UserID}) of [] -> ignore; - [#user_info{password_hash = PasswordHash, salt = Salt0, superuser = Superuser}] -> + [#user_info{password_hash = PasswordHash, salt = Salt0, is_superuser = IsSuperuser}] -> Salt = case Algorithm of bcrypt -> PasswordHash; _ -> Salt0 end, case PasswordHash =:= hash(Algorithm, Password, Salt) of - true -> {ok, #{superuser => Superuser}}; + true -> {ok, #{is_superuser => IsSuperuser}}; false -> {error, bad_username_or_password} end end. @@ -187,9 +197,9 @@ add_user(#{user_id := UserID, case mnesia:read(?TAB, {UserGroup, UserID}, write) of [] -> {PasswordHash, Salt} = hash(Password, State), - Superuser = maps:get(superuser, UserInfo, false), - insert_user(UserGroup, UserID, PasswordHash, Salt, Superuser), - {ok, #{user_id => UserID, superuser => Superuser}}; + IsSuperuser = maps:get(is_superuser, UserInfo, false), + insert_user(UserGroup, UserID, PasswordHash, Salt, IsSuperuser), + {ok, #{user_id => UserID, is_superuser => IsSuperuser}}; [_] -> {error, already_exist} end @@ -215,8 +225,8 @@ update_user(UserID, UserInfo, {error, not_found}; [#user_info{ password_hash = PasswordHash , salt = Salt - , superuser = Superuser}] -> - NSuperuser = maps:get(superuser, UserInfo, Superuser), + , is_superuser = IsSuperuser}] -> + NSuperuser = maps:get(is_superuser, UserInfo, IsSuperuser), {NPasswordHash, NSalt} = case maps:get(password, UserInfo, undefined) of undefined -> {PasswordHash, Salt}; @@ -224,7 +234,7 @@ update_user(UserID, UserInfo, hash(Password, State) end, insert_user(UserGroup, UserID, NPasswordHash, NSalt, NSuperuser), - {ok, #{user_id => UserID, superuser => NSuperuser}} + {ok, #{user_id => UserID, is_superuser => NSuperuser}} end end). @@ -280,8 +290,8 @@ import(UserGroup, [#{<<"user_id">> := UserID, <<"password_hash">> := PasswordHash} = UserInfo | More]) when is_binary(UserID) andalso is_binary(PasswordHash) -> Salt = maps:get(<<"salt">>, UserInfo, <<>>), - Superuser = maps:get(<<"superuser">>, UserInfo, false), - insert_user(UserGroup, UserID, PasswordHash, Salt, Superuser), + IsSuperuser = maps:get(<<"is_superuser">>, UserInfo, false), + insert_user(UserGroup, UserID, PasswordHash, Salt, IsSuperuser), import(UserGroup, More); import(_UserGroup, [_ | _More]) -> {error, bad_format}. @@ -295,8 +305,8 @@ import(UserGroup, File, Seq) -> {ok, #{user_id := UserID, password_hash := PasswordHash} = UserInfo} -> Salt = maps:get(salt, UserInfo, <<>>), - Superuser = maps:get(superuser, UserInfo, false), - insert_user(UserGroup, UserID, PasswordHash, Salt, Superuser), + IsSuperuser = maps:get(is_superuser, UserInfo, false), + insert_user(UserGroup, UserID, PasswordHash, Salt, IsSuperuser), import(UserGroup, File, Seq); {error, Reason} -> {error, Reason} @@ -331,10 +341,10 @@ get_user_info_by_seq([PasswordHash | More1], [<<"password_hash">> | More2], Acc) get_user_info_by_seq(More1, More2, Acc#{password_hash => PasswordHash}); get_user_info_by_seq([Salt | More1], [<<"salt">> | More2], Acc) -> get_user_info_by_seq(More1, More2, Acc#{salt => Salt}); -get_user_info_by_seq([<<"true">> | More1], [<<"superuser">> | More2], Acc) -> - get_user_info_by_seq(More1, More2, Acc#{superuser => true}); -get_user_info_by_seq([<<"false">> | More1], [<<"superuser">> | More2], Acc) -> - get_user_info_by_seq(More1, More2, Acc#{superuser => false}); +get_user_info_by_seq([<<"true">> | More1], [<<"is_superuser">> | More2], Acc) -> + get_user_info_by_seq(More1, More2, Acc#{is_superuser => true}); +get_user_info_by_seq([<<"false">> | More1], [<<"is_superuser">> | More2], Acc) -> + get_user_info_by_seq(More1, More2, Acc#{is_superuser => false}); get_user_info_by_seq(_, _, _) -> {error, bad_format}. @@ -358,11 +368,11 @@ hash(Password, #{password_hash_algorithm := Algorithm} = State) -> PasswordHash = hash(Algorithm, Password, Salt), {PasswordHash, Salt}. -insert_user(UserGroup, UserID, PasswordHash, Salt, Superuser) -> +insert_user(UserGroup, UserID, PasswordHash, Salt, IsSuperuser) -> UserInfo = #user_info{user_id = {UserGroup, UserID}, password_hash = PasswordHash, salt = Salt, - superuser = Superuser}, + is_superuser = IsSuperuser}, mnesia:write(?TAB, UserInfo, write). delete_user2(UserInfo) -> @@ -390,5 +400,5 @@ to_binary(B) when is_binary(B) -> to_binary(L) when is_list(L) -> iolist_to_binary(L). -serialize_user_info(#user_info{user_id = {_, UserID}, superuser = Superuser}) -> - #{user_id => UserID, superuser => Superuser}. +serialize_user_info(#user_info{user_id = {_, UserID}, is_superuser = IsSuperuser}) -> + #{user_id => UserID, is_superuser => IsSuperuser}. diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl index 11411b70f..9d77c673c 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl @@ -21,12 +21,15 @@ -include_lib("typerefl/include/types.hrl"). -behaviour(hocon_schema). +-behaviour(emqx_authentication). --export([ roots/0 +-export([ namespace/0 + , roots/0 , fields/1 ]). --export([ create/1 +-export([ refs/0 + , create/1 , update/2 , authenticate/2 , destroy/1 @@ -36,10 +39,12 @@ %% Hocon Schema %%------------------------------------------------------------------------------ +namespace() -> "authn:password-based:mongodb". + roots() -> - [ {config, {union, [ hoconsc:t(standalone) - , hoconsc:t('replica-set') - , hoconsc:t('sharded-cluster') + [ {config, {union, [ hoconsc:mk(standalone) + , hoconsc:mk('replica-set') + , hoconsc:mk('sharded-cluster') ]}} ]. @@ -53,16 +58,16 @@ fields('sharded-cluster') -> common_fields() ++ emqx_connector_mongo:fields(sharded). common_fields() -> - [ {name, fun emqx_authn_schema:authenticator_name/1} - , {mechanism, {enum, ['password-based']}} - , {server_type, {enum, [mongodb]}} + [ {mechanism, {enum, ['password-based']}} + , {backend, {enum, [mongodb]}} , {collection, fun collection/1} , {selector, fun selector/1} , {password_hash_field, fun password_hash_field/1} , {salt_field, fun salt_field/1} + , {is_superuser_field, fun is_superuser_field/1} , {password_hash_algorithm, fun password_hash_algorithm/1} , {salt_position, fun salt_position/1} - ]. + ] ++ emqx_authn_schema:common_fields(). collection(type) -> binary(); collection(nullable) -> false; @@ -80,6 +85,10 @@ salt_field(type) -> binary(); salt_field(nullable) -> true; salt_field(_) -> undefined. +is_superuser_field(type) -> binary(); +is_superuser_field(nullable) -> true; +is_superuser_field(_) -> undefined. + password_hash_algorithm(type) -> {enum, [plain, md5, sha, sha256, sha512, bcrypt]}; password_hash_algorithm(default) -> sha256; password_hash_algorithm(_) -> undefined. @@ -92,6 +101,12 @@ salt_position(_) -> undefined. %% APIs %%------------------------------------------------------------------------------ +refs() -> + [ hoconsc:ref(?MODULE, standalone) + , hoconsc:ref(?MODULE, 'replica-set') + , hoconsc:ref(?MODULE, 'sharded-cluster') + ]. + create(#{ selector := Selector , '_unique' := Unique } = Config) -> @@ -99,6 +114,7 @@ create(#{ selector := Selector State = maps:with([ collection , password_hash_field , salt_field + , is_superuser_field , password_hash_algorithm , salt_position , '_unique'], Config), @@ -139,7 +155,7 @@ authenticate(#{password := Password} = Credential, Doc -> case check_password(Password, Doc, State) of ok -> - {ok, #{superuser => superuser(Doc, State)}}; + {ok, #{is_superuser => is_superuser(Doc, State)}}; {error, {cannot_find_password_hash_field, PasswordHashField}} -> ?LOG(error, "['~s'] Can't find password hash field: ~s", [Unique, PasswordHashField]), {error, bad_username_or_password}; @@ -220,9 +236,9 @@ check_password(Password, end end. -superuser(Doc, #{superuser_field := SuperuserField}) -> - maps:get(SuperuserField, Doc, false); -superuser(_, _) -> +is_superuser(Doc, #{is_superuser_field := IsSuperuserField}) -> + maps:get(IsSuperuserField, Doc, false); +is_superuser(_, _) -> false. hash(Algorithm, Password, Salt, prefix) -> diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl index 3cafdb94e..991bb6aee 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl @@ -21,12 +21,15 @@ -include_lib("typerefl/include/types.hrl"). -behaviour(hocon_schema). +-behaviour(emqx_authentication). --export([ roots/0 +-export([ namespace/0 + , roots/0 , fields/1 ]). --export([ create/1 +-export([ refs/0 + , create/1 , update/2 , authenticate/2 , destroy/1 @@ -36,17 +39,19 @@ %% Hocon Schema %%------------------------------------------------------------------------------ +namespace() -> "authn:password-based:mysql". + roots() -> [config]. fields(config) -> - [ {name, fun emqx_authn_schema:authenticator_name/1} - , {mechanism, {enum, ['password-based']}} - , {server_type, {enum, [mysql]}} + [ {mechanism, {enum, ['password-based']}} + , {backend, {enum, [mysql]}} , {password_hash_algorithm, fun password_hash_algorithm/1} , {salt_position, fun salt_position/1} , {query, fun query/1} , {query_timeout, fun query_timeout/1} - ] ++ emqx_connector_schema_lib:relational_db_fields() + ] ++ emqx_authn_schema:common_fields() + ++ emqx_connector_schema_lib:relational_db_fields() ++ emqx_connector_schema_lib:ssl_fields(). password_hash_algorithm(type) -> {enum, [plain, md5, sha, sha256, sha512, bcrypt]}; @@ -69,6 +74,9 @@ query_timeout(_) -> undefined. %% APIs %%------------------------------------------------------------------------------ +refs() -> + [hoconsc:ref(?MODULE, config)]. + create(#{ password_hash_algorithm := Algorithm , salt_position := SaltPosition , query := Query0 @@ -115,7 +123,7 @@ authenticate(#{password := Password} = Credential, Selected = maps:from_list(lists:zip(Columns, Rows)), case check_password(Password, Selected, State) of ok -> - {ok, #{superuser => maps:get(<<"superuser">>, Selected, false)}}; + {ok, #{is_superuser => maps:get(<<"is_superuser">>, Selected, false)}}; {error, Reason} -> {error, Reason} end; diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl index 5c21d3d6c..c497074de 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl @@ -22,10 +22,15 @@ -include_lib("typerefl/include/types.hrl"). -behaviour(hocon_schema). +-behaviour(emqx_authentication). --export([ roots/0, fields/1 ]). +-export([ namespace/0 + , roots/0 + , fields/1 + ]). --export([ create/1 +-export([ refs/0 + , create/1 , update/2 , authenticate/2 , destroy/1 @@ -35,16 +40,18 @@ %% Hocon Schema %%------------------------------------------------------------------------------ +namespace() -> "authn:password-based:postgresql". + roots() -> [config]. fields(config) -> - [ {name, fun emqx_authn_schema:authenticator_name/1} - , {mechanism, {enum, ['password-based']}} - , {server_type, {enum, [pgsql]}} + [ {mechanism, {enum, ['password-based']}} + , {backend, {enum, [postgresql]}} , {password_hash_algorithm, fun password_hash_algorithm/1} , {salt_position, {enum, [prefix, suffix]}} , {query, fun query/1} - ] ++ emqx_connector_schema_lib:relational_db_fields() + ] ++ emqx_authn_schema:common_fields() + ++ emqx_connector_schema_lib:relational_db_fields() ++ emqx_connector_schema_lib:ssl_fields(). password_hash_algorithm(type) -> {enum, [plain, md5, sha, sha256, sha512, bcrypt]}; @@ -59,6 +66,9 @@ query(_) -> undefined. %% APIs %%------------------------------------------------------------------------------ +refs() -> + [hoconsc:ref(?MODULE, config)]. + create(#{ query := Query0 , password_hash_algorithm := Algorithm , salt_position := SaltPosition @@ -103,7 +113,7 @@ authenticate(#{password := Password} = Credential, Selected = maps:from_list(lists:zip(NColumns, Rows)), case check_password(Password, Selected, State) of ok -> - {ok, #{superuser => maps:get(<<"superuser">>, Selected, false)}}; + {ok, #{is_superuser => maps:get(<<"is_superuser">>, Selected, false)}}; {error, Reason} -> {error, Reason} end; diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl index 1b090b007..949aeeaea 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl @@ -21,12 +21,15 @@ -include_lib("typerefl/include/types.hrl"). -behaviour(hocon_schema). +-behaviour(emqx_authentication). --export([ roots/0 +-export([ namespace/0 + , roots/0 , fields/1 ]). --export([ create/1 +-export([ refs/0 + , create/1 , update/2 , authenticate/2 , destroy/1 @@ -36,10 +39,12 @@ %% Hocon Schema %%------------------------------------------------------------------------------ +namespace() -> "authn:password-based:redis". + roots() -> - [ {config, {union, [ hoconsc:t(standalone) - , hoconsc:t(cluster) - , hoconsc:t(sentinel) + [ {config, {union, [ hoconsc:mk(standalone) + , hoconsc:mk(cluster) + , hoconsc:mk(sentinel) ]}} ]. @@ -53,13 +58,12 @@ fields(sentinel) -> common_fields() ++ emqx_connector_redis:fields(sentinel). common_fields() -> - [ {name, fun emqx_authn_schema:authenticator_name/1} - , {mechanism, {enum, ['password-based']}} - , {server_type, {enum, [redis]}} + [ {mechanism, {enum, ['password-based']}} + , {backend, {enum, [redis]}} , {query, fun query/1} , {password_hash_algorithm, fun password_hash_algorithm/1} , {salt_position, fun salt_position/1} - ]. + ] ++ emqx_authn_schema:common_fields(). query(type) -> string(); query(nullable) -> false; @@ -77,6 +81,12 @@ salt_position(_) -> undefined. %% APIs %%------------------------------------------------------------------------------ +refs() -> + [ hoconsc:ref(?MODULE, standalone) + , hoconsc:ref(?MODULE, cluster) + , hoconsc:ref(?MODULE, sentinel) + ]. + create(#{ query := Query , '_unique' := Unique } = Config) -> @@ -125,7 +135,7 @@ authenticate(#{password := Password} = Credential, Selected = merge(Fields, Values), case check_password(Password, Selected, State) of ok -> - {ok, #{superuser => maps:get("superuser", Selected, false)}}; + {ok, #{is_superuser => maps:get("is_superuser", Selected, false)}}; {error, Reason} -> {error, Reason} end; @@ -170,7 +180,7 @@ check_fields(["password_hash" | More], false) -> check_fields(More, true); check_fields(["salt" | More], HasPassHash) -> check_fields(More, HasPassHash); -check_fields(["superuser" | More], HasPassHash) -> +check_fields(["is_superuser" | More], HasPassHash) -> check_fields(More, HasPassHash); check_fields([Field | _], _) -> error({unsupported_field, Field}). diff --git a/apps/emqx_authn/test/data/user-credentials.csv b/apps/emqx_authn/test/data/user-credentials.csv index 0548308b7..cbadaefbc 100644 --- a/apps/emqx_authn/test/data/user-credentials.csv +++ b/apps/emqx_authn/test/data/user-credentials.csv @@ -1,3 +1,3 @@ -user_id,password_hash,salt,superuser +user_id,password_hash,salt,is_superuser myuser3,b6c743545a7817ae8c8f624371d5f5f0373234bb0ff36b8ffbf19bce0e06ab75,de1024f462fb83910fd13151bd4bd235,true myuser4,ee68c985a69208b6eda8c6c9b4c7c2d2b15ee2352cdd64a903171710a99182e8,ad773b5be9dd0613fe6c2f4d8c403139,false diff --git a/apps/emqx_authn/test/data/user-credentials.json b/apps/emqx_authn/test/data/user-credentials.json index e54501233..94375df22 100644 --- a/apps/emqx_authn/test/data/user-credentials.json +++ b/apps/emqx_authn/test/data/user-credentials.json @@ -3,12 +3,12 @@ "user_id":"myuser1", "password_hash":"c5e46903df45e5dc096dc74657610dbee8deaacae656df88a1788f1847390242", "salt": "e378187547bf2d6f0545a3f441aa4d8a", - "superuser": true + "is_superuser": true }, { "user_id":"myuser2", "password_hash":"f4d17f300b11e522fd33f497c11b126ef1ea5149c74d2220f9a16dc876d4567b", "salt": "6d3f9bd5b54d94b98adbcfe10b6d181f", - "superuser": false + "is_superuser": false } ] diff --git a/apps/emqx_authn/test/emqx_authn_SUITE.erl b/apps/emqx_authn/test/emqx_authn_SUITE.erl index eb7f0291a..74ec397cc 100644 --- a/apps/emqx_authn/test/emqx_authn_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_SUITE.erl @@ -19,97 +19,4 @@ -compile(export_all). -compile(nowarn_export_all). --include_lib("common_test/include/ct.hrl"). --include_lib("eunit/include/eunit.hrl"). - --include("emqx_authn.hrl"). - --define(AUTH, emqx_authn). - -all() -> - emqx_ct:all(?MODULE). - -init_per_suite(Config) -> - application:set_env(ekka, strict_mode, true), - emqx_ct_helpers:start_apps([emqx_authn]), - Config. - -end_per_suite(_) -> - emqx_ct_helpers:stop_apps([emqx_authn]), - ok. - -t_chain(_) -> - ?assertMatch({ok, #{id := ?CHAIN, authenticators := []}}, ?AUTH:lookup_chain(?CHAIN)), - - ChainID = <<"mychain">>, - Chain = #{id => ChainID}, - ?assertMatch({ok, #{id := ChainID, authenticators := []}}, ?AUTH:create_chain(Chain)), - ?assertEqual({error, {already_exists, {chain, ChainID}}}, ?AUTH:create_chain(Chain)), - ?assertMatch({ok, #{id := ChainID, authenticators := []}}, ?AUTH:lookup_chain(ChainID)), - ?assertEqual(ok, ?AUTH:delete_chain(ChainID)), - ?assertMatch({error, {not_found, {chain, ChainID}}}, ?AUTH:lookup_chain(ChainID)), - ok. - -t_authenticator(_) -> - AuthenticatorName1 = <<"myauthenticator1">>, - AuthenticatorConfig1 = #{name => AuthenticatorName1, - mechanism => 'password-based', - server_type => 'built-in-database', - user_id_type => username, - password_hash_algorithm => #{ - name => sha256 - }}, - {ok, #{name := AuthenticatorName1, id := ID1}} = ?AUTH:create_authenticator(?CHAIN, AuthenticatorConfig1), - ?assertMatch({ok, #{name := AuthenticatorName1}}, ?AUTH:lookup_authenticator(?CHAIN, ID1)), - ?assertMatch({ok, [#{name := AuthenticatorName1}]}, ?AUTH:list_authenticators(?CHAIN)), - ?assertEqual({error, name_has_be_used}, ?AUTH:create_authenticator(?CHAIN, AuthenticatorConfig1)), - - AuthenticatorConfig2 = #{name => AuthenticatorName1, - mechanism => jwt, - use_jwks => false, - algorithm => 'hmac-based', - secret => <<"abcdef">>, - secret_base64_encoded => false, - verify_claims => []}, - {ok, #{name := AuthenticatorName1, id := ID1}} = ?AUTH:update_authenticator(?CHAIN, ID1, AuthenticatorConfig2), - - ID2 = <<"random">>, - ?assertEqual({error, {not_found, {authenticator, ID2}}}, ?AUTH:update_authenticator(?CHAIN, ID2, AuthenticatorConfig2)), - ?assertEqual({error, name_has_be_used}, ?AUTH:update_or_create_authenticator(?CHAIN, ID2, AuthenticatorConfig2)), - - AuthenticatorName2 = <<"myauthenticator2">>, - AuthenticatorConfig3 = AuthenticatorConfig2#{name => AuthenticatorName2}, - {ok, #{name := AuthenticatorName2, id := ID2}} = ?AUTH:update_or_create_authenticator(?CHAIN, ID2, AuthenticatorConfig3), - ?assertMatch({ok, #{name := AuthenticatorName2}}, ?AUTH:lookup_authenticator(?CHAIN, ID2)), - {ok, #{name := AuthenticatorName2, id := ID2}} = ?AUTH:update_or_create_authenticator(?CHAIN, ID2, AuthenticatorConfig3#{secret := <<"fedcba">>}), - - ?assertMatch({ok, #{id := ?CHAIN, authenticators := [#{name := AuthenticatorName1}, #{name := AuthenticatorName2}]}}, ?AUTH:lookup_chain(?CHAIN)), - ?assertMatch({ok, [#{name := AuthenticatorName1}, #{name := AuthenticatorName2}]}, ?AUTH:list_authenticators(?CHAIN)), - - ?assertEqual(ok, ?AUTH:move_authenticator(?CHAIN, ID2, top)), - ?assertMatch({ok, [#{name := AuthenticatorName2}, #{name := AuthenticatorName1}]}, ?AUTH:list_authenticators(?CHAIN)), - - ?assertEqual(ok, ?AUTH:move_authenticator(?CHAIN, ID2, bottom)), - ?assertMatch({ok, [#{name := AuthenticatorName1}, #{name := AuthenticatorName2}]}, ?AUTH:list_authenticators(?CHAIN)), - - ?assertEqual(ok, ?AUTH:move_authenticator(?CHAIN, ID2, {before, ID1})), - - ?assertMatch({ok, [#{name := AuthenticatorName2}, #{name := AuthenticatorName1}]}, ?AUTH:list_authenticators(?CHAIN)), - - ?assertEqual({error, {not_found, {authenticator, <<"nonexistent">>}}}, ?AUTH:move_authenticator(?CHAIN, ID2, {before, <<"nonexistent">>})), - - ?assertEqual(ok, ?AUTH:delete_authenticator(?CHAIN, ID1)), - ?assertEqual(ok, ?AUTH:delete_authenticator(?CHAIN, ID2)), - ?assertEqual({ok, []}, ?AUTH:list_authenticators(?CHAIN)), - ok. - -t_authenticate(_) -> - ClientInfo = #{zone => default, - listener => {tcp, default}, - username => <<"myuser">>, - password => <<"mypass">>}, - ?assertEqual({ok, #{superuser => false}}, emqx_access_control:authenticate(ClientInfo)), - ?assertEqual(false, emqx_authn:is_enabled()), - emqx_authn:enable(), - ?assertEqual(true, emqx_authn:is_enabled()), - ?assertEqual({error, not_authorized}, emqx_access_control:authenticate(ClientInfo)). +all() -> emqx_ct:all(?MODULE). \ No newline at end of file diff --git a/apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl b/apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl index ddb2bb209..16c04771d 100644 --- a/apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl @@ -19,140 +19,140 @@ -compile(export_all). -compile(nowarn_export_all). --include_lib("common_test/include/ct.hrl"). --include_lib("eunit/include/eunit.hrl"). +% -include_lib("common_test/include/ct.hrl"). +% -include_lib("eunit/include/eunit.hrl"). --include("emqx_authn.hrl"). +% -include("emqx_authn.hrl"). --define(AUTH, emqx_authn). +% -define(AUTH, emqx_authn). all() -> emqx_ct:all(?MODULE). -init_per_suite(Config) -> - emqx_ct_helpers:start_apps([emqx_authn]), - Config. +% init_per_suite(Config) -> +% emqx_ct_helpers:start_apps([emqx_authn]), +% Config. -end_per_suite(_) -> - emqx_ct_helpers:stop_apps([emqx_authn]), - ok. +% end_per_suite(_) -> +% emqx_ct_helpers:stop_apps([emqx_authn]), +% ok. -t_jwt_authenticator(_) -> - AuthenticatorName = <<"myauthenticator">>, - Config = #{name => AuthenticatorName, - mechanism => jwt, - use_jwks => false, - algorithm => 'hmac-based', - secret => <<"abcdef">>, - secret_base64_encoded => false, - verify_claims => []}, - {ok, #{name := AuthenticatorName, id := ID}} = ?AUTH:create_authenticator(?CHAIN, Config), +% t_jwt_authenticator(_) -> +% AuthenticatorName = <<"myauthenticator">>, +% Config = #{name => AuthenticatorName, +% mechanism => jwt, +% use_jwks => false, +% algorithm => 'hmac-based', +% secret => <<"abcdef">>, +% secret_base64_encoded => false, +% verify_claims => []}, +% {ok, #{name := AuthenticatorName, id := ID}} = ?AUTH:create_authenticator(?CHAIN, Config), - Payload = #{<<"username">> => <<"myuser">>}, - JWS = generate_jws('hmac-based', Payload, <<"abcdef">>), - ClientInfo = #{username => <<"myuser">>, - password => JWS}, - ?assertEqual({stop, {ok, #{superuser => false}}}, ?AUTH:authenticate(ClientInfo, ignored)), +% Payload = #{<<"username">> => <<"myuser">>}, +% JWS = generate_jws('hmac-based', Payload, <<"abcdef">>), +% ClientInfo = #{username => <<"myuser">>, +% password => JWS}, +% ?assertEqual({stop, {ok, #{is_superuser => false}}}, ?AUTH:authenticate(ClientInfo, ignored)), - Payload1 = #{<<"username">> => <<"myuser">>, <<"superuser">> => true}, - JWS1 = generate_jws('hmac-based', Payload1, <<"abcdef">>), - ClientInfo1 = #{username => <<"myuser">>, - password => JWS1}, - ?assertEqual({stop, {ok, #{superuser => true}}}, ?AUTH:authenticate(ClientInfo1, ignored)), +% Payload1 = #{<<"username">> => <<"myuser">>, <<"is_superuser">> => true}, +% JWS1 = generate_jws('hmac-based', Payload1, <<"abcdef">>), +% ClientInfo1 = #{username => <<"myuser">>, +% password => JWS1}, +% ?assertEqual({stop, {ok, #{is_superuser => true}}}, ?AUTH:authenticate(ClientInfo1, ignored)), - BadJWS = generate_jws('hmac-based', Payload, <<"bad_secret">>), - ClientInfo2 = ClientInfo#{password => BadJWS}, - ?assertEqual({stop, {error, not_authorized}}, ?AUTH:authenticate(ClientInfo2, ignored)), +% BadJWS = generate_jws('hmac-based', Payload, <<"bad_secret">>), +% ClientInfo2 = ClientInfo#{password => BadJWS}, +% ?assertEqual({stop, {error, not_authorized}}, ?AUTH:authenticate(ClientInfo2, ignored)), - %% secret_base64_encoded - Config2 = Config#{secret => base64:encode(<<"abcdef">>), - secret_base64_encoded => true}, - ?assertMatch({ok, _}, ?AUTH:update_authenticator(?CHAIN, ID, Config2)), - ?assertEqual({stop, {ok, #{superuser => false}}}, ?AUTH:authenticate(ClientInfo, ignored)), +% %% secret_base64_encoded +% Config2 = Config#{secret => base64:encode(<<"abcdef">>), +% secret_base64_encoded => true}, +% ?assertMatch({ok, _}, ?AUTH:update_authenticator(?CHAIN, ID, Config2)), +% ?assertEqual({stop, {ok, #{is_superuser => false}}}, ?AUTH:authenticate(ClientInfo, ignored)), - Config3 = Config#{verify_claims => [{<<"username">>, <<"${mqtt-username}">>}]}, - ?assertMatch({ok, _}, ?AUTH:update_authenticator(?CHAIN, ID, Config3)), - ?assertEqual({stop, {ok, #{superuser => false}}}, ?AUTH:authenticate(ClientInfo, ignored)), - ?assertEqual({stop, {error, bad_username_or_password}}, ?AUTH:authenticate(ClientInfo#{username => <<"otheruser">>}, ok)), +% Config3 = Config#{verify_claims => [{<<"username">>, <<"${mqtt-username}">>}]}, +% ?assertMatch({ok, _}, ?AUTH:update_authenticator(?CHAIN, ID, Config3)), +% ?assertEqual({stop, {ok, #{is_superuser => false}}}, ?AUTH:authenticate(ClientInfo, ignored)), +% ?assertEqual({stop, {error, bad_username_or_password}}, ?AUTH:authenticate(ClientInfo#{username => <<"otheruser">>}, ok)), - %% Expiration - Payload3 = #{ <<"username">> => <<"myuser">> - , <<"exp">> => erlang:system_time(second) - 60}, - JWS3 = generate_jws('hmac-based', Payload3, <<"abcdef">>), - ClientInfo3 = ClientInfo#{password => JWS3}, - ?assertEqual({stop, {error, bad_username_or_password}}, ?AUTH:authenticate(ClientInfo3, ignored)), +% %% Expiration +% Payload3 = #{ <<"username">> => <<"myuser">> +% , <<"exp">> => erlang:system_time(second) - 60}, +% JWS3 = generate_jws('hmac-based', Payload3, <<"abcdef">>), +% ClientInfo3 = ClientInfo#{password => JWS3}, +% ?assertEqual({stop, {error, bad_username_or_password}}, ?AUTH:authenticate(ClientInfo3, ignored)), - Payload4 = #{ <<"username">> => <<"myuser">> - , <<"exp">> => erlang:system_time(second) + 60}, - JWS4 = generate_jws('hmac-based', Payload4, <<"abcdef">>), - ClientInfo4 = ClientInfo#{password => JWS4}, - ?assertEqual({stop, {ok, #{superuser => false}}}, ?AUTH:authenticate(ClientInfo4, ignored)), +% Payload4 = #{ <<"username">> => <<"myuser">> +% , <<"exp">> => erlang:system_time(second) + 60}, +% JWS4 = generate_jws('hmac-based', Payload4, <<"abcdef">>), +% ClientInfo4 = ClientInfo#{password => JWS4}, +% ?assertEqual({stop, {ok, #{is_superuser => false}}}, ?AUTH:authenticate(ClientInfo4, ignored)), - %% Issued At - Payload5 = #{ <<"username">> => <<"myuser">> - , <<"iat">> => erlang:system_time(second) - 60}, - JWS5 = generate_jws('hmac-based', Payload5, <<"abcdef">>), - ClientInfo5 = ClientInfo#{password => JWS5}, - ?assertEqual({stop, {ok, #{superuser => false}}}, ?AUTH:authenticate(ClientInfo5, ignored)), +% %% Issued At +% Payload5 = #{ <<"username">> => <<"myuser">> +% , <<"iat">> => erlang:system_time(second) - 60}, +% JWS5 = generate_jws('hmac-based', Payload5, <<"abcdef">>), +% ClientInfo5 = ClientInfo#{password => JWS5}, +% ?assertEqual({stop, {ok, #{is_superuser => false}}}, ?AUTH:authenticate(ClientInfo5, ignored)), - Payload6 = #{ <<"username">> => <<"myuser">> - , <<"iat">> => erlang:system_time(second) + 60}, - JWS6 = generate_jws('hmac-based', Payload6, <<"abcdef">>), - ClientInfo6 = ClientInfo#{password => JWS6}, - ?assertEqual({stop, {error, bad_username_or_password}}, ?AUTH:authenticate(ClientInfo6, ignored)), +% Payload6 = #{ <<"username">> => <<"myuser">> +% , <<"iat">> => erlang:system_time(second) + 60}, +% JWS6 = generate_jws('hmac-based', Payload6, <<"abcdef">>), +% ClientInfo6 = ClientInfo#{password => JWS6}, +% ?assertEqual({stop, {error, bad_username_or_password}}, ?AUTH:authenticate(ClientInfo6, ignored)), - %% Not Before - Payload7 = #{ <<"username">> => <<"myuser">> - , <<"nbf">> => erlang:system_time(second) - 60}, - JWS7 = generate_jws('hmac-based', Payload7, <<"abcdef">>), - ClientInfo7 = ClientInfo#{password => JWS7}, - ?assertEqual({stop, {ok, #{superuser => false}}}, ?AUTH:authenticate(ClientInfo7, ignored)), +% %% Not Before +% Payload7 = #{ <<"username">> => <<"myuser">> +% , <<"nbf">> => erlang:system_time(second) - 60}, +% JWS7 = generate_jws('hmac-based', Payload7, <<"abcdef">>), +% ClientInfo7 = ClientInfo#{password => JWS7}, +% ?assertEqual({stop, {ok, #{is_superuser => false}}}, ?AUTH:authenticate(ClientInfo7, ignored)), - Payload8 = #{ <<"username">> => <<"myuser">> - , <<"nbf">> => erlang:system_time(second) + 60}, - JWS8 = generate_jws('hmac-based', Payload8, <<"abcdef">>), - ClientInfo8 = ClientInfo#{password => JWS8}, - ?assertEqual({stop, {error, bad_username_or_password}}, ?AUTH:authenticate(ClientInfo8, ignored)), +% Payload8 = #{ <<"username">> => <<"myuser">> +% , <<"nbf">> => erlang:system_time(second) + 60}, +% JWS8 = generate_jws('hmac-based', Payload8, <<"abcdef">>), +% ClientInfo8 = ClientInfo#{password => JWS8}, +% ?assertEqual({stop, {error, bad_username_or_password}}, ?AUTH:authenticate(ClientInfo8, ignored)), - ?assertEqual(ok, ?AUTH:delete_authenticator(?CHAIN, ID)), - ok. +% ?assertEqual(ok, ?AUTH:delete_authenticator(?CHAIN, ID)), +% ok. -t_jwt_authenticator2(_) -> - Dir = code:lib_dir(emqx_authn, test), - PublicKey = list_to_binary(filename:join([Dir, "data/public_key.pem"])), - PrivateKey = list_to_binary(filename:join([Dir, "data/private_key.pem"])), - AuthenticatorName = <<"myauthenticator">>, - Config = #{name => AuthenticatorName, - mechanism => jwt, - use_jwks => false, - algorithm => 'public-key', - certificate => PublicKey, - verify_claims => []}, - {ok, #{name := AuthenticatorName, id := ID}} = ?AUTH:create_authenticator(?CHAIN, Config), +% t_jwt_authenticator2(_) -> +% Dir = code:lib_dir(emqx_authn, test), +% PublicKey = list_to_binary(filename:join([Dir, "data/public_key.pem"])), +% PrivateKey = list_to_binary(filename:join([Dir, "data/private_key.pem"])), +% AuthenticatorName = <<"myauthenticator">>, +% Config = #{name => AuthenticatorName, +% mechanism => jwt, +% use_jwks => false, +% algorithm => 'public-key', +% certificate => PublicKey, +% verify_claims => []}, +% {ok, #{name := AuthenticatorName, id := ID}} = ?AUTH:create_authenticator(?CHAIN, Config), - Payload = #{<<"username">> => <<"myuser">>}, - JWS = generate_jws('public-key', Payload, PrivateKey), - ClientInfo = #{username => <<"myuser">>, - password => JWS}, - ?assertEqual({stop, {ok, #{superuser => false}}}, ?AUTH:authenticate(ClientInfo, ignored)), - ?assertEqual({stop, {error, not_authorized}}, ?AUTH:authenticate(ClientInfo#{password => <<"badpassword">>}, ignored)), +% Payload = #{<<"username">> => <<"myuser">>}, +% JWS = generate_jws('public-key', Payload, PrivateKey), +% ClientInfo = #{username => <<"myuser">>, +% password => JWS}, +% ?assertEqual({stop, {ok, #{is_superuser => false}}}, ?AUTH:authenticate(ClientInfo, ignored)), +% ?assertEqual({stop, {error, not_authorized}}, ?AUTH:authenticate(ClientInfo#{password => <<"badpassword">>}, ignored)), - ?assertEqual(ok, ?AUTH:delete_authenticator(?CHAIN, ID)), - ok. +% ?assertEqual(ok, ?AUTH:delete_authenticator(?CHAIN, ID)), +% ok. -generate_jws('hmac-based', Payload, Secret) -> - JWK = jose_jwk:from_oct(Secret), - Header = #{ <<"alg">> => <<"HS256">> - , <<"typ">> => <<"JWT">> - }, - Signed = jose_jwt:sign(JWK, Header, Payload), - {_, JWS} = jose_jws:compact(Signed), - JWS; -generate_jws('public-key', Payload, PrivateKey) -> - JWK = jose_jwk:from_pem_file(PrivateKey), - Header = #{ <<"alg">> => <<"RS256">> - , <<"typ">> => <<"JWT">> - }, - Signed = jose_jwt:sign(JWK, Header, Payload), - {_, JWS} = jose_jws:compact(Signed), - JWS. +% generate_jws('hmac-based', Payload, Secret) -> +% JWK = jose_jwk:from_oct(Secret), +% Header = #{ <<"alg">> => <<"HS256">> +% , <<"typ">> => <<"JWT">> +% }, +% Signed = jose_jwt:sign(JWK, Header, Payload), +% {_, JWS} = jose_jws:compact(Signed), +% JWS; +% generate_jws('public-key', Payload, PrivateKey) -> +% JWK = jose_jwk:from_pem_file(PrivateKey), +% Header = #{ <<"alg">> => <<"RS256">> +% , <<"typ">> => <<"JWT">> +% }, +% Signed = jose_jwt:sign(JWK, Header, Payload), +% {_, JWS} = jose_jws:compact(Signed), +% JWS. diff --git a/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl b/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl index d6425a89c..959cf0323 100644 --- a/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl @@ -19,146 +19,146 @@ -compile(export_all). -compile(nowarn_export_all). --include_lib("common_test/include/ct.hrl"). --include_lib("eunit/include/eunit.hrl"). +% -include_lib("common_test/include/ct.hrl"). +% -include_lib("eunit/include/eunit.hrl"). --include("emqx_authn.hrl"). +% -include("emqx_authn.hrl"). --define(AUTH, emqx_authn). +% -define(AUTH, emqx_authn). all() -> emqx_ct:all(?MODULE). -init_per_suite(Config) -> - emqx_ct_helpers:start_apps([emqx_authn]), - Config. +% init_per_suite(Config) -> +% emqx_ct_helpers:start_apps([emqx_authn]), +% Config. -end_per_suite(_) -> - emqx_ct_helpers:stop_apps([emqx_authn]), - ok. +% end_per_suite(_) -> +% emqx_ct_helpers:stop_apps([emqx_authn]), +% ok. -t_mnesia_authenticator(_) -> - AuthenticatorName = <<"myauthenticator">>, - AuthenticatorConfig = #{name => AuthenticatorName, - mechanism => 'password-based', - server_type => 'built-in-database', - user_id_type => username, - password_hash_algorithm => #{ - name => sha256 - }}, - {ok, #{name := AuthenticatorName, id := ID}} = ?AUTH:create_authenticator(?CHAIN, AuthenticatorConfig), +% t_mnesia_authenticator(_) -> +% AuthenticatorName = <<"myauthenticator">>, +% AuthenticatorConfig = #{name => AuthenticatorName, +% mechanism => 'password-based', +% server_type => 'built-in-database', +% user_id_type => username, +% password_hash_algorithm => #{ +% name => sha256 +% }}, +% {ok, #{name := AuthenticatorName, id := ID}} = ?AUTH:create_authenticator(?CHAIN, AuthenticatorConfig), - UserInfo = #{user_id => <<"myuser">>, - password => <<"mypass">>}, - ?assertMatch({ok, #{user_id := <<"myuser">>}}, ?AUTH:add_user(?CHAIN, ID, UserInfo)), - ?assertMatch({ok, #{user_id := <<"myuser">>}}, ?AUTH:lookup_user(?CHAIN, ID, <<"myuser">>)), +% UserInfo = #{user_id => <<"myuser">>, +% password => <<"mypass">>}, +% ?assertMatch({ok, #{user_id := <<"myuser">>}}, ?AUTH:add_user(?CHAIN, ID, UserInfo)), +% ?assertMatch({ok, #{user_id := <<"myuser">>}}, ?AUTH:lookup_user(?CHAIN, ID, <<"myuser">>)), - ClientInfo = #{zone => external, - username => <<"myuser">>, - password => <<"mypass">>}, - ?assertEqual({stop, {ok, #{superuser => false}}}, ?AUTH:authenticate(ClientInfo, ignored)), - ?AUTH:enable(), - ?assertEqual({ok, #{superuser => false}}, emqx_access_control:authenticate(ClientInfo)), +% ClientInfo = #{zone => external, +% username => <<"myuser">>, +% password => <<"mypass">>}, +% ?assertEqual({stop, {ok, #{is_superuser => false}}}, ?AUTH:authenticate(ClientInfo, ignored)), +% ?AUTH:enable(), +% ?assertEqual({ok, #{is_superuser => false}}, emqx_access_control:authenticate(ClientInfo)), - ClientInfo2 = ClientInfo#{username => <<"baduser">>}, - ?assertEqual({stop, {error, not_authorized}}, ?AUTH:authenticate(ClientInfo2, ignored)), - ?assertEqual({error, not_authorized}, emqx_access_control:authenticate(ClientInfo2)), +% ClientInfo2 = ClientInfo#{username => <<"baduser">>}, +% ?assertEqual({stop, {error, not_authorized}}, ?AUTH:authenticate(ClientInfo2, ignored)), +% ?assertEqual({error, not_authorized}, emqx_access_control:authenticate(ClientInfo2)), - ClientInfo3 = ClientInfo#{password => <<"badpass">>}, - ?assertEqual({stop, {error, bad_username_or_password}}, ?AUTH:authenticate(ClientInfo3, ignored)), - ?assertEqual({error, bad_username_or_password}, emqx_access_control:authenticate(ClientInfo3)), +% ClientInfo3 = ClientInfo#{password => <<"badpass">>}, +% ?assertEqual({stop, {error, bad_username_or_password}}, ?AUTH:authenticate(ClientInfo3, ignored)), +% ?assertEqual({error, bad_username_or_password}, emqx_access_control:authenticate(ClientInfo3)), - UserInfo2 = UserInfo#{password => <<"mypass2">>}, - ?assertMatch({ok, #{user_id := <<"myuser">>}}, ?AUTH:update_user(?CHAIN, ID, <<"myuser">>, UserInfo2)), - ClientInfo4 = ClientInfo#{password => <<"mypass2">>}, - ?assertEqual({stop, {ok, #{superuser => false}}}, ?AUTH:authenticate(ClientInfo4, ignored)), +% UserInfo2 = UserInfo#{password => <<"mypass2">>}, +% ?assertMatch({ok, #{user_id := <<"myuser">>}}, ?AUTH:update_user(?CHAIN, ID, <<"myuser">>, UserInfo2)), +% ClientInfo4 = ClientInfo#{password => <<"mypass2">>}, +% ?assertEqual({stop, {ok, #{is_superuser => false}}}, ?AUTH:authenticate(ClientInfo4, ignored)), - ?assertMatch({ok, #{user_id := <<"myuser">>}}, ?AUTH:update_user(?CHAIN, ID, <<"myuser">>, #{superuser => true})), - ?assertEqual({stop, {ok, #{superuser => true}}}, ?AUTH:authenticate(ClientInfo4, ignored)), +% ?assertMatch({ok, #{user_id := <<"myuser">>}}, ?AUTH:update_user(?CHAIN, ID, <<"myuser">>, #{is_superuser => true})), +% ?assertEqual({stop, {ok, #{is_superuser => true}}}, ?AUTH:authenticate(ClientInfo4, ignored)), - ?assertEqual(ok, ?AUTH:delete_user(?CHAIN, ID, <<"myuser">>)), - ?assertEqual({error, not_found}, ?AUTH:lookup_user(?CHAIN, ID, <<"myuser">>)), +% ?assertEqual(ok, ?AUTH:delete_user(?CHAIN, ID, <<"myuser">>)), +% ?assertEqual({error, not_found}, ?AUTH:lookup_user(?CHAIN, ID, <<"myuser">>)), - ?assertMatch({ok, #{user_id := <<"myuser">>}}, ?AUTH:add_user(?CHAIN, ID, UserInfo)), - ?assertMatch({ok, #{user_id := <<"myuser">>}}, ?AUTH:lookup_user(?CHAIN, ID, <<"myuser">>)), - ?assertEqual(ok, ?AUTH:delete_authenticator(?CHAIN, ID)), +% ?assertMatch({ok, #{user_id := <<"myuser">>}}, ?AUTH:add_user(?CHAIN, ID, UserInfo)), +% ?assertMatch({ok, #{user_id := <<"myuser">>}}, ?AUTH:lookup_user(?CHAIN, ID, <<"myuser">>)), +% ?assertEqual(ok, ?AUTH:delete_authenticator(?CHAIN, ID)), - {ok, #{name := AuthenticatorName, id := ID1}} = ?AUTH:create_authenticator(?CHAIN, AuthenticatorConfig), - ?assertMatch({error, not_found}, ?AUTH:lookup_user(?CHAIN, ID1, <<"myuser">>)), - ?assertEqual(ok, ?AUTH:delete_authenticator(?CHAIN, ID1)), - ok. +% {ok, #{name := AuthenticatorName, id := ID1}} = ?AUTH:create_authenticator(?CHAIN, AuthenticatorConfig), +% ?assertMatch({error, not_found}, ?AUTH:lookup_user(?CHAIN, ID1, <<"myuser">>)), +% ?assertEqual(ok, ?AUTH:delete_authenticator(?CHAIN, ID1)), +% ok. -t_import(_) -> - AuthenticatorName = <<"myauthenticator">>, - AuthenticatorConfig = #{name => AuthenticatorName, - mechanism => 'password-based', - server_type => 'built-in-database', - user_id_type => username, - password_hash_algorithm => #{ - name => sha256 - }}, - {ok, #{name := AuthenticatorName, id := ID}} = ?AUTH:create_authenticator(?CHAIN, AuthenticatorConfig), +% t_import(_) -> +% AuthenticatorName = <<"myauthenticator">>, +% AuthenticatorConfig = #{name => AuthenticatorName, +% mechanism => 'password-based', +% server_type => 'built-in-database', +% user_id_type => username, +% password_hash_algorithm => #{ +% name => sha256 +% }}, +% {ok, #{name := AuthenticatorName, id := ID}} = ?AUTH:create_authenticator(?CHAIN, AuthenticatorConfig), - Dir = code:lib_dir(emqx_authn, test), - ?assertEqual(ok, ?AUTH:import_users(?CHAIN, ID, filename:join([Dir, "data/user-credentials.json"]))), - ?assertEqual(ok, ?AUTH:import_users(?CHAIN, ID, filename:join([Dir, "data/user-credentials.csv"]))), - ?assertMatch({ok, #{user_id := <<"myuser1">>}}, ?AUTH:lookup_user(?CHAIN, ID, <<"myuser1">>)), - ?assertMatch({ok, #{user_id := <<"myuser3">>}}, ?AUTH:lookup_user(?CHAIN, ID, <<"myuser3">>)), +% Dir = code:lib_dir(emqx_authn, test), +% ?assertEqual(ok, ?AUTH:import_users(?CHAIN, ID, filename:join([Dir, "data/user-credentials.json"]))), +% ?assertEqual(ok, ?AUTH:import_users(?CHAIN, ID, filename:join([Dir, "data/user-credentials.csv"]))), +% ?assertMatch({ok, #{user_id := <<"myuser1">>}}, ?AUTH:lookup_user(?CHAIN, ID, <<"myuser1">>)), +% ?assertMatch({ok, #{user_id := <<"myuser3">>}}, ?AUTH:lookup_user(?CHAIN, ID, <<"myuser3">>)), - ClientInfo1 = #{username => <<"myuser1">>, - password => <<"mypassword1">>}, - ?assertEqual({stop, {ok, #{superuser => true}}}, ?AUTH:authenticate(ClientInfo1, ignored)), +% ClientInfo1 = #{username => <<"myuser1">>, +% password => <<"mypassword1">>}, +% ?assertEqual({stop, {ok, #{is_superuser => true}}}, ?AUTH:authenticate(ClientInfo1, ignored)), - ClientInfo2 = ClientInfo1#{username => <<"myuser2">>, - password => <<"mypassword2">>}, - ?assertEqual({stop, {ok, #{superuser => false}}}, ?AUTH:authenticate(ClientInfo2, ignored)), +% ClientInfo2 = ClientInfo1#{username => <<"myuser2">>, +% password => <<"mypassword2">>}, +% ?assertEqual({stop, {ok, #{is_superuser => false}}}, ?AUTH:authenticate(ClientInfo2, ignored)), - ClientInfo3 = ClientInfo1#{username => <<"myuser3">>, - password => <<"mypassword3">>}, - ?assertEqual({stop, {ok, #{superuser => true}}}, ?AUTH:authenticate(ClientInfo3, ignored)), +% ClientInfo3 = ClientInfo1#{username => <<"myuser3">>, +% password => <<"mypassword3">>}, +% ?assertEqual({stop, {ok, #{is_superuser => true}}}, ?AUTH:authenticate(ClientInfo3, ignored)), - ?assertEqual(ok, ?AUTH:delete_authenticator(?CHAIN, ID)), - ok. +% ?assertEqual(ok, ?AUTH:delete_authenticator(?CHAIN, ID)), +% ok. -t_multi_mnesia_authenticator(_) -> - AuthenticatorName1 = <<"myauthenticator1">>, - AuthenticatorConfig1 = #{name => AuthenticatorName1, - mechanism => 'password-based', - server_type => 'built-in-database', - user_id_type => username, - password_hash_algorithm => #{ - name => sha256 - }}, - AuthenticatorName2 = <<"myauthenticator2">>, - AuthenticatorConfig2 = #{name => AuthenticatorName2, - mechanism => 'password-based', - server_type => 'built-in-database', - user_id_type => clientid, - password_hash_algorithm => #{ - name => sha256 - }}, - {ok, #{name := AuthenticatorName1, id := ID1}} = ?AUTH:create_authenticator(?CHAIN, AuthenticatorConfig1), - {ok, #{name := AuthenticatorName2, id := ID2}} = ?AUTH:create_authenticator(?CHAIN, AuthenticatorConfig2), +% t_multi_mnesia_authenticator(_) -> +% AuthenticatorName1 = <<"myauthenticator1">>, +% AuthenticatorConfig1 = #{name => AuthenticatorName1, +% mechanism => 'password-based', +% server_type => 'built-in-database', +% user_id_type => username, +% password_hash_algorithm => #{ +% name => sha256 +% }}, +% AuthenticatorName2 = <<"myauthenticator2">>, +% AuthenticatorConfig2 = #{name => AuthenticatorName2, +% mechanism => 'password-based', +% server_type => 'built-in-database', +% user_id_type => clientid, +% password_hash_algorithm => #{ +% name => sha256 +% }}, +% {ok, #{name := AuthenticatorName1, id := ID1}} = ?AUTH:create_authenticator(?CHAIN, AuthenticatorConfig1), +% {ok, #{name := AuthenticatorName2, id := ID2}} = ?AUTH:create_authenticator(?CHAIN, AuthenticatorConfig2), - ?assertMatch({ok, #{user_id := <<"myuser">>}}, - ?AUTH:add_user(?CHAIN, ID1, - #{user_id => <<"myuser">>, - password => <<"mypass1">>})), - ?assertMatch({ok, #{user_id := <<"myclient">>}}, - ?AUTH:add_user(?CHAIN, ID2, - #{user_id => <<"myclient">>, - password => <<"mypass2">>})), +% ?assertMatch({ok, #{user_id := <<"myuser">>}}, +% ?AUTH:add_user(?CHAIN, ID1, +% #{user_id => <<"myuser">>, +% password => <<"mypass1">>})), +% ?assertMatch({ok, #{user_id := <<"myclient">>}}, +% ?AUTH:add_user(?CHAIN, ID2, +% #{user_id => <<"myclient">>, +% password => <<"mypass2">>})), - ClientInfo1 = #{username => <<"myuser">>, - clientid => <<"myclient">>, - password => <<"mypass1">>}, - ?assertEqual({stop, {ok, #{superuser => false}}}, ?AUTH:authenticate(ClientInfo1, ignored)), - ?assertEqual(ok, ?AUTH:move_authenticator(?CHAIN, ID2, top)), +% ClientInfo1 = #{username => <<"myuser">>, +% clientid => <<"myclient">>, +% password => <<"mypass1">>}, +% ?assertEqual({stop, {ok, #{is_superuser => false}}}, ?AUTH:authenticate(ClientInfo1, ignored)), +% ?assertEqual(ok, ?AUTH:move_authenticator(?CHAIN, ID2, top)), - ?assertEqual({stop, {error, bad_username_or_password}}, ?AUTH:authenticate(ClientInfo1, ignored)), - ClientInfo2 = ClientInfo1#{password => <<"mypass2">>}, - ?assertEqual({stop, {ok, #{superuser => false}}}, ?AUTH:authenticate(ClientInfo2, ignored)), +% ?assertEqual({stop, {error, bad_username_or_password}}, ?AUTH:authenticate(ClientInfo1, ignored)), +% ClientInfo2 = ClientInfo1#{password => <<"mypass2">>}, +% ?assertEqual({stop, {ok, #{is_superuser => false}}}, ?AUTH:authenticate(ClientInfo2, ignored)), - ?assertEqual(ok, ?AUTH:delete_authenticator(?CHAIN, ID1)), - ?assertEqual(ok, ?AUTH:delete_authenticator(?CHAIN, ID2)), - ok. +% ?assertEqual(ok, ?AUTH:delete_authenticator(?CHAIN, ID1)), +% ?assertEqual(ok, ?AUTH:delete_authenticator(?CHAIN, ID2)), +% ok. diff --git a/apps/emqx_authz/etc/authorization_rules.conf b/apps/emqx_authz/etc/acl.conf similarity index 100% rename from apps/emqx_authz/etc/authorization_rules.conf rename to apps/emqx_authz/etc/acl.conf diff --git a/apps/emqx_authz/etc/emqx_authz.conf b/apps/emqx_authz/etc/emqx_authz.conf index 8eadab38b..c2856f0b5 100644 --- a/apps/emqx_authz/etc/emqx_authz.conf +++ b/apps/emqx_authz/etc/emqx_authz.conf @@ -2,72 +2,62 @@ authorization { sources = [ # { # type: http - # config: { - # url: "https://emqx.com" - # headers: { - # Accept: "application/json" - # Content-Type: "application/json" - # } + # url: "https://emqx.com" + # headers: { + # Accept: "application/json" + # Content-Type: "application/json" # } # }, # { # type: mysql - # config: { - # server: "127.0.0.1:3306" - # database: mqtt - # pool_size: 1 - # username: root - # password: public - # auto_reconnect: true - # ssl: { - # enable: true - # cacertfile: "{{ platform_etc_dir }}/certs/cacert.pem" - # certfile: "{{ platform_etc_dir }}/certs/client-cert.pem" - # keyfile: "{{ platform_etc_dir }}/certs/client-key.pem" - # } + # server: "127.0.0.1:3306" + # database: mqtt + # pool_size: 1 + # username: root + # password: public + # auto_reconnect: true + # ssl: { + # enable: true + # cacertfile: "{{ platform_etc_dir }}/certs/cacert.pem" + # certfile: "{{ platform_etc_dir }}/certs/client-cert.pem" + # keyfile: "{{ platform_etc_dir }}/certs/client-key.pem" # } - # sql: "select ipaddress, username, clientid, action, permission, topic from mqtt_authz where ipaddr = '%a' or username = '%u' or clientid = '%c'" + # query: "select ipaddress, username, clientid, action, permission, topic from mqtt_authz where ipaddr = '%a' or username = '%u' or clientid = '%c'" # }, # { # type: pgsql - # config: { - # server: "127.0.0.1:5432" - # database: mqtt - # pool_size: 1 - # username: root - # password: public - # auto_reconnect: true - # ssl: {enable: false} - # } - # sql: "select ipaddress, username, clientid, action, permission, topic from mqtt_authz where ipaddr = '%a' or username = '%u' or username = '$all' or clientid = '%c'" + # server: "127.0.0.1:5432" + # database: mqtt + # pool_size: 1 + # username: root + # password: public + # auto_reconnect: true + # ssl: {enable: false} + # query: "select ipaddress, username, clientid, action, permission, topic from mqtt_authz where ipaddr = '%a' or username = '%u' or username = '$all' or clientid = '%c'" # }, # { # type: redis - # config: { - # server: "127.0.0.1:6379" - # database: 0 - # pool_size: 1 - # password: public - # auto_reconnect: true - # ssl: {enable: false} - # } + # server: "127.0.0.1:6379" + # database: 0 + # pool_size: 1 + # password: public + # auto_reconnect: true + # ssl: {enable: false} # cmd: "HGETALL mqtt_authz:%u" # }, # { # type: mongo - # config: { - # mongo_type: single - # server: "127.0.0.1:27017" - # pool_size: 1 - # database: mqtt - # ssl: {enable: false} - # } + # mongo_type: single + # server: "127.0.0.1:27017" + # pool_size: 1 + # database: mqtt + # ssl: {enable: false} # collection: mqtt_authz - # find: { "$or": [ { "username": "%u" }, { "clientid": "%c" } ] } + # selector: { "$or": [ { "username": "%u" }, { "clientid": "%c" } ] } # }, { type: file - path: "{{ platform_etc_dir }}/authorization_rules.conf" + path: "{{ platform_etc_dir }}/acl.conf" } ] } diff --git a/apps/emqx_authz/src/emqx_authz.erl b/apps/emqx_authz/src/emqx_authz.erl index 4a6d7033e..6fe2d7565 100644 --- a/apps/emqx_authz/src/emqx_authz.erl +++ b/apps/emqx_authz/src/emqx_authz.erl @@ -30,13 +30,16 @@ , lookup/0 , lookup/1 , move/2 + , move/3 , update/2 + , update/3 , authorize/5 ]). -export([post_config_update/4, pre_config_update/2]). -define(CONF_KEY_PATH, [authorization, sources]). +-define(SOURCE_TYPES, [file, http, mongo, mysql, pgsql, redis]). -spec(register_metrics() -> ok). register_metrics() -> @@ -45,7 +48,9 @@ register_metrics() -> init() -> ok = register_metrics(), emqx_config_handler:add_handler(?CONF_KEY_PATH, ?MODULE), - NSources = [init_source(Source) || Source <- emqx:get_config(?CONF_KEY_PATH, [])], + Sources = emqx:get_config(?CONF_KEY_PATH, []), + ok = check_dup_types(Sources), + NSources = [init_source(Source) || Source <- Sources], ok = emqx_hooks:add('client.authorize', {?MODULE, authorize, [NSources]}, -1). lookup() -> @@ -58,29 +63,39 @@ lookup(Type) -> error:Reason -> {error, Reason} end. -move(Type, #{<<"before">> := Before}) -> - emqx:update_config(?CONF_KEY_PATH, {move, atom(Type), #{<<"before">> => atom(Before)}}); -move(Type, #{<<"after">> := After}) -> - emqx:update_config(?CONF_KEY_PATH, {move, atom(Type), #{<<"after">> => atom(After)}}); -move(Type, Position) -> - emqx:update_config(?CONF_KEY_PATH, {move, atom(Type), Position}). +move(Type, Cmd) -> + move(Type, Cmd, #{}). + +move(Type, #{<<"before">> := Before}, Opts) -> + emqx:update_config(?CONF_KEY_PATH, {move, atom(Type), #{<<"before">> => atom(Before)}}, Opts); +move(Type, #{<<"after">> := After}, Opts) -> + emqx:update_config(?CONF_KEY_PATH, {move, atom(Type), #{<<"after">> => atom(After)}}, Opts); +move(Type, Position, Opts) -> + emqx:update_config(?CONF_KEY_PATH, {move, atom(Type), Position}, Opts). -update({replace_once, Type}, Sources) -> - emqx:update_config(?CONF_KEY_PATH, {{replace_once, atom(Type)}, Sources}); -update({delete_once, Type}, Sources) -> - emqx:update_config(?CONF_KEY_PATH, {{delete_once, atom(Type)}, Sources}); update(Cmd, Sources) -> - emqx:update_config(?CONF_KEY_PATH, {Cmd, Sources}). + update(Cmd, Sources, #{}). + +update({replace_once, Type}, Sources, Opts) -> + emqx:update_config(?CONF_KEY_PATH, {{replace_once, atom(Type)}, Sources}, Opts); +update({delete_once, Type}, Sources, Opts) -> + emqx:update_config(?CONF_KEY_PATH, {{delete_once, atom(Type)}, Sources}, Opts); +update(Cmd, Sources, Opts) -> + emqx:update_config(?CONF_KEY_PATH, {Cmd, Sources}, Opts). pre_config_update({move, Type, <<"top">>}, Conf) when is_list(Conf) -> {Index, _} = find_source_by_type(Type), {List1, List2} = lists:split(Index, Conf), - {ok, [lists:nth(Index, Conf)] ++ lists:droplast(List1) ++ List2}; + NConf = [lists:nth(Index, Conf)] ++ lists:droplast(List1) ++ List2, + ok = check_dup_types(NConf), + {ok, NConf}; pre_config_update({move, Type, <<"bottom">>}, Conf) when is_list(Conf) -> {Index, _} = find_source_by_type(Type), {List1, List2} = lists:split(Index, Conf), - {ok, lists:droplast(List1) ++ List2 ++ [lists:nth(Index, Conf)]}; + NConf = lists:droplast(List1) ++ List2 ++ [lists:nth(Index, Conf)], + ok = check_dup_types(NConf), + {ok, NConf}; pre_config_update({move, Type, #{<<"before">> := Before}}, Conf) when is_list(Conf) -> {Index1, _} = find_source_by_type(Type), @@ -89,9 +104,11 @@ pre_config_update({move, Type, #{<<"before">> := Before}}, Conf) when is_list(Co Conf2 = lists:nth(Index2, Conf), {List1, List2} = lists:split(Index2, Conf), - {ok, lists:delete(Conf1, lists:droplast(List1)) - ++ [Conf1] ++ [Conf2] - ++ lists:delete(Conf1, List2)}; + NConf = lists:delete(Conf1, lists:droplast(List1)) + ++ [Conf1] ++ [Conf2] + ++ lists:delete(Conf1, List2), + ok = check_dup_types(NConf), + {ok, NConf}; pre_config_update({move, Type, #{<<"after">> := After}}, Conf) when is_list(Conf) -> {Index1, _} = find_source_by_type(Type), @@ -99,21 +116,31 @@ pre_config_update({move, Type, #{<<"after">> := After}}, Conf) when is_list(Conf {Index2, _} = find_source_by_type(After), {List1, List2} = lists:split(Index2, Conf), - {ok, lists:delete(Conf1, List1) - ++ [Conf1] - ++ lists:delete(Conf1, List2)}; + NConf = lists:delete(Conf1, List1) + ++ [Conf1] + ++ lists:delete(Conf1, List2), + ok = check_dup_types(NConf), + {ok, NConf}; pre_config_update({head, Sources}, Conf) when is_list(Sources), is_list(Conf) -> + NConf = Sources ++ Conf, + ok = check_dup_types(NConf), {ok, Sources ++ Conf}; pre_config_update({tail, Sources}, Conf) when is_list(Sources), is_list(Conf) -> + NConf = Conf ++ Sources, + ok = check_dup_types(NConf), {ok, Conf ++ Sources}; pre_config_update({{replace_once, Type}, Source}, Conf) when is_map(Source), is_list(Conf) -> {Index, _} = find_source_by_type(Type), {List1, List2} = lists:split(Index, Conf), - {ok, lists:droplast(List1) ++ [Source] ++ List2}; + NConf = lists:droplast(List1) ++ [Source] ++ List2, + ok = check_dup_types(NConf), + {ok, NConf}; pre_config_update({{delete_once, Type}, _Source}, Conf) when is_list(Conf) -> {_, Source} = find_source_by_type(Type), - {ok, lists:delete(Source, Conf)}; + NConf = lists:delete(Source, Conf), + ok = check_dup_types(NConf), + {ok, NConf}; pre_config_update({_, Sources}, _Conf) when is_list(Sources)-> %% overwrite the entire config! {ok, Sources}. @@ -171,6 +198,7 @@ post_config_update({{replace_once, Type}, #{type := Type} = Source}, _NewSources {Index, OldSource} = find_source_by_type(Type, OldInitedSources), case maps:get(type, OldSource, undefined) of undefined -> ok; + file -> ok; _ -> #{annotations := #{id := Id}} = OldSource, ok = emqx_resource:remove(Id) @@ -204,6 +232,27 @@ post_config_update(_, NewSources, _OldConf, _AppEnvs) -> %% Initialize source %%-------------------------------------------------------------------- +check_dup_types(Sources) -> + check_dup_types(Sources, ?SOURCE_TYPES). +check_dup_types(_Sources, []) -> ok; +check_dup_types(Sources, [T0 | Tail]) -> + case lists:foldl(fun (#{type := T1}, AccIn) -> + case T0 =:= T1 of + true -> AccIn + 1; + false -> AccIn + end; + (#{<<"type">> := T1}, AccIn) -> + case T0 =:= atom(T1) of + true -> AccIn + 1; + false -> AccIn + end + end, 0, Sources) > 1 of + true -> + ?LOG(error, "The type is duplicated in the Authorization source"), + {error, authz_source_dup}; + false -> check_dup_types(Sources, Tail) + end. + init_source(#{enable := true, type := file, path := Path @@ -224,10 +273,10 @@ init_source(#{enable := true, Source#{annotations => #{rules => Rules}}; init_source(#{enable := true, type := http, - config := #{url := Url} = Config + url := Url } = Source) -> - NConfig = maps:merge(Config, #{base_url => maps:remove(query, Url)}), - case create_resource(Source#{config := NConfig}) of + NSource= maps:put(base_url, maps:remove(query, Url), Source), + case create_resource(NSource) of {error, Reason} -> error({load_config_error, Reason}); Id -> Source#{annotations => #{id => Id}} end; @@ -241,7 +290,7 @@ init_source(#{enable := true, end; init_source(#{enable := true, type := DB, - sql := SQL + query := SQL } = Source) when DB =:= mysql; DB =:= pgsql -> Mod = authz_module(DB), @@ -249,7 +298,7 @@ init_source(#{enable := true, {error, Reason} -> error({load_config_error, Reason}); Id -> Source#{annotations => #{id => Id, - sql => Mod:parse_query(SQL) + query => Mod:parse_query(SQL) } } end; @@ -302,14 +351,14 @@ do_authorize(Client, PubSub, Topic, %%-------------------------------------------------------------------- check_sources(RawSources) -> - {ok, Conf} = hocon:binary(jsx:encode(#{<<"authorization">> => #{<<"sources">> => RawSources}}), #{format => richmap}), - CheckConf = hocon_schema:check(emqx_authz_schema, Conf, #{atom_key => true}), - #{authorization:= #{sources := Sources}} = hocon_schema:richmap_to_map(CheckConf), + Schema = #{roots => emqx_authz_schema:fields("authorization"), fields => #{}}, + Conf = #{<<"sources">> => RawSources}, + #{sources := Sources} = hocon_schema:check_plain(Schema, Conf, #{atom_key => true}), Sources. find_source_by_type(Type) -> find_source_by_type(Type, lookup()). find_source_by_type(Type, Sources) -> find_source_by_type(Type, Sources, 1). -find_source_by_type(_, [], _N) -> error(not_found_rule); +find_source_by_type(_, [], _N) -> error(not_found_source); find_source_by_type(Type, [ Source = #{type := T} | Tail], N) -> case Type =:= T of true -> {N, Source}; @@ -325,16 +374,14 @@ gen_id(Type) -> iolist_to_binary([io_lib:format("~s_~s",[?APP, Type])]). create_resource(#{type := DB, - config := Config, - annotations := #{id := ResourceID}}) -> - case emqx_resource:update(ResourceID, connector_module(DB), Config, []) of + annotations := #{id := ResourceID}} = Source) -> + case emqx_resource:update(ResourceID, connector_module(DB), Source, []) of {ok, _} -> ResourceID; {error, Reason} -> {error, Reason} end; -create_resource(#{type := DB, - config := Config}) -> +create_resource(#{type := DB} = Source) -> ResourceID = gen_id(DB), - case emqx_resource:create(ResourceID, connector_module(DB), Config) of + case emqx_resource:create(ResourceID, connector_module(DB), Source) of {ok, already_created} -> ResourceID; {ok, _} -> ResourceID; {error, Reason} -> {error, Reason} diff --git a/apps/emqx_authz/src/emqx_authz_api_schema.erl b/apps/emqx_authz/src/emqx_authz_api_schema.erl index 4c17cd0b6..158e26eab 100644 --- a/apps/emqx_authz/src/emqx_authz_api_schema.erl +++ b/apps/emqx_authz/src/emqx_authz_api_schema.erl @@ -26,6 +26,9 @@ definitions() -> type => object, required => [status], properties => #{ + id => #{ + type => string + }, status => #{ type => string, example => <<"healthy">> @@ -38,12 +41,332 @@ definitions() -> ] }, Sources = #{ - oneOf => [ minirest:ref(<<"connector_redis">>) + oneOf => [ minirest:ref(<<"http">>) + , minirest:ref(<<"mongo_single">>) + , minirest:ref(<<"mongo_rs">>) + , minirest:ref(<<"mongo_sharded">>) + , minirest:ref(<<"mysql">>) + , minirest:ref(<<"pgsql">>) + , minirest:ref(<<"redis_single">>) + , minirest:ref(<<"redis_sentinel">>) + , minirest:ref(<<"redis_cluster">>) + , minirest:ref(<<"file">>) ] }, - ConnectorRedis= #{ + SSL = #{ + type => object, + required => [enable], + properties => #{ + enable => #{type => boolean, example => true}, + cacertfile => #{type => string}, + keyfile => #{type => string}, + certfile => #{type => string}, + verify => #{type => boolean, example => false} + } + }, + HTTP = #{ type => object, - required => [type, enable, config, cmd], + required => [ type + , enable + , method + , headers + , request_timeout + , connect_timeout + , max_retries + , retry_interval + , pool_type + , pool_size + , enable_pipelining + , ssl + ], + properties => #{ + type => #{ + type => string, + enum => [<<"http">>], + example => <<"http">> + }, + enable => #{ + type => boolean, + example => true + }, + url => #{ + type => string, + example => <<"https://emqx.com">> + }, + method => #{ + type => string, + enum => [<<"get">>, <<"post">>], + example => <<"get">> + }, + headers => #{type => object}, + body => #{type => object}, + connect_timeout => #{type => integer}, + max_retries => #{type => integer}, + retry_interval => #{type => integer}, + pool_type => #{ + type => string, + enum => [<<"random">>, <<"hash">>], + example => <<"random">> + }, + pool_size => #{type => integer}, + enable_pipelining => #{type => boolean}, + ssl => minirest:ref(<<"ssl">>) + } + }, + MongoSingle= #{ + type => object, + required => [ type + , enable + , collection + , selector + , mongo_type + , server + , pool_size + , username + , password + , auth_source + , database + , topology + , ssl + ], + properties => #{ + type => #{ + type => string, + enum => [<<"mongo">>], + example => <<"mongo">> + }, + enable => #{ + type => boolean, + example => true + }, + collection => #{type => string}, + selector => #{type => object}, + mongo_type => #{type => string, + enum => [<<"single">>], + example => <<"single">>}, + server => #{type => string, example => <<"127.0.0.1:27017">>}, + pool_size => #{type => integer}, + username => #{type => string}, + password => #{type => string}, + auth_source => #{type => string}, + database => #{type => string}, + topology => #{type => object, + properties => #{ + pool_size => #{type => integer}, + max_overflow => #{type => integer}, + overflow_ttl => #{type => integer}, + overflow_check_period => #{type => integer}, + local_threshold_ms => #{type => integer}, + connect_timeout_ms => #{type => integer}, + socket_timeout_ms => #{type => integer}, + server_selection_timeout_ms => #{type => integer}, + wait_queue_timeout_ms => #{type => integer}, + heartbeat_frequency_ms => #{type => integer}, + min_heartbeat_frequency_ms => #{type => integer} + } + }, + ssl => minirest:ref(<<"ssl">>) + } + }, + MongoRs= #{ + type => object, + required => [ type + , enable + , collection + , selector + , mongo_type + , servers + , replica_set_name + , pool_size + , username + , password + , auth_source + , database + , topology + , ssl + ], + properties => #{ + type => #{ + type => string, + enum => [<<"mongo">>], + example => <<"mongo">> + }, + enable => #{ + type => boolean, + example => true + }, + collection => #{type => string}, + selector => #{type => object}, + mongo_type => #{type => string, + enum => [<<"rs">>], + example => <<"rs">>}, + servers => #{type => array, + items => #{type => string,example => <<"127.0.0.1:27017">>}}, + replica_set_name => #{type => string}, + pool_size => #{type => integer}, + username => #{type => string}, + password => #{type => string}, + auth_source => #{type => string}, + database => #{type => string}, + topology => #{type => object, + properties => #{ + pool_size => #{type => integer}, + max_overflow => #{type => integer}, + overflow_ttl => #{type => integer}, + overflow_check_period => #{type => integer}, + local_threshold_ms => #{type => integer}, + connect_timeout_ms => #{type => integer}, + socket_timeout_ms => #{type => integer}, + server_selection_timeout_ms => #{type => integer}, + wait_queue_timeout_ms => #{type => integer}, + heartbeat_frequency_ms => #{type => integer}, + min_heartbeat_frequency_ms => #{type => integer} + } + }, + ssl => minirest:ref(<<"ssl">>) + } + }, + MongoSharded = #{ + type => object, + required => [ type + , enable + , collection + , selector + , mongo_type + , servers + , pool_size + , username + , password + , auth_source + , database + , topology + , ssl + ], + properties => #{ + type => #{ + type => string, + enum => [<<"mongo">>], + example => <<"mongo">> + }, + enable => #{ + type => boolean, + example => true + }, + collection => #{type => string}, + selector => #{type => object}, + mongo_type => #{type => string, + enum => [<<"sharded">>], + example => <<"sharded">>}, + servers => #{type => array, + items => #{type => string,example => <<"127.0.0.1:27017">>}}, + pool_size => #{type => integer}, + username => #{type => string}, + password => #{type => string}, + auth_source => #{type => string}, + database => #{type => string}, + topology => #{type => object, + properties => #{ + pool_size => #{type => integer}, + max_overflow => #{type => integer}, + overflow_ttl => #{type => integer}, + overflow_check_period => #{type => integer}, + local_threshold_ms => #{type => integer}, + connect_timeout_ms => #{type => integer}, + socket_timeout_ms => #{type => integer}, + server_selection_timeout_ms => #{type => integer}, + wait_queue_timeout_ms => #{type => integer}, + heartbeat_frequency_ms => #{type => integer}, + min_heartbeat_frequency_ms => #{type => integer} + } + }, + ssl => minirest:ref(<<"ssl">>) + } + }, + Mysql = #{ + type => object, + required => [ type + , enable + , query + , server + , database + , pool_size + , username + , password + , auto_reconnect + , ssl + ], + properties => #{ + type => #{ + type => string, + enum => [<<"mysql">>], + example => <<"mysql">> + }, + enable => #{ + type => boolean, + example => true + }, + query => #{type => string}, + server => #{type => string, + example => <<"127.0.0.1:3306">> + }, + database => #{type => string}, + pool_size => #{type => integer}, + username => #{type => string}, + password => #{type => string}, + auto_reconnect => #{type => boolean, + example => true + }, + ssl => minirest:ref(<<"ssl">>) + } + }, + Pgsql = #{ + type => object, + required => [ type + , enable + , query + , server + , database + , pool_size + , username + , password + , auto_reconnect + , ssl + ], + properties => #{ + type => #{ + type => string, + enum => [<<"pgsql">>], + example => <<"pgsql">> + }, + enable => #{ + type => boolean, + example => true + }, + query => #{type => string}, + server => #{type => string, + example => <<"127.0.0.1:5432">> + }, + database => #{type => string}, + pool_size => #{type => integer}, + username => #{type => string}, + password => #{type => string}, + auto_reconnect => #{type => boolean, + example => true + }, + ssl => minirest:ref(<<"ssl">>) + } + }, + RedisSingle = #{ + type => object, + required => [ type + , enable + , cmd + , server + , redis_type + , pool_size + , auto_reconnect + , ssl + ], properties => #{ type => #{ type => string, @@ -54,59 +377,133 @@ definitions() -> type => boolean, example => true }, - config => #{ - oneOf => [ #{type => object, - required => [server, redis_type, pool_size, auto_reconnect], - properties => #{ - server => #{type => string, example => <<"127.0.0.1:3306">>}, - redis_type => #{type => string, - enum => [<<"single">>], - example => <<"single">>}, - pool_size => #{type => integer}, - auto_reconnect => #{type => boolean, example => true}, - password => #{type => string}, - database => #{type => string, example => mqtt} - } - } - , #{type => object, - required => [servers, redis_type, sentinel, pool_size, auto_reconnect], - properties => #{ - servers => #{type => array, - items => #{type => string,example => <<"127.0.0.1:3306">>}}, - redis_type => #{type => string, - enum => [<<"sentinel">>], - example => <<"sentinel">>}, - sentinel => #{type => string}, - pool_size => #{type => integer}, - auto_reconnect => #{type => boolean, example => true}, - password => #{type => string}, - database => #{type => string, example => mqtt} - } - } - , #{type => object, - required => [servers, redis_type, pool_size, auto_reconnect], - properties => #{ - servers => #{type => array, - items => #{type => string, example => <<"127.0.0.1:3306">>}}, - redis_type => #{type => string, - enum => [<<"cluster">>], - example => <<"cluster">>}, - pool_size => #{type => integer}, - auto_reconnect => #{type => boolean, example => true}, - password => #{type => string}, - database => #{type => string, example => mqtt} - } - } - ], - type => object + cmd => #{ + type => string, + example => <<"HGETALL mqtt_authz">> + }, + server => #{type => string, example => <<"127.0.0.1:3306">>}, + redis_type => #{type => string, + enum => [<<"single">>], + example => <<"single">>}, + pool_size => #{type => integer}, + auto_reconnect => #{type => boolean, example => true}, + password => #{type => string}, + database => #{type => integer}, + ssl => minirest:ref(<<"ssl">>) + } + }, + RedisSentinel= #{ + type => object, + required => [ type + , enable + , cmd + , servers + , redis_type + , sentinel + , pool_size + , auto_reconnect + , ssl + ], + properties => #{ + type => #{ + type => string, + enum => [<<"redis">>], + example => <<"redis">> + }, + enable => #{ + type => boolean, + example => true }, cmd => #{ type => string, example => <<"HGETALL mqtt_authz">> + }, + servers => #{type => array, + items => #{type => string,example => <<"127.0.0.1:3306">>}}, + redis_type => #{type => string, + enum => [<<"sentinel">>], + example => <<"sentinel">>}, + sentinel => #{type => string}, + pool_size => #{type => integer}, + auto_reconnect => #{type => boolean, example => true}, + password => #{type => string}, + database => #{type => integer}, + ssl => minirest:ref(<<"ssl">>) + } + }, + RedisCluster= #{ + type => object, + required => [ type + , enable + , cmd + , servers + , redis_type + , pool_size + , auto_reconnect + , ssl], + properties => #{ + type => #{ + type => string, + enum => [<<"redis">>], + example => <<"redis">> + }, + enable => #{ + type => boolean, + example => true + }, + cmd => #{ + type => string, + example => <<"HGETALL mqtt_authz">> + }, + servers => #{type => array, + items => #{type => string, example => <<"127.0.0.1:3306">>}}, + redis_type => #{type => string, + enum => [<<"cluster">>], + example => <<"cluster">>}, + pool_size => #{type => integer}, + auto_reconnect => #{type => boolean, example => true}, + password => #{type => string}, + database => #{type => integer}, + ssl => minirest:ref(<<"ssl">>) + } + }, + File = #{ + type => object, + required => [type, enable, rules], + properties => #{ + type => #{ + type => string, + enum => [<<"redis">>], + example => <<"redis">> + }, + enable => #{ + type => boolean, + example => true + }, + rules => #{ + type => array, + items => #{ + type => string, + example => <<"{allow,{username,\"^dashboard?\"},subscribe,[\"$SYS/#\"]}.\n{allow,{ipaddr,\"127.0.0.1\"},all,[\"$SYS/#\",\"#\"]}.">> + } + }, + path => #{ + type => string, + example => <<"/path/to/authorizaiton_rules.conf">> } } }, [ #{<<"returned_sources">> => RetruenedSources} , #{<<"sources">> => Sources} - , #{<<"connector_redis">> => ConnectorRedis} + , #{<<"ssl">> => SSL} + , #{<<"http">> => HTTP} + , #{<<"mongo_single">> => MongoSingle} + , #{<<"mongo_rs">> => MongoRs} + , #{<<"mongo_sharded">> => MongoSharded} + , #{<<"mysql">> => Mysql} + , #{<<"pgsql">> => Pgsql} + , #{<<"redis_single">> => RedisSingle} + , #{<<"redis_sentinel">> => RedisSentinel} + , #{<<"redis_cluster">> => RedisCluster} + , #{<<"file">> => File} ]. diff --git a/apps/emqx_authz/src/emqx_authz_api_sources.erl b/apps/emqx_authz/src/emqx_authz_api_sources.erl index 2ad5db1da..a3f198a30 100644 --- a/apps/emqx_authz/src/emqx_authz_api_sources.erl +++ b/apps/emqx_authz/src/emqx_authz_api_sources.erl @@ -19,21 +19,32 @@ -behavior(minirest_api). -include("emqx_authz.hrl"). +-include_lib("emqx/include/logger.hrl"). -define(EXAMPLE_REDIS, #{type=> redis, - config => #{server => <<"127.0.0.1:3306">>, - redis_type => single, - pool_size => 1, - auto_reconnect => true - }, + enable => true, + server => <<"127.0.0.1:3306">>, + redis_type => single, + pool_size => 1, + auto_reconnect => true, cmd => <<"HGETALL mqtt_authz">>}). +-define(EXAMPLE_FILE, + #{type=> file, + enable => true, + rules => <<"{allow,{username,\"^dashboard?\"},subscribe,[\"$SYS/#\"]}.\n{allow,{ipaddr,\"127.0.0.1\"},all,[\"$SYS/#\",\"#\"]}.">> + }). + -define(EXAMPLE_RETURNED_REDIS, maps:put(annotations, #{status => healthy}, ?EXAMPLE_REDIS) ). +-define(EXAMPLE_RETURNED_FILE, + maps:put(annotations, #{status => healthy}, ?EXAMPLE_FILE) + ). --define(EXAMPLE_RETURNED_RULES, - #{sources => [?EXAMPLE_RETURNED_REDIS +-define(EXAMPLE_RETURNED, + #{sources => [ ?EXAMPLE_RETURNED_REDIS + , ?EXAMPLE_RETURNED_FILE ] }). @@ -55,24 +66,6 @@ sources_api() -> Metadata = #{ get => #{ description => "List authorization sources", - parameters => [ - #{ - name => page, - in => query, - schema => #{ - type => integer - }, - required => false - }, - #{ - name => limit, - in => query, - schema => #{ - type => integer - }, - required => false - } - ], responses => #{ <<"200">> => #{ description => <<"OK">>, @@ -90,7 +83,7 @@ sources_api() -> examples => #{ sources => #{ summary => <<"Sources">>, - value => jsx:encode(?EXAMPLE_RETURNED_RULES) + value => jsx:encode(?EXAMPLE_RETURNED) } } } @@ -108,6 +101,10 @@ sources_api() -> redis => #{ summary => <<"Redis">>, value => jsx:encode(?EXAMPLE_REDIS) + }, + file => #{ + summary => <<"File">>, + value => jsx:encode(?EXAMPLE_FILE) } } } @@ -130,7 +127,11 @@ sources_api() -> examples => #{ redis => #{ summary => <<"Redis">>, - value => jsx:encode([?EXAMPLE_REDIS]) + value => jsx:encode(?EXAMPLE_REDIS) + }, + file => #{ + summary => <<"File">>, + value => jsx:encode(?EXAMPLE_FILE) } } } @@ -165,9 +166,13 @@ source_api() -> 'application/json' => #{ schema => minirest:ref(<<"returned_sources">>), examples => #{ - sources => #{ - summary => <<"Sources">>, + redis => #{ + summary => <<"Redis">>, value => jsx:encode(?EXAMPLE_RETURNED_REDIS) + }, + file => #{ + summary => <<"File">>, + value => jsx:encode(?EXAMPLE_RETURNED_FILE) } } } @@ -196,6 +201,10 @@ source_api() -> redis => #{ summary => <<"Redis">>, value => jsx:encode(?EXAMPLE_REDIS) + }, + file => #{ + summary => <<"File">>, + value => jsx:encode(?EXAMPLE_FILE) } } } @@ -287,53 +296,65 @@ move_source_api() -> }, {"/authorization/sources/:type/move", Metadata, move_source}. -sources(get, #{query_string := Query}) -> - Sources = lists:foldl(fun (#{type := _Type, enable := true, config := #{server := Server} = Config, annotations := #{id := Id}} = Source, AccIn) -> - NSource = case emqx_resource:health_check(Id) of - ok -> - Source#{config => Config#{server => emqx_connector_schema_lib:ip_port_to_string(Server)}, - annotations => #{id => Id, - status => healthy}}; - _ -> - Source#{config => Config#{server => emqx_connector_schema_lib:ip_port_to_string(Server)}, - annotations => #{id => Id, - status => unhealthy}} - end, - lists:append(AccIn, [NSource]); - (#{type := _Type, enable := true, annotations := #{id := Id}} = Source, AccIn) -> - NSource = case emqx_resource:health_check(Id) of - ok -> - Source#{annotations => #{status => healthy}}; - _ -> - Source#{annotations => #{status => unhealthy}} - end, - lists:append(AccIn, [NSource]); - (Source, AccIn) -> - lists:append(AccIn, [Source]) +sources(get, _) -> + Sources = lists:foldl(fun (#{type := file, enable := Enable, path := Path}, AccIn) -> + {ok, Rules} = file:consult(Path), + lists:append(AccIn, [#{type => file, + enable => Enable, + rules => [ iolist_to_binary(io_lib:format("~p.", [R])) || R <- Rules], + annotations => #{status => healthy} + }]); + (#{enable := false} = Source, AccIn) -> + lists:append(AccIn, [Source#{annotations => #{status => unhealthy}}]); + (#{type := _Type, annotations := #{id := Id}} = Source, AccIn) -> + NSource0 = case maps:get(server, Source, undefined) of + undefined -> Source; + Server -> + Source#{server => emqx_connector_schema_lib:ip_port_to_string(Server)} + end, + NSource1 = case maps:get(servers, Source, undefined) of + undefined -> NSource0; + Servers -> + NSource0#{servers => [emqx_connector_schema_lib:ip_port_to_string(Server) || Server <- Servers]} + end, + NSource2 = case emqx_resource:health_check(Id) of + ok -> + NSource1#{annotations => #{status => healthy}}; + _ -> + NSource1#{annotations => #{status => unhealthy}} + end, + lists:append(AccIn, [read_cert(NSource2)]); + (Source, AccIn) -> + lists:append(AccIn, [Source#{annotations => #{status => healthy}}]) end, [], emqx_authz:lookup()), - case maps:is_key(<<"page">>, Query) andalso maps:is_key(<<"limit">>, Query) of - true -> - Page = maps:get(<<"page">>, Query), - Limit = maps:get(<<"limit">>, Query), - Index = (binary_to_integer(Page) - 1) * binary_to_integer(Limit), - {_, Sources1} = lists:split(Index, Sources), - case binary_to_integer(Limit) < length(Sources1) of - true -> - {Sources2, _} = lists:split(binary_to_integer(Limit), Sources1), - {200, #{sources => Sources2}}; - false -> {200, #{sources => Sources1}} - end; - false -> {200, #{sources => Sources}} - end; -sources(post, #{body := RawConfig}) -> - case emqx_authz:update(head, [RawConfig]) of + {200, #{sources => Sources}}; +sources(post, #{body := #{<<"type">> := <<"file">>, <<"rules">> := Rules, <<"enable">> := Enable}}) when is_list(Rules) -> + {ok, Filename} = write_file(filename:join([emqx:get_config([node, data_dir]), "acl.conf"]), + erlang:list_to_bitstring([<> || Rule <- Rules]) + ), + case emqx_authz:update(head, [#{type => file, enable => Enable, path => Filename}]) of {ok, _} -> {204}; {error, Reason} -> {400, #{code => <<"BAD_REQUEST">>, messgae => atom_to_binary(Reason)}} end; -sources(put, #{body := RawConfig}) -> - case emqx_authz:update(replace, RawConfig) of +sources(post, #{body := Body}) when is_map(Body) -> + case emqx_authz:update(head, [write_cert(Body)]) of + {ok, _} -> {204}; + {error, Reason} -> + {400, #{code => <<"BAD_REQUEST">>, + messgae => atom_to_binary(Reason)}} + end; +sources(put, #{body := Body}) when is_list(Body) -> + NBody = [ begin + case Source of + #{<<"type">> := <<"file">>, <<"rules">> := Rules, <<"enable">> := Enable} -> + {ok, Filename} = write_file(filename:join([emqx:get_config([node, data_dir]), "acl.conf"]), Rules), + #{type => file, enable => Enable, path => Filename}; + _ -> write_cert(Source) + end + end || Source <- Body], + case emqx_authz:update(replace, NBody) of {ok, _} -> {204}; {error, Reason} -> {400, #{code => <<"BAD_REQUEST">>, @@ -343,29 +364,44 @@ sources(put, #{body := RawConfig}) -> source(get, #{bindings := #{type := Type}}) -> case emqx_authz:lookup(Type) of {error, Reason} -> {404, #{messgae => atom_to_binary(Reason)}}; - #{enable := false} = Source -> {200, Source}; - #{type := file} = Source -> {200, Source}; - #{config := #{server := Server, - annotations := #{id := Id} - } = Config} = Source -> - case emqx_resource:health_check(Id) of + #{type := file, enable := Enable, path := Path}-> + {ok, Rules} = file:consult(Path), + {200, #{type => file, + enable => Enable, + rules => [ iolist_to_binary(io_lib:format("~p.", [R])) || R <- Rules], + annotations => #{status => healthy} + } + }; + #{enable := false} = Source -> {200, Source#{annotations => #{status => unhealthy}}}; + #{annotations := #{id := Id}} = Source -> + NSource0 = case maps:get(server, Source, undefined) of + undefined -> Source; + Server -> + Source#{server => emqx_connector_schema_lib:ip_port_to_string(Server)} + end, + NSource1 = case maps:get(servers, Source, undefined) of + undefined -> NSource0; + Servers -> + NSource0#{servers => [emqx_connector_schema_lib:ip_port_to_string(Server) || Server <- Servers]} + end, + NSource2 = case emqx_resource:health_check(Id) of ok -> - {200, Source#{config => Config#{server => emqx_connector_schema_lib:ip_port_to_string(Server)}, - annotations => #{status => healthy}}}; + NSource1#{annotations => #{status => healthy}}; _ -> - {200, Source#{config => Config#{server => emqx_connector_schema_lib:ip_port_to_string(Server)}, - annotations => #{status => unhealthy}}} - end; - #{config := #{annotations := #{id := Id}}} = Source -> - case emqx_resource:health_check(Id) of - ok -> - {200, Source#{annotations => #{status => healthy}}}; - _ -> - {200, Source#{annotations => #{status => unhealthy}}} - end + NSource1#{annotations => #{status => unhealthy}} + end, + {200, read_cert(NSource2)} end; -source(put, #{bindings := #{type := Type}, body := RawConfig}) -> - case emqx_authz:update({replace_once, Type}, RawConfig) of +source(put, #{bindings := #{type := <<"file">>}, body := #{<<"type">> := <<"file">>, <<"rules">> := Rules, <<"enable">> := Enable}}) -> + {ok, Filename} = write_file(maps:get(path, emqx_authz:lookup(file), ""), Rules), + case emqx_authz:update({replace_once, file}, #{type => file, enable => Enable, path => Filename}) of + {ok, _} -> {204}; + {error, Reason} -> + {400, #{code => <<"BAD_REQUEST">>, + messgae => atom_to_binary(Reason)}} + end; +source(put, #{bindings := #{type := Type}, body := Body}) when is_map(Body) -> + case emqx_authz:update({replace_once, Type}, write_cert(Body)) of {ok, _} -> {204}; {error, not_found_source} -> {404, #{code => <<"NOT_FOUND">>, @@ -391,3 +427,62 @@ move_source(post, #{bindings := #{type := Type}, body := #{<<"position">> := Pos {400, #{code => <<"BAD_REQUEST">>, messgae => atom_to_binary(Reason)}} end. + +read_cert(#{ssl := #{enable := true} = SSL} = Source) -> + CaCert = case file:read_file(maps:get(cacertfile, SSL, "")) of + {ok, CaCert0} -> CaCert0; + _ -> "" + end, + Cert = case file:read_file(maps:get(certfile, SSL, "")) of + {ok, Cert0} -> Cert0; + _ -> "" + end, + Key = case file:read_file(maps:get(keyfile, SSL, "")) of + {ok, Key0} -> Key0; + _ -> "" + end, + Source#{ssl => SSL#{cacertfile => CaCert, + certfile => Cert, + keyfile => Key + } + }; +read_cert(Source) -> Source. + +write_cert(#{<<"ssl">> := #{<<"enable">> := true} = SSL} = Source) -> + CertPath = filename:join([emqx:get_config([node, data_dir]), "certs"]), + CaCert = case maps:is_key(<<"cacertfile">>, SSL) of + true -> + {ok, CaCertFile} = write_file(filename:join([CertPath, "cacert-" ++ emqx_plugin_libs_id:gen() ++".pem"]), + maps:get(<<"cacertfile">>, SSL)), + CaCertFile; + false -> "" + end, + Cert = case maps:is_key(<<"certfile">>, SSL) of + true -> + {ok, CertFile} = write_file(filename:join([CertPath, "cert-" ++ emqx_plugin_libs_id:gen() ++".pem"]), + maps:get(<<"certfile">>, SSL)), + CertFile; + false -> "" + end, + Key = case maps:is_key(<<"keyfile">>, SSL) of + true -> + {ok, KeyFile} = write_file(filename:join([CertPath, "key-" ++ emqx_plugin_libs_id:gen() ++".pem"]), + maps:get(<<"keyfile">>, SSL)), + KeyFile; + false -> "" + end, + Source#{<<"ssl">> => SSL#{<<"cacertfile">> => CaCert, + <<"certfile">> => Cert, + <<"keyfile">> => Key + } + }; +write_cert(Source) -> Source. + +write_file(Filename, Bytes) -> + ok = filelib:ensure_dir(Filename), + case file:write_file(Filename, Bytes) of + ok -> {ok, iolist_to_binary(Filename)}; + {error, Reason} -> + ?LOG(error, "Write File ~p Error: ~p", [Filename, Reason]), + error(Reason) + end. diff --git a/apps/emqx_authz/src/emqx_authz_http.erl b/apps/emqx_authz/src/emqx_authz_http.erl index c95d200e1..93aa634f3 100644 --- a/apps/emqx_authz/src/emqx_authz_http.erl +++ b/apps/emqx_authz/src/emqx_authz_http.erl @@ -35,12 +35,12 @@ description() -> authorize(Client, PubSub, Topic, #{type := http, - config := #{url := #{path := Path} = Url, - headers := Headers, - method := Method, - request_timeout := RequestTimeout} = Config, + url := #{path := Path} = Url, + headers := Headers, + method := Method, + request_timeout := RequestTimeout, annotations := #{id := ResourceID} - }) -> + } = Source) -> Request = case Method of get -> Query = maps:get(query, Url, ""), @@ -49,7 +49,7 @@ authorize(Client, PubSub, Topic, _ -> Body0 = serialize_body( maps:get('Accept', Headers, <<"application/json">>), - maps:get(body, Config, #{}) + maps:get(body, Source, #{}) ), Body1 = replvar(Body0, PubSub, Topic, Client), Path1 = replvar(Path, PubSub, Topic, Client), diff --git a/apps/emqx_authz/src/emqx_authz_mongo.erl b/apps/emqx_authz/src/emqx_authz_mongo.erl index 25a787b8f..68808c20b 100644 --- a/apps/emqx_authz/src/emqx_authz_mongo.erl +++ b/apps/emqx_authz/src/emqx_authz_mongo.erl @@ -35,10 +35,10 @@ description() -> authorize(Client, PubSub, Topic, #{collection := Collection, - find := Find, + selector := Selector, annotations := #{id := ResourceID} }) -> - case emqx_resource:query(ResourceID, {find, Collection, replvar(Find, Client), #{}}) of + case emqx_resource:query(ResourceID, {find, Collection, replvar(Selector, Client), #{}}) of {error, Reason} -> ?LOG(error, "[AuthZ] Query mongo error: ~p", [Reason]), nomatch; @@ -57,7 +57,7 @@ do_authorize(Client, PubSub, Topic, [Rule | Tail]) -> nomatch -> do_authorize(Client, PubSub, Topic, Tail) end. -replvar(Find, #{clientid := Clientid, +replvar(Selector, #{clientid := Clientid, username := Username, peerhost := IpAddress }) -> @@ -76,7 +76,7 @@ replvar(Find, #{clientid := Clientid, maps:put(K, V3, AccIn); _Fun(K, V, AccIn) -> maps:put(K, V, AccIn) end, - maps:fold(Fun, #{}, Find). + maps:fold(Fun, #{}, Selector). bin(A) when is_atom(A) -> atom_to_binary(A, utf8); bin(B) when is_binary(B) -> B; diff --git a/apps/emqx_authz/src/emqx_authz_mysql.erl b/apps/emqx_authz/src/emqx_authz_mysql.erl index d5550b2fb..ac8f04f32 100644 --- a/apps/emqx_authz/src/emqx_authz_mysql.erl +++ b/apps/emqx_authz/src/emqx_authz_mysql.erl @@ -47,10 +47,10 @@ parse_query(Sql) -> authorize(Client, PubSub, Topic, #{annotations := #{id := ResourceID, - sql := {SQL, Params} + query := {Query, Params} } }) -> - case emqx_resource:query(ResourceID, {sql, SQL, replvar(Params, Client)}) of + case emqx_resource:query(ResourceID, {sql, Query, replvar(Params, Client)}) of {ok, _Columns, []} -> nomatch; {ok, Columns, Rows} -> do_authorize(Client, PubSub, Topic, Columns, Rows); diff --git a/apps/emqx_authz/src/emqx_authz_pgsql.erl b/apps/emqx_authz/src/emqx_authz_pgsql.erl index d9555b85d..3e1f40fb2 100644 --- a/apps/emqx_authz/src/emqx_authz_pgsql.erl +++ b/apps/emqx_authz/src/emqx_authz_pgsql.erl @@ -51,10 +51,10 @@ parse_query(Sql) -> authorize(Client, PubSub, Topic, #{annotations := #{id := ResourceID, - sql := {SQL, Params} + query := {Query, Params} } }) -> - case emqx_resource:query(ResourceID, {sql, SQL, replvar(Params, Client)}) of + case emqx_resource:query(ResourceID, {sql, Query, replvar(Params, Client)}) of {ok, _Columns, []} -> nomatch; {ok, Columns, Rows} -> do_authorize(Client, PubSub, Topic, Columns, Rows); diff --git a/apps/emqx_authz/src/emqx_authz_schema.erl b/apps/emqx_authz/src/emqx_authz_schema.erl index 7fb60bae2..e17a55d0a 100644 --- a/apps/emqx_authz/src/emqx_authz_schema.erl +++ b/apps/emqx_authz/src/emqx_authz_schema.erl @@ -13,14 +13,32 @@ -type permission() :: allow | deny. -type url() :: emqx_http_lib:uri_map(). --export([ roots/0 +-export([ namespace/0 + , roots/0 , fields/1 ]). -roots() -> ["authorization"]. +namespace() -> authz. + +%% @doc authorization schema is not exported +%% but directly used by emqx_schema +roots() -> []. fields("authorization") -> - [ {sources, sources()} + [ {sources, #{type => union_array( + [ hoconsc:ref(?MODULE, file) + , hoconsc:ref(?MODULE, http_get) + , hoconsc:ref(?MODULE, http_post) + , hoconsc:ref(?MODULE, mongo_single) + , hoconsc:ref(?MODULE, mongo_rs) + , hoconsc:ref(?MODULE, mongo_sharded) + , hoconsc:ref(?MODULE, mysql) + , hoconsc:ref(?MODULE, pgsql) + , hoconsc:ref(?MODULE, redis_single) + , hoconsc:ref(?MODULE, redis_sentinel) + , hoconsc:ref(?MODULE, redis_cluster) + ])} + } ]; fields(file) -> [ {type, #{type => file}} @@ -34,17 +52,12 @@ fields(file) -> end }} ]; -fields(http) -> +fields(http_get) -> [ {type, #{type => http}} , {enable, #{type => boolean(), default => true}} - , {config, #{type => hoconsc:union([ hoconsc:ref(?MODULE, http_get) - , hoconsc:ref(?MODULE, http_post) - ])} - } - ]; -fields(http_get) -> - [ {url, #{type => url()}} + , {url, #{type => url()}} + , {method, #{type => get, default => get }} , {headers, #{type => map(), default => #{ <<"accept">> => <<"application/json">> , <<"cache-control">> => <<"no-cache">> @@ -53,7 +66,7 @@ fields(http_get) -> }, converter => fun (Headers0) -> Headers1 = maps:fold(fun(K0, V, AccIn) -> - K1 = iolist_to_binary(string:to_lower(binary_to_list(K0))), + K1 = iolist_to_binary(string:to_lower(to_list(K0))), maps:put(K1, V, AccIn) end, #{}, Headers0), maps:merge(#{ <<"accept">> => <<"application/json">> @@ -64,11 +77,15 @@ fields(http_get) -> end } } - , {method, #{type => get, default => get }} , {request_timeout, #{type => timeout(), default => 30000 }} ] ++ proplists:delete(base_url, emqx_connector_http:fields(config)); fields(http_post) -> - [ {url, #{type => url()}} + [ {type, #{type => http}} + , {enable, #{type => boolean(), + default => true}} + , {url, #{type => url()}} + , {method, #{type => post, + default => get}} , {headers, #{type => map(), default => #{ <<"accept">> => <<"application/json">> , <<"cache-control">> => <<"no-cache">> @@ -90,54 +107,42 @@ fields(http_post) -> end } } - , {method, #{type => hoconsc:enum([post, put]), - default => get}} + , {request_timeout, #{type => timeout(), default => 30000 }} , {body, #{type => map(), nullable => true } } ] ++ proplists:delete(base_url, emqx_connector_http:fields(config)); -fields(mongo) -> - connector_fields(mongo) ++ +fields(mongo_single) -> + connector_fields(mongo, single) ++ [ {collection, #{type => atom()}} - , {find, #{type => map()}} + , {selector, #{type => map()}} + ]; +fields(mongo_rs) -> + connector_fields(mongo, rs) ++ + [ {collection, #{type => atom()}} + , {selector, #{type => map()}} + ]; +fields(mongo_sharded) -> + connector_fields(mongo, sharded) ++ + [ {collection, #{type => atom()}} + , {selector, #{type => map()}} ]; -fields(redis) -> - connector_fields(redis) ++ - [ {cmd, query()} ]; fields(mysql) -> connector_fields(mysql) ++ - [ {sql, query()} ]; + [ {query, query()} ]; fields(pgsql) -> connector_fields(pgsql) ++ - [ {sql, query()} ]; -fields(username) -> - [{username, #{type => binary()}}]; -fields(clientid) -> - [{clientid, #{type => binary()}}]; -fields(ipaddress) -> - [{ipaddress, #{type => string()}}]; -fields(andlist) -> - [{'and', #{type => union_array( - [ hoconsc:ref(?MODULE, username) - , hoconsc:ref(?MODULE, clientid) - , hoconsc:ref(?MODULE, ipaddress) - ]) - } - } - ]; -fields(orlist) -> - [{'or', #{type => union_array( - [ hoconsc:ref(?MODULE, username) - , hoconsc:ref(?MODULE, clientid) - , hoconsc:ref(?MODULE, ipaddress) - ]) - } - } - ]; -fields(eq_topic) -> - [{eq, #{type => binary()}}]. - + [ {query, query()} ]; +fields(redis_single) -> + connector_fields(redis, single) ++ + [ {cmd, query()} ]; +fields(redis_sentinel) -> + connector_fields(redis, sentinel) ++ + [ {cmd, query()} ]; +fields(redis_cluster) -> + connector_fields(redis, cluster) ++ + [ {cmd, query()} ]. %%-------------------------------------------------------------------- %% Internal functions @@ -146,17 +151,6 @@ fields(eq_topic) -> union_array(Item) when is_list(Item) -> hoconsc:array(hoconsc:union(Item)). -sources() -> - #{type => union_array( - [ hoconsc:ref(?MODULE, file) - , hoconsc:ref(?MODULE, http) - , hoconsc:ref(?MODULE, mysql) - , hoconsc:ref(?MODULE, pgsql) - , hoconsc:ref(?MODULE, redis) - , hoconsc:ref(?MODULE, mongo) - ]) - }. - query() -> #{type => binary(), validator => fun(S) -> @@ -168,6 +162,8 @@ query() -> }. connector_fields(DB) -> + connector_fields(DB, config). +connector_fields(DB, Fields) -> Mod0 = io_lib:format("~s_~s",[emqx_connector, DB]), Mod = try list_to_existing_atom(Mod0) @@ -180,4 +176,9 @@ connector_fields(DB) -> [ {type, #{type => DB}} , {enable, #{type => boolean(), default => true}} - ] ++ Mod:roots(). + ] ++ Mod:fields(Fields). + +to_list(A) when is_atom(A) -> + atom_to_list(A); +to_list(B) when is_binary(B) -> + binary_to_list(B). \ No newline at end of file diff --git a/apps/emqx_authz/test/emqx_authz_SUITE.erl b/apps/emqx_authz/test/emqx_authz_SUITE.erl index cee83cd30..fe1f04bd2 100644 --- a/apps/emqx_authz/test/emqx_authz_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_SUITE.erl @@ -62,58 +62,58 @@ init_per_testcase(_, Config) -> -define(SOURCE1, #{<<"type">> => <<"http">>, <<"enable">> => true, - <<"config">> => #{ - <<"url">> => <<"https://fake.com:443/">>, - <<"headers">> => #{}, - <<"method">> => <<"get">>, - <<"request_timeout">> => 5000} + <<"url">> => <<"https://fake.com:443/">>, + <<"headers">> => #{}, + <<"method">> => <<"get">>, + <<"request_timeout">> => 5000 }). -define(SOURCE2, #{<<"type">> => <<"mongo">>, <<"enable">> => true, - <<"config">> => #{ - <<"mongo_type">> => <<"single">>, - <<"server">> => <<"127.0.0.1:27017">>, - <<"pool_size">> => 1, - <<"database">> => <<"mqtt">>, - <<"ssl">> => #{<<"enable">> => false}}, + <<"mongo_type">> => <<"single">>, + <<"server">> => <<"127.0.0.1:27017">>, + <<"pool_size">> => 1, + <<"database">> => <<"mqtt">>, + <<"ssl">> => #{<<"enable">> => false}, <<"collection">> => <<"fake">>, - <<"find">> => #{<<"a">> => <<"b">>} + <<"selector">> => #{<<"a">> => <<"b">>} }). -define(SOURCE3, #{<<"type">> => <<"mysql">>, <<"enable">> => true, - <<"config">> => #{ - <<"server">> => <<"127.0.0.1:27017">>, - <<"pool_size">> => 1, - <<"database">> => <<"mqtt">>, - <<"username">> => <<"xx">>, - <<"password">> => <<"ee">>, - <<"auto_reconnect">> => true, - <<"ssl">> => #{<<"enable">> => false}}, - <<"sql">> => <<"abcb">> + <<"server">> => <<"127.0.0.1:27017">>, + <<"pool_size">> => 1, + <<"database">> => <<"mqtt">>, + <<"username">> => <<"xx">>, + <<"password">> => <<"ee">>, + <<"auto_reconnect">> => true, + <<"ssl">> => #{<<"enable">> => false}, + <<"query">> => <<"abcb">> }). -define(SOURCE4, #{<<"type">> => <<"pgsql">>, <<"enable">> => true, - <<"config">> => #{ - <<"server">> => <<"127.0.0.1:27017">>, - <<"pool_size">> => 1, - <<"database">> => <<"mqtt">>, - <<"username">> => <<"xx">>, - <<"password">> => <<"ee">>, - <<"auto_reconnect">> => true, - <<"ssl">> => #{<<"enable">> => false}}, - <<"sql">> => <<"abcb">> + <<"server">> => <<"127.0.0.1:27017">>, + <<"pool_size">> => 1, + <<"database">> => <<"mqtt">>, + <<"username">> => <<"xx">>, + <<"password">> => <<"ee">>, + <<"auto_reconnect">> => true, + <<"ssl">> => #{<<"enable">> => false}, + <<"query">> => <<"abcb">> }). -define(SOURCE5, #{<<"type">> => <<"redis">>, <<"enable">> => true, - <<"config">> => #{ - <<"server">> => <<"127.0.0.1:27017">>, - <<"pool_size">> => 1, - <<"database">> => 0, - <<"password">> => <<"ee">>, - <<"auto_reconnect">> => true, - <<"ssl">> => #{<<"enable">> => false}}, + <<"server">> => <<"127.0.0.1:27017">>, + <<"pool_size">> => 1, + <<"database">> => 0, + <<"password">> => <<"ee">>, + <<"auto_reconnect">> => true, + <<"ssl">> => #{<<"enable">> => false}, <<"cmd">> => <<"HGETALL mqtt_authz:%u">> }). +-define(SOURCE6, #{<<"type">> => <<"file">>, + <<"enable">> => true, + <<"path">> => emqx_ct_helpers:deps_path(emqx_authz, "etc/acl.conf") + }). + %%------------------------------------------------------------------------------ %% Testcases @@ -125,12 +125,14 @@ t_update_source(_) -> {ok, _} = emqx_authz:update(head, [?SOURCE1]), {ok, _} = emqx_authz:update(tail, [?SOURCE4]), {ok, _} = emqx_authz:update(tail, [?SOURCE5]), + {ok, _} = emqx_authz:update(tail, [?SOURCE6]), ?assertMatch([ #{type := http, enable := true} , #{type := mongo, enable := true} , #{type := mysql, enable := true} , #{type := pgsql, enable := true} , #{type := redis, enable := true} + , #{type := file, enable := true} ], emqx:get_config([authorization, sources], [])), {ok, _} = emqx_authz:update({replace_once, http}, ?SOURCE1#{<<"enable">> := false}), @@ -138,23 +140,26 @@ t_update_source(_) -> {ok, _} = emqx_authz:update({replace_once, mysql}, ?SOURCE3#{<<"enable">> := false}), {ok, _} = emqx_authz:update({replace_once, pgsql}, ?SOURCE4#{<<"enable">> := false}), {ok, _} = emqx_authz:update({replace_once, redis}, ?SOURCE5#{<<"enable">> := false}), + {ok, _} = emqx_authz:update({replace_once, file}, ?SOURCE6#{<<"enable">> := false}), ?assertMatch([ #{type := http, enable := false} , #{type := mongo, enable := false} , #{type := mysql, enable := false} , #{type := pgsql, enable := false} , #{type := redis, enable := false} + , #{type := file, enable := false} ], emqx:get_config([authorization, sources], [])), {ok, _} = emqx_authz:update(replace, []). t_move_source(_) -> - {ok, _} = emqx_authz:update(replace, [?SOURCE1, ?SOURCE2, ?SOURCE3, ?SOURCE4, ?SOURCE5]), + {ok, _} = emqx_authz:update(replace, [?SOURCE1, ?SOURCE2, ?SOURCE3, ?SOURCE4, ?SOURCE5, ?SOURCE6]), ?assertMatch([ #{type := http} , #{type := mongo} , #{type := mysql} , #{type := pgsql} , #{type := redis} + , #{type := file} ], emqx_authz:lookup()), {ok, _} = emqx_authz:move(pgsql, <<"top">>), @@ -163,6 +168,7 @@ t_move_source(_) -> , #{type := mongo} , #{type := mysql} , #{type := redis} + , #{type := file} ], emqx_authz:lookup()), {ok, _} = emqx_authz:move(http, <<"bottom">>), @@ -170,6 +176,7 @@ t_move_source(_) -> , #{type := mongo} , #{type := mysql} , #{type := redis} + , #{type := file} , #{type := http} ], emqx_authz:lookup()), @@ -178,6 +185,7 @@ t_move_source(_) -> , #{type := pgsql} , #{type := mongo} , #{type := redis} + , #{type := file} , #{type := http} ], emqx_authz:lookup()), @@ -185,6 +193,7 @@ t_move_source(_) -> ?assertMatch([ #{type := mysql} , #{type := pgsql} , #{type := redis} + , #{type := file} , #{type := http} , #{type := mongo} ], emqx_authz:lookup()), diff --git a/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl b/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl index 55185de78..b4b7b87c9 100644 --- a/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl @@ -39,58 +39,61 @@ -define(SOURCE1, #{<<"type">> => <<"http">>, <<"enable">> => true, - <<"config">> => #{ - <<"url">> => <<"https://fake.com:443/">>, - <<"headers">> => #{}, - <<"method">> => <<"get">>, - <<"request_timeout">> => 5000} + <<"url">> => <<"https://fake.com:443/">>, + <<"headers">> => #{}, + <<"method">> => <<"get">>, + <<"request_timeout">> => 5000 }). -define(SOURCE2, #{<<"type">> => <<"mongo">>, <<"enable">> => true, - <<"config">> => #{ - <<"mongo_type">> => <<"single">>, - <<"server">> => <<"127.0.0.1:27017">>, - <<"pool_size">> => 1, - <<"database">> => <<"mqtt">>, - <<"ssl">> => #{<<"enable">> => false}}, + <<"mongo_type">> => <<"sharded">>, + <<"servers">> => [<<"127.0.0.1:27017">>, + <<"192.168.0.1:27017">> + ], + <<"pool_size">> => 1, + <<"database">> => <<"mqtt">>, + <<"ssl">> => #{<<"enable">> => false}, <<"collection">> => <<"fake">>, - <<"find">> => #{<<"a">> => <<"b">>} + <<"selector">> => #{<<"a">> => <<"b">>} }). -define(SOURCE3, #{<<"type">> => <<"mysql">>, <<"enable">> => true, - <<"config">> => #{ - <<"server">> => <<"127.0.0.1:27017">>, - <<"pool_size">> => 1, - <<"database">> => <<"mqtt">>, - <<"username">> => <<"xx">>, - <<"password">> => <<"ee">>, - <<"auto_reconnect">> => true, - <<"ssl">> => #{<<"enable">> => false}}, - <<"sql">> => <<"abcb">> + <<"server">> => <<"127.0.0.1:3306">>, + <<"pool_size">> => 1, + <<"database">> => <<"mqtt">>, + <<"username">> => <<"xx">>, + <<"password">> => <<"ee">>, + <<"auto_reconnect">> => true, + <<"ssl">> => #{<<"enable">> => false}, + <<"query">> => <<"abcb">> }). -define(SOURCE4, #{<<"type">> => <<"pgsql">>, <<"enable">> => true, - <<"config">> => #{ - <<"server">> => <<"127.0.0.1:27017">>, - <<"pool_size">> => 1, - <<"database">> => <<"mqtt">>, - <<"username">> => <<"xx">>, - <<"password">> => <<"ee">>, - <<"auto_reconnect">> => true, - <<"ssl">> => #{<<"enable">> => false}}, - <<"sql">> => <<"abcb">> + <<"server">> => <<"127.0.0.1:5432">>, + <<"pool_size">> => 1, + <<"database">> => <<"mqtt">>, + <<"username">> => <<"xx">>, + <<"password">> => <<"ee">>, + <<"auto_reconnect">> => true, + <<"ssl">> => #{<<"enable">> => false}, + <<"query">> => <<"abcb">> }). -define(SOURCE5, #{<<"type">> => <<"redis">>, <<"enable">> => true, - <<"config">> => #{ - <<"server">> => <<"127.0.0.1:27017">>, - <<"pool_size">> => 1, - <<"database">> => 0, - <<"password">> => <<"ee">>, - <<"auto_reconnect">> => true, - <<"ssl">> => #{<<"enable">> => false}}, + <<"servers">> => [<<"127.0.0.1:6379">>, + <<"127.0.0.1:6380">> + ], + <<"pool_size">> => 1, + <<"database">> => 0, + <<"password">> => <<"ee">>, + <<"auto_reconnect">> => true, + <<"ssl">> => #{<<"enable">> => false}, <<"cmd">> => <<"HGETALL mqtt_authz:%u">> }). +-define(SOURCE6, #{<<"type">> => <<"file">>, + <<"enable">> => true, + <<"rules">> => <<"{allow,{username,\"^dashboard?\"},subscribe,[\"$SYS/#\"]}.\n{allow,{ipaddr,\"127.0.0.1\"},all,[\"$SYS/#\",\"#\"]}.">> + }). all() -> emqx_ct:all(?MODULE). @@ -144,6 +147,26 @@ set_special_configs(emqx_authz) -> set_special_configs(_App) -> ok. +init_per_testcase(t_api, Config) -> + meck:new(emqx_plugin_libs_id, [non_strict, passthrough, no_history, no_link]), + meck:expect(emqx_plugin_libs_id, gen, fun() -> "fake" end), + + meck:new(emqx, [non_strict, passthrough, no_history, no_link]), + meck:expect(emqx, get_config, fun([node, data_dir]) -> + % emqx_ct_helpers:deps_path(emqx_authz, "test"); + {data_dir, Data} = lists:keyfind(data_dir, 1, Config), + Data; + (C) -> meck:passthrough([C]) + end), + Config; +init_per_testcase(_, Config) -> Config. + +end_per_testcase(t_api, _Config) -> + meck:unload(emqx_plugin_libs_id), + meck:unload(emqx), + ok; +end_per_testcase(_, _Config) -> ok. + %%------------------------------------------------------------------------------ %% Testcases %%------------------------------------------------------------------------------ @@ -152,33 +175,44 @@ t_api(_) -> {ok, 200, Result1} = request(get, uri(["authorization", "sources"]), []), ?assertEqual([], get_sources(Result1)), - lists:foreach(fun(_) -> - {ok, 204, _} = request(post, uri(["authorization", "sources"]), ?SOURCE1) - end, lists:seq(1, 20)), + {ok, 204, _} = request(put, uri(["authorization", "sources"]), [?SOURCE2, ?SOURCE3, ?SOURCE4, ?SOURCE5, ?SOURCE6]), + {ok, 204, _} = request(post, uri(["authorization", "sources"]), ?SOURCE1), + {ok, 200, Result2} = request(get, uri(["authorization", "sources"]), []), - ?assertEqual(20, length(get_sources(Result2))), - - lists:foreach(fun(Page) -> - Query = "?page=" ++ integer_to_list(Page) ++ "&&limit=10", - Url = uri(["authorization/sources" ++ Query]), - {ok, 200, Result} = request(get, Url, []), - ?assertEqual(10, length(get_sources(Result))) - end, lists:seq(1, 2)), - - {ok, 204, _} = request(put, uri(["authorization", "sources"]), [?SOURCE1, ?SOURCE2, ?SOURCE3, ?SOURCE4]), - - {ok, 200, Result3} = request(get, uri(["authorization", "sources"]), []), - Sources = get_sources(Result3), + Sources = get_sources(Result2), ?assertMatch([ #{<<"type">> := <<"http">>} , #{<<"type">> := <<"mongo">>} , #{<<"type">> := <<"mysql">>} , #{<<"type">> := <<"pgsql">>} + , #{<<"type">> := <<"redis">>} + , #{<<"type">> := <<"file">>} ], Sources), + ?assert(filelib:is_file(filename:join([emqx:get_config([node, data_dir]), "acl.conf"]))), {ok, 204, _} = request(put, uri(["authorization", "sources", "http"]), ?SOURCE1#{<<"enable">> := false}), + {ok, 200, Result3} = request(get, uri(["authorization", "sources", "http"]), []), + ?assertMatch(#{<<"type">> := <<"http">>, <<"enable">> := false}, jsx:decode(Result3)), - {ok, 200, Result4} = request(get, uri(["authorization", "sources", "http"]), []), - ?assertMatch(#{<<"type">> := <<"http">>, <<"enable">> := false}, jsx:decode(Result4)), + {ok, 204, _} = request(put, uri(["authorization", "sources", "mongo"]), + ?SOURCE2#{<<"ssl">> := #{ + <<"enable">> => true, + <<"cacertfile">> => <<"fake cacert file">>, + <<"certfile">> => <<"fake cert file">>, + <<"keyfile">> => <<"fake key file">>, + <<"verify">> => false + }}), + {ok, 200, Result4} = request(get, uri(["authorization", "sources", "mongo"]), []), + ?assertMatch(#{<<"type">> := <<"mongo">>, + <<"ssl">> := #{<<"enable">> := true, + <<"cacertfile">> := <<"fake cacert file">>, + <<"certfile">> := <<"fake cert file">>, + <<"keyfile">> := <<"fake key file">>, + <<"verify">> := false + } + }, jsx:decode(Result4)), + ?assert(filelib:is_file(filename:join([emqx:get_config([node, data_dir]), "certs", "cacert-fake.pem"]))), + ?assert(filelib:is_file(filename:join([emqx:get_config([node, data_dir]), "certs", "cert-fake.pem"]))), + ?assert(filelib:is_file(filename:join([emqx:get_config([node, data_dir]), "certs", "key-fake.pem"]))), lists:foreach(fun(#{<<"type">> := Type}) -> {ok, 204, _} = request(delete, uri(["authorization", "sources", binary_to_list(Type)]), []) diff --git a/apps/emqx_authz/test/emqx_authz_http_SUITE.erl b/apps/emqx_authz/test/emqx_authz_http_SUITE.erl index fad5e9580..17763d993 100644 --- a/apps/emqx_authz/test/emqx_authz_http_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_http_SUITE.erl @@ -47,12 +47,11 @@ init_per_suite(Config) -> {ok, _} = emqx:update_config([authorization, cache, enable], false), {ok, _} = emqx:update_config([authorization, no_match], deny), Rules = [#{<<"type">> => <<"http">>, - <<"config">> => #{ - <<"url">> => <<"https://fake.com:443/">>, - <<"headers">> => #{}, - <<"method">> => <<"get">>, - <<"request_timeout">> => 5000 - }} + <<"url">> => <<"https://fake.com:443/">>, + <<"headers">> => #{}, + <<"method">> => <<"get">>, + <<"request_timeout">> => 5000 + } ], {ok, _} = emqx_authz:update(replace, Rules), Config. diff --git a/apps/emqx_authz/test/emqx_authz_mongo_SUITE.erl b/apps/emqx_authz/test/emqx_authz_mongo_SUITE.erl index db111ce83..ec4c4f384 100644 --- a/apps/emqx_authz/test/emqx_authz_mongo_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_mongo_SUITE.erl @@ -47,14 +47,13 @@ init_per_suite(Config) -> {ok, _} = emqx:update_config([authorization, cache, enable], false), {ok, _} = emqx:update_config([authorization, no_match], deny), Rules = [#{<<"type">> => <<"mongo">>, - <<"config">> => #{ - <<"mongo_type">> => <<"single">>, - <<"server">> => <<"127.0.0.1:27017">>, - <<"pool_size">> => 1, - <<"database">> => <<"mqtt">>, - <<"ssl">> => #{<<"enable">> => false}}, + <<"mongo_type">> => <<"single">>, + <<"server">> => <<"127.0.0.1:27017">>, + <<"pool_size">> => 1, + <<"database">> => <<"mqtt">>, + <<"ssl">> => #{<<"enable">> => false}, <<"collection">> => <<"fake">>, - <<"find">> => #{<<"a">> => <<"b">>} + <<"selector">> => #{<<"a">> => <<"b">>} }], {ok, _} = emqx_authz:update(replace, Rules), Config. diff --git a/apps/emqx_authz/test/emqx_authz_mysql_SUITE.erl b/apps/emqx_authz/test/emqx_authz_mysql_SUITE.erl index 0675e1caf..32e52e7c0 100644 --- a/apps/emqx_authz/test/emqx_authz_mysql_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_mysql_SUITE.erl @@ -48,15 +48,14 @@ init_per_suite(Config) -> {ok, _} = emqx:update_config([authorization, cache, enable], false), {ok, _} = emqx:update_config([authorization, no_match], deny), Rules = [#{<<"type">> => <<"mysql">>, - <<"config">> => #{ - <<"server">> => <<"127.0.0.1:27017">>, - <<"pool_size">> => 1, - <<"database">> => <<"mqtt">>, - <<"username">> => <<"xx">>, - <<"password">> => <<"ee">>, - <<"auto_reconnect">> => true, - <<"ssl">> => #{<<"enable">> => false}}, - <<"sql">> => <<"abcb">> + <<"server">> => <<"127.0.0.1:27017">>, + <<"pool_size">> => 1, + <<"database">> => <<"mqtt">>, + <<"username">> => <<"xx">>, + <<"password">> => <<"ee">>, + <<"auto_reconnect">> => true, + <<"ssl">> => #{<<"enable">> => false}, + <<"query">> => <<"abcb">> }], {ok, _} = emqx_authz:update(replace, Rules), Config. diff --git a/apps/emqx_authz/test/emqx_authz_pgsql_SUITE.erl b/apps/emqx_authz/test/emqx_authz_pgsql_SUITE.erl index 6880ab405..570ea0e77 100644 --- a/apps/emqx_authz/test/emqx_authz_pgsql_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_pgsql_SUITE.erl @@ -48,15 +48,14 @@ init_per_suite(Config) -> {ok, _} = emqx:update_config([authorization, cache, enable], false), {ok, _} = emqx:update_config([authorization, no_match], deny), Rules = [#{<<"type">> => <<"pgsql">>, - <<"config">> => #{ - <<"server">> => <<"127.0.0.1:27017">>, - <<"pool_size">> => 1, - <<"database">> => <<"mqtt">>, - <<"username">> => <<"xx">>, - <<"password">> => <<"ee">>, - <<"auto_reconnect">> => true, - <<"ssl">> => #{<<"enable">> => false}}, - <<"sql">> => <<"abcb">> + <<"server">> => <<"127.0.0.1:27017">>, + <<"pool_size">> => 1, + <<"database">> => <<"mqtt">>, + <<"username">> => <<"xx">>, + <<"password">> => <<"ee">>, + <<"auto_reconnect">> => true, + <<"ssl">> => #{<<"enable">> => false}, + <<"query">> => <<"abcb">> }], {ok, _} = emqx_authz:update(replace, Rules), Config. diff --git a/apps/emqx_authz/test/emqx_authz_redis_SUITE.erl b/apps/emqx_authz/test/emqx_authz_redis_SUITE.erl index 09682761d..9949e8b51 100644 --- a/apps/emqx_authz/test/emqx_authz_redis_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_redis_SUITE.erl @@ -47,13 +47,12 @@ init_per_suite(Config) -> {ok, _} = emqx:update_config([authorization, cache, enable], false), {ok, _} = emqx:update_config([authorization, no_match], deny), Rules = [#{<<"type">> => <<"redis">>, - <<"config">> => #{ - <<"server">> => <<"127.0.0.1:27017">>, - <<"pool_size">> => 1, - <<"database">> => 0, - <<"password">> => <<"ee">>, - <<"auto_reconnect">> => true, - <<"ssl">> => #{<<"enable">> => false}}, + <<"server">> => <<"127.0.0.1:27017">>, + <<"pool_size">> => 1, + <<"database">> => 0, + <<"password">> => <<"ee">>, + <<"auto_reconnect">> => true, + <<"ssl">> => #{<<"enable">> => false}, <<"cmd">> => <<"HGETALL mqtt_authz:%u">> }], {ok, _} = emqx_authz:update(replace, Rules), diff --git a/apps/emqx_auto_subscribe/src/emqx_auto_subscribe_schema.erl b/apps/emqx_auto_subscribe/src/emqx_auto_subscribe_schema.erl index 92420e217..5b781455d 100644 --- a/apps/emqx_auto_subscribe/src/emqx_auto_subscribe_schema.erl +++ b/apps/emqx_auto_subscribe/src/emqx_auto_subscribe_schema.erl @@ -19,25 +19,30 @@ -include_lib("typerefl/include/types.hrl"). --export([ roots/0 +-export([ namespace/0 + , roots/0 , fields/1]). +namespace() -> "auto_subscribe". + roots() -> ["auto_subscribe"]. fields("auto_subscribe") -> - [ {topics, hoconsc:array(hoconsc:ref(?MODULE, "topic"))}]; + [ {topics, hoconsc:array(hoconsc:ref(?MODULE, "topic"))} + ]; fields("topic") -> - [ {topic, emqx_schema:t(binary())} - , {qos, t(hoconsc:union([0, 1, 2]), 0)} - , {rh, t(hoconsc:union([0, 1, 2]), 0)} - , {rap, t(hoconsc:union([0, 1]), 0)} - , {nl, t(hoconsc:union([0, 1]), 0)} + [ {topic, sc(binary(), #{})} + , {qos, sc(typerefl:union([0, 1, 2]), #{default => 0})} + , {rh, sc(typerefl:union([0, 1, 2]), #{default => 0})} + , {rap, sc(typerefl:union([0, 1]), #{default => 0})} + , {nl, sc(typerefl:union([0, 1]), #{default => 0})} ]. %%-------------------------------------------------------------------- %% Internal functions %%-------------------------------------------------------------------- -t(Type, Default) -> - hoconsc:t(Type, #{default => Default}). + +sc(Type, Meta) -> + hoconsc:mk(Type, Meta). diff --git a/apps/emqx_data_bridge/.gitignore b/apps/emqx_bridge/.gitignore similarity index 100% rename from apps/emqx_data_bridge/.gitignore rename to apps/emqx_bridge/.gitignore diff --git a/apps/emqx_data_bridge/README.md b/apps/emqx_bridge/README.md similarity index 95% rename from apps/emqx_data_bridge/README.md rename to apps/emqx_bridge/README.md index 8f76f17a5..0f274eea1 100644 --- a/apps/emqx_data_bridge/README.md +++ b/apps/emqx_bridge/README.md @@ -1,4 +1,4 @@ -# emqx_data_bridge +# emqx_bridge EMQ X Data Bridge is an application that managing the resources (see emqx_resource) used by emqx rule engine. diff --git a/apps/emqx_bridge/etc/emqx_bridge.conf b/apps/emqx_bridge/etc/emqx_bridge.conf new file mode 100644 index 000000000..bfba34e7c --- /dev/null +++ b/apps/emqx_bridge/etc/emqx_bridge.conf @@ -0,0 +1,49 @@ +##-------------------------------------------------------------------- +## EMQ X Bridge +##-------------------------------------------------------------------- + +#bridges.mqtt.my_mqtt_bridge_to_aws { +# server = "127.0.0.1:1883" +# proto_ver = "v4" +# username = "username1" +# password = "" +# clean_start = true +# keepalive = 300 +# retry_interval = "30s" +# max_inflight = 32 +# reconnect_interval = "30s" +# bridge_mode = true +# replayq { +# dir = "{{ platform_data_dir }}/replayq/bridge_mqtt/" +# seg_bytes = "100MB" +# offload = false +# max_total_bytes = "1GB" +# } +# ssl { +# enable = false +# keyfile = "{{ platform_etc_dir }}/certs/client-key.pem" +# certfile = "{{ platform_etc_dir }}/certs/client-cert.pem" +# cacertfile = "{{ platform_etc_dir }}/certs/cacert.pem" +# } +# ## we will create one MQTT connection for each element of the `message_in` +# message_in: [{ +# ## the `id` will be used as part of the clientid +# id = "pull_msgs_from_aws" +# subscribe_remote_topic = "aws/#" +# subscribe_qos = 1 +# local_topic = "from_aws/${topic}" +# payload = "${payload}" +# qos = "${qos}" +# retain = "${retain}" +# }] +# ## we will create one MQTT connection for each element of the `message_out` +# message_out: [{ +# ## the `id` will be used as part of the clientid +# id = "push_msgs_to_aws" +# subscribe_local_topic = "emqx/#" +# remote_topic = "from_emqx/${topic}" +# payload = "${payload}" +# qos = 1 +# retain = false +# }] +#} diff --git a/apps/emqx_data_bridge/rebar.config b/apps/emqx_bridge/rebar.config similarity index 73% rename from apps/emqx_data_bridge/rebar.config rename to apps/emqx_bridge/rebar.config index cf4cfcf1b..3fd6b41e0 100644 --- a/apps/emqx_data_bridge/rebar.config +++ b/apps/emqx_bridge/rebar.config @@ -3,5 +3,5 @@ {shell, [ % {config, "config/sys.config"}, - {apps, [emqx_data_bridge]} + {apps, [emqx_bridge]} ]}. diff --git a/apps/emqx_data_bridge/src/emqx_data_bridge.app.src b/apps/emqx_bridge/src/emqx_bridge.app.src similarity index 69% rename from apps/emqx_data_bridge/src/emqx_data_bridge.app.src rename to apps/emqx_bridge/src/emqx_bridge.app.src index 84486da19..9c0f6b779 100644 --- a/apps/emqx_data_bridge/src/emqx_data_bridge.app.src +++ b/apps/emqx_bridge/src/emqx_bridge.app.src @@ -1,12 +1,13 @@ -{application, emqx_data_bridge, +{application, emqx_bridge, [{description, "An OTP application"}, {vsn, "0.1.0"}, {registered, []}, - {mod, {emqx_data_bridge_app, []}}, + {mod, {emqx_bridge_app, []}}, {applications, [kernel, stdlib, - emqx + emqx, + emqx_connector ]}, {env,[]}, {modules, []}, diff --git a/apps/emqx_data_bridge/src/emqx_data_bridge.erl b/apps/emqx_bridge/src/emqx_bridge.erl similarity index 78% rename from apps/emqx_data_bridge/src/emqx_data_bridge.erl rename to apps/emqx_bridge/src/emqx_bridge.erl index 52cea80fb..75ebfac0c 100644 --- a/apps/emqx_data_bridge/src/emqx_data_bridge.erl +++ b/apps/emqx_bridge/src/emqx_bridge.erl @@ -13,7 +13,7 @@ %% See the License for the specific language governing permissions and %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_data_bridge). +-module(emqx_bridge). -export([ load_bridges/0 , resource_type/1 @@ -27,15 +27,17 @@ ]). load_bridges() -> - Bridges = emqx:get_config([emqx_data_bridge, bridges], []), - emqx_data_bridge_monitor:ensure_all_started(Bridges). + Bridges = emqx:get_config([bridges], #{}), + emqx_bridge_monitor:ensure_all_started(Bridges). +resource_type(mqtt) -> emqx_connector_mqtt; resource_type(mysql) -> emqx_connector_mysql; resource_type(pgsql) -> emqx_connector_pgsql; resource_type(mongo) -> emqx_connector_mongo; resource_type(redis) -> emqx_connector_redis; resource_type(ldap) -> emqx_connector_ldap. +bridge_type(emqx_connector_mqtt) -> mqtt; bridge_type(emqx_connector_mysql) -> mysql; bridge_type(emqx_connector_pgsql) -> pgsql; bridge_type(emqx_connector_mongo) -> mongo; @@ -43,13 +45,14 @@ bridge_type(emqx_connector_redis) -> redis; bridge_type(emqx_connector_ldap) -> ldap. name_to_resource_id(BridgeName) -> - <<"bridge:", BridgeName/binary>>. + Name = bin(BridgeName), + <<"bridge:", Name/binary>>. resource_id_to_name(<<"bridge:", BridgeName/binary>> = _ResourceId) -> BridgeName. list_bridges() -> - emqx_resource_api:list_instances(fun emqx_data_bridge:is_bridge/1). + emqx_resource_api:list_instances(fun emqx_bridge:is_bridge/1). is_bridge(#{id := <<"bridge:", _/binary>>}) -> true; @@ -57,7 +60,11 @@ is_bridge(_Data) -> false. config_key_path() -> - [emqx_data_bridge, bridges]. + [emqx_bridge, bridges]. update_config(ConfigReq) -> emqx:update_config(config_key_path(), ConfigReq). + +bin(Bin) when is_binary(Bin) -> Bin; +bin(Str) when is_list(Str) -> list_to_binary(Str); +bin(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8). diff --git a/apps/emqx_data_bridge/src/emqx_data_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl similarity index 83% rename from apps/emqx_data_bridge/src/emqx_data_bridge_api.erl rename to apps/emqx_bridge/src/emqx_bridge_api.erl index 6fe75e4ce..c10875e55 100644 --- a/apps/emqx_data_bridge/src/emqx_data_bridge_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_api.erl @@ -13,7 +13,7 @@ %% See the License for the specific language governing permissions and %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_data_bridge_api). +-module(emqx_bridge_api). -rest_api(#{ name => list_data_bridges , method => 'GET' @@ -61,10 +61,10 @@ list_bridges(_Binding, _Params) -> {200, #{code => 0, data => [format_api_reply(Data) || - Data <- emqx_data_bridge:list_bridges()]}}. + Data <- emqx_bridge:list_bridges()]}}. get_bridge(#{name := Name}, _Params) -> - case emqx_resource:get_instance(emqx_data_bridge:name_to_resource_id(Name)) of + case emqx_resource:get_instance(emqx_bridge:name_to_resource_id(Name)) of {ok, Data} -> {200, #{code => 0, data => format_api_reply(emqx_resource_api:format_data(Data))}}; {error, not_found} -> @@ -75,8 +75,8 @@ create_bridge(#{name := Name}, Params) -> Config = proplists:get_value(<<"config">>, Params), BridgeType = proplists:get_value(<<"type">>, Params), case emqx_resource:check_and_create( - emqx_data_bridge:name_to_resource_id(Name), - emqx_data_bridge:resource_type(atom(BridgeType)), maps:from_list(Config)) of + emqx_bridge:name_to_resource_id(Name), + emqx_bridge:resource_type(atom(BridgeType)), maps:from_list(Config)) of {ok, already_created} -> {400, #{code => 102, message => <<"bridge already created: ", Name/binary>>}}; {ok, Data} -> @@ -91,8 +91,8 @@ update_bridge(#{name := Name}, Params) -> Config = proplists:get_value(<<"config">>, Params), BridgeType = proplists:get_value(<<"type">>, Params), case emqx_resource:check_and_update( - emqx_data_bridge:name_to_resource_id(Name), - emqx_data_bridge:resource_type(atom(BridgeType)), maps:from_list(Config), []) of + emqx_bridge:name_to_resource_id(Name), + emqx_bridge:resource_type(atom(BridgeType)), maps:from_list(Config), []) of {ok, Data} -> update_config_and_reply(Name, BridgeType, Config, Data); {error, not_found} -> @@ -104,26 +104,26 @@ update_bridge(#{name := Name}, Params) -> end. delete_bridge(#{name := Name}, _Params) -> - case emqx_resource:remove(emqx_data_bridge:name_to_resource_id(Name)) of + case emqx_resource:remove(emqx_bridge:name_to_resource_id(Name)) of ok -> delete_config_and_reply(Name); {error, Reason} -> {500, #{code => 102, message => emqx_resource_api:stringnify(Reason)}} end. format_api_reply(#{resource_type := Type, id := Id, config := Conf, status := Status}) -> - #{type => emqx_data_bridge:bridge_type(Type), - name => emqx_data_bridge:resource_id_to_name(Id), + #{type => emqx_bridge:bridge_type(Type), + name => emqx_bridge:resource_id_to_name(Id), config => Conf, status => Status}. % format_conf(#{resource_type := Type, id := Id, config := Conf}) -> -% #{type => Type, name => emqx_data_bridge:resource_id_to_name(Id), +% #{type => Type, name => emqx_bridge:resource_id_to_name(Id), % config => Conf}. % get_all_configs() -> -% [format_conf(Data) || Data <- emqx_data_bridge:list_bridges()]. +% [format_conf(Data) || Data <- emqx_bridge:list_bridges()]. update_config_and_reply(Name, BridgeType, Config, Data) -> - case emqx_data_bridge:update_config({update, ?BRIDGE(Name, BridgeType, Config)}) of + case emqx_bridge:update_config({update, ?BRIDGE(Name, BridgeType, Config)}) of {ok, _} -> {200, #{code => 0, data => format_api_reply( emqx_resource_api:format_data(Data))}}; @@ -132,7 +132,7 @@ update_config_and_reply(Name, BridgeType, Config, Data) -> end. delete_config_and_reply(Name) -> - case emqx_data_bridge:update_config({delete, Name}) of + case emqx_bridge:update_config({delete, Name}) of {ok, _} -> {200, #{code => 0, data => #{}}}; {error, Reason} -> {500, #{code => 102, message => emqx_resource_api:stringnify(Reason)}} diff --git a/apps/emqx_data_bridge/src/emqx_data_bridge_app.erl b/apps/emqx_bridge/src/emqx_bridge_app.erl similarity index 87% rename from apps/emqx_data_bridge/src/emqx_data_bridge_app.erl rename to apps/emqx_bridge/src/emqx_bridge_app.erl index 859952480..cfefe118f 100644 --- a/apps/emqx_data_bridge/src/emqx_data_bridge_app.erl +++ b/apps/emqx_bridge/src/emqx_bridge_app.erl @@ -13,7 +13,7 @@ %% See the License for the specific language governing permissions and %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_data_bridge_app). +-module(emqx_bridge_app). -behaviour(application). @@ -22,9 +22,9 @@ -export([start/2, stop/1, pre_config_update/2]). start(_StartType, _StartArgs) -> - {ok, Sup} = emqx_data_bridge_sup:start_link(), - ok = emqx_data_bridge:load_bridges(), - emqx_config_handler:add_handler(emqx_data_bridge:config_key_path(), ?MODULE), + {ok, Sup} = emqx_bridge_sup:start_link(), + ok = emqx_bridge:load_bridges(), + emqx_config_handler:add_handler(emqx_bridge:config_key_path(), ?MODULE), {ok, Sup}. stop(_State) -> diff --git a/apps/emqx_data_bridge/src/emqx_data_bridge_monitor.erl b/apps/emqx_bridge/src/emqx_bridge_monitor.erl similarity index 84% rename from apps/emqx_data_bridge/src/emqx_data_bridge_monitor.erl rename to apps/emqx_bridge/src/emqx_bridge_monitor.erl index 4917833ec..d76af5fb9 100644 --- a/apps/emqx_data_bridge/src/emqx_data_bridge_monitor.erl +++ b/apps/emqx_bridge/src/emqx_bridge_monitor.erl @@ -15,7 +15,7 @@ %%-------------------------------------------------------------------- %% This process monitors all the data bridges, and try to restart a bridge %% when one of it stopped. --module(emqx_data_bridge_monitor). +-module(emqx_bridge_monitor). -behaviour(gen_server). @@ -65,14 +65,18 @@ code_change(_OldVsn, State, _Extra) -> %%============================================================================ load_bridges(Configs) -> - lists:foreach(fun load_bridge/1, Configs). + lists:foreach(fun({Type, NamedConf}) -> + lists:foreach(fun({Name, Conf}) -> + load_bridge(Name, Type, Conf) + end, maps:to_list(NamedConf)) + end, maps:to_list(Configs)). %% TODO: move this monitor into emqx_resource %% emqx_resource:check_and_create_local(ResourceId, ResourceType, Config, #{keep_retry => true}). -load_bridge(#{name := Name, type := Type, config := Config}) -> +load_bridge(Name, Type, Config) -> case emqx_resource:create_local( - emqx_data_bridge:name_to_resource_id(Name), - emqx_data_bridge:resource_type(Type), Config) of + emqx_bridge:name_to_resource_id(Name), + emqx_bridge:resource_type(Type), Config) of {ok, already_created} -> ok; {ok, _} -> ok; {error, Reason} -> diff --git a/apps/emqx_bridge/src/emqx_bridge_schema.erl b/apps/emqx_bridge/src/emqx_bridge_schema.erl new file mode 100644 index 000000000..beb0f282c --- /dev/null +++ b/apps/emqx_bridge/src/emqx_bridge_schema.erl @@ -0,0 +1,17 @@ +-module(emqx_bridge_schema). + +-export([roots/0, fields/1]). + +%%====================================================================================== +%% Hocon Schema Definitions + +roots() -> ["bridges"]. + +fields("bridges") -> + [{mqtt, hoconsc:ref(?MODULE, "mqtt")}]; + +fields("mqtt") -> + [{"$name", hoconsc:ref(?MODULE, "mqtt_bridge")}]; + +fields("mqtt_bridge") -> + emqx_connector_mqtt:fields("config"). diff --git a/apps/emqx_data_bridge/src/emqx_data_bridge_sup.erl b/apps/emqx_bridge/src/emqx_bridge_sup.erl similarity index 86% rename from apps/emqx_data_bridge/src/emqx_data_bridge_sup.erl rename to apps/emqx_bridge/src/emqx_bridge_sup.erl index a699a72a0..fd12b1a99 100644 --- a/apps/emqx_data_bridge/src/emqx_data_bridge_sup.erl +++ b/apps/emqx_bridge/src/emqx_bridge_sup.erl @@ -13,7 +13,7 @@ %% See the License for the specific language governing permissions and %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_data_bridge_sup). +-module(emqx_bridge_sup). -behaviour(supervisor). @@ -31,11 +31,11 @@ init([]) -> intensity => 10, period => 10}, ChildSpecs = [ - #{id => emqx_data_bridge_monitor, - start => {emqx_data_bridge_monitor, start_link, []}, + #{id => emqx_bridge_monitor, + start => {emqx_bridge_monitor, start_link, []}, restart => permanent, type => worker, - modules => [emqx_data_bridge_monitor]} + modules => [emqx_bridge_monitor]} ], {ok, {SupFlags, ChildSpecs}}. diff --git a/apps/emqx_bridge_mqtt/.gitignore b/apps/emqx_bridge_mqtt/.gitignore deleted file mode 100644 index bf9523be5..000000000 --- a/apps/emqx_bridge_mqtt/.gitignore +++ /dev/null @@ -1,21 +0,0 @@ -.eunit -deps -*.o -*.beam -*.plt -erl_crash.dump -ebin/*.beam -rel -_build -.concrete/DEV_MODE -.rebar -.erlang.mk -data -ebin -emqx_bridge_mqtt.d -*.rendered -.rebar3/ -*.coverdata -rebar.lock -.DS_Store -Mnesia.*/ \ No newline at end of file diff --git a/apps/emqx_bridge_mqtt/README.md b/apps/emqx_bridge_mqtt/README.md deleted file mode 100644 index 812645627..000000000 --- a/apps/emqx_bridge_mqtt/README.md +++ /dev/null @@ -1,265 +0,0 @@ -# EMQ Bridge MQTT - -The concept of **Bridge** means that EMQ X supports forwarding messages -of one of its own topics to another MQTT Broker in some way. - -**Bridge** differs from **Cluster** in that the bridge does not -replicate the topic trie and routing tables and only forwards MQTT -messages based on bridging rules. - -At present, the bridging methods supported by EMQ X are as follows: - -- RPC bridge: RPC Bridge only supports message forwarding and does not - support subscribing to the topic of remote nodes to synchronize - data; -- MQTT Bridge: MQTT Bridge supports both forwarding and data - synchronization through subscription topic. - -These concepts are shown below: - -![bridge](docs/images/bridge.png) - -In addition, the EMQ X message broker supports multi-node bridge mode interconnection - -``` - --------- --------- --------- -Publisher --> | Node1 | --Bridge Forward--> | Node2 | --Bridge Forward--> | Node3 | --> Subscriber - --------- --------- --------- -``` - -In EMQ X, bridge is configured by modifying `etc/emqx.conf`. EMQ X distinguishes between different bridges based on different names. E.g - -``` -## Bridge address: node name for local bridge, host:port for remote. -bridge.mqtt.aws.address = 127.0.0.1:1883 -``` - -This configuration declares a bridge named `aws` and specifies that it is bridged to the MQTT broker of 127.0.0.1:1883 by MQTT mode. - -In case of creating multiple bridges, it is convenient to replicate all configuration items of the first bridge, and modify the bridge name and other configuration items if necessary (such as bridge.$name.address, where $name refers to the name of bridge) - -The next two sections describe how to create a bridge in RPC and MQTT mode respectively and create a forwarding rule that forwards the messages from sensors. Assuming that two EMQ X nodes are running on two hosts: - - -| Name | Node | MQTT Port | -|------|-------------------|-----------| -| emqx1| emqx1@192.168.1.1.| 1883 | -| emqx2| emqx2@192.168.1.2 | 1883 | - - -## EMQ X RPC Bridge Configuration - -The following is the basic configuration of RPC bridging. A simplest RPC bridging only requires the following three items - -``` -## Bridge Address: Use node name (nodename@host) for rpc bridging, and host:port for mqtt connection -bridge.mqtt.emqx2.address = "emqx2@192.168.1.2" - -## Forwarding topics of the message -bridge.mqtt.emqx2.forwards = "sensor1/#,sensor2/#" - -## bridged mountpoint -bridge.mqtt.emqx2.mountpoint = "bridge/emqx2/${node}/" -``` - -If the messages received by the local node emqx1 matches the topic `sersor1/#` or `sensor2/#`, these messages will be forwarded to the `sensor1/#` or `sensor2/#` topic of the remote node emqx2. - -`forwards` is used to specify topics. Messages of the in `forwards` specified topics on local node are forwarded to the remote node. - -`mountpoint` is used to add a topic prefix when forwarding a message. To use `mountpoint`, the `forwards` directive must be set. In the above example, a message with the topic `sensor1/hello` received by the local node will be forwarded to the remote node with the topic `bridge/emqx2/emqx1@192.168.1.1/sensor1/hello`. - -Limitations of RPC bridging: - -1. The RPC bridge of emqx can only forward local messages to the remote node, and cannot synchronize the messages of the remote node to the local node; - -2. RPC bridge can only bridge two EMQ X broker together and cannot bridge EMQ X broker to other MQTT brokers. - -## EMQ X MQTT Bridge Configuration - -EMQ X 3.0 officially introduced MQTT bridge, so that EMQ X can bridge any MQTT broker. Because of the characteristics of the MQTT protocol, EMQ X can subscribe to the remote mqtt broker's topic through MQTT bridge, and then synchronize the remote MQTT broker's message to the local. - -EMQ X MQTT bridging principle: Create an MQTT client on the EMQ X broker, and connect this MQTT client to the remote MQTT broker. Therefore, in the MQTT bridge configuration, following fields may be set for the EMQ X to connect to the remote broker as an mqtt client - -``` -## Bridge Address: Use node name for rpc bridging, use host:port for mqtt connection -bridge.mqtt.emqx2.address = "192.168.1.2:1883" - -## Bridged Protocol Version -## Enumeration value: mqttv3 | mqttv4 | mqttv5 -bridge.mqtt.emqx2.proto_ver = "mqttv4" - -## mqtt client's clientid -bridge.mqtt.emqx2.clientid = "bridge_emq" - -## mqtt client's clean_start field -## Note: Some MQTT Brokers need to set the clean_start value as `true` -bridge.mqtt.emqx2.clean_start = true - -## mqtt client's username field -bridge.mqtt.emqx2.username = "user" - -## mqtt client's password field -bridge.mqtt.emqx2.password = "passwd" - -## Whether the mqtt client uses ssl to connect to a remote serve or not -bridge.mqtt.emqx2.ssl = off - -## CA Certificate of Client SSL Connection (PEM format) -bridge.mqtt.emqx2.cacertfile = "etc/certs/cacert.pem" - -## SSL certificate of Client SSL connection -bridge.mqtt.emqx2.certfile = "etc/certs/client-cert.pem" - -## Key file of Client SSL connection -bridge.mqtt.emqx2.keyfile = "etc/certs/client-key.pem" - -## SSL encryption -bridge.mqtt.emqx2.ciphers = "ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384" - -## TTLS PSK password -## Note 'listener.ssl.external.ciphers' and 'listener.ssl.external.psk_ciphers' cannot be configured at the same time -## -## See 'https://tools.ietf.org/html/rfc4279#section-2'. -## bridge.mqtt.emqx2.psk_ciphers = "PSK-AES128-CBC-SHA,PSK-AES256-CBC-SHA,PSK-3DES-EDE-CBC-SHA,PSK-RC4-SHA" - -## Client's heartbeat interval -bridge.mqtt.emqx2.keepalive = 60s - -## Supported TLS version -bridge.mqtt.emqx2.tls_versions = "tlsv1.3,tlsv1.2,tlsv1.1,tlsv1" - -## Forwarding topics of the message -bridge.mqtt.emqx2.forwards = "sensor1/#,sensor2/#" - -## Bridged mountpoint -bridge.mqtt.emqx2.mountpoint = "bridge/emqx2/${node}/" - -## Subscription topic for bridging -bridge.mqtt.emqx2.subscription.1.topic = "cmd/topic1" - -## Subscription qos for bridging -bridge.mqtt.emqx2.subscription.1.qos = 1 - -## Subscription topic for bridging -bridge.mqtt.emqx2.subscription.2.topic = "cmd/topic2" - -## Subscription qos for bridging -bridge.mqtt.emqx2.subscription.2.qos = 1 - -## Bridging reconnection interval -## Default: 30s -bridge.mqtt.emqx2.reconnect_interval = 30s - -## QoS1 message retransmission interval -bridge.mqtt.emqx2.retry_interval = 20s - -## Inflight Size. -bridge.mqtt.emqx2.max_inflight_batches = 32 -``` - -## Bridge Cache Configuration - -The bridge of EMQ X has a message caching mechanism. The caching mechanism is applicable to both RPC bridging and MQTT bridging. When the bridge is disconnected (such as when the network connection is unstable), the messages with a topic specified in `forwards` can be cached to the local message queue. Until the bridge is restored, these messages are re-forwarded to the remote node. The configuration of the cache queue is as follows - -``` -## emqx_bridge internal number of messages used for batch -bridge.mqtt.emqx2.queue.batch_count_limit = 32 - -## emqx_bridge internal number of message bytes used for batch -bridge.mqtt.emqx2.queue.batch_bytes_limit = 1000MB - -## The path for placing replayq queue. If it is not specified, then replayq will run in `mem-only` mode and messages will not be cached on disk. -bridge.mqtt.emqx2.queue.replayq_dir = data/emqx_emqx2_bridge/ - -## Replayq data segment size -bridge.mqtt.emqx2.queue.replayq_seg_bytes = 10MB -``` - -`bridge.mqtt.emqx2.queue.replayq_dir` is a configuration parameter for specifying the path of the bridge storage queue. - -`bridge.mqtt.emqx2.queue.replayq_seg_bytes` is used to specify the size of the largest single file of the message queue that is cached on disk. If the message queue size exceeds the specified value, a new file is created to store the message queue. - -## CLI for EMQ X Bridge MQTT - -CLI for EMQ X Bridge MQTT: - -``` bash -$ cd emqx1/ && ./bin/emqx_ctl bridges -bridges list # List bridges -bridges start # Start a bridge -bridges stop # Stop a bridge -bridges forwards # Show a bridge forward topic -bridges add-forward # Add bridge forward topic -bridges del-forward # Delete bridge forward topic -bridges subscriptions # Show a bridge subscriptions topic -bridges add-subscription # Add bridge subscriptions topic -``` - -List all bridge states - -``` bash -$ ./bin/emqx_ctl bridges list -name: emqx status: Stopped $ ./bin/emqx_ctl bridges list -name: emqx status: Stopped -``` - -Start the specified bridge - -``` bash -$ ./bin/emqx_ctl bridges start emqx -Start bridge successfully. -``` - -Stop the specified bridge - -``` bash -$ ./bin/emqx_ctl bridges stop emqx -Stop bridge successfully. -``` -List the forwarding topics for the specified bridge - -``` bash -$ ./bin/emqx_ctl bridges forwards emqx -topic: topic1/# -topic: topic2/# -``` - -Add a forwarding topic for the specified bridge - -``` bash -$ ./bin/emqx_ctl bridges add-forwards emqx topic3/# -Add-forward topic successfully. -``` - -Delete the forwarding topic for the specified bridge - - -``` bash -$ ./bin/emqx_ctl bridges del-forwards emqx topic3/# -Del-forward topic successfully. -``` - -List subscriptions for the specified bridge - -``` bash -$ ./bin/emqx_ctl bridges subscriptions emqx -topic: cmd/topic1, qos: 1 -topic: cmd/topic2, qos: 1 -``` - -Add a subscription topic for the specified bridge - -``` bash -$ ./bin/emqx_ctl bridges add-subscription emqx cmd/topic3 1 -Add-subscription topic successfully. -``` - -Delete the subscription topic for the specified bridge - -``` bash -$ ./bin/emqx_ctl bridges del-subscription emqx cmd/topic3 -Del-subscription topic successfully. -``` - -Note: In case of creating multiple bridges, it is convenient to replicate all configuration items of the first bridge, and modify the bridge name and other configuration items if necessary. - diff --git a/apps/emqx_bridge_mqtt/docs/guide.rst b/apps/emqx_bridge_mqtt/docs/guide.rst deleted file mode 100644 index a1c2a9126..000000000 --- a/apps/emqx_bridge_mqtt/docs/guide.rst +++ /dev/null @@ -1,286 +0,0 @@ - -EMQ Bridge MQTT -=============== - -The concept of **Bridge** means that EMQ X supports forwarding messages -of one of its own topics to another MQTT Broker in some way. - -**Bridge** differs from **Cluster** in that the bridge does not -replicate the topic trie and routing tables and only forwards MQTT -messages based on bridging rules. - -At present, the bridging methods supported by EMQ X are as follows: - - -* RPC bridge: RPC Bridge only supports message forwarding and does not - support subscribing to the topic of remote nodes to synchronize - data; -* MQTT Bridge: MQTT Bridge supports both forwarding and data - synchronization through subscription topic. - -These concepts are shown below: - - -.. image:: images/bridge.png - :target: images/bridge.png - :alt: bridge - - -In addition, the EMQ X message broker supports multi-node bridge mode interconnection - -.. code-block:: - - --------- --------- --------- - Publisher --> | Node1 | --Bridge Forward--> | Node2 | --Bridge Forward--> | Node3 | --> Subscriber - --------- --------- --------- - -In EMQ X, bridge is configured by modifying ``etc/emqx.conf``. EMQ X distinguishes between different bridges based on different names. E.g - -.. code-block:: - - ## Bridge address: node name for local bridge, host:port for remote. - bridge.mqtt.aws.address = "127.0.0.1:1883" - -This configuration declares a bridge named ``aws`` and specifies that it is bridged to the MQTT broker of 127.0.0.1:1883 by MQTT mode. - -In case of creating multiple bridges, it is convenient to replicate all configuration items of the first bridge, and modify the bridge name and other configuration items if necessary (such as bridge.$name.address, where $name refers to the name of bridge) - -The next two sections describe how to create a bridge in RPC and MQTT mode respectively and create a forwarding rule that forwards the messages from sensors. Assuming that two EMQ X nodes are running on two hosts: - -.. list-table:: - :header-rows: 1 - - * - Name - - Node - - MQTT Port - * - emqx1 - - emqx1@192.168.1.1. - - 1883 - * - emqx2 - - emqx2@192.168.1.2 - - 1883 - - -EMQ X RPC Bridge Configuration ------------------------------- - -The following is the basic configuration of RPC bridging. A simplest RPC bridging only requires the following three items - -.. code-block:: - - ## Bridge Address: Use node name (nodename@host) for rpc bridging, and host:port for mqtt connection - bridge.mqtt.emqx2.address = "emqx2@192.168.1.2" - - ## Forwarding topics of the message - bridge.mqtt.emqx2.forwards = "sensor1/#,sensor2/#" - - ## bridged mountpoint - bridge.mqtt.emqx2.mountpoint = "bridge/emqx2/${node}/" - -If the messages received by the local node emqx1 matches the topic ``sersor1/#`` or ``sensor2/#``\ , these messages will be forwarded to the ``sensor1/#`` or ``sensor2/#`` topic of the remote node emqx2. - -``forwards`` is used to specify topics. Messages of the in ``forwards`` specified topics on local node are forwarded to the remote node. - -``mountpoint`` is used to add a topic prefix when forwarding a message. To use ``mountpoint``\ , the ``forwards`` directive must be set. In the above example, a message with the topic ``sensor1/hello`` received by the local node will be forwarded to the remote node with the topic ``bridge/emqx2/emqx1@192.168.1.1/sensor1/hello``. - -Limitations of RPC bridging: - - -#. - The RPC bridge of emqx can only forward local messages to the remote node, and cannot synchronize the messages of the remote node to the local node; - -#. - RPC bridge can only bridge two EMQ X broker together and cannot bridge EMQ X broker to other MQTT brokers. - -EMQ X MQTT Bridge Configuration -------------------------------- - -EMQ X 3.0 officially introduced MQTT bridge, so that EMQ X can bridge any MQTT broker. Because of the characteristics of the MQTT protocol, EMQ X can subscribe to the remote mqtt broker's topic through MQTT bridge, and then synchronize the remote MQTT broker's message to the local. - -EMQ X MQTT bridging principle: Create an MQTT client on the EMQ X broker, and connect this MQTT client to the remote MQTT broker. Therefore, in the MQTT bridge configuration, following fields may be set for the EMQ X to connect to the remote broker as an mqtt client - -.. code-block:: - - ## Bridge Address: Use node name for rpc bridging, use host:port for mqtt connection - bridge.mqtt.emqx2.address = "192.168.1.2:1883" - - ## Bridged Protocol Version - ## Enumeration value: mqttv3 | mqttv4 | mqttv5 - bridge.mqtt.emqx2.proto_ver = "mqttv4" - - ## mqtt client's clientid - bridge.mqtt.emqx2.clientid = "bridge_emq" - - ## mqtt client's clean_start field - ## Note: Some MQTT Brokers need to set the clean_start value as `true` - bridge.mqtt.emqx2.clean_start = true - - ## mqtt client's username field - bridge.mqtt.emqx2.username = "user" - - ## mqtt client's password field - bridge.mqtt.emqx2.password = "passwd" - - ## Whether the mqtt client uses ssl to connect to a remote serve or not - bridge.mqtt.emqx2.ssl = off - - ## CA Certificate of Client SSL Connection (PEM format) - bridge.mqtt.emqx2.cacertfile = "etc/certs/cacert.pem" - - ## SSL certificate of Client SSL connection - bridge.mqtt.emqx2.certfile = "etc/certs/client-cert.pem" - - ## Key file of Client SSL connection - bridge.mqtt.emqx2.keyfile = "etc/certs/client-key.pem" - - ## TTLS PSK password - ## Note 'listener.ssl.external.ciphers' and 'listener.ssl.external.psk_ciphers' cannot be configured at the same time - ## - ## See 'https://tools.ietf.org/html/rfc4279#section-2'. - ## bridge.mqtt.emqx2.psk_ciphers = "PSK-AES128-CBC-SHA,PSK-AES256-CBC-SHA,PSK-3DES-EDE-CBC-SHA,PSK-RC4-SHA" - - ## Client's heartbeat interval - bridge.mqtt.emqx2.keepalive = 60s - - ## Supported TLS version - bridge.mqtt.emqx2.tls_versions = "tlsv1.2" - - ## SSL encryption - bridge.mqtt.emqx2.ciphers = "ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384" - - ## Forwarding topics of the message - bridge.mqtt.emqx2.forwards = "sensor1/#,sensor2/#" - - ## Bridged mountpoint - bridge.mqtt.emqx2.mountpoint = "bridge/emqx2/${node}/" - - ## Subscription topic for bridging - bridge.mqtt.emqx2.subscription.1.topic = "cmd/topic1" - - ## Subscription qos for bridging - bridge.mqtt.emqx2.subscription.1.qos = 1 - - ## Subscription topic for bridging - bridge.mqtt.emqx2.subscription.2.topic = "cmd/topic2" - - ## Subscription qos for bridging - bridge.mqtt.emqx2.subscription.2.qos = 1 - - ## Bridging reconnection interval - ## Default: 30s - bridge.mqtt.emqx2.reconnect_interval = 30s - - ## QoS1 message retransmission interval - bridge.mqtt.emqx2.retry_interval = 20s - - ## Inflight Size. - bridge.mqtt.emqx2.max_inflight_batches = 32 - -Bridge Cache Configuration --------------------------- - -The bridge of EMQ X has a message caching mechanism. The caching mechanism is applicable to both RPC bridging and MQTT bridging. When the bridge is disconnected (such as when the network connection is unstable), the messages with a topic specified in ``forwards`` can be cached to the local message queue. Until the bridge is restored, these messages are re-forwarded to the remote node. The configuration of the cache queue is as follows - -.. code-block:: - - ## emqx_bridge internal number of messages used for batch - bridge.mqtt.emqx2.queue.batch_count_limit = 32 - - ## emqx_bridge internal number of message bytes used for batch - bridge.mqtt.emqx2.queue.batch_bytes_limit = 1000MB - - ## The path for placing replayq queue. If it is not specified, then replayq will run in `mem-only` mode and messages will not be cached on disk. - bridge.mqtt.emqx2.queue.replayq_dir = "data/emqx_emqx2_bridge/" - - ## Replayq data segment size - bridge.mqtt.emqx2.queue.replayq_seg_bytes = 10MB - -``bridge.mqtt.emqx2.queue.replayq_dir`` is a configuration parameter for specifying the path of the bridge storage queue. - -``bridge.mqtt.emqx2.queue.replayq_seg_bytes`` is used to specify the size of the largest single file of the message queue that is cached on disk. If the message queue size exceeds the specified value, a new file is created to store the message queue. - -CLI for EMQ X Bridge MQTT -------------------------- - -CLI for EMQ X Bridge MQTT: - -.. code-block:: bash - - $ cd emqx1/ && ./bin/emqx_ctl bridges - bridges list # List bridges - bridges start # Start a bridge - bridges stop # Stop a bridge - bridges forwards # Show a bridge forward topic - bridges add-forward # Add bridge forward topic - bridges del-forward # Delete bridge forward topic - bridges subscriptions # Show a bridge subscriptions topic - bridges add-subscription # Add bridge subscriptions topic - -List all bridge states - -.. code-block:: bash - - $ ./bin/emqx_ctl bridges list - name: emqx status: Stopped $ ./bin/emqx_ctl bridges list - name: emqx status: Stopped - -Start the specified bridge - -.. code-block:: bash - - $ ./bin/emqx_ctl bridges start emqx - Start bridge successfully. - -Stop the specified bridge - -.. code-block:: bash - - $ ./bin/emqx_ctl bridges stop emqx - Stop bridge successfully. - -List the forwarding topics for the specified bridge - -.. code-block:: bash - - $ ./bin/emqx_ctl bridges forwards emqx - topic: topic1/# - topic: topic2/# - -Add a forwarding topic for the specified bridge - -.. code-block:: bash - - $ ./bin/emqx_ctl bridges add-forwards emqx topic3/# - Add-forward topic successfully. - -Delete the forwarding topic for the specified bridge - -.. code-block:: bash - - $ ./bin/emqx_ctl bridges del-forwards emqx topic3/# - Del-forward topic successfully. - -List subscriptions for the specified bridge - -.. code-block:: bash - - $ ./bin/emqx_ctl bridges subscriptions emqx - topic: cmd/topic1, qos: 1 - topic: cmd/topic2, qos: 1 - -Add a subscription topic for the specified bridge - -.. code-block:: bash - - $ ./bin/emqx_ctl bridges add-subscription emqx cmd/topic3 1 - Add-subscription topic successfully. - -Delete the subscription topic for the specified bridge - -.. code-block:: bash - - $ ./bin/emqx_ctl bridges del-subscription emqx cmd/topic3 - Del-subscription topic successfully. - -Note: In case of creating multiple bridges, it is convenient to replicate all configuration items of the first bridge, and modify the bridge name and other configuration items if necessary. - diff --git a/apps/emqx_bridge_mqtt/docs/images/bridge.png b/apps/emqx_bridge_mqtt/docs/images/bridge.png deleted file mode 100644 index 9bb9c024c..000000000 Binary files a/apps/emqx_bridge_mqtt/docs/images/bridge.png and /dev/null differ diff --git a/apps/emqx_bridge_mqtt/etc/emqx_bridge_mqtt.conf b/apps/emqx_bridge_mqtt/etc/emqx_bridge_mqtt.conf deleted file mode 100644 index 247c09d8d..000000000 --- a/apps/emqx_bridge_mqtt/etc/emqx_bridge_mqtt.conf +++ /dev/null @@ -1,56 +0,0 @@ -##==================================================================== -## Configuration for EMQ X MQTT Broker Bridge -##==================================================================== - -bridge_mqtt: [ - # { - # name: "mqtt1" - # start_type: auto - # forwards: ["test/#"], - # forward_mountpoint: "" - # reconnect_interval: "30s" - # batch_size: 100 - # queue { - # replayq_dir: "{{ platform_data_dir }}/replayq/bridge_mqtt/" - # replayq_seg_bytes: "100MB" - # replayq_offload_mode: false - # replayq_max_total_bytes: "1GB" - # }, - # config { - # conn_type: mqtt - # address: "127.0.0.1:1883" - # proto_ver: v4 - # bridge_mode: true - # clientid: "client1" - # clean_start: true - # username: "username1" - # password: "" - # keepalive: 300 - # subscriptions: [{ - # topic: "t/#" - # qos: 1 - # }] - # receive_mountpoint: "" - # retry_interval: "30s" - # max_inflight: 32 - # } - # }, - # { - # name: "rpc1" - # start_type: auto - # forwards: ["test/#"], - # forward_mountpoint: "" - # reconnect_interval: "30s" - # batch_size: 100 - # queue { - # replayq_dir: "{{ platform_data_dir }}/replayq/bridge_mqtt/" - # replayq_seg_bytes: "100MB" - # replayq_offload_mode: false - # replayq_max_total_bytes: "1GB" - # }, - # config { - # conn_type: rpc - # node: "emqx@127.0.0.1" - # } - # } -] diff --git a/apps/emqx_bridge_mqtt/include/emqx_bridge_mqtt.hrl b/apps/emqx_bridge_mqtt/include/emqx_bridge_mqtt.hrl deleted file mode 100644 index 531518668..000000000 --- a/apps/emqx_bridge_mqtt/include/emqx_bridge_mqtt.hrl +++ /dev/null @@ -1,18 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --define(APP, emqx_bridge_mqtt). - diff --git a/apps/emqx_bridge_mqtt/rebar.config b/apps/emqx_bridge_mqtt/rebar.config deleted file mode 100644 index 37ac5b034..000000000 --- a/apps/emqx_bridge_mqtt/rebar.config +++ /dev/null @@ -1,19 +0,0 @@ -{deps, []}. -{edoc_opts, [{preprocess, true}]}. -{erl_opts, [warn_unused_vars, - warn_shadow_vars, - warn_unused_import, - warn_obsolete_guard, - debug_info]}. - -{xref_checks, [undefined_function_calls, undefined_functions, - locals_not_used, deprecated_function_calls, - warnings_as_errors, deprecated_functions]}. -{cover_enabled, true}. -{cover_opts, [verbose]}. -{cover_export_enabled, true}. - -{shell, [ - % {config, "config/sys.config"}, - {apps, [emqx, emqx_bridge_mqtt]} -]}. diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src deleted file mode 100644 index afac20404..000000000 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src +++ /dev/null @@ -1,14 +0,0 @@ -{application, emqx_bridge_mqtt, - [{description, "EMQ X Bridge to MQTT Broker"}, - {vsn, "5.0.0"}, % strict semver, bump manually! - {modules, []}, - {registered, []}, - {applications, [kernel,stdlib,replayq,emqtt,emqx]}, - {mod, {emqx_bridge_mqtt_app, []}}, - {env, []}, - {licenses, ["Apache-2.0"]}, - {maintainers, ["EMQ X Team "]}, - {links, [{"Homepage", "https://emqx.io/"}, - {"Github", "https://github.com/emqx/emqx-bridge-mqtt"} - ]} - ]}. diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_app.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_app.erl deleted file mode 100644 index a145009c9..000000000 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_app.erl +++ /dev/null @@ -1,31 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_bridge_mqtt_app). - --behaviour(application). - --export([start/2, stop/1]). - -start(_StartType, _StartArgs) -> - emqx_ctl:register_command(bridges, {emqx_bridge_mqtt_cli, cli}, []), - emqx_bridge_worker:register_metrics(), - emqx_bridge_mqtt_sup:start_link(). - -stop(_State) -> - emqx_ctl:unregister_command(bridges), - ok. - diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_cli.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_cli.erl deleted file mode 100644 index a76ea3a8c..000000000 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_cli.erl +++ /dev/null @@ -1,92 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_bridge_mqtt_cli). - --include("emqx_bridge_mqtt.hrl"). - --import(lists, [foreach/2]). - --export([cli/1]). - -cli(["list"]) -> - foreach(fun({Name, State0}) -> - State = case State0 of - connected -> <<"Running">>; - _ -> <<"Stopped">> - end, - emqx_ctl:print("name: ~s status: ~s~n", [Name, State]) - end, emqx_bridge_mqtt_sup:bridges()); - -cli(["start", Name]) -> - emqx_ctl:print("~s.~n", [try emqx_bridge_worker:ensure_started(Name) of - ok -> <<"Start bridge successfully">>; - connected -> <<"Bridge already started">>; - _ -> <<"Start bridge failed">> - catch - _Error:_Reason -> - <<"Start bridge failed">> - end]); - -cli(["stop", Name]) -> - emqx_ctl:print("~s.~n", [try emqx_bridge_worker:ensure_stopped(Name) of - ok -> <<"Stop bridge successfully">>; - _ -> <<"Stop bridge failed">> - catch - _Error:_Reason -> - <<"Stop bridge failed">> - end]); - -cli(["forwards", Name]) -> - foreach(fun(Topic) -> - emqx_ctl:print("topic: ~s~n", [Topic]) - end, emqx_bridge_worker:get_forwards(Name)); - -cli(["add-forward", Name, Topic]) -> - ok = emqx_bridge_worker:ensure_forward_present(Name, iolist_to_binary(Topic)), - emqx_ctl:print("Add-forward topic successfully.~n"); - -cli(["del-forward", Name, Topic]) -> - ok = emqx_bridge_worker:ensure_forward_absent(Name, iolist_to_binary(Topic)), - emqx_ctl:print("Del-forward topic successfully.~n"); - -cli(["subscriptions", Name]) -> - foreach(fun({Topic, Qos}) -> - emqx_ctl:print("topic: ~s, qos: ~p~n", [Topic, Qos]) - end, emqx_bridge_worker:get_subscriptions(Name)); - -cli(["add-subscription", Name, Topic, Qos]) -> - case emqx_bridge_worker:ensure_subscription_present(Name, Topic, list_to_integer(Qos)) of - ok -> emqx_ctl:print("Add-subscription topic successfully.~n"); - {error, Reason} -> emqx_ctl:print("Add-subscription failed reason: ~p.~n", [Reason]) - end; - -cli(["del-subscription", Name, Topic]) -> - ok = emqx_bridge_worker:ensure_subscription_absent(Name, Topic), - emqx_ctl:print("Del-subscription topic successfully.~n"); - -cli(_) -> - emqx_ctl:usage([{"bridges list", "List bridges"}, - {"bridges start ", "Start a bridge"}, - {"bridges stop ", "Stop a bridge"}, - {"bridges forwards ", "Show a bridge forward topic"}, - {"bridges add-forward ", "Add bridge forward topic"}, - {"bridges del-forward ", "Delete bridge forward topic"}, - {"bridges subscriptions ", "Show a bridge subscriptions topic"}, - {"bridges add-subscription ", "Add bridge subscriptions topic"}, - {"bridges del-subscription ", "Delete bridge subscriptions topic"}]). - - diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_schema.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_schema.erl deleted file mode 100644 index 925bfa403..000000000 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_schema.erl +++ /dev/null @@ -1,87 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_bridge_mqtt_schema). - --include_lib("typerefl/include/types.hrl"). - --behaviour(hocon_schema). - --export([ roots/0 - , fields/1]). - -roots() -> [array("bridge_mqtt")]. - -array(Name) -> {Name, hoconsc:array(hoconsc:ref(Name))}. - -fields("bridge_mqtt") -> - [ {name, emqx_schema:t(string(), undefined, true)} - , {start_type, fun start_type/1} - , {forwards, fun forwards/1} - , {forward_mountpoint, emqx_schema:t(string())} - , {reconnect_interval, emqx_schema:t(emqx_schema:duration_ms(), undefined, "30s")} - , {batch_size, emqx_schema:t(integer(), undefined, 100)} - , {queue, emqx_schema:t(hoconsc:ref(?MODULE, "queue"))} - , {config, hoconsc:union([hoconsc:ref(?MODULE, "mqtt"), hoconsc:ref(?MODULE, "rpc")])} - ]; - -fields("mqtt") -> - [ {conn_type, fun conn_type/1} - , {address, emqx_schema:t(string(), undefined, "127.0.0.1:1883")} - , {proto_ver, fun proto_ver/1} - , {bridge_mode, emqx_schema:t(boolean(), undefined, true)} - , {clientid, emqx_schema:t(string())} - , {username, emqx_schema:t(string())} - , {password, emqx_schema:t(string())} - , {clean_start, emqx_schema:t(boolean(), undefined, true)} - , {keepalive, emqx_schema:t(integer(), undefined, 300)} - , {subscriptions, hoconsc:array("subscriptions")} - , {receive_mountpoint, emqx_schema:t(string())} - , {retry_interval, emqx_schema:t(emqx_schema:duration_ms(), undefined, "30s")} - , {max_inflight, emqx_schema:t(integer(), undefined, 32)} - ]; - -fields("rpc") -> - [ {conn_type, fun conn_type/1} - , {node, emqx_schema:t(atom(), undefined, 'emqx@127.0.0.1')} - ]; - -fields("subscriptions") -> - [ {topic, #{type => binary(), nullable => false}} - , {qos, emqx_schema:t(integer(), undefined, 1)} - ]; - -fields("queue") -> - [ {replayq_dir, hoconsc:union([boolean(), string()])} - , {replayq_seg_bytes, emqx_schema:t(emqx_schema:bytesize(), undefined, "100MB")} - , {replayq_offload_mode, emqx_schema:t(boolean(), undefined, false)} - , {replayq_max_total_bytes, emqx_schema:t(emqx_schema:bytesize(), undefined, "1024MB")} - ]. - -conn_type(type) -> hoconsc:enum([mqtt, rpc]); -conn_type(_) -> undefined. - -proto_ver(type) -> hoconsc:enum([v3, v4, v5]); -proto_ver(default) -> v4; -proto_ver(_) -> undefined. - -start_type(type) -> hoconsc:enum([auto, manual]); -start_type(default) -> auto; -start_type(_) -> undefined. - -forwards(type) -> hoconsc:array(binary()); -forwards(default) -> []; -forwards(_) -> undefined. diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_sup.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_sup.erl deleted file mode 100644 index 4207067fe..000000000 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_sup.erl +++ /dev/null @@ -1,72 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_bridge_mqtt_sup). --behaviour(supervisor). - --include("emqx_bridge_mqtt.hrl"). --include_lib("emqx/include/logger.hrl"). - - -%% APIs --export([ start_link/0 - ]). - --export([ create_bridge/1 - , drop_bridge/1 - , bridges/0 - ]). - -%% supervisor callbacks --export([init/1]). - --define(WORKER_SUP, emqx_bridge_worker_sup). - -start_link() -> - supervisor:start_link({local, ?MODULE}, ?MODULE, []). - -init([]) -> - BridgesConf = emqx:get_config([?APP, bridges], []), - BridgeSpec = lists:map(fun bridge_spec/1, BridgesConf), - SupFlag = #{strategy => one_for_one, - intensity => 100, - period => 10}, - {ok, {SupFlag, BridgeSpec}}. - -bridge_spec(Config) -> - Name = list_to_atom(maps:get(name, Config)), - #{id => Name, - start => {emqx_bridge_worker, start_link, [Config]}, - restart => permanent, - shutdown => 5000, - type => worker, - modules => [emqx_bridge_worker]}. - --spec(bridges() -> [{node(), map()}]). -bridges() -> - [{Name, emqx_bridge_worker:status(Name)} || {Name, _Pid, _, _} <- supervisor:which_children(?MODULE)]. - -create_bridge(Config) -> - supervisor:start_child(?MODULE, bridge_spec(Config)). - -drop_bridge(Name) -> - case supervisor:terminate_child(?MODULE, Name) of - ok -> - supervisor:delete_child(?MODULE, Name); - {error, Error} -> - ?LOG(error, "Delete bridge failed, error : ~p", [Error]), - {error, Error} - end. diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_msg.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_msg.erl deleted file mode 100644 index 91fd18bf4..000000000 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_msg.erl +++ /dev/null @@ -1,99 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_bridge_msg). - --export([ to_binary/1 - , from_binary/1 - , to_export/3 - , to_broker_msgs/1 - , to_broker_msg/1 - , to_broker_msg/2 - , estimate_size/1 - ]). - --export_type([msg/0]). - --include_lib("emqx/include/emqx.hrl"). - --include_lib("emqx_bridge_mqtt/include/emqx_bridge_mqtt.hrl"). --include_lib("emqtt/include/emqtt.hrl"). - - --type msg() :: emqx_types:message(). --type exp_msg() :: emqx_types:message() | #mqtt_msg{}. - -%% @doc Make export format: -%% 1. Mount topic to a prefix -%% 2. Fix QoS to 1 -%% @end -%% Shame that we have to know the callback module here -%% would be great if we can get rid of #mqtt_msg{} record -%% and use #message{} in all places. --spec to_export(emqx_bridge_rpc | emqx_bridge_worker, - undefined | binary(), msg()) -> exp_msg(). -to_export(emqx_bridge_mqtt, Mountpoint, - #message{topic = Topic, - payload = Payload, - flags = Flags, - qos = QoS - }) -> - Retain = maps:get(retain, Flags, false), - #mqtt_msg{qos = QoS, - retain = Retain, - topic = topic(Mountpoint, Topic), - props = #{}, - payload = Payload}; -to_export(_Module, Mountpoint, - #message{topic = Topic} = Msg) -> - Msg#message{topic = topic(Mountpoint, Topic)}. - -%% @doc Make `binary()' in order to make iodata to be persisted on disk. --spec to_binary(msg()) -> binary(). -to_binary(Msg) -> term_to_binary(Msg). - -%% @doc Unmarshal binary into `msg()'. --spec from_binary(binary()) -> msg(). -from_binary(Bin) -> binary_to_term(Bin). - -%% @doc Estimate the size of a message. -%% Count only the topic length + payload size --spec estimate_size(msg()) -> integer(). -estimate_size(#message{topic = Topic, payload = Payload}) -> - size(Topic) + size(Payload). - -%% @doc By message/batch receiver, transform received batch into -%% messages to deliver to local brokers. -to_broker_msgs(Batch) -> lists:map(fun to_broker_msg/1, Batch). - -to_broker_msg(#message{} = Msg) -> - %% internal format from another EMQX node via rpc - Msg; -to_broker_msg(Msg) -> - to_broker_msg(Msg, undefined). -to_broker_msg(#{qos := QoS, dup := Dup, retain := Retain, topic := Topic, - properties := Props, payload := Payload}, Mountpoint) -> - %% published from remote node over a MQTT connection - set_headers(Props, - emqx_message:set_flags(#{dup => Dup, retain => Retain}, - emqx_message:make(bridge, QoS, topic(Mountpoint, Topic), Payload))). - -set_headers(undefined, Msg) -> - Msg; -set_headers(Val, Msg) -> - emqx_message:set_headers(Val, Msg). -topic(undefined, Topic) -> Topic; -topic(Prefix, Topic) -> emqx_topic:prepend(Prefix, Topic). diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_rpc.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_rpc.erl deleted file mode 100644 index 33511cc03..000000000 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_rpc.erl +++ /dev/null @@ -1,95 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - -%% @doc This module implements EMQX Bridge transport layer based on gen_rpc. - --module(emqx_bridge_rpc). - --export([ start/1 - , send/2 - , stop/1 - ]). - -%% Internal exports --export([ handle_send/1 - , heartbeat/2 - ]). - --type ack_ref() :: emqx_bridge_worker:ack_ref(). --type batch() :: emqx_bridge_worker:batch(). --define(HEARTBEAT_INTERVAL, timer:seconds(1)). - --define(RPC, emqx_rpc). - -start(#{node := RemoteNode}) -> - case poke(RemoteNode) of - ok -> - Pid = proc_lib:spawn_link(?MODULE, heartbeat, [self(), RemoteNode]), - {ok, #{client_pid => Pid, remote_node => RemoteNode}}; - Error -> - Error - end. - -stop(#{client_pid := Pid}) when is_pid(Pid) -> - Ref = erlang:monitor(process, Pid), - unlink(Pid), - Pid ! stop, - receive - {'DOWN', Ref, process, Pid, _Reason} -> - ok - after - 1000 -> - exit(Pid, kill) - end, - ok. - -%% @doc Callback for `emqx_bridge_connect' behaviour --spec send(#{remote_node := atom(), _ => _}, batch()) -> {ok, ack_ref()} | {error, any()}. -send(#{remote_node := RemoteNode}, Batch) -> - case ?RPC:call(RemoteNode, ?MODULE, handle_send, [Batch]) of - ok -> - Ref = make_ref(), - self() ! {batch_ack, Ref}, - {ok, Ref}; - {badrpc, Reason} -> {error, Reason} - end. - -%% @doc Handle send on receiver side. --spec handle_send(batch()) -> ok. -handle_send(Batch) -> - lists:foreach(fun(Msg) -> emqx_broker:publish(Msg) end, Batch). - -%% @hidden Heartbeat loop -heartbeat(Parent, RemoteNode) -> - Interval = ?HEARTBEAT_INTERVAL, - receive - stop -> exit(normal) - after - Interval -> - case poke(RemoteNode) of - ok -> - ?MODULE:heartbeat(Parent, RemoteNode); - {error, Reason} -> - Parent ! {disconnected, self(), Reason}, - exit(normal) - end - end. - -poke(RemoteNode) -> - case ?RPC:call(RemoteNode, erlang, node, []) of - RemoteNode -> ok; - {badrpc, Reason} -> {error, Reason} - end. diff --git a/apps/emqx_bridge_mqtt/test/emqx_bridge_rpc_tests.erl b/apps/emqx_bridge_mqtt/test/emqx_bridge_rpc_tests.erl deleted file mode 100644 index cbd80ba3d..000000000 --- a/apps/emqx_bridge_mqtt/test/emqx_bridge_rpc_tests.erl +++ /dev/null @@ -1,42 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_bridge_rpc_tests). --include_lib("eunit/include/eunit.hrl"). - -send_and_ack_test() -> - %% delegate from emqx_rpc to rpc for unit test - meck:new(emqx_rpc, [passthrough, no_history]), - meck:expect(emqx_rpc, call, 4, - fun(Node, Module, Fun, Args) -> - rpc:call(Node, Module, Fun, Args) - end), - meck:expect(emqx_rpc, cast, 4, - fun(Node, Module, Fun, Args) -> - rpc:cast(Node, Module, Fun, Args) - end), - meck:new(emqx_bridge_worker, [passthrough, no_history]), - try - {ok, #{client_pid := Pid, remote_node := Node}} = emqx_bridge_rpc:start(#{node => node()}), - {ok, Ref} = emqx_bridge_rpc:send(#{remote_node => Node}, []), - receive - {batch_ack, Ref} -> - ok - end, - ok = emqx_bridge_rpc:stop( #{client_pid => Pid}) - after - meck:unload(emqx_rpc) - end. diff --git a/apps/emqx_bridge_mqtt/test/emqx_bridge_stub_conn.erl b/apps/emqx_bridge_mqtt/test/emqx_bridge_stub_conn.erl deleted file mode 100644 index 4c2fde6dd..000000000 --- a/apps/emqx_bridge_mqtt/test/emqx_bridge_stub_conn.erl +++ /dev/null @@ -1,38 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_bridge_stub_conn). - --export([ start/1 - , send/2 - , stop/1 - ]). - --type ack_ref() :: emqx_bridge_worker:ack_ref(). --type batch() :: emqx_bridge_worker:batch(). - -start(#{client_pid := Pid} = Cfg) -> - Pid ! {self(), ?MODULE, ready}, - {ok, Cfg}. - -stop(_) -> ok. - -%% @doc Callback for `emqx_bridge_connect' behaviour --spec send(_, batch()) -> {ok, ack_ref()} | {error, any()}. -send(#{client_pid := Pid}, Batch) -> - Ref = make_ref(), - Pid ! {stub_message, self(), Ref, Batch}, - {ok, Ref}. diff --git a/apps/emqx_bridge_mqtt/test/emqx_bridge_worker_SUITE.erl b/apps/emqx_bridge_mqtt/test/emqx_bridge_worker_SUITE.erl deleted file mode 100644 index f3f5d5ceb..000000000 --- a/apps/emqx_bridge_mqtt/test/emqx_bridge_worker_SUITE.erl +++ /dev/null @@ -1,372 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_bridge_worker_SUITE). - --compile(export_all). --compile(nowarn_export_all). - --include_lib("eunit/include/eunit.hrl"). --include_lib("common_test/include/ct.hrl"). --include_lib("emqx/include/emqx_mqtt.hrl"). --include_lib("emqx/include/emqx.hrl"). --include_lib("snabbkaffe/include/snabbkaffe.hrl"). - --define(wait(For, Timeout), emqx_ct_helpers:wait_for(?FUNCTION_NAME, ?LINE, fun() -> For end, Timeout)). - --define(SNK_WAIT(WHAT), ?assertMatch({ok, _}, ?block_until(#{?snk_kind := WHAT}, 2000, 1000))). - -receive_messages(Count) -> - receive_messages(Count, []). - -receive_messages(0, Msgs) -> - Msgs; -receive_messages(Count, Msgs) -> - receive - {publish, Msg} -> - receive_messages(Count-1, [Msg|Msgs]); - _Other -> - receive_messages(Count, Msgs) - after 1000 -> - Msgs - end. - -all() -> - lists:filtermap( - fun({FunName, _Arity}) -> - case atom_to_list(FunName) of - "t_" ++ _ -> {true, FunName}; - _ -> false - end - end, - ?MODULE:module_info(exports)). - -init_per_suite(Config) -> - case node() of - nonode@nohost -> net_kernel:start(['emqx@127.0.0.1', longnames]); - _ -> ok - end, - emqx_ct_helpers:start_apps([emqx_bridge_mqtt]), - emqx_logger:set_log_level(error), - [{log_level, error} | Config]. - -end_per_suite(_Config) -> - emqx_ct_helpers:stop_apps([emqx_bridge_mqtt]). - -init_per_testcase(_TestCase, Config) -> - ok = snabbkaffe:start_trace(), - Config. - -end_per_testcase(_TestCase, _Config) -> - ok = snabbkaffe:stop(). - -t_rpc_mngr(_Config) -> - Name = "rpc_name", - Cfg = #{ - name => Name, - forwards => [<<"mngr">>], - forward_mountpoint => <<"forwarded">>, - start_type => auto, - config => #{ - conn_type => rpc, - node => node() - } - }, - {ok, Pid} = emqx_bridge_mqtt_sup:create_bridge(Cfg), - ?assertEqual([<<"mngr">>], emqx_bridge_worker:get_forwards(Name)), - ?assertEqual(ok, emqx_bridge_worker:ensure_forward_present(Name, "mngr")), - ?assertEqual(ok, emqx_bridge_worker:ensure_forward_present(Name, "mngr2")), - ?assertEqual([<<"mngr2">>, <<"mngr">>], emqx_bridge_worker:get_forwards(Name)), - ?assertEqual(ok, emqx_bridge_worker:ensure_forward_absent(Name, "mngr2")), - ?assertEqual(ok, emqx_bridge_worker:ensure_forward_absent(Name, "mngr3")), - ?assertEqual([<<"mngr">>], emqx_bridge_worker:get_forwards(Name)), - ?assertEqual({error, no_remote_subscription_support}, - emqx_bridge_worker:ensure_subscription_present(Name, <<"t">>, 0)), - ?assertEqual({error, no_remote_subscription_support}, - emqx_bridge_worker:ensure_subscription_absent(Name, <<"t">>)), - ok = emqx_bridge_worker:stop(Pid). - -t_mqtt_mngr(_Config) -> - Name = "mqtt_name", - Cfg = #{ - name => Name, - forwards => [<<"mngr">>], - forward_mountpoint => <<"forwarded">>, - start_type => auto, - config => #{ - address => "127.0.0.1:1883", - conn_type => mqtt, - clientid => <<"client1">>, - keepalive => 300, - subscriptions => [#{topic => <<"t/#">>, qos => 1}] - } - }, - {ok, Pid} = emqx_bridge_mqtt_sup:create_bridge(Cfg), - ?assertEqual([<<"mngr">>], emqx_bridge_worker:get_forwards(Name)), - ?assertEqual(ok, emqx_bridge_worker:ensure_forward_present(Name, "mngr")), - ?assertEqual(ok, emqx_bridge_worker:ensure_forward_present(Name, "mngr2")), - ?assertEqual([<<"mngr2">>, <<"mngr">>], emqx_bridge_worker:get_forwards(Name)), - ?assertEqual(ok, emqx_bridge_worker:ensure_forward_absent(Name, "mngr2")), - ?assertEqual(ok, emqx_bridge_worker:ensure_forward_absent(Name, "mngr3")), - ?assertEqual([<<"mngr">>], emqx_bridge_worker:get_forwards(Name)), - ?assertEqual(ok, emqx_bridge_worker:ensure_subscription_present(Name, <<"t">>, 0)), - ?assertEqual(ok, emqx_bridge_worker:ensure_subscription_absent(Name, <<"t">>)), - ?assertEqual([{<<"t/#">>,1}], emqx_bridge_worker:get_subscriptions(Name)), - ok = emqx_bridge_worker:stop(Pid). - -%% A loopback RPC to local node -t_rpc(_Config) -> - Name = "rpc", - Cfg = #{ - name => Name, - forwards => [<<"t_rpc/#">>], - forward_mountpoint => <<"forwarded">>, - start_type => auto, - config => #{ - conn_type => rpc, - node => node() - } - }, - {ok, Pid} = emqx_bridge_mqtt_sup:create_bridge(Cfg), - {ok, ConnPid} = emqtt:start_link([{clientid, <<"ClientId">>}]), - {ok, _Props} = emqtt:connect(ConnPid), - {ok, _Props, [1]} = emqtt:subscribe(ConnPid, {<<"forwarded/t_rpc/one">>, ?QOS_1}), - timer:sleep(100), - {ok, _PacketId} = emqtt:publish(ConnPid, <<"t_rpc/one">>, <<"hello">>, ?QOS_1), - timer:sleep(100), - ?assertEqual(1, length(receive_messages(1))), - emqtt:disconnect(ConnPid), - emqx_bridge_worker:stop(Pid). - -%% Full data loopback flow explained: -%% mqtt-client ----> local-broker ---(local-subscription)---> -%% bridge(export) --- (mqtt-connection)--> local-broker ---(remote-subscription) --> -%% bridge(import) --> mqtt-client -t_mqtt(_Config) -> - SendToTopic = <<"t_mqtt/one">>, - SendToTopic2 = <<"t_mqtt/two">>, - SendToTopic3 = <<"t_mqtt/three">>, - Mountpoint = <<"forwarded/${node}/">>, - Name = "mqtt", - Cfg = #{ - name => Name, - forwards => [SendToTopic], - forward_mountpoint => Mountpoint, - start_type => auto, - config => #{ - address => "127.0.0.1:1883", - conn_type => mqtt, - clientid => <<"client1">>, - keepalive => 300, - subscriptions => [#{topic => SendToTopic2, qos => 1}], - receive_mountpoint => <<"receive/aws/">> - }, - queue => #{ - replayq_dir => "data/t_mqtt/", - replayq_seg_bytes => 10000, - batch_bytes_limit => 1000, - batch_count_limit => 10 - } - }, - {ok, Pid} = emqx_bridge_mqtt_sup:create_bridge(Cfg), - ?assertEqual([{SendToTopic2, 1}], emqx_bridge_worker:get_subscriptions(Name)), - ok = emqx_bridge_worker:ensure_subscription_present(Name, SendToTopic3, _QoS = 1), - ?assertEqual([{SendToTopic3, 1},{SendToTopic2, 1}], - emqx_bridge_worker:get_subscriptions(Name)), - {ok, ConnPid} = emqtt:start_link([{clientid, <<"client-1">>}]), - {ok, _Props} = emqtt:connect(ConnPid), - emqtt:subscribe(ConnPid, <<"forwarded/+/t_mqtt/one">>, 1), - %% message from a different client, to avoid getting terminated by no-local - Max = 10, - Msgs = lists:seq(1, Max), - lists:foreach(fun(I) -> - {ok, _PacketId} = emqtt:publish(ConnPid, SendToTopic, integer_to_binary(I), ?QOS_1) - end, Msgs), - ?assertEqual(10, length(receive_messages(200))), - - emqtt:subscribe(ConnPid, <<"receive/aws/t_mqtt/two">>, 1), - %% message from a different client, to avoid getting terminated by no-local - Max = 10, - Msgs = lists:seq(1, Max), - lists:foreach(fun(I) -> - {ok, _PacketId} = emqtt:publish(ConnPid, SendToTopic2, integer_to_binary(I), ?QOS_1) - end, Msgs), - ?assertEqual(10, length(receive_messages(200))), - - emqtt:disconnect(ConnPid), - ok = emqx_bridge_worker:stop(Pid). - -t_stub_normal(Config) when is_list(Config) -> - Name = "stub_normal", - Cfg = #{ - name => Name, - forwards => [<<"t_stub_normal/#">>], - forward_mountpoint => <<"forwarded">>, - start_type => auto, - config => #{ - conn_type => emqx_bridge_stub_conn, - client_pid => self() - } - }, - {ok, Pid} = emqx_bridge_mqtt_sup:create_bridge(Cfg), - receive - {Pid, emqx_bridge_stub_conn, ready} -> ok - after - 5000 -> - error(timeout) - end, - {ok, ConnPid} = emqtt:start_link([{clientid, <<"ClientId">>}]), - {ok, _} = emqtt:connect(ConnPid), - {ok, _PacketId} = emqtt:publish(ConnPid, <<"t_stub_normal/one">>, <<"hello">>, ?QOS_1), - receive - {stub_message, WorkerPid, BatchRef, _Batch} -> - WorkerPid ! {batch_ack, BatchRef}, - ok - after - 5000 -> - error(timeout) - end, - ?SNK_WAIT(inflight_drained), - ?SNK_WAIT(replayq_drained), - emqtt:disconnect(ConnPid), - ok = emqx_bridge_worker:stop(Pid). - -t_stub_overflow(_Config) -> - Topic = <<"t_stub_overflow/one">>, - MaxInflight = 20, - Name = "stub_overflow", - Cfg = #{ - name => Name, - forwards => [<<"t_stub_overflow/one">>], - forward_mountpoint => <<"forwarded">>, - start_type => auto, - max_inflight => MaxInflight, - config => #{ - conn_type => emqx_bridge_stub_conn, - client_pid => self() - } - }, - {ok, Worker} = emqx_bridge_mqtt_sup:create_bridge(Cfg), - {ok, ConnPid} = emqtt:start_link([{clientid, <<"ClientId">>}]), - {ok, _} = emqtt:connect(ConnPid), - lists:foreach( - fun(I) -> - Data = integer_to_binary(I), - _ = emqtt:publish(ConnPid, Topic, Data, ?QOS_1) - end, lists:seq(1, MaxInflight * 2)), - ?SNK_WAIT(inflight_full), - Acks = stub_receive(MaxInflight), - lists:foreach(fun({Pid, Ref}) -> Pid ! {batch_ack, Ref} end, Acks), - Acks2 = stub_receive(MaxInflight), - lists:foreach(fun({Pid, Ref}) -> Pid ! {batch_ack, Ref} end, Acks2), - ?SNK_WAIT(inflight_drained), - ?SNK_WAIT(replayq_drained), - emqtt:disconnect(ConnPid), - ok = emqx_bridge_worker:stop(Worker). - -t_stub_random_order(_Config) -> - Topic = <<"t_stub_random_order/a">>, - MaxInflight = 10, - Name = "stub_random_order", - Cfg = #{ - name => Name, - forwards => [Topic], - forward_mountpoint => <<"forwarded">>, - start_type => auto, - max_inflight => MaxInflight, - config => #{ - conn_type => emqx_bridge_stub_conn, - client_pid => self() - } - }, - {ok, Worker} = emqx_bridge_mqtt_sup:create_bridge(Cfg), - ClientId = <<"ClientId">>, - {ok, ConnPid} = emqtt:start_link([{clientid, ClientId}]), - {ok, _} = emqtt:connect(ConnPid), - lists:foreach( - fun(I) -> - Data = integer_to_binary(I), - _ = emqtt:publish(ConnPid, Topic, Data, ?QOS_1) - end, lists:seq(1, MaxInflight)), - Acks = stub_receive(MaxInflight), - lists:foreach(fun({Pid, Ref}) -> Pid ! {batch_ack, Ref} end, - lists:reverse(Acks)), - ?SNK_WAIT(inflight_drained), - ?SNK_WAIT(replayq_drained), - emqtt:disconnect(ConnPid), - ok = emqx_bridge_worker:stop(Worker). - -t_stub_retry_inflight(_Config) -> - Topic = <<"to_stub_retry_inflight/a">>, - MaxInflight = 10, - Name = "stub_retry_inflight", - Cfg = #{ - name => Name, - forwards => [Topic], - forward_mountpoint => <<"forwarded">>, - reconnect_interval => 10, - start_type => auto, - max_inflight => MaxInflight, - config => #{ - conn_type => emqx_bridge_stub_conn, - client_pid => self() - } - }, - {ok, Worker} = emqx_bridge_mqtt_sup:create_bridge(Cfg), - ClientId = <<"ClientId2">>, - case ?block_until(#{?snk_kind := connected, inflight := 0}, 2000, 1000) of - {ok, #{inflight := 0}} -> ok; - Other -> ct:fail("~p", [Other]) - end, - {ok, ConnPid} = emqtt:start_link([{clientid, ClientId}]), - {ok, _} = emqtt:connect(ConnPid), - lists:foreach( - fun(I) -> - Data = integer_to_binary(I), - _ = emqtt:publish(ConnPid, Topic, Data, ?QOS_1) - end, lists:seq(1, MaxInflight)), - %% receive acks but do not ack - Acks1 = stub_receive(MaxInflight), - ?assertEqual(MaxInflight, length(Acks1)), - %% simulate a disconnect - Worker ! {disconnected, self(), test}, - ?SNK_WAIT(disconnected), - case ?block_until(#{?snk_kind := connected, inflight := MaxInflight}, 2000, 20) of - {ok, _} -> ok; - Error -> ct:fail("~p", [Error]) - end, - %% expect worker to retry inflight, so to receive acks again - Acks2 = stub_receive(MaxInflight), - ?assertEqual(MaxInflight, length(Acks2)), - lists:foreach(fun({Pid, Ref}) -> Pid ! {batch_ack, Ref} end, - lists:reverse(Acks2)), - ?SNK_WAIT(inflight_drained), - ?SNK_WAIT(replayq_drained), - emqtt:disconnect(ConnPid), - ok = emqx_bridge_worker:stop(Worker). - -stub_receive(N) -> - stub_receive(N, []). - -stub_receive(0, Acc) -> lists:reverse(Acc); -stub_receive(N, Acc) -> - receive - {stub_message, WorkerPid, BatchRef, _Batch} -> - stub_receive(N - 1, [{WorkerPid, BatchRef} | Acc]) - after - 5000 -> - lists:reverse(Acc) - end. diff --git a/apps/emqx_connector/rebar.config b/apps/emqx_connector/rebar.config index cbeff37eb..fd2329cbd 100644 --- a/apps/emqx_connector/rebar.config +++ b/apps/emqx_connector/rebar.config @@ -17,7 +17,8 @@ %% By accident, We have always been using the upstream fork due to %% eredis_cluster's dependency getting resolved earlier. %% Here we pin 1.5.2 to avoid surprises in the future. - {poolboy, {git, "https://github.com/emqx/poolboy.git", {tag, "1.5.2"}}} + {poolboy, {git, "https://github.com/emqx/poolboy.git", {tag, "1.5.2"}}}, + {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.4.3"}}} ]}. {shell, [ diff --git a/apps/emqx_connector/src/emqx_connector.app.src b/apps/emqx_connector/src/emqx_connector.app.src index 5e1ca2ca8..f4481dc2c 100644 --- a/apps/emqx_connector/src/emqx_connector.app.src +++ b/apps/emqx_connector/src/emqx_connector.app.src @@ -13,7 +13,8 @@ epgsql, mysql, mongodb, - emqx + emqx, + emqtt ]}, {env,[]}, {modules, []}, diff --git a/apps/emqx_connector/src/emqx_connector_app.erl b/apps/emqx_connector/src/emqx_connector_app.erl index 64e6b8109..4de078076 100644 --- a/apps/emqx_connector/src/emqx_connector_app.erl +++ b/apps/emqx_connector/src/emqx_connector_app.erl @@ -21,6 +21,7 @@ -export([start/2, stop/1]). start(_StartType, _StartArgs) -> + emqx_connector_mqtt_worker:register_metrics(), emqx_connector_sup:start_link(). stop(_State) -> diff --git a/apps/emqx_connector/src/emqx_connector_http.erl b/apps/emqx_connector/src/emqx_connector_http.erl index 572b2a4e8..73412f388 100644 --- a/apps/emqx_connector/src/emqx_connector_http.erl +++ b/apps/emqx_connector/src/emqx_connector_http.erl @@ -58,16 +58,7 @@ fields(config) -> , {pool_type, fun pool_type/1} , {pool_size, fun pool_size/1} , {enable_pipelining, fun enable_pipelining/1} - , {ssl_opts, #{type => hoconsc:ref(?MODULE, ssl_opts), - default => #{}}} - ]; - -fields(ssl_opts) -> - [ {cacertfile, fun cacertfile/1} - , {keyfile, fun keyfile/1} - , {certfile, fun certfile/1} - , {verify, fun verify/1} - ]. + ] ++ emqx_connector_schema_lib:ssl_fields(). validations() -> [ {check_ssl_opts, fun check_ssl_opts/1} ]. @@ -104,23 +95,6 @@ enable_pipelining(type) -> boolean(); enable_pipelining(default) -> true; enable_pipelining(_) -> undefined. -cacertfile(type) -> string(); -cacertfile(nullable) -> true; -cacertfile(_) -> undefined. - -keyfile(type) -> string(); -keyfile(nullable) -> true; -keyfile(_) -> undefined. - -%% TODO: certfile is required -certfile(type) -> string(); -certfile(nullable) -> true; -certfile(_) -> undefined. - -verify(type) -> boolean(); -verify(default) -> false; -verify(_) -> undefined. - %% =================================================================== on_start(InstId, #{base_url := #{scheme := Scheme, host := Host, @@ -200,12 +174,11 @@ check_ssl_opts(Conf) -> check_ssl_opts(URLFrom, Conf) -> #{schema := Scheme} = hocon_schema:get_value(URLFrom, Conf), - SSLOpts = hocon_schema:get_value("ssl_opts", Conf), - case {Scheme, maps:size(SSLOpts)} of - {http, 0} -> true; - {http, _} -> false; - {https, 0} -> false; - {https, _} -> true + SSL= hocon_schema:get_value("ssl", Conf), + case {Scheme, maps:get(enable, SSL, false)} of + {http, false} -> true; + {https, true} -> true; + {_, _} -> false end. update_path(BasePath, {Path, Headers}) -> diff --git a/apps/emqx_connector/src/emqx_connector_mongo.erl b/apps/emqx_connector/src/emqx_connector_mongo.erl index 88dfb2b72..906b57fb3 100644 --- a/apps/emqx_connector/src/emqx_connector_mongo.erl +++ b/apps/emqx_connector/src/emqx_connector_mongo.erl @@ -83,9 +83,7 @@ mongo_fields() -> nullable => true}} , {database, fun emqx_connector_schema_lib:database/1} , {topology, #{type => hoconsc:ref(?MODULE, topology), - default => #{}}} - %% TODO: Does the ref type support nullable=ture ? - % nullable => true}} + nullable => true}} ] ++ emqx_connector_schema_lib:ssl_fields(). @@ -178,7 +176,7 @@ do_start(InstId, Opts0, Config = #{mongo_type := Type, ]; false -> [{ssl, false}] end, - Topology= maps:get(topology, Config, #{}), + Topology= maps:get(topology, Config, #{}), Opts = Opts0 ++ [{pool_size, PoolSize}, {options, init_topology_options(maps:to_list(Topology), [])}, @@ -244,15 +242,8 @@ init_worker_options([_ | R], Acc) -> init_worker_options(R, Acc); init_worker_options([], Acc) -> Acc. -host_port(HostPort) -> - case string:split(HostPort, ":") of - [Host, Port] -> - {ok, Host1} = inet:parse_address(Host), - [{host, Host1}, {port, list_to_integer(Port)}]; - [Host] -> - {ok, Host1} = inet:parse_address(Host), - [{host, Host1}] - end. +host_port({Host, Port}) -> + [{host, Host}, {port, Port}]. server(type) -> server(); server(validator) -> [?NOT_EMPTY("the value of the field 'server' cannot be empty")]; diff --git a/apps/emqx_connector/src/emqx_connector_mqtt.erl b/apps/emqx_connector/src/emqx_connector_mqtt.erl new file mode 100644 index 000000000..708bcdeb9 --- /dev/null +++ b/apps/emqx_connector/src/emqx_connector_mqtt.erl @@ -0,0 +1,213 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_connector_mqtt). + +-include_lib("typerefl/include/types.hrl"). +-include_lib("emqx_resource/include/emqx_resource_behaviour.hrl"). + +-behaviour(supervisor). + +%% API and callbacks for supervisor +-export([ start_link/0 + , init/1 + , create_bridge/1 + , drop_bridge/1 + , bridges/0 + ]). + +%% callbacks of behaviour emqx_resource +-export([ on_start/2 + , on_stop/2 + , on_query/4 + , on_health_check/2 + ]). + +-behaviour(hocon_schema). + +-export([ roots/0 + , fields/1]). + +%%===================================================================== +%% Hocon schema +roots() -> + [{config, #{type => hoconsc:ref(?MODULE, "config")}}]. + +fields("config") -> + emqx_connector_mqtt_schema:fields("config"). + +%% =================================================================== +%% supervisor APIs +start_link() -> + supervisor:start_link({local, ?MODULE}, ?MODULE, []). + +init([]) -> + SupFlag = #{strategy => one_for_one, + intensity => 100, + period => 10}, + {ok, {SupFlag, []}}. + +bridge_spec(Config) -> + #{id => maps:get(name, Config), + start => {emqx_connector_mqtt_worker, start_link, [Config]}, + restart => permanent, + shutdown => 5000, + type => worker, + modules => [emqx_connector_mqtt_worker]}. + +-spec(bridges() -> [{node(), map()}]). +bridges() -> + [{Name, emqx_connector_mqtt_worker:status(Name)} + || {Name, _Pid, _, _} <- supervisor:which_children(?MODULE)]. + +create_bridge(Config) -> + supervisor:start_child(?MODULE, bridge_spec(Config)). + +drop_bridge(Name) -> + case supervisor:terminate_child(?MODULE, Name) of + ok -> + supervisor:delete_child(?MODULE, Name); + {error, Error} -> + {error, Error} + end. + +%% =================================================================== +on_start(InstId, Conf) -> + logger:info("starting mqtt connector: ~p, ~p", [InstId, Conf]), + NamePrefix = binary_to_list(InstId), + BasicConf = basic_config(Conf), + InitRes = {ok, #{name_prefix => NamePrefix, baisc_conf => BasicConf, sub_bridges => []}}, + InOutConfigs = check_channel_id_dup(maps:get(message_in, Conf, []) + ++ maps:get(message_out, Conf, [])), + lists:foldl(fun + (_InOutConf, {error, Reason}) -> + {error, Reason}; + (InOutConf, {ok, #{sub_bridges := SubBridges} = Res}) -> + case create_channel(InOutConf, NamePrefix, BasicConf) of + {error, Reason} -> {error, Reason}; + {ok, Name} -> {ok, Res#{sub_bridges => [Name | SubBridges]}} + end + end, InitRes, InOutConfigs). + +on_stop(InstId, #{}) -> + logger:info("stopping mqtt connector: ~p", [InstId]), + case ?MODULE:drop_bridge(InstId) of + ok -> ok; + {error, not_found} -> ok; + {error, Reason} -> + logger:error("stop bridge failed, error: ~p", [Reason]) + end. + +%% TODO: let the emqx_resource trigger on_query/4 automatically according to the +%% `message_in` and `message_out` config +on_query(InstId, {create_channel, Conf}, _AfterQuery, #{name_prefix := Prefix, + baisc_conf := BasicConf}) -> + logger:debug("create channel to connector: ~p, conf: ~p", [InstId, Conf]), + create_channel(Conf, Prefix, BasicConf); +on_query(InstId, {publish_to_local, Msg}, _AfterQuery, _State) -> + logger:debug("publish to local node, connector: ~p, msg: ~p", [InstId, Msg]); +on_query(InstId, {publish_to_remote, Msg}, _AfterQuery, _State) -> + logger:debug("publish to remote node, connector: ~p, msg: ~p", [InstId, Msg]). + +on_health_check(_InstId, #{sub_bridges := NameList} = State) -> + Results = [{Name, emqx_connector_mqtt_worker:ping(Name)} || Name <- NameList], + case lists:all(fun({_, pong}) -> true; ({_, _}) -> false end, Results) of + true -> {ok, State}; + false -> {error, {some_sub_bridge_down, Results}, State} + end. + +check_channel_id_dup(Confs) -> + lists:foreach(fun(#{id := Id}) -> + case length([Id || #{id := Id0} <- Confs, Id0 == Id]) of + 1 -> ok; + L when L > 1 -> error({mqtt_bridge_conf, {duplicate_id_found, Id}}) + end + end, Confs), + Confs. + +%% this is an `message_in` bridge +create_channel(#{subscribe_remote_topic := _, id := Id} = InConf, NamePrefix, BasicConf) -> + logger:info("creating 'message_in' channel for: ~p", [Id]), + create_sub_bridge(BasicConf#{ + name => bridge_name(NamePrefix, Id), + clientid => clientid(Id), + subscriptions => InConf, forwards => undefined}); +%% this is an `message_out` bridge +create_channel(#{subscribe_local_topic := _, id := Id} = OutConf, NamePrefix, BasicConf) -> + logger:info("creating 'message_out' channel for: ~p", [Id]), + create_sub_bridge(BasicConf#{ + name => bridge_name(NamePrefix, Id), + clientid => clientid(Id), + subscriptions => undefined, forwards => OutConf}). + +create_sub_bridge(#{name := Name} = Conf) -> + case ?MODULE:create_bridge(Conf) of + {ok, _Pid} -> + start_sub_bridge(Name); + {error, {already_started, _Pid}} -> + ok; + {error, Reason} -> + {error, Reason} + end. + +start_sub_bridge(Name) -> + case emqx_connector_mqtt_worker:ensure_started(Name) of + ok -> {ok, Name}; + {error, Reason} -> {error, Reason} + end. + +basic_config(#{ + server := Server, + reconnect_interval := ReconnIntv, + proto_ver := ProtoVer, + bridge_mode := BridgeMod, + username := User, + password := Password, + clean_start := CleanStart, + keepalive := KeepAlive, + retry_interval := RetryIntv, + max_inflight := MaxInflight, + replayq := ReplayQ, + ssl := #{enable := EnableSsl} = Ssl}) -> + #{ + replayq => ReplayQ, + %% connection opts + server => Server, + reconnect_interval => ReconnIntv, + proto_ver => ProtoVer, + bridge_mode => BridgeMod, + username => User, + password => Password, + clean_start => CleanStart, + keepalive => KeepAlive, + retry_interval => RetryIntv, + max_inflight => MaxInflight, + ssl => EnableSsl, + ssl_opts => maps:to_list(maps:remove(enable, Ssl)), + if_record_metrics => true + }. + +bridge_name(Prefix, Id) -> + list_to_atom(str(Prefix) ++ ":" ++ str(Id)). + +clientid(Id) -> + list_to_binary(str(Id) ++ ":" ++ emqx_plugin_libs_id:gen(4)). + +str(A) when is_atom(A) -> + atom_to_list(A); +str(B) when is_binary(B) -> + binary_to_list(B); +str(S) when is_list(S) -> + S. diff --git a/apps/emqx_connector/src/emqx_connector_redis.erl b/apps/emqx_connector/src/emqx_connector_redis.erl index 4fe26381e..44b036f39 100644 --- a/apps/emqx_connector/src/emqx_connector_redis.erl +++ b/apps/emqx_connector/src/emqx_connector_redis.erl @@ -19,9 +19,13 @@ -include_lib("typerefl/include/types.hrl"). -include_lib("emqx_resource/include/emqx_resource_behaviour.hrl"). --type server() :: emqx_schema:ip_port(). +-type server() :: tuple(). + -reflect_type([server/0]). --typerefl_from_string({server/0, emqx_connector_schema_lib, to_ip_port}). + +-typerefl_from_string({server/0, ?MODULE, to_server}). + +-export([to_server/1]). -export([roots/0, fields/1]). @@ -168,3 +172,9 @@ redis_fields() -> default => 0}} , {auto_reconnect, fun emqx_connector_schema_lib:auto_reconnect/1} ]. + +to_server(Server) -> + case string:tokens(Server, ":") of + [Host, Port] -> {ok, {Host, list_to_integer(Port)}}; + _ -> {error, Server} + end. \ No newline at end of file diff --git a/apps/emqx_connector/src/emqx_connector_schema_lib.erl b/apps/emqx_connector/src/emqx_connector_schema_lib.erl index 5f9472cca..6dcc564af 100644 --- a/apps/emqx_connector/src/emqx_connector_schema_lib.erl +++ b/apps/emqx_connector/src/emqx_connector_schema_lib.erl @@ -53,24 +53,18 @@ -export([roots/0, fields/1]). -roots() -> [ssl_on, ssl_off]. +roots() -> ["ssl"]. -fields(ssl_on) -> - [ {enable, #{type => true}} +fields("ssl") -> + [ {enable, #{type => boolean(), default => false}} , {cacertfile, fun cacertfile/1} , {keyfile, fun keyfile/1} , {certfile, fun certfile/1} , {verify, fun verify/1} - ]; - -fields(ssl_off) -> - [ {enable, #{type => false}} ]. + ]. ssl_fields() -> - [ {ssl, #{type => hoconsc:union( - [ hoconsc:ref(?MODULE, ssl_on) - , hoconsc:ref(?MODULE, ssl_off) - ]), + [ {ssl, #{type => hoconsc:ref(?MODULE, "ssl"), default => #{<<"enable">> => false} } } @@ -142,7 +136,9 @@ to_ip_port(Str) -> _ -> {error, Str} end. -ip_port_to_string({Ip, Port}) -> +ip_port_to_string({Ip, Port}) when is_list(Ip) -> + iolist_to_binary([Ip, ":", integer_to_list(Port)]); +ip_port_to_string({Ip, Port}) when is_tuple(Ip) -> iolist_to_binary([inet:ntoa(Ip), ":", integer_to_list(Port)]). to_servers(Str) -> diff --git a/apps/emqx_connector/src/emqx_connector_sup.erl b/apps/emqx_connector/src/emqx_connector_sup.erl index 603b9a8ad..a24a97b8f 100644 --- a/apps/emqx_connector/src/emqx_connector_sup.erl +++ b/apps/emqx_connector/src/emqx_connector_sup.erl @@ -28,9 +28,19 @@ start_link() -> init([]) -> SupFlags = #{strategy => one_for_all, - intensity => 0, - period => 1}, - ChildSpecs = [], + intensity => 5, + period => 20}, + ChildSpecs = [ + child_spec(emqx_connector_mqtt) + ], {ok, {SupFlags, ChildSpecs}}. +child_spec(Mod) -> + #{id => Mod, + start => {Mod, start_link, []}, + restart => permanent, + shutdown => 3000, + type => supervisor, + modules => [Mod]}. + %% internal functions diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_mod.erl similarity index 78% rename from apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.erl rename to apps/emqx_connector/src/mqtt/emqx_connector_mqtt_mod.erl index 8d442463b..3de7feac4 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.erl +++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_mod.erl @@ -16,11 +16,12 @@ %% @doc This module implements EMQX Bridge transport layer on top of MQTT protocol --module(emqx_bridge_mqtt). +-module(emqx_connector_mqtt_mod). -export([ start/1 , send/2 , stop/1 + , ping/1 ]). -export([ ensure_subscribed/3 @@ -33,9 +34,6 @@ , handle_disconnected/2 ]). --export([ check_subscriptions/1 - ]). - -include_lib("emqx/include/logger.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). @@ -50,15 +48,11 @@ start(Config) -> Parent = self(), - Address = maps:get(address, Config), + {Host, Port} = maps:get(server, Config), Mountpoint = maps:get(receive_mountpoint, Config, undefined), - Subscriptions = maps:get(subscriptions, Config, []), - Subscriptions1 = check_subscriptions(Subscriptions), - Handlers = make_hdlr(Parent, Mountpoint), - {Host, Port} = case string:tokens(Address, ":") of - [H] -> {H, 1883}; - [H, P] -> {H, list_to_integer(P)} - end, + Subscriptions = maps:get(subscriptions, Config, undefined), + Vars = emqx_connector_mqtt_msg:make_pub_vars(Mountpoint, Subscriptions), + Handlers = make_hdlr(Parent, Vars), Config1 = Config#{ msg_handler => Handlers, host => Host, @@ -66,13 +60,13 @@ start(Config) -> force_ping => true, proto_ver => maps:get(proto_ver, Config, v4) }, - case emqtt:start_link(without_config(Config1)) of + case emqtt:start_link(process_config(Config1)) of {ok, Pid} -> case emqtt:connect(Pid) of {ok, _} -> try - Subscriptions2 = subscribe_remote_topics(Pid, Subscriptions1), - {ok, #{client_pid => Pid, subscriptions => Subscriptions2}} + ok = subscribe_remote_topics(Pid, Subscriptions), + {ok, #{client_pid => Pid, subscriptions => Subscriptions}} catch throw : Reason -> ok = stop(#{client_pid => Pid}), @@ -90,6 +84,9 @@ stop(#{client_pid := Pid}) -> safe_stop(Pid, fun() -> emqtt:stop(Pid) end, 1000), ok. +ping(#{client_pid := Pid}) -> + emqtt:ping(Pid). + ensure_subscribed(#{client_pid := Pid, subscriptions := Subs} = Conn, Topic, QoS) when is_pid(Pid) -> case emqtt:subscribe(Pid, Topic, QoS) of {ok, _, _} -> Conn#{subscriptions => [{Topic, QoS}|Subs]}; @@ -158,33 +155,29 @@ handle_puback(#{packet_id := PktId, reason_code := RC}, Parent) RC =:= ?RC_NO_MATCHING_SUBSCRIBERS -> Parent ! {batch_ack, PktId}, ok; handle_puback(#{packet_id := PktId, reason_code := RC}, _Parent) -> - ?LOG(warning, "Publish ~p to remote node falied, reason_code: ~p", [PktId, RC]). + ?LOG(warning, "publish ~p to remote node falied, reason_code: ~p", [PktId, RC]). -handle_publish(Msg, Mountpoint) -> - emqx_broker:publish(emqx_bridge_msg:to_broker_msg(Msg, Mountpoint)). +handle_publish(Msg, undefined) -> + ?LOG(error, "cannot publish to local broker as 'bridge.mqtt..in' not configured, msg: ~p", [Msg]); +handle_publish(Msg, Vars) -> + ?LOG(debug, "publish to local broker, msg: ~p, vars: ~p", [Msg, Vars]), + emqx_broker:publish(emqx_connector_mqtt_msg:to_broker_msg(Msg, Vars)). handle_disconnected(Reason, Parent) -> Parent ! {disconnected, self(), Reason}. -make_hdlr(Parent, Mountpoint) -> +make_hdlr(Parent, Vars) -> #{puback => {fun ?MODULE:handle_puback/2, [Parent]}, - publish => {fun ?MODULE:handle_publish/2, [Mountpoint]}, + publish => {fun ?MODULE:handle_publish/2, [Vars]}, disconnected => {fun ?MODULE:handle_disconnected/2, [Parent]} }. -subscribe_remote_topics(ClientPid, Subscriptions) -> - lists:map(fun({Topic, Qos}) -> - case emqtt:subscribe(ClientPid, Topic, Qos) of - {ok, _, _} -> {Topic, Qos}; - Error -> throw(Error) - end - end, Subscriptions). +subscribe_remote_topics(_ClientPid, undefined) -> ok; +subscribe_remote_topics(ClientPid, #{subscribe_remote_topic := FromTopic, subscribe_qos := QoS}) -> + case emqtt:subscribe(ClientPid, FromTopic, QoS) of + {ok, _, _} -> ok; + Error -> throw(Error) + end. -without_config(Config) -> - maps:without([conn_type, address, receive_mountpoint, subscriptions], Config). - -check_subscriptions(Subscriptions) -> - lists:map(fun(#{qos := QoS, topic := Topic}) -> - true = emqx_topic:validate({filter, Topic}), - {Topic, QoS} - end, Subscriptions). +process_config(Config) -> + maps:without([conn_type, address, receive_mountpoint, subscriptions, name], Config). diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl new file mode 100644 index 000000000..5f076ed9e --- /dev/null +++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl @@ -0,0 +1,125 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_connector_mqtt_msg). + +-export([ to_binary/1 + , from_binary/1 + , make_pub_vars/2 + , to_remote_msg/2 + , to_broker_msg/2 + , estimate_size/1 + ]). + +-export_type([msg/0]). + +-include_lib("emqx/include/emqx.hrl"). + +-include_lib("emqtt/include/emqtt.hrl"). + + +-type msg() :: emqx_types:message(). +-type exp_msg() :: emqx_types:message() | #mqtt_msg{}. + +-type variables() :: #{ + mountpoint := undefined | binary(), + topic := binary(), + qos := original | integer(), + retain := original | boolean(), + payload := binary() +}. + +make_pub_vars(_, undefined) -> undefined; +make_pub_vars(Mountpoint, #{payload := _, qos := _, retain := _, remote_topic := Topic} = Conf) -> + Conf#{topic => Topic, mountpoint => Mountpoint}; +make_pub_vars(Mountpoint, #{payload := _, qos := _, retain := _, local_topic := Topic} = Conf) -> + Conf#{topic => Topic, mountpoint => Mountpoint}. + +%% @doc Make export format: +%% 1. Mount topic to a prefix +%% 2. Fix QoS to 1 +%% @end +%% Shame that we have to know the callback module here +%% would be great if we can get rid of #mqtt_msg{} record +%% and use #message{} in all places. +-spec to_remote_msg(msg() | map(), variables()) + -> exp_msg(). +to_remote_msg(#message{flags = Flags0} = Msg, Vars) -> + Retain0 = maps:get(retain, Flags0, false), + MapMsg = maps:put(retain, Retain0, emqx_message:to_map(Msg)), + to_remote_msg(MapMsg, Vars); +to_remote_msg(MapMsg, #{topic := TopicToken, payload := PayloadToken, + qos := QoSToken, retain := RetainToken, mountpoint := Mountpoint}) when is_map(MapMsg) -> + Topic = replace_vars_in_str(TopicToken, MapMsg), + Payload = replace_vars_in_str(PayloadToken, MapMsg), + QoS = replace_simple_var(QoSToken, MapMsg), + Retain = replace_simple_var(RetainToken, MapMsg), + #mqtt_msg{qos = QoS, + retain = Retain, + topic = topic(Mountpoint, Topic), + props = #{}, + payload = Payload}; +to_remote_msg(#message{topic = Topic} = Msg, #{mountpoint := Mountpoint}) -> + Msg#message{topic = topic(Mountpoint, Topic)}. + +%% published from remote node over a MQTT connection +to_broker_msg(#{dup := Dup, properties := Props} = MapMsg, + #{topic := TopicToken, payload := PayloadToken, + qos := QoSToken, retain := RetainToken, mountpoint := Mountpoint}) -> + Topic = replace_vars_in_str(TopicToken, MapMsg), + Payload = replace_vars_in_str(PayloadToken, MapMsg), + QoS = replace_simple_var(QoSToken, MapMsg), + Retain = replace_simple_var(RetainToken, MapMsg), + set_headers(Props, + emqx_message:set_flags(#{dup => Dup, retain => Retain}, + emqx_message:make(bridge, QoS, topic(Mountpoint, Topic), Payload))). + +%% Replace a string contains vars to another string in which the placeholders are replace by the +%% corresponding values. For example, given "a: ${var}", if the var=1, the result string will be: +%% "a: 1". +replace_vars_in_str(Tokens, Data) when is_list(Tokens) -> + emqx_plugin_libs_rule:proc_tmpl(Tokens, Data, #{return => full_binary}); +replace_vars_in_str(Val, _Data) -> + Val. + +%% Replace a simple var to its value. For example, given "${var}", if the var=1, then the result +%% value will be an integer 1. +replace_simple_var(Tokens, Data) when is_list(Tokens) -> + [Var] = emqx_plugin_libs_rule:proc_tmpl(Tokens, Data, #{return => rawlist}), + Var; +replace_simple_var(Val, _Data) -> + Val. + +%% @doc Make `binary()' in order to make iodata to be persisted on disk. +-spec to_binary(msg()) -> binary(). +to_binary(Msg) -> term_to_binary(Msg). + +%% @doc Unmarshal binary into `msg()'. +-spec from_binary(binary()) -> msg(). +from_binary(Bin) -> binary_to_term(Bin). + +%% @doc Estimate the size of a message. +%% Count only the topic length + payload size +-spec estimate_size(msg()) -> integer(). +estimate_size(#message{topic = Topic, payload = Payload}) -> + size(Topic) + size(Payload). + +set_headers(undefined, Msg) -> + Msg; +set_headers(Val, Msg) -> + emqx_message:set_headers(Val, Msg). +topic(undefined, Topic) -> Topic; +topic(Prefix, Topic) -> emqx_topic:prepend(Prefix, Topic). diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl new file mode 100644 index 000000000..184a8610c --- /dev/null +++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl @@ -0,0 +1,77 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_connector_mqtt_schema). + +-include_lib("typerefl/include/types.hrl"). + +-behaviour(hocon_schema). + +-export([ roots/0 + , fields/1]). + +roots() -> + [{config, #{type => hoconsc:ref(?MODULE, "config")}}]. + +fields("config") -> + [ {server, hoconsc:mk(emqx_schema:ip_port(), #{default => "127.0.0.1:1883"})} + , {reconnect_interval, hoconsc:mk(emqx_schema:duration_ms(), #{default => "30s"})} + , {proto_ver, fun proto_ver/1} + , {bridge_mode, hoconsc:mk(boolean(), #{default => true})} + , {username, hoconsc:mk(string())} + , {password, hoconsc:mk(string())} + , {clean_start, hoconsc:mk(boolean(), #{default => true})} + , {keepalive, hoconsc:mk(integer(), #{default => 300})} + , {retry_interval, hoconsc:mk(emqx_schema:duration_ms(), #{default => "30s"})} + , {max_inflight, hoconsc:mk(integer(), #{default => 32})} + , {replayq, hoconsc:mk(hoconsc:ref(?MODULE, "replayq"))} + , {message_in, hoconsc:mk(hoconsc:array(hoconsc:ref(?MODULE, "message_in")), #{default => []})} + , {message_out, hoconsc:mk(hoconsc:array(hoconsc:ref(?MODULE, "message_out")), #{default => []})} + ] ++ emqx_connector_schema_lib:ssl_fields(); + +fields("message_in") -> + [ {subscribe_remote_topic, #{type => binary(), nullable => false}} + , {local_topic, hoconsc:mk(binary(), #{default => <<"${topic}">>})} + , {subscribe_qos, hoconsc:mk(qos(), #{default => 1})} + ] ++ common_inout_confs(); + +fields("message_out") -> + [ {subscribe_local_topic, #{type => binary(), nullable => false}} + , {remote_topic, hoconsc:mk(binary(), #{default => <<"${topic}">>})} + ] ++ common_inout_confs(); + +fields("replayq") -> + [ {dir, hoconsc:union([boolean(), string()])} + , {seg_bytes, hoconsc:mk(emqx_schema:bytesize(), #{default => "100MB"})} + , {offload, hoconsc:mk(boolean(), #{default => false})} + , {max_total_bytes, hoconsc:mk(emqx_schema:bytesize(), #{default => "1024MB"})} + ]. + +common_inout_confs() -> + [{id, #{type => binary(), nullable => false}}] ++ publish_confs(). + +publish_confs() -> + [ {qos, hoconsc:mk(qos(), #{default => <<"${qos}">>})} + , {retain, hoconsc:mk(hoconsc:union([boolean(), binary()]), #{default => <<"${retain}">>})} + , {payload, hoconsc:mk(binary(), #{default => <<"${payload}">>})} + ]. + +qos() -> + hoconsc:union([typerefl:integer(0), typerefl:integer(1), typerefl:integer(2), binary()]). + +proto_ver(type) -> hoconsc:enum([v3, v4, v5]); +proto_ver(default) -> v4; +proto_ver(_) -> undefined. diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_worker.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl similarity index 68% rename from apps/emqx_bridge_mqtt/src/emqx_bridge_worker.erl rename to apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl index 630fb4443..6ced719df 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_worker.erl +++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl @@ -19,7 +19,7 @@ %% to remote MQTT node/cluster via `connection' transport layer. %% In case `REMOTE' is also an EMQX node, `connection' is recommended to be %% the `gen_rpc' based implementation `emqx_bridge_rpc'. Otherwise `connection' -%% has to be `emqx_bridge_mqtt'. +%% has to be `emqx_connector_mqtt_mod'. %% %% ``` %% +------+ +--------+ @@ -59,7 +59,7 @@ %% NOTES: %% * Local messages are all normalised to QoS-1 when exporting to remote --module(emqx_bridge_worker). +-module(emqx_connector_mqtt_worker). -behaviour(gen_statem). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). @@ -86,16 +86,13 @@ -export([ ensure_started/1 , ensure_stopped/1 , status/1 + , ping/1 ]). -export([ get_forwards/1 - , ensure_forward_present/2 - , ensure_forward_absent/2 ]). -export([ get_subscriptions/1 - , ensure_subscription_present/3 - , ensure_subscription_absent/2 ]). %% Internal @@ -109,7 +106,7 @@ -type id() :: atom() | string() | pid(). -type qos() :: emqx_mqtt_types:qos(). -type config() :: map(). --type batch() :: [emqx_bridge_msg:exp_msg()]. +-type batch() :: [emqx_connector_mqtt_msg:exp_msg()]. -type ack_ref() :: term(). -type topic() :: emqx_topic:topic(). @@ -135,12 +132,12 @@ %% mountpoint: The topic mount point for messages sent to remote node/cluster %% `undefined', `<<>>' or `""' to disable %% forwards: Local topics to subscribe. -%% queue.batch_bytes_limit: Max number of bytes to collect in a batch for each +%% replayq.batch_bytes_limit: Max number of bytes to collect in a batch for each %% send call towards emqx_bridge_connect -%% queue.batch_count_limit: Max number of messages to collect in a batch for +%% replayq.batch_count_limit: Max number of messages to collect in a batch for %% each send call towards emqx_bridge_connect -%% queue.replayq_dir: Directory where replayq should persist messages -%% queue.replayq_seg_bytes: Size in bytes for each replayq segment file +%% replayq.dir: Directory where replayq should persist messages +%% replayq.seg_bytes: Size in bytes for each replayq segment file %% %% Find more connection specific configs in the callback modules %% of emqx_bridge_connect behaviour. @@ -169,6 +166,11 @@ status(Pid) when is_pid(Pid) -> status(Name) -> gen_statem:call(name(Name), status). +ping(Pid) when is_pid(Pid) -> + gen_statem:call(Pid, ping); +ping(Name) -> + gen_statem:call(name(Name), ping). + %% @doc Return all forwards (local subscriptions). -spec get_forwards(id()) -> [topic()]. get_forwards(Name) -> gen_statem:call(name(Name), get_forwards, timer:seconds(1000)). @@ -177,47 +179,21 @@ get_forwards(Name) -> gen_statem:call(name(Name), get_forwards, timer:seconds(10 -spec get_subscriptions(id()) -> [{emqx_topic:topic(), qos()}]. get_subscriptions(Name) -> gen_statem:call(name(Name), get_subscriptions). -%% @doc Add a new forward (local topic subscription). --spec ensure_forward_present(id(), topic()) -> ok. -ensure_forward_present(Name, Topic) -> - gen_statem:call(name(Name), {ensure_forward_present, topic(Topic)}). - -%% @doc Ensure a forward topic is deleted. --spec ensure_forward_absent(id(), topic()) -> ok. -ensure_forward_absent(Name, Topic) -> - gen_statem:call(name(Name), {ensure_forward_absent, topic(Topic)}). - -%% @doc Ensure subscribed to remote topic. -%% NOTE: only applicable when connection module is emqx_bridge_mqtt -%% return `{error, no_remote_subscription_support}' otherwise. --spec ensure_subscription_present(id(), topic(), qos()) -> ok | {error, any()}. -ensure_subscription_present(Name, Topic, QoS) -> - gen_statem:call(name(Name), {ensure_subscription_present, topic(Topic), QoS}). - -%% @doc Ensure unsubscribed from remote topic. -%% NOTE: only applicable when connection module is emqx_bridge_mqtt --spec ensure_subscription_absent(id(), topic()) -> ok. -ensure_subscription_absent(Name, Topic) -> - gen_statem:call(name(Name), {ensure_subscription_absent, topic(Topic)}). - callback_mode() -> [state_functions]. %% @doc Config should be a map(). -init(Opts) -> +init(#{name := Name} = ConnectOpts) -> + ?LOG(info, "starting bridge worker for ~p", [Name]), erlang:process_flag(trap_exit, true), - ConnectOpts = maps:get(config, Opts), - ConnectModule = conn_type(maps:get(conn_type, ConnectOpts)), - Forwards = maps:get(forwards, Opts, []), - Queue = open_replayq(maps:get(queue, Opts, #{})), - State = init_opts(Opts), + Queue = open_replayq(Name, maps:get(replayq, ConnectOpts, #{})), + State = init_state(ConnectOpts), self() ! idle, - {ok, idle, State#{connect_module => ConnectModule, - connect_opts => ConnectOpts, - forwards => Forwards, - replayq => Queue - }}. + {ok, idle, State#{ + connect_opts => pre_process_opts(ConnectOpts), + replayq => Queue + }}. -init_opts(Opts) -> +init_state(Opts) -> IfRecordMetrics = maps:get(if_record_metrics, Opts, true), ReconnDelayMs = maps:get(reconnect_interval, Opts, ?DEFAULT_RECONNECT_DELAY_MS), StartType = maps:get(start_type, Opts, manual), @@ -235,17 +211,39 @@ init_opts(Opts) -> if_record_metrics => IfRecordMetrics, name => Name}. -open_replayq(QCfg) -> - Dir = maps:get(replayq_dir, QCfg, undefined), - SegBytes = maps:get(replayq_seg_bytes, QCfg, ?DEFAULT_SEG_BYTES), +open_replayq(Name, QCfg) -> + Dir = maps:get(dir, QCfg, undefined), + SegBytes = maps:get(seg_bytes, QCfg, ?DEFAULT_SEG_BYTES), MaxTotalSize = maps:get(max_total_size, QCfg, ?DEFAULT_MAX_TOTAL_SIZE), QueueConfig = case Dir =:= undefined orelse Dir =:= "" of true -> #{mem_only => true}; - false -> #{dir => Dir, seg_bytes => SegBytes, max_total_size => MaxTotalSize} + false -> #{dir => filename:join([Dir, node(), Name]), + seg_bytes => SegBytes, max_total_size => MaxTotalSize} end, - replayq:open(QueueConfig#{sizer => fun emqx_bridge_msg:estimate_size/1, + replayq:open(QueueConfig#{sizer => fun emqx_connector_mqtt_msg:estimate_size/1, marshaller => fun ?MODULE:msg_marshaller/1}). +pre_process_opts(#{subscriptions := InConf, forwards := OutConf} = ConnectOpts) -> + ConnectOpts#{subscriptions => pre_process_in_out(InConf), + forwards => pre_process_in_out(OutConf)}. + +pre_process_in_out(undefined) -> undefined; +pre_process_in_out(Conf) when is_map(Conf) -> + Conf1 = pre_process_conf(local_topic, Conf), + Conf2 = pre_process_conf(remote_topic, Conf1), + Conf3 = pre_process_conf(payload, Conf2), + Conf4 = pre_process_conf(qos, Conf3), + pre_process_conf(retain, Conf4). + +pre_process_conf(Key, Conf) -> + case maps:find(Key, Conf) of + error -> Conf; + {ok, Val} when is_binary(Val) -> + Conf#{Key => emqx_plugin_libs_rule:preproc_tmpl(Val)}; + {ok, Val} -> + Conf#{Key => Val} + end. + code_change(_Vsn, State, Data, _Extra) -> {ok, State, Data}. @@ -311,28 +309,18 @@ connected(Type, Content, State) -> %% Common handlers common(StateName, {call, From}, status, _State) -> {keep_state_and_data, [{reply, From, StateName}]}; +common(_StateName, {call, From}, ping, #{connection := Conn} =_State) -> + Reply = emqx_connector_mqtt_mod:ping(Conn), + {keep_state_and_data, [{reply, From, Reply}]}; common(_StateName, {call, From}, ensure_stopped, #{connection := undefined} = _State) -> {keep_state_and_data, [{reply, From, ok}]}; -common(_StateName, {call, From}, ensure_stopped, #{connection := Conn, - connect_module := ConnectModule} = State) -> - Reply = ConnectModule:stop(Conn), +common(_StateName, {call, From}, ensure_stopped, #{connection := Conn} = State) -> + Reply = emqx_connector_mqtt_mod:stop(Conn), {next_state, idle, State#{connection => undefined}, [{reply, From, Reply}]}; -common(_StateName, {call, From}, get_forwards, #{forwards := Forwards}) -> +common(_StateName, {call, From}, get_forwards, #{connect_opts := #{forwards := Forwards}}) -> {keep_state_and_data, [{reply, From, Forwards}]}; common(_StateName, {call, From}, get_subscriptions, #{connection := Connection}) -> - {keep_state_and_data, [{reply, From, maps:get(subscriptions, Connection, [])}]}; -common(_StateName, {call, From}, {ensure_forward_present, Topic}, State) -> - {Result, NewState} = do_ensure_forward_present(Topic, State), - {keep_state, NewState, [{reply, From, Result}]}; -common(_StateName, {call, From}, {ensure_subscription_present, Topic, QoS}, State) -> - {Result, NewState} = do_ensure_subscription_present(Topic, QoS, State), - {keep_state, NewState, [{reply, From, Result}]}; -common(_StateName, {call, From}, {ensure_forward_absent, Topic}, State) -> - {Result, NewState} = do_ensure_forward_absent(Topic, State), - {keep_state, NewState, [{reply, From, Result}]}; -common(_StateName, {call, From}, {ensure_subscription_absent, Topic}, State) -> - {Result, NewState} = do_ensure_subscription_absent(Topic, State), - {keep_state, NewState, [{reply, From, Result}]}; + {keep_state_and_data, [{reply, From, maps:get(subscriptions, Connection, #{})}]}; common(_StateName, info, {deliver, _, Msg}, State = #{replayq := Q, if_record_metrics := IfRecordMetric}) -> Msgs = collect([Msg]), @@ -349,76 +337,21 @@ common(StateName, Type, Content, #{name := Name} = State) -> [Name, Type, StateName, Content]), {keep_state, State}. -do_ensure_forward_present(Topic, #{forwards := Forwards, name := Name} = State) -> - case is_topic_present(Topic, Forwards) of - true -> - {ok, State}; - false -> - R = subscribe_local_topic(Topic, Name), - {R, State#{forwards => [Topic | Forwards]}} - end. - -do_ensure_subscription_present(_Topic, _QoS, #{connection := undefined} = State) -> - {{error, no_connection}, State}; -do_ensure_subscription_present(_Topic, _QoS, #{connect_module := emqx_bridge_rpc} = State) -> - {{error, no_remote_subscription_support}, State}; -do_ensure_subscription_present(Topic, QoS, #{connect_module := ConnectModule, - connection := Conn} = State) -> - case is_topic_present(Topic, maps:get(subscriptions, Conn, [])) of - true -> - {ok, State}; - false -> - case ConnectModule:ensure_subscribed(Conn, Topic, QoS) of - {error, Error} -> - {{error, Error}, State}; - Conn1 -> - {ok, State#{connection => Conn1}} - end - end. - -do_ensure_forward_absent(Topic, #{forwards := Forwards} = State) -> - case is_topic_present(Topic, Forwards) of - true -> - R = do_unsubscribe(Topic), - {R, State#{forwards => lists:delete(Topic, Forwards)}}; - false -> - {ok, State} - end. -do_ensure_subscription_absent(_Topic, #{connection := undefined} = State) -> - {{error, no_connection}, State}; -do_ensure_subscription_absent(_Topic, #{connect_module := emqx_bridge_rpc} = State) -> - {{error, no_remote_subscription_support}, State}; -do_ensure_subscription_absent(Topic, #{connect_module := ConnectModule, - connection := Conn} = State) -> - case is_topic_present(Topic, maps:get(subscriptions, Conn, [])) of - true -> - case ConnectModule:ensure_unsubscribed(Conn, Topic) of - {error, Error} -> - {{error, Error}, State}; - Conn1 -> - {ok, State#{connection => Conn1}} - end; - false -> - {ok, State} - end. - -is_topic_present(Topic, Topics) -> - lists:member(Topic, Topics) orelse false =/= lists:keyfind(Topic, 1, Topics). - -do_connect(#{forwards := Forwards, - connect_module := ConnectModule, - connect_opts := ConnectOpts, +do_connect(#{connect_opts := ConnectOpts = #{forwards := Forwards}, inflight := Inflight, name := Name} = State) -> - ok = subscribe_local_topics(Forwards, Name), - case ConnectModule:start(ConnectOpts) of + case Forwards of + undefined -> ok; + #{subscribe_local_topic := Topic} -> subscribe_local_topic(Topic, Name) + end, + case emqx_connector_mqtt_mod:start(ConnectOpts) of {ok, Conn} -> ?tp(info, connected, #{name => Name, inflight => length(Inflight)}), {ok, State#{connection => Conn}}; {error, Reason} -> ConnectOpts1 = obfuscate(ConnectOpts), - ?LOG(error, "Failed to connect with module=~p\n" - "config=~p\nreason:~p", [ConnectModule, ConnectOpts1, Reason]), + ?LOG(error, "Failed to connect \n" + "config=~p\nreason:~p", [ConnectOpts1, Reason]), {error, Reason, State} end. @@ -441,22 +374,19 @@ retry_inflight(State, [#{q_ack_ref := QAckRef, batch := Batch} | Rest] = OldInf) {error, State1#{inflight := NewInf ++ OldInf}} end. -pop_and_send(#{inflight := Inflight, max_inflight := Max } = State) -> +pop_and_send(#{inflight := Inflight, max_inflight := Max} = State) -> pop_and_send_loop(State, Max - length(Inflight)). pop_and_send_loop(State, 0) -> ?tp(debug, inflight_full, #{}), {ok, State}; -pop_and_send_loop(#{replayq := Q, connect_module := Module} = State, N) -> +pop_and_send_loop(#{replayq := Q} = State, N) -> case replayq:is_empty(Q) of true -> ?tp(debug, replayq_drained, #{}), {ok, State}; false -> - BatchSize = case Module of - emqx_bridge_rpc -> maps:get(batch_size, State); - _ -> 1 - end, + BatchSize = 1, Opts = #{count_limit => BatchSize, bytes_limit => 999999999}, {Q1, QAckRef, Batch} = replayq:pop(Q, Opts), case do_send(State#{replayq := Q1}, QAckRef, Batch) of @@ -466,16 +396,20 @@ pop_and_send_loop(#{replayq := Q, connect_module := Module} = State, N) -> end. %% Assert non-empty batch because we have a is_empty check earlier. +do_send(#{connect_opts := #{forwards := undefined}}, _QAckRef, Batch) -> + ?LOG(error, "cannot forward messages to remote broker as 'bridge.mqtt..in' not configured, msg: ~p", [Batch]); do_send(#{inflight := Inflight, - connect_module := Module, connection := Connection, mountpoint := Mountpoint, + connect_opts := #{forwards := Forwards}, if_record_metrics := IfRecordMetrics} = State, QAckRef, [_ | _] = Batch) -> + Vars = emqx_connector_mqtt_msg:make_pub_vars(Mountpoint, Forwards), ExportMsg = fun(Message) -> bridges_metrics_inc(IfRecordMetrics, 'bridge.mqtt.message_sent'), - emqx_bridge_msg:to_export(Module, Mountpoint, Message) + emqx_connector_mqtt_msg:to_remote_msg(Message, Vars) end, - case Module:send(Connection, [ExportMsg(M) || M <- Batch]) of + ?LOG(debug, "publish to remote broker, msg: ~p, vars: ~p", [Batch, Vars]), + case emqx_connector_mqtt_mod:send(Connection, [ExportMsg(M) || M <- Batch]) of {ok, Refs} -> {ok, State#{inflight := Inflight ++ [#{q_ack_ref => QAckRef, send_ack_ref => map_set(Refs), @@ -530,9 +464,6 @@ drop_acked_batches(Q, [#{send_ack_ref := Refs, All end. -subscribe_local_topics(Topics, Name) -> - lists:foreach(fun(Topic) -> subscribe_local_topic(Topic, Name) end, Topics). - subscribe_local_topic(Topic, Name) -> do_subscribe(Topic, Name). @@ -549,32 +480,25 @@ validate(RawTopic) -> do_subscribe(RawTopic, Name) -> TopicFilter = validate(RawTopic), - {Topic, SubOpts} = emqx_topic:parse(TopicFilter, #{qos => ?QOS_1}), + {Topic, SubOpts} = emqx_topic:parse(TopicFilter, #{qos => ?QOS_2}), emqx_broker:subscribe(Topic, Name, SubOpts). -do_unsubscribe(RawTopic) -> - TopicFilter = validate(RawTopic), - {Topic, _SubOpts} = emqx_topic:parse(TopicFilter), - emqx_broker:unsubscribe(Topic). - -disconnect(#{connection := Conn, - connect_module := Module - } = State) when Conn =/= undefined -> - Module:stop(Conn), +disconnect(#{connection := Conn} = State) when Conn =/= undefined -> + emqx_connector_mqtt_mod:stop(Conn), State#{connection => undefined}; disconnect(State) -> State. %% Called only when replayq needs to dump it to disk. -msg_marshaller(Bin) when is_binary(Bin) -> emqx_bridge_msg:from_binary(Bin); -msg_marshaller(Msg) -> emqx_bridge_msg:to_binary(Msg). +msg_marshaller(Bin) when is_binary(Bin) -> emqx_connector_mqtt_msg:from_binary(Bin); +msg_marshaller(Msg) -> emqx_connector_mqtt_msg:to_binary(Msg). format_mountpoint(undefined) -> undefined; format_mountpoint(Prefix) -> binary:replace(iolist_to_binary(Prefix), <<"${node}">>, atom_to_binary(node(), utf8)). -name(Id) -> list_to_atom(lists:concat([?MODULE, "_", Id])). +name(Id) -> list_to_atom(str(Id)). register_metrics() -> lists:foreach(fun emqx_metrics:ensure/1, @@ -603,9 +527,9 @@ obfuscate(Map) -> is_sensitive(password) -> true; is_sensitive(_) -> false. -conn_type(rpc) -> - emqx_bridge_rpc; -conn_type(mqtt) -> - emqx_bridge_mqtt; -conn_type(Mod) when is_atom(Mod) -> - Mod. +str(A) when is_atom(A) -> + atom_to_list(A); +str(B) when is_binary(B) -> + binary_to_list(B); +str(S) when is_list(S) -> + S. \ No newline at end of file diff --git a/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_tests.erl b/apps/emqx_connector/test/emqx_connector_mqtt_tests.erl similarity index 87% rename from apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_tests.erl rename to apps/emqx_connector/test/emqx_connector_mqtt_tests.erl index 5babe0ed9..0f4d651c9 100644 --- a/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_tests.erl +++ b/apps/emqx_connector/test/emqx_connector_mqtt_tests.erl @@ -14,7 +14,7 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_bridge_mqtt_tests). +-module(emqx_connector_mqtt_tests). -include_lib("eunit/include/eunit.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). @@ -37,11 +37,11 @@ send_and_ack_test() -> try Max = 1, Batch = lists:seq(1, Max), - {ok, Conn} = emqx_bridge_mqtt:start(#{address => "127.0.0.1:1883"}), + {ok, Conn} = emqx_connector_mqtt_mod:start(#{server => {{127,0,0,1}, 1883}}), % %% return last packet id as batch reference - {ok, _AckRef} = emqx_bridge_mqtt:send(Conn, Batch), + {ok, _AckRef} = emqx_connector_mqtt_mod:send(Conn, Batch), - ok = emqx_bridge_mqtt:stop(Conn) + ok = emqx_connector_mqtt_mod:stop(Conn) after meck:unload(emqtt) end. diff --git a/apps/emqx_bridge_mqtt/test/emqx_bridge_worker_tests.erl b/apps/emqx_connector/test/emqx_connector_mqtt_worker_tests.erl similarity index 55% rename from apps/emqx_bridge_mqtt/test/emqx_bridge_worker_tests.erl rename to apps/emqx_connector/test/emqx_connector_mqtt_worker_tests.erl index ffa2e9ee5..090106cef 100644 --- a/apps/emqx_bridge_mqtt/test/emqx_bridge_worker_tests.erl +++ b/apps/emqx_connector/test/emqx_connector_mqtt_worker_tests.erl @@ -14,14 +14,14 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_bridge_worker_tests). +-module(emqx_connector_mqtt_worker_tests). -include_lib("eunit/include/eunit.hrl"). -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). -define(BRIDGE_NAME, test). --define(BRIDGE_REG_NAME, emqx_bridge_worker_test). +-define(BRIDGE_REG_NAME, emqx_connector_mqtt_worker_test). -define(WAIT(PATTERN, TIMEOUT), receive PATTERN -> @@ -31,7 +31,6 @@ error(timeout) end). -%% stub callbacks -export([start/1, send/2, stop/1]). start(#{connect_result := Result, test_pid := Pid, test_ref := Ref}) -> @@ -49,33 +48,41 @@ stop(_Pid) -> ok. %% bridge worker should retry connecting remote node indefinitely % reconnect_test() -> % emqx_metrics:start_link(), -% emqx_bridge_worker:register_metrics(), +% emqx_connector_mqtt_worker:register_metrics(), % Ref = make_ref(), % Config = make_config(Ref, self(), {error, test}), -% {ok, Pid} = emqx_bridge_worker:start_link(?BRIDGE_NAME, Config), +% {ok, Pid} = emqx_connector_mqtt_worker:start_link(?BRIDGE_NAME, Config), % %% assert name registered % ?assertEqual(Pid, whereis(?BRIDGE_REG_NAME)), % ?WAIT({connection_start_attempt, Ref}, 1000), % %% expect same message again % ?WAIT({connection_start_attempt, Ref}, 1000), -% ok = emqx_bridge_worker:stop(?BRIDGE_REG_NAME), +% ok = emqx_connector_mqtt_worker:stop(?BRIDGE_REG_NAME), % emqx_metrics:stop(), % ok. %% connect first, disconnect, then connect again disturbance_test() -> - emqx_metrics:start_link(), - emqx_bridge_worker:register_metrics(), - Ref = make_ref(), - TestPid = self(), - Config = make_config(Ref, TestPid, {ok, #{client_pid => TestPid}}), - {ok, Pid} = emqx_bridge_worker:start_link(Config#{name => disturbance}), - ?assertEqual(Pid, whereis(emqx_bridge_worker_disturbance)), - ?WAIT({connection_start_attempt, Ref}, 1000), - Pid ! {disconnected, TestPid, test}, - ?WAIT({connection_start_attempt, Ref}, 1000), - emqx_metrics:stop(), - ok = emqx_bridge_worker:stop(Pid). + meck:new(emqx_connector_mqtt_mod, [passthrough, no_history]), + meck:expect(emqx_connector_mqtt_mod, start, 1, fun(Conf) -> start(Conf) end), + meck:expect(emqx_connector_mqtt_mod, send, 2, fun(SendFun, Batch) -> send(SendFun, Batch) end), + meck:expect(emqx_connector_mqtt_mod, stop, 1, fun(Pid) -> stop(Pid) end), + try + emqx_metrics:start_link(), + emqx_connector_mqtt_worker:register_metrics(), + Ref = make_ref(), + TestPid = self(), + Config = make_config(Ref, TestPid, {ok, #{client_pid => TestPid}}), + {ok, Pid} = emqx_connector_mqtt_worker:start_link(Config#{name => bridge_disturbance}), + ?assertEqual(Pid, whereis(bridge_disturbance)), + ?WAIT({connection_start_attempt, Ref}, 1000), + Pid ! {disconnected, TestPid, test}, + ?WAIT({connection_start_attempt, Ref}, 1000), + emqx_metrics:stop(), + ok = emqx_connector_mqtt_worker:stop(Pid) + after + meck:unload(emqx_connector_mqtt_mod) + end. % % %% buffer should continue taking in messages when disconnected % buffer_when_disconnected_test_() -> @@ -96,40 +103,47 @@ disturbance_test() -> % Config0 = make_config(Ref, false, {ok, #{client_pid => undefined}}), % Config = Config0#{reconnect_delay_ms => 100}, % emqx_metrics:start_link(), -% emqx_bridge_worker:register_metrics(), -% {ok, Pid} = emqx_bridge_worker:start_link(?BRIDGE_NAME, Config), +% emqx_connector_mqtt_worker:register_metrics(), +% {ok, Pid} = emqx_connector_mqtt_worker:start_link(?BRIDGE_NAME, Config), % Sender ! {bridge, Pid}, % Receiver ! {bridge, Pid}, % ?assertEqual(Pid, whereis(?BRIDGE_REG_NAME)), % Pid ! {disconnected, Ref, test}, % ?WAIT({'DOWN', SenderMref, process, Sender, normal}, 5000), % ?WAIT({'DOWN', ReceiverMref, process, Receiver, normal}, 1000), -% ok = emqx_bridge_worker:stop(?BRIDGE_REG_NAME), +% ok = emqx_connector_mqtt_worker:stop(?BRIDGE_REG_NAME), % emqx_metrics:stop(). manual_start_stop_test() -> - emqx_metrics:start_link(), - emqx_bridge_worker:register_metrics(), - Ref = make_ref(), - TestPid = self(), - BridgeName = manual_start_stop, - Config0 = make_config(Ref, TestPid, {ok, #{client_pid => TestPid}}), - Config = Config0#{start_type := manual}, - {ok, Pid} = emqx_bridge_worker:start_link(Config#{name => BridgeName}), - %% call ensure_started again should yeld the same result - ok = emqx_bridge_worker:ensure_started(BridgeName), - emqx_bridge_worker:ensure_stopped(BridgeName), - emqx_metrics:stop(), - ok = emqx_bridge_worker:stop(Pid). + meck:new(emqx_connector_mqtt_mod, [passthrough, no_history]), + meck:expect(emqx_connector_mqtt_mod, start, 1, fun(Conf) -> start(Conf) end), + meck:expect(emqx_connector_mqtt_mod, send, 2, fun(SendFun, Batch) -> send(SendFun, Batch) end), + meck:expect(emqx_connector_mqtt_mod, stop, 1, fun(Pid) -> stop(Pid) end), + try + emqx_metrics:start_link(), + emqx_connector_mqtt_worker:register_metrics(), + Ref = make_ref(), + TestPid = self(), + BridgeName = manual_start_stop, + Config0 = make_config(Ref, TestPid, {ok, #{client_pid => TestPid}}), + Config = Config0#{start_type := manual}, + {ok, Pid} = emqx_connector_mqtt_worker:start_link(Config#{name => BridgeName}), + %% call ensure_started again should yeld the same result + ok = emqx_connector_mqtt_worker:ensure_started(BridgeName), + emqx_connector_mqtt_worker:ensure_stopped(BridgeName), + emqx_metrics:stop(), + ok = emqx_connector_mqtt_worker:stop(Pid) + after + meck:unload(emqx_connector_mqtt_mod) + end. make_config(Ref, TestPid, Result) -> #{ start_type => auto, + subscriptions => undefined, + forwards => undefined, reconnect_interval => 50, - config => #{ - test_pid => TestPid, - test_ref => Ref, - conn_type => ?MODULE, - connect_result => Result - } + test_pid => TestPid, + test_ref => Ref, + connect_result => Result }. diff --git a/apps/emqx_dashboard/src/emqx_dashboard.erl b/apps/emqx_dashboard/src/emqx_dashboard.erl index 603d8009b..d109dd445 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard.erl @@ -127,6 +127,16 @@ listener_name(Protocol, Port) -> authorize_appid(Req) -> case cowboy_req:parse_header(<<"authorization">>, Req) of + {basic, Username, Password} -> + case emqx_dashboard_admin:check(Username, Password) of + ok -> + ok; + {error, _} -> + {401, #{<<"WWW-Authenticate">> => + <<"Basic Realm=\"minirest-server\"">>}, + #{code => <<"ERROR_USERNAME_OR_PWD">>, + message => <<"Check your username and password">>}} + end; {bearer, Token} -> case emqx_dashboard_admin:verify_token(Token) of ok -> @@ -135,8 +145,7 @@ authorize_appid(Req) -> {401, #{<<"WWW-Authenticate">> => <<"Bearer Realm=\"minirest-server\"">>}, #{code => <<"TOKEN_TIME_OUT">>, - message => <<"POST '/login', get your new token">>} - }; + message => <<"POST '/login', get your new token">>}}; {error, not_found} -> {401, #{<<"WWW-Authenticate">> => <<"Bearer Realm=\"minirest-server\"">>}, @@ -145,7 +154,7 @@ authorize_appid(Req) -> end; _ -> {401, #{<<"WWW-Authenticate">> => - <<"Bearer Realm=\"minirest-server\"">>}, - #{code => <<"UNAUTHORIZED">>, - message => <<"POST '/login'">>}} + <<"Basic Realm=\"minirest-server\"">>}, + #{code => <<"ERROR_USERNAME_OR_PWD">>, + message => <<"Check your username and password">>}} end. diff --git a/apps/emqx_dashboard/src/emqx_dashboard_admin.erl b/apps/emqx_dashboard/src/emqx_dashboard_admin.erl index 8a1306e94..b477bd779 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_admin.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_admin.erl @@ -40,7 +40,7 @@ -export([ sign_token/2 , verify_token/1 - , destroy_token_by_username/1 + , destroy_token_by_username/2 ]). -export([add_default_user/0]). @@ -177,8 +177,13 @@ sign_token(Username, Password) -> verify_token(Token) -> emqx_dashboard_token:verify(Token). -destroy_token_by_username(Username) -> - emqx_dashboard_token:destroy_by_username(Username). +destroy_token_by_username(Username, Token) -> + case emqx_dashboard_token:lookup(Token) of + {ok, #mqtt_admin_jwt{username = Username}} -> + emqx_dashboard_token:destroy(Token); + _ -> + {error, not_found} + end. %%-------------------------------------------------------------------- %% Internal functions diff --git a/apps/emqx_dashboard/src/emqx_dashboard_api.erl b/apps/emqx_dashboard/src/emqx_dashboard_api.erl index 88ae85d9d..68c737488 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_api.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_api.erl @@ -49,6 +49,8 @@ -define(EMPTY(V), (V == undefined orelse V == <<>>)). +-define(ERROR_USERNAME_OR_PWD, 'ERROR_USERNAME_OR_PWD'). + api_spec() -> {[ login_api() , logout_api() @@ -164,14 +166,18 @@ login(post, #{body := Params}) -> {ok, Token} -> Version = iolist_to_binary(proplists:get_value(version, emqx_sys:info())), {200, #{token => Token, version => Version, license => #{edition => ?RELEASE}}}; - {error, Code} -> - {401, #{code => Code, message => <<"Auth filed">>}} + {error, _} -> + {401, #{code => ?ERROR_USERNAME_OR_PWD, message => <<"Auth filed">>}} end. -logout(_, #{body := Params}) -> - Username = maps:get(<<"username">>, Params), - emqx_dashboard_admin:destroy_token_by_username(Username), - {200}. +logout(_, #{body := #{<<"username">> := Username}, + headers := #{<<"authorization">> := <<"Bearer ", Token/binary>>}}) -> + case emqx_dashboard_admin:destroy_token_by_username(Username, Token) of + ok -> + 200; + _R -> + {401, 'BAD_TOKEN_OR_USERNAME', <<"Ensure your token & username">>} + end. users(get, _Request) -> {200, [row(User) || User <- emqx_dashboard_admin:all_users()]}; @@ -233,7 +239,7 @@ parameters() -> unauthorized_request() -> object_schema( properties([{message, string}, - {code, string, <<"Resp Code">>, ['PASSWORD_ERROR','USERNAME_ERROR']} + {code, string, <<"Resp Code">>, [?ERROR_USERNAME_OR_PWD]} ]), <<"Unauthorized">> ). diff --git a/apps/emqx_dashboard/src/emqx_dashboard_schema.erl b/apps/emqx_dashboard/src/emqx_dashboard_schema.erl index 7dfbc923b..3ba3dc803 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_schema.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_schema.erl @@ -27,19 +27,19 @@ fields("emqx_dashboard") -> hoconsc:ref(?MODULE, "https")]))} , {default_username, fun default_username/1} , {default_password, fun default_password/1} - , {sample_interval, emqx_schema:t(emqx_schema:duration_s(), undefined, "10s")} - , {token_expired_time, emqx_schema:t(emqx_schema:duration(), undefined, "30m")} + , {sample_interval, sc(emqx_schema:duration_s(), #{default => "10s"})} + , {token_expired_time, sc(emqx_schema:duration(), #{default => "30m"})} ]; fields("http") -> [ {"protocol", hoconsc:enum([http, https])} - , {"port", emqx_schema:t(integer(), undefined, 18083)} - , {"num_acceptors", emqx_schema:t(integer(), undefined, 4)} - , {"max_connections", emqx_schema:t(integer(), undefined, 512)} - , {"backlog", emqx_schema:t(integer(), undefined, 1024)} - , {"send_timeout", emqx_schema:t(emqx_schema:duration(), undefined, "5s")} - , {"inet6", emqx_schema:t(boolean(), undefined, false)} - , {"ipv6_v6only", emqx_schema:t(boolean(), undefined, false)} + , {"port", hoconsc:mk(integer(), #{default => 18083})} + , {"num_acceptors", sc(integer(), #{default => 4})} + , {"max_connections", sc(integer(), #{default => 512})} + , {"backlog", sc(integer(), #{default => 1024})} + , {"send_timeout", sc(emqx_schema:duration(), #{default => "5s"})} + , {"inet6", sc(boolean(), #{default => false})} + , {"ipv6_v6only", sc(boolean(), #{dfeault => false})} ]; fields("https") -> @@ -54,3 +54,5 @@ default_password(type) -> string(); default_password(default) -> "public"; default_password(nullable) -> false; default_password(_) -> undefined. + +sc(Type, Meta) -> hoconsc:mk(Type, Meta). diff --git a/apps/emqx_dashboard/src/emqx_dashboard_token.erl b/apps/emqx_dashboard/src/emqx_dashboard_token.erl index 9086b4c2e..2acf00f13 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_token.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_token.erl @@ -22,6 +22,7 @@ -export([ sign/2 , verify/1 + , lookup/1 , destroy/1 , destroy_by_username/1 ]). @@ -121,14 +122,15 @@ do_verify(Token)-> do_destroy(Token) -> Fun = fun mnesia:delete/1, - ekka_mnesia:transaction(?DASHBOARD_SHARD, Fun, [{?TAB, Token}]). + {atomic, ok} = ekka_mnesia:transaction(?DASHBOARD_SHARD, Fun, [{?TAB, Token}]), + ok. do_destroy_by_username(Username) -> gen_server:cast(?MODULE, {destroy, Username}). %%-------------------------------------------------------------------- %% jwt internal util function - +-spec(lookup(Token :: binary()) -> {ok, #mqtt_admin_jwt{}} | {error, not_found}). lookup(Token) -> case mnesia:dirty_read(?TAB, Token) of [JWT] -> {ok, JWT}; diff --git a/apps/emqx_data_bridge/etc/emqx_data_bridge.conf b/apps/emqx_data_bridge/etc/emqx_data_bridge.conf deleted file mode 100644 index 99a49dba3..000000000 --- a/apps/emqx_data_bridge/etc/emqx_data_bridge.conf +++ /dev/null @@ -1,129 +0,0 @@ -##-------------------------------------------------------------------- -## EMQ X Bridge Plugin -##-------------------------------------------------------------------- - -emqx_data_bridge { - bridges:[ - # {name: "mysql_bridge_1" - # type: mysql - # config: { - # server: "192.168.0.172:3306" - # database: mqtt - # pool_size: 1 - # username: root - # password: public - # auto_reconnect: true - # ssl: false - # } - # } - # , {name: "pgsql_bridge_1" - # type: pgsql - # config: { - # server: "192.168.0.172:5432" - # database: mqtt - # pool_size: 1 - # username: root - # password: public - # auto_reconnect: true - # ssl: false - # } - # } - # , {name: "mongodb_bridge_single" - # type: mongo - # config: { - # servers: "192.168.0.172:27017" - # mongo_type: single - # pool_size: 1 - # login: root - # password: public - # auth_source: mqtt - # database: mqtt - # ssl: false - # } - # } - # ,{name: "mongodb_bridge_rs" - # type: mongo - # config: { - # servers: "127.0.0.1:27017" - # mongo_type: rs - # rs_set_name: rs_name - # pool_size: 1 - # login: root - # password: public - # auth_source: mqtt - # database: mqtt - # ssl: false - # } - # } - # ,{name: "mongodb_bridge_shared" - # type: mongo - # config: { - # servers: "127.0.0.1:27017" - # mongo_type: shared - # pool_size: 1 - # login: root - # password: public - # auth_source: mqtt - # database: mqtt - # ssl: false - # max_overflow: 1 - # overflow_ttl: - # overflow_check_period: 10s - # local_threshold_ms: 10s - # connect_timeout_ms: 10s - # socket_timeout_ms: 10s - # server_selection_timeout_ms: 10s - # wait_queue_timeout_ms: 10s - # heartbeat_frequency_ms: 10s - # min_heartbeat_frequency_ms: 10s - # } - # } - # , {name: "redis_bridge_single" - # type: redis - # config: { - # servers: "192.168.0.172:6379" - # redis_type: single - # pool_size: 1 - # database: 0 - # password: public - # auto_reconnect: true - # ssl: false - # } - # } - # ,{name: "redis_bridge_sentinel" - # type: redis - # config: { - # servers: "127.0.0.1:6379, 127.0.0.2:6379, 127.0.0.3:6379" - # redis_type: sentinel - # sentinel_name: mymaster - # pool_size: 1 - # database: 0 - # ssl: false - # } - # } - # ,{name: "redis_bridge_cluster" - # type: redis - # config: { - # servers: "127.0.0.1:6379, 127.0.0.2:6379, 127.0.0.3:6379" - # redis_type: cluster - # pool_size: 1 - # database: 0 - # password: "public" - # ssl: false - # } - # } - # , {name: "ldap_bridge_1" - # type: ldap - # config: { - # servers: "192.168.0.172" - # port: 389 - # bind_dn: "cn=root,dc=emqx,dc=io" - # bind_password: "public" - # timeout: 30s - # pool_size: 1 - # ssl: false - # } - # } - - ] -} diff --git a/apps/emqx_data_bridge/src/emqx_data_bridge_schema.erl b/apps/emqx_data_bridge/src/emqx_data_bridge_schema.erl index 69f53d6c1..fe2d3947d 100644 --- a/apps/emqx_data_bridge/src/emqx_data_bridge_schema.erl +++ b/apps/emqx_data_bridge/src/emqx_data_bridge_schema.erl @@ -14,13 +14,12 @@ fields("emqx_data_bridge") -> [{bridges, #{type => hoconsc:array(hoconsc:union(?BRIDGES)), default => []}}]; -fields(mysql) -> connector_fields(mysql); -fields(pgsql) -> connector_fields(pgsql); -fields(mongo) -> connector_fields(mongo); -fields(redis) -> connector_fields(redis); -fields(ldap) -> connector_fields(ldap). +fields(mysql) -> connector_fields(emqx_connector_mysql, mysql); +fields(pgsql) -> connector_fields(emqx_connector_pgsql, pgsql); +fields(mongo) -> connector_fields(emqx_connector_mongo, mongo); +fields(redis) -> connector_fields(emqx_connector_redis, redis); +fields(ldap) -> connector_fields(emqx_connector_ldap, ldap). -connector_fields(DB) -> - Mod = list_to_existing_atom(io_lib:format("~s_~s",[emqx_connector, DB])), - [{name, hoconsc:t(typerefl:binary())}, - {type, #{type => DB}}] ++ Mod:roots(). +connector_fields(ConnectModule, DB) -> + [{name, hoconsc:mk(typerefl:binary())}, + {type, #{type => DB}}] ++ ConnectModule:roots(). diff --git a/apps/emqx_exhook/src/emqx_exhook_schema.erl b/apps/emqx_exhook/src/emqx_exhook_schema.erl index 16fd93fa0..64d39eb52 100644 --- a/apps/emqx_exhook/src/emqx_exhook_schema.erl +++ b/apps/emqx_exhook/src/emqx_exhook_schema.erl @@ -32,43 +32,57 @@ -reflect_type([duration/0]). --export([roots/0, fields/1]). +-export([namespace/0, roots/0, fields/1]). --export([t/1, t/3, t/4, ref/1]). +namespace() -> exhook. roots() -> [exhook]. fields(exhook) -> - [ {request_failed_action, t(union([deny, ignore]), undefined, deny)} - , {request_timeout, t(duration(), undefined, "5s")} - , {auto_reconnect, t(union([false, duration()]), undefined, "60s")} - , {servers, t(hoconsc:array(ref(servers)), undefined, [])} + [ {request_failed_action, + sc(union([deny, ignore]), + #{default => deny})} + , {request_timeout, + sc(duration(), + #{default => "5s"})} + , {auto_reconnect, + sc(union([false, duration()]), + #{ default => "60s" + })} + , {servers, + sc(hoconsc:array(ref(servers)), + #{default => []})} ]; fields(servers) -> - [ {name, string()} - , {url, string()} - , {ssl, t(ref(ssl_conf_group))} + [ {name, + sc(string(), + #{})} + , {url, + sc(string(), + #{})} + , {ssl, + sc(ref(ssl_conf), + #{})} ]; -fields(ssl_conf_group) -> - [ {cacertfile, string()} - , {certfile, string()} - , {keyfile, string()} +fields(ssl_conf) -> + [ {cacertfile, + sc(string(), + #{}) + } + , {certfile, + sc(string(), + #{}) + } + , {keyfile, + sc(string(), + #{})} ]. %% types -t(Type) -> #{type => Type}. - -t(Type, Mapping, Default) -> - hoconsc:t(Type, #{mapping => Mapping, default => Default}). - -t(Type, Mapping, Default, OverrideEnv) -> - hoconsc:t(Type, #{ mapping => Mapping - , default => Default - , override_env => OverrideEnv - }). +sc(Type, Meta) -> Meta#{type => Type}. ref(Field) -> hoconsc:ref(?MODULE, Field). diff --git a/apps/emqx_gateway/etc/emqx_gateway.conf b/apps/emqx_gateway/etc/emqx_gateway.conf index 206c54b93..2ce48bf75 100644 --- a/apps/emqx_gateway/etc/emqx_gateway.conf +++ b/apps/emqx_gateway/etc/emqx_gateway.conf @@ -29,19 +29,59 @@ gateway.stomp { password = "${Packet.headers.passcode}" } - authentication { - name = "authenticator1" - mechanism = password-based - server_type = built-in-database - user_id_type = clientid - } + authentication: [ + # { + # name = "authenticator1" + # type = "password-based:built-in-database" + # user_id_type = clientid + # } + ] listeners.tcp.default { bind = 61613 acceptors = 16 max_connections = 1024000 max_conn_rate = 1000 - active_n = 100 + + access_rules = [ + "allow all" + ] + + ## TCP options + ## See ${example_common_tcp_options} for more information + tcp.active_n = 100 + tcp.backlog = 1024 + tcp.buffer = 4KB + } + + listeners.ssl.default { + bind = 61614 + acceptors = 16 + max_connections = 1024000 + max_conn_rate = 1000 + + ## TCP options + ## See ${example_common_tcp_options} for more information + tcp.active_n = 100 + tcp.backlog = 1024 + tcp.buffer = 4KB + + ## SSL options + ## See ${example_common_ssl_options} for more information + ssl.versions = ["tlsv1.3", "tlsv1.2", "tlsv1.1", "tlsv1"] + ssl.keyfile = "{{ platform_etc_dir }}/certs/key.pem" + ssl.certfile = "{{ platform_etc_dir }}/certs/cert.pem" + ssl.cacertfile = "{{ platform_etc_dir }}/certs/cacert.pem" + #ssl.verify = verify_none + #ssl.fail_if_no_peer_cert = false + #ssl.server_name_indication = disable + #ssl.secure_renegotiate = false + #ssl.reuse_sessions = false + #ssl.honor_cipher_order = false + #ssl.handshake_timeout = 15s + #ssl.depth = 10 + #ssl.password = foo + #ssl.dhfile = path-to-your-file } } @@ -58,20 +98,41 @@ gateway.coap { ## When publishing or subscribing, prefix all topics with a mountpoint string. mountpoint = "" - heartbeat = 30s notify_type = qos + + ## if true, you need to establish a connection before use + connection_required = false subscribe_qos = qos0 publish_qos = qos1 - authentication { - name = "authenticator1" - mechanism = password-based - server_type = built-in-database - user_id_type = clientid - } - listeners.udp.default { bind = 5683 + acceptors = 4 + max_connections = 102400 + max_conn_rate = 1000 + + ## UDP Options + ## See ${example_common_udp_options} for more information + udp.active_n = 100 + udp.buffer = 16KB + } + listeners.dtls.default { + bind = 5684 + acceptors = 4 + max_connections = 102400 + max_conn_rate = 1000 + + ## UDP Options + ## See ${example_common_udp_options} for more information + udp.active_n = 100 + udp.buffer = 16KB + + ## DTLS Options + ## See #{example_common_dtls_options} for more information + dtls.versions = ["dtlsv1.2", "dtlsv1"] + dtls.keyfile = "{{ platform_etc_dir }}/certs/key.pem" + dtls.certfile = "{{ platform_etc_dir }}/certs/cert.pem" + dtls.cacertfile = "{{ platform_etc_dir }}/certs/cacert.pem" } } @@ -121,6 +182,26 @@ gateway.mqttsn { max_connections = 10240000 max_conn_rate = 1000 } + + listeners.dtls.default { + bind = 1885 + acceptors = 4 + max_connections = 102400 + max_conn_rate = 1000 + + ## UDP Options + ## See ${example_common_udp_options} for more information + udp.active_n = 100 + udp.buffer = 16KB + + ## DTLS Options + ## See #{example_common_dtls_options} for more information + dtls.versions = ["dtlsv1.2", "dtlsv1"] + dtls.keyfile = "{{ platform_etc_dir }}/certs/key.pem" + dtls.certfile = "{{ platform_etc_dir }}/certs/cert.pem" + dtls.cacertfile = "{{ platform_etc_dir }}/certs/cacert.pem" + } + } gateway.lwm2m { @@ -134,7 +215,7 @@ gateway.lwm2m { enable_stats = true ## When publishing or subscribing, prefix all topics with a mountpoint string. - mountpoint = "lwm2m/%e/" + mountpoint = "lwm2m/%u" xml_dir = "{{ platform_etc_dir }}/lwm2m_xml" @@ -146,12 +227,32 @@ gateway.lwm2m { ## always | contains_object_list update_msg_publish_condition = contains_object_list + translators { - command = "dn/#" - response = "up/resp" - notify = "up/notify" - register = "up/resp" - update = "up/resp" + command { + topic = "/dn/#" + qos = 0 + } + + response { + topic = "/up/resp" + qos = 0 + } + + notify { + topic = "/up/notify" + qos = 0 + } + + register { + topic = "/up/resp" + qos = 0 + } + + update { + topic = "/up/resp" + qos = 0 + } } listeners.udp.default { diff --git a/apps/emqx_gateway/src/bhvrs/emqx_gateway_channel.erl b/apps/emqx_gateway/src/bhvrs/emqx_gateway_channel.erl index 06efe4fd0..c4fd114e4 100644 --- a/apps/emqx_gateway/src/bhvrs/emqx_gateway_channel.erl +++ b/apps/emqx_gateway/src/bhvrs/emqx_gateway_channel.erl @@ -44,6 +44,8 @@ -type conn_state() :: idle | connecting | connected | disconnected | atom(). +-type gen_server_from() :: {pid(), Tag :: term()}. + -type reply() :: {outgoing, emqx_gateway_frame:packet()} | {outgoing, [emqx_gateway_frame:packet()]} | {event, conn_state() | updated} @@ -71,11 +73,13 @@ | {shutdown, Reason :: any(), channel()}. %% @doc Handle the custom gen_server:call/2 for its connection process --callback handle_call(Req :: any(), channel()) +-callback handle_call(Req :: any(), From :: gen_server_from(), channel()) -> {reply, Reply :: any(), channel()} %% Reply to caller and trigger an event(s) | {reply, Reply :: any(), - EventOrEvents:: tuple() | list(tuple()), channel()} + EventOrEvents :: tuple() | list(tuple()), channel()} + | {noreply, channel()} + | {noreply, EventOrEvents :: tuple() | list(tuple()), channel()} | {shutdown, Reason :: any(), Reply :: any(), channel()} %% Shutdown the process, reply to caller and write a packet to client | {shutdown, Reason :: any(), Reply :: any(), diff --git a/apps/emqx_gateway/src/bhvrs/emqx_gateway_conn.erl b/apps/emqx_gateway/src/bhvrs/emqx_gateway_conn.erl index fa0a830e5..543b2e169 100644 --- a/apps/emqx_gateway/src/bhvrs/emqx_gateway_conn.erl +++ b/apps/emqx_gateway/src/bhvrs/emqx_gateway_conn.erl @@ -20,7 +20,6 @@ -include_lib("emqx/include/types.hrl"). -include_lib("emqx/include/logger.hrl"). - %% API -export([ start_link/3 , stop/1 @@ -48,7 +47,6 @@ %% Internal callback -export([wakeup_from_hib/2, recvloop/2]). - -record(state, { %% TCP/SSL/UDP/DTLS Wrapped Socket socket :: {esockd_transport, esockd:socket()} | {udp, _, _}, @@ -226,6 +224,9 @@ esockd_send(Data, #state{socket = {udp, _SockPid, Sock}, esockd_send(Data, #state{socket = {esockd_transport, Sock}}) -> esockd_transport:async_send(Sock, Data). +is_datadram_socket({esockd_transport, _}) -> false; +is_datadram_socket({udp, _, _}) -> true. + %%-------------------------------------------------------------------- %% callbacks %%-------------------------------------------------------------------- @@ -393,6 +394,10 @@ append_msg(Q, Msg) -> handle_msg({'$gen_call', From, Req}, State) -> case handle_call(From, Req, State) of + {noreply, NState} -> + {ok, NState}; + {noreply, Msgs, NState} -> + {ok, next_msgs(Msgs), NState}; {reply, Reply, NState} -> gen_server:reply(From, Reply), {ok, NState}; @@ -544,10 +549,14 @@ handle_call(_From, info, State) -> handle_call(_From, stats, State) -> {reply, stats(State), State}; -handle_call(_From, Req, State = #state{ +handle_call(From, Req, State = #state{ chann_mod = ChannMod, channel = Channel}) -> - case ChannMod:handle_call(Req, Channel) of + case ChannMod:handle_call(Req, From, Channel) of + {noreply, NChannel} -> + {noreply, State#state{channel = NChannel}}; + {noreply, Msgs, NChannel} -> + {noreply, Msgs, State#state{channel = NChannel}}; {reply, Reply, NChannel} -> {reply, Reply, State#state{channel = NChannel}}; {reply, Reply, Msgs, NChannel} -> @@ -558,8 +567,6 @@ handle_call(_From, Req, State = #state{ NState = State#state{channel = NChannel}, ok = handle_outgoing(Packet, NState), shutdown(Reason, Reply, NState) - - end. %%-------------------------------------------------------------------- @@ -678,8 +685,20 @@ with_channel(Fun, Args, State = #state{ %%-------------------------------------------------------------------- %% Handle outgoing packets -handle_outgoing(Packets, State) when is_list(Packets) -> - send(lists:map(serialize_and_inc_stats_fun(State), Packets), State); +handle_outgoing(_Packets = [], _State) -> + ok; +handle_outgoing(Packets, + State = #state{socket = Socket}) when is_list(Packets) -> + case is_datadram_socket(Socket) of + false -> + send( + lists:map(serialize_and_inc_stats_fun(State), Packets), + State); + _ -> + lists:foreach(fun(Packet) -> + handle_outgoing(Packet, State) + end, Packets) + end; handle_outgoing(Packet, State) -> send((serialize_and_inc_stats_fun(State))(Packet), State). @@ -816,7 +835,7 @@ inc_incoming_stats(Ctx, FrameMod, Packet) -> ok end, Name = list_to_atom( - lists:concat(["packets.", FrameMod:type(Packet), ".recevied"])), + lists:concat(["packets.", FrameMod:type(Packet), ".received"])), emqx_gateway_ctx:metrics_inc(Ctx, Name). inc_outgoing_stats(Ctx, FrameMod, Packet) -> diff --git a/apps/emqx_gateway/src/coap/README.md b/apps/emqx_gateway/src/coap/README.md index 12b5ac5b7..54f0fde84 100644 --- a/apps/emqx_gateway/src/coap/README.md +++ b/apps/emqx_gateway/src/coap/README.md @@ -9,6 +9,7 @@ 4. [Query String](#org9a6b996) 2. [Implementation](#org9985dfe) 1. [Request/Response flow](#orge94210c) + 3. [Example](#ref_example) @@ -401,3 +402,42 @@ CoAP gateway uses some options in query string to conversion between MQTT CoAP. + + + +## Example +1. Create Connection +``` +coap-client -m post -e "" "coap://127.0.0.1/mqtt/connection?clientid=123&username=admin&password=public" +``` +Server will return token **X** in payload + +2. Update Connection +``` +coap-client -m put -e "" "coap://127.0.0.1/mqtt/connection?clientid=123&token=X" +``` + +3. Publish +``` +coap-client -m post -e "Hellow" "obstoken" "coap://127.0.0.1/ps/coap/test?clientid=123&username=admin&password=public" +``` +if you want to publish with auth, you must first establish a connection, and then post publish request on the same socket, so libcoap client can't simulation publish with a token + +``` +coap-client -m post -e "Hellow" "coap://127.0.0.1/ps/coap/test?clientid=123&token=X" +``` + +4. Subscribe +``` +coap-client -m get -s 60 -O 6,0x00 -o - -T "obstoken" "coap://127.0.0.1/ps/coap/test?clientid=123&username=admin&password=public" +``` +**Or** + +``` +coap-client -m get -s 60 -O 6,0x00 -o - -T "obstoken" "coap://127.0.0.1/ps/coap/test?clientid=123&token=X" +``` +5. Close Connection +``` +coap-client -m delete -e "" "coap://127.0.0.1/mqtt/connection?clientid=123&token=X +``` + diff --git a/apps/emqx_gateway/src/coap/doc/flow.png b/apps/emqx_gateway/src/coap/doc/flow.png index 5c7288348..bb9b775a5 100644 Binary files a/apps/emqx_gateway/src/coap/doc/flow.png and b/apps/emqx_gateway/src/coap/doc/flow.png differ diff --git a/apps/emqx_gateway/src/coap/emqx_coap_api.erl b/apps/emqx_gateway/src/coap/emqx_coap_api.erl new file mode 100644 index 000000000..4d0e8aff8 --- /dev/null +++ b/apps/emqx_gateway/src/coap/emqx_coap_api.erl @@ -0,0 +1,145 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2017-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_coap_api). + +-behaviour(minirest_api). + +-include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). + +%% API +-export([api_spec/0]). + +-export([request/2]). + +-define(PREFIX, "/gateway/coap/:clientid"). +-define(DEF_WAIT_TIME, 10). + +-import(emqx_mgmt_util, [ schema/1 + , schema/2 + , object_schema/1 + , object_schema/2 + , error_schema/2 + , properties/1]). + +%%-------------------------------------------------------------------- +%% API +%%-------------------------------------------------------------------- +api_spec() -> + {[request_api()], []}. + +request_api() -> + Metadata = #{post => request_method_meta()}, + {?PREFIX ++ "/request", Metadata, request}. + +request(post, #{body := Body, bindings := Bindings}) -> + ClientId = maps:get(clientid, Bindings, undefined), + + Method = maps:get(<<"method">>, Body, <<"get">>), + CT = maps:get(<<"content_type">>, Body, <<"text/plain">>), + Token = maps:get(<<"token">>, Body, <<>>), + Payload = maps:get(<<"payload">>, Body, <<>>), + WaitTime = maps:get(<<"timeout">>, Body, ?DEF_WAIT_TIME), + + Payload2 = parse_payload(CT, Payload), + ReqType = erlang:binary_to_atom(Method), + + Msg = emqx_coap_message:request(con, + ReqType, Payload2, #{content_format => CT}), + + Msg2 = Msg#coap_message{token = Token}, + + case call_client(ClientId, Msg2, timer:seconds(WaitTime)) of + timeout -> + {504, #{code => 'CLIENT_NOT_RESPONSE'}}; + not_found -> + {404, #{code => 'CLIENT_NOT_FOUND'}}; + Response -> + {200, format_to_response(CT, Response)} + end. + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- +request_parameters() -> + [#{name => clientid, + in => path, + schema => #{type => string}, + required => true}]. + +request_properties() -> + properties([ {token, string, "message token, can be empty"} + , {method, string, "request method type", ["get", "put", "post", "delete"]} + , {timeout, integer, "timespan for response"} + , {content_type, string, "payload type", + [<<"text/plain">>, <<"application/json">>, <<"application/octet-stream">>]} + , {payload, string, "payload"}]). + +coap_message_properties() -> + properties([ {id, integer, "message id"} + , {token, string, "message token, can be empty"} + , {method, string, "response code"} + , {payload, string, "payload"}]). + +request_method_meta() -> + #{description => <<"lookup matching messages">>, + parameters => request_parameters(), + 'requestBody' => object_schema(request_properties(), + <<"request payload, binary must encode by base64">>), + responses => #{ + <<"200">> => object_schema(coap_message_properties()), + <<"404">> => error_schema("client not found error", ['CLIENT_NOT_FOUND']), + <<"504">> => error_schema("timeout", ['CLIENT_NOT_RESPONSE']) + }}. + + +format_to_response(ContentType, #coap_message{id = Id, + token = Token, + method = Method, + payload = Payload}) -> + #{id => Id, + token => Token, + method => format_to_binary(Method), + payload => format_payload(ContentType, Payload)}. + +format_to_binary(Obj) -> + erlang:list_to_binary(io_lib:format("~p", [Obj])). + +format_payload(<<"application/octet-stream">>, Payload) -> + base64:encode(Payload); + +format_payload(_, Payload) -> + Payload. + +parse_payload(<<"application/octet-stream">>, Body) -> + base64:decode(Body); + +parse_payload(_, Body) -> + Body. + +call_client(ClientId, Msg, Timeout) -> + case emqx_gateway_cm_registry:lookup_channels(coap, ClientId) of + [Channel | _] -> + RequestId = emqx_coap_channel:send_request(Channel, Msg), + case gen_server:wait_response(RequestId, Timeout) of + {reply, Reply} -> + Reply; + _ -> + timeout + end; + _ -> + not_found + end. diff --git a/apps/emqx_gateway/src/coap/emqx_coap_channel.erl b/apps/emqx_gateway/src/coap/emqx_coap_channel.erl index 510432441..112efdc44 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_channel.erl +++ b/apps/emqx_gateway/src/coap/emqx_coap_channel.erl @@ -22,17 +22,13 @@ -include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). %% API --export([]). - -export([ info/1 , info/2 , stats/1 - , validator/3 - , get_clientinfo/1 - , get_config/2 - , get_config/3 - , result_keys/0 - , transfer_result/3]). + , validator/4 + , metrics_inc/2 + , run_hooks/3 + , send_request/2]). -export([ init/2 , handle_in/2 @@ -41,7 +37,7 @@ , terminate/2 ]). --export([ handle_call/2 +-export([ handle_call/3 , handle_cast/2 , handle_info/2 ]). @@ -61,20 +57,21 @@ keepalive :: emqx_keepalive:keepalive() | undefined, %% Timer timers :: #{atom() => disable | undefined | reference()}, - token :: binary() | undefined, - config :: hocon:config() + + connection_required :: boolean(), + + conn_state :: idle | connected | disconnected, + + token :: binary() | undefined }). -%% the execuate context for session call --record(exec_ctx, { config :: hocon:config(), - ctx :: emqx_gateway_ctx:context(), - clientinfo :: emqx_types:clientinfo() - }). - -type channel() :: #channel{}. --define(DISCONNECT_WAIT_TIME, timer:seconds(10)). +-define(TOKEN_MAXIMUM, 4294967295). -define(INFO_KEYS, [conninfo, conn_state, clientinfo, session]). +-define(DEF_IDLE_TIME, timer:seconds(30)). +-define(GET_IDLE_TIME(Cfg), maps:get(idle_timeout, Cfg, ?DEF_IDLE_TIME)). +-import(emqx_coap_medium, [reply/2, reply/3, reply/4, iter/3, iter/4]). %%-------------------------------------------------------------------- %% API %%-------------------------------------------------------------------- @@ -87,8 +84,8 @@ info(Keys, Channel) when is_list(Keys) -> info(conninfo, #channel{conninfo = ConnInfo}) -> ConnInfo; -info(conn_state, _) -> - connected; +info(conn_state, #channel{conn_state = CState}) -> + CState; info(clientinfo, #channel{clientinfo = ClientInfo}) -> ClientInfo; info(session, #channel{session = Session}) -> @@ -105,19 +102,14 @@ init(ConnInfo = #{peername := {PeerHost, _}, sockname := {_, SockPort}}, #{ctx := Ctx} = Config) -> Peercert = maps:get(peercert, ConnInfo, undefined), - Mountpoint = maps:get(mountpoint, Config, undefined), - EnableAuth = is_authentication_enabled(Config), + Mountpoint = maps:get(mountpoint, Config, <<>>), ClientInfo = set_peercert_infos( Peercert, #{ zone => default , protocol => 'coap' , peerhost => PeerHost , sockport => SockPort - , clientid => if EnableAuth -> - undefined; - true -> - emqx_guid:to_base62(emqx_guid:gen()) - end + , clientid => emqx_guid:to_base62(emqx_guid:gen()) , username => undefined , is_bridge => false , is_superuser => false @@ -125,63 +117,42 @@ init(ConnInfo = #{peername := {PeerHost, _}, } ), + Heartbeat = ?GET_IDLE_TIME(Config), #channel{ ctx = Ctx , conninfo = ConnInfo , clientinfo = ClientInfo , timers = #{} - , config = Config , session = emqx_coap_session:new() - , keepalive = emqx_keepalive:init(maps:get(heartbeat, Config)) + , keepalive = emqx_keepalive:init(Heartbeat) + , connection_required = maps:get(connection_required, Config, false) + , conn_state = idle }. -is_authentication_enabled(Cfg) -> - case maps:get(authentication, Cfg, #{enable => false}) of - AuthCfg when is_map(AuthCfg) -> - maps:get(enable, AuthCfg, true); - _ -> false - end. - -validator(Type, Topic, #exec_ctx{ctx = Ctx, - clientinfo = ClientInfo}) -> +validator(Type, Topic, Ctx, ClientInfo) -> emqx_gateway_ctx:authorize(Ctx, ClientInfo, Type, Topic). -get_clientinfo(#exec_ctx{clientinfo = ClientInfo}) -> - ClientInfo. - -get_config(Key, Ctx) -> - get_config(Key, Ctx, undefined). - -get_config(Key, #exec_ctx{config = Cfg}, Def) -> - maps:get(Key, Cfg, Def). - -result_keys() -> - [out, connection]. - -transfer_result(From, Value, Result) -> - ?TRANSFER_RESULT(From, Value, Result). +-spec send_request(pid(), emqx_coap_message()) -> any(). +send_request(Channel, Request) -> + gen_server:send_request(Channel, {?FUNCTION_NAME, Request}). %%-------------------------------------------------------------------- %% Handle incoming packet %%-------------------------------------------------------------------- handle_in(Msg, ChannleT) -> Channel = ensure_keepalive_timer(ChannleT), - case convert_queries(Msg) of - {ok, Msg2} -> - case emqx_coap_message:is_request(Msg2) of - true -> - check_auth_state(Msg2, Channel); - _ -> - call_session(handle_response, Msg2, Channel) - end; + case emqx_coap_message:is_request(Msg) of + true -> + check_auth_state(Msg, Channel); _ -> - response({error, bad_request}, <<"bad uri_query">>, Msg, Channel) + call_session(handle_response, Msg, Channel) end. %%-------------------------------------------------------------------- %% Handle Delivers from broker to client %%-------------------------------------------------------------------- -handle_deliver(Delivers, Channel) -> - call_session(deliver, Delivers, Channel). +handle_deliver(Delivers, #channel{session = Session, + ctx = Ctx} = Channel) -> + handle_result(emqx_coap_session:deliver(Delivers, Ctx, Session), Channel). %%-------------------------------------------------------------------- %% Handle timeout @@ -192,7 +163,7 @@ handle_timeout(_, {keepalive, NewVal}, #channel{keepalive = KeepAlive} = Channel Channel2 = ensure_keepalive_timer(fun make_timer/4, Channel), {ok, Channel2#channel{keepalive = NewKeepAlive}}; {error, timeout} -> - {shutdown, timeout, Channel} + {shutdown, timeout, ensure_disconnected(keepalive_timeout, Channel)} end; handle_timeout(_, {transport, Msg}, Channel) -> @@ -207,7 +178,11 @@ handle_timeout(_, _, Channel) -> %%-------------------------------------------------------------------- %% Handle call %%-------------------------------------------------------------------- -handle_call(Req, Channel) -> +handle_call({send_request, Msg}, From, Channel) -> + Result = call_session(handle_out, {{send_request, From}, Msg}, Channel), + erlang:setelement(1, Result, noreply); + +handle_call(Req, _From, Channel) -> ?LOG(error, "Unexpected call: ~p", [Req]), {reply, ignored, Channel}. @@ -221,6 +196,9 @@ handle_cast(Req, Channel) -> %%-------------------------------------------------------------------- %% Handle Info %%-------------------------------------------------------------------- +handle_info({subscribe, _}, Channel) -> + {ok, Channel}; + handle_info(Info, Channel) -> ?LOG(error, "Unexpected info: ~p", [Info]), {ok, Channel}. @@ -228,8 +206,10 @@ handle_info(Info, Channel) -> %%-------------------------------------------------------------------- %% Terminate %%-------------------------------------------------------------------- -terminate(_Reason, _Channel) -> - ok. +terminate(Reason, #channel{clientinfo = ClientInfo, + ctx = Ctx, + session = Session}) -> + run_hooks(Ctx, 'session.terminated', [ClientInfo, Reason, Session]). %%-------------------------------------------------------------------- %% Internal functions @@ -258,94 +238,56 @@ make_timer(Name, Time, Msg, Channel = #channel{timers = Timers}) -> ensure_keepalive_timer(Channel) -> ensure_keepalive_timer(fun ensure_timer/4, Channel). -ensure_keepalive_timer(Fun, #channel{config = Cfg} = Channel) -> - Interval = maps:get(heartbeat, Cfg), - Fun(keepalive, Interval, keepalive, Channel). +ensure_keepalive_timer(Fun, #channel{keepalive = KeepAlive} = Channel) -> + Heartbeat = emqx_keepalive:info(interval, KeepAlive), + Fun(keepalive, Heartbeat, keepalive, Channel). -call_session(Fun, - Msg, - #channel{session = Session} = Channel) -> - Ctx = new_exec_ctx(Channel), - Result = erlang:apply(emqx_coap_session, Fun, [Msg, Ctx, Session]), - process_result([session, connection, out], Result, Msg, Channel). - -process_result([Key | T], Result, Msg, Channel) -> - case handle_result(Key, Result, Msg, Channel) of - {ok, Channel2} -> - process_result(T, Result, Msg, Channel2); - Other -> - Other - end; - -process_result(_, _, _, Channel) -> - {ok, Channel}. - -handle_result(session, #{session := Session}, _, Channel) -> - {ok, Channel#channel{session = Session}}; - -handle_result(connection, #{connection := open}, Msg, Channel) -> - do_connect(Msg, Channel); - -handle_result(connection, #{connection := close}, Msg, Channel) -> - Reply = emqx_coap_message:piggyback({ok, deleted}, Msg), - {shutdown, close, {outgoing, Reply}, Channel}; - -handle_result(out, #{out := Out}, _, Channel) -> - {ok, {outgoing, Out}, Channel}; - -handle_result(_, _, _, Channel) -> - {ok, Channel}. - -check_auth_state(Msg, #channel{config = Cfg} = Channel) -> - Enable = is_authentication_enabled(Cfg), - check_token(Enable, Msg, Channel). +check_auth_state(Msg, #channel{connection_required = Required} = Channel) -> + check_token(Required, Msg, Channel). check_token(true, - #coap_message{options = Options} = Msg, + Msg, #channel{token = Token, - clientinfo = ClientInfo} = Channel) -> + clientinfo = ClientInfo, + conn_state = CState} = Channel) -> #{clientid := ClientId} = ClientInfo, - case maps:get(uri_query, Options, undefined) of + case emqx_coap_message:get_option(uri_query, Msg) of #{<<"clientid">> := ClientId, <<"token">> := Token} -> call_session(handle_request, Msg, Channel); #{<<"clientid">> := DesireId} -> - try_takeover(ClientId, DesireId, Msg, Channel); + try_takeover(CState, DesireId, Msg, Channel); _ -> - response({error, unauthorized}, Msg, Channel) + Reply = emqx_coap_message:piggyback({error, unauthorized}, Msg), + {ok, {outgoing, Reply}, Channel} end; -check_token(false, - #coap_message{options = Options} = Msg, - Channel) -> - case maps:get(uri_query, Options, undefined) of +check_token(false, Msg, Channel) -> + case emqx_coap_message:get_option(uri_query, Msg) of #{<<"clientid">> := _} -> - response({error, unauthorized}, Msg, Channel); + Reply = emqx_coap_message:piggyback({error, unauthorized}, Msg), + {ok, {outgoing, Reply}, Channel}; #{<<"token">> := _} -> - response({error, unauthorized}, Msg, Channel); + Reply = emqx_coap_message:piggyback({error, unauthorized}, Msg), + {ok, {outgoing, Reply}, Channel}; _ -> call_session(handle_request, Msg, Channel) end. -response(Method, Req, Channel) -> - response(Method, <<>>, Req, Channel). - -response(Method, Payload, Req, Channel) -> - Reply = emqx_coap_message:piggyback(Method, Payload, Req), - call_session(handle_out, Reply, Channel). - -try_takeover(undefined, - DesireId, - #coap_message{options = Opts} = Msg, - Channel) -> - case maps:get(uri_path, Opts, []) of - [<<"mqtt">>, <<"connection">> | _] -> +try_takeover(idle, DesireId, Msg, Channel) -> + case emqx_coap_message:get_option(uri_path, Msg, []) of + [<<"mqtt">>, <<"connection">> | _] -> %% may be is a connect request %% TODO need check repeat connect, unless we implement the %% udp connection baseon the clientid call_session(handle_request, Msg, Channel); _ -> - do_takeover(DesireId, Msg, Channel) + case emqx:get_config([gateway, coap, authentication], undefined) of + undefined -> + call_session(handle_request, Msg, Channel); + _ -> + do_takeover(DesireId, Msg, Channel) + end end; try_takeover(_, DesireId, Msg, Channel) -> @@ -354,31 +296,7 @@ try_takeover(_, DesireId, Msg, Channel) -> do_takeover(_DesireId, Msg, Channel) -> %% TODO completed the takeover, now only reset the message Reset = emqx_coap_message:reset(Msg), - call_session(handle_out, Reset, Channel). - -new_exec_ctx(#channel{config = Cfg, - ctx = Ctx, - clientinfo = ClientInfo}) -> - #exec_ctx{config = Cfg, - ctx = Ctx, - clientinfo = ClientInfo}. - -do_connect(#coap_message{options = Opts} = Req, Channel) -> - Queries = maps:get(uri_query, Opts), - case emqx_misc:pipeline( - [ fun run_conn_hooks/2 - , fun enrich_clientinfo/2 - , fun set_log_meta/2 - , fun auth_connect/2 - ], - {Queries, Req}, - Channel) of - {ok, _Input, NChannel} -> - process_connect(ensure_connected(NChannel), Req); - {error, ReasonCode, NChannel} -> - ErrMsg = io_lib:format("Login Failed: ~s", [ReasonCode]), - response({error, bad_request}, ErrMsg, Req, NChannel) - end. + {ok, {outgoing, Reset}, Channel}. run_conn_hooks(Input, Channel = #channel{ctx = Ctx, conninfo = ConnInfo}) -> @@ -421,11 +339,9 @@ auth_connect(_Input, Channel = #channel{ctx = Ctx, {error, Reason} end. -fix_mountpoint(_Packet, #{mountpoint := undefined} = ClientInfo) -> +fix_mountpoint(_Packet, #{mountpoint := <<>>} = ClientInfo) -> {ok, ClientInfo}; fix_mountpoint(_Packet, ClientInfo = #{mountpoint := Mountpoint}) -> - %% TODO: Enrich the varibale replacement???? - %% i.e: ${ClientInfo.auth_result.productKey} Mountpoint1 = emqx_mountpoint:replvar(Mountpoint, ClientInfo), {ok, ClientInfo#{mountpoint := Mountpoint1}}. @@ -437,13 +353,14 @@ ensure_connected(Channel = #channel{ctx = Ctx, , proto_ver => <<"1">> }, ok = run_hooks(Ctx, 'client.connected', [ClientInfo, NConnInfo]), + _ = run_hooks(Ctx, 'client.connack', [NConnInfo, connection_accepted, []]), Channel#channel{conninfo = NConnInfo}. -process_connect(Channel = #channel{ctx = Ctx, - session = Session, - conninfo = ConnInfo, - clientinfo = ClientInfo}, - Msg) -> +process_connect(#channel{ctx = Ctx, + session = Session, + conninfo = ConnInfo, + clientinfo = ClientInfo} = Channel, + Msg, Result, Iter) -> %% inherit the old session SessFun = fun(_,_) -> Session end, case emqx_gateway_ctx:open_session( @@ -455,10 +372,14 @@ process_connect(Channel = #channel{ctx = Ctx, emqx_coap_session ) of {ok, _Sess} -> - response({ok, created}, <<"connected">>, Msg, Channel); + RandVal = rand:uniform(?TOKEN_MAXIMUM), + Token = erlang:list_to_binary(erlang:integer_to_list(RandVal)), + iter(Iter, + reply({ok, created}, Token, Msg, Result), + Channel#channel{token = Token}); {error, Reason} -> ?LOG(error, "Failed to open session du to ~p", [Reason]), - response({error, bad_request}, Msg, Channel) + iter(Iter, reply({error, bad_request}, Msg, Result), Channel) end. run_hooks(Ctx, Name, Args) -> @@ -469,20 +390,110 @@ run_hooks(Ctx, Name, Args, Acc) -> emqx_gateway_ctx:metrics_inc(Ctx, Name), emqx_hooks:run_fold(Name, Args, Acc). -convert_queries(#coap_message{options = Opts} = Msg) -> - case maps:get(uri_query, Opts, undefined) of - undefined -> - {ok, Msg#coap_message{options = Opts#{uri_query => #{}}}}; - Queries -> - convert_queries(Queries, #{}, Msg) - end. +metrics_inc(Name, Ctx) -> + emqx_gateway_ctx:metrics_inc(Ctx, Name). -convert_queries([H | T], Queries, Msg) -> - case re:split(H, "=") of - [Key, Val] -> - convert_queries(T, Queries#{Key => Val}, Msg); - _ -> - error +ensure_disconnected(Reason, Channel = #channel{ + ctx = Ctx, + conninfo = ConnInfo, + clientinfo = ClientInfo}) -> + NConnInfo = ConnInfo#{disconnected_at => erlang:system_time(millisecond)}, + ok = run_hooks(Ctx, 'client.disconnected', [ClientInfo, Reason, NConnInfo]), + Channel#channel{conninfo = NConnInfo, conn_state = disconnected}. + +%%-------------------------------------------------------------------- +%% Call Chain +%%-------------------------------------------------------------------- +call_session(Fun, Msg, #channel{session = Session} = Channel) -> + Result = emqx_coap_session:Fun(Msg, Session), + handle_result(Result, Channel). + +handle_result(Result, Channel) -> + iter([ session, fun process_session/4 + , proto, fun process_protocol/4 + , reply, fun process_reply/4 + , out, fun process_out/4 + , fun process_nothing/3 + ], + Result, + Channel). + +call_handler(request, Msg, Result, + #channel{ctx = Ctx, + clientinfo = ClientInfo} = Channel, Iter) -> + HandlerResult = + case emqx_coap_message:get_option(uri_path, Msg) of + [<<"ps">> | RestPath] -> + emqx_coap_pubsub_handler:handle_request(RestPath, Msg, Ctx, ClientInfo); + [<<"mqtt">> | RestPath] -> + emqx_coap_mqtt_handler:handle_request(RestPath, Msg, Ctx, ClientInfo); + _ -> + reply({error, bad_request}, Msg) + end, + iter([ connection, fun process_connection/4 + , subscribe, fun process_subscribe/4 | Iter], + maps:merge(Result, HandlerResult), + Channel); + +call_handler(response, {{send_request, From}, Response}, Result, Channel, Iter) -> + gen_server:reply(From, Response), + iter(Iter, Result, Channel); + +call_handler(_, _, Result, Channel, Iter) -> + iter(Iter, Result, Channel). + +process_session(Session, Result, Channel, Iter) -> + iter(Iter, Result, Channel#channel{session = Session}). + +process_protocol({Type, Msg}, Result, Channel, Iter) -> + call_handler(Type, Msg, Result, Channel, Iter). + +%% leaf node +process_out(Outs, Result, Channel, _) -> + Outs2 = lists:reverse(Outs), + Outs3 = case maps:get(reply, Result, undefined) of + undefined -> + Outs2; + Reply -> + [Reply | Outs2] + end, + {ok, {outgoing, Outs3}, Channel}. + +%% leaf node +process_nothing(_, _, Channel) -> + {ok, Channel}. + +process_connection({open, Req}, Result, Channel, Iter) -> + Queries = emqx_coap_message:get_option(uri_query, Req), + case emqx_misc:pipeline( + [ fun run_conn_hooks/2 + , fun enrich_clientinfo/2 + , fun set_log_meta/2 + , fun auth_connect/2 + ], + {Queries, Req}, + Channel) of + {ok, _Input, NChannel} -> + process_connect(ensure_connected(NChannel), Req, Result, Iter); + {error, ReasonCode, NChannel} -> + ErrMsg = io_lib:format("Login Failed: ~s", [ReasonCode]), + Payload = erlang:list_to_binary(lists:flatten(ErrMsg)), + iter(Iter, + reply({error, bad_request}, Payload, Req, Result), + NChannel) end; -convert_queries([], Queries, #coap_message{options = Opts} = Msg) -> - {ok, Msg#coap_message{options = Opts#{uri_query => Queries}}}. + +process_connection({close, Msg}, _, Channel, _) -> + Reply = emqx_coap_message:piggyback({ok, deleted}, Msg), + {shutdown, close, Reply, Channel}. + +process_subscribe({Sub, Msg}, Result, #channel{session = Session} = Channel, Iter) -> + Result2 = emqx_coap_session:process_subscribe(Sub, Msg, Result, Session), + iter([session, fun process_session/4 | Iter], Result2, Channel). + +%% leaf node +process_reply(Reply, Result, #channel{session = Session} = Channel, _) -> + Session2 = emqx_coap_session:set_reply(Reply, Session), + Outs = maps:get(out, Result, []), + Outs2 = lists:reverse(Outs), + {ok, {outgoing, [Reply | Outs2]}, Channel#channel{session = Session2}}. diff --git a/apps/emqx_gateway/src/coap/emqx_coap_frame.erl b/apps/emqx_gateway/src/coap/emqx_coap_frame.erl index c1bc08928..4d12997a7 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_frame.erl +++ b/apps/emqx_gateway/src/coap/emqx_coap_frame.erl @@ -103,11 +103,7 @@ flatten_options([{OptId, OptVal} | T], Acc) -> false -> [encode_option(OptId, OptVal) | Acc]; _ -> - lists:foldl(fun(undefined, InnerAcc) -> - InnerAcc; - (E, InnerAcc) -> - [encode_option(OptId, E) | InnerAcc] - end, Acc, OptVal) + try_encode_repeatable(OptId, OptVal) ++ Acc end); flatten_options([], Acc) -> @@ -141,6 +137,19 @@ encode_option_list([], _LastNum, Acc, <<>>) -> encode_option_list([], _, Acc, Payload) -> <>. +try_encode_repeatable(uri_query, Val) when is_map(Val) -> + maps:fold(fun(K, V, Acc) -> + [encode_option(uri_query, <>) | Acc] + end, + [], Val); + +try_encode_repeatable(K, Val) -> + lists:foldr(fun(undefined, Acc) -> + Acc; + (E, Acc) -> + [encode_option(K, E) | Acc] + end, [], Val). + %% RFC 7252 encode_option(if_match, OptVal) -> {?OPTION_IF_MATCH, OptVal}; encode_option(uri_host, OptVal) -> {?OPTION_URI_HOST, OptVal}; @@ -188,6 +197,8 @@ content_format_to_code(<<"application/octet-stream">>) -> 42; content_format_to_code(<<"application/exi">>) -> 47; content_format_to_code(<<"application/json">>) -> 50; content_format_to_code(<<"application/cbor">>) -> 60; +content_format_to_code(<<"application/vnd.oma.lwm2m+tlv">>) -> 11542; +content_format_to_code(<<"application/vnd.oma.lwm2m+json">>) -> 11543; content_format_to_code(_) -> 42. %% use octet-stream as default method_to_class_code(get) -> {0, 01}; @@ -235,12 +246,7 @@ parse(< {Options, Payload} = decode_option_list(Tail), Options2 = maps:fold(fun(K, V, Acc) -> - case is_repeatable_option(K) of - true -> - Acc#{K => lists:reverse(V)}; - _ -> - Acc#{K => V} - end + Acc#{K => get_option_val(K, V)} end, #{}, Options), @@ -255,6 +261,24 @@ parse(<>, ParseState}. +get_option_val(uri_query, V) -> + KVList = lists:foldl(fun(E, Acc) -> + [Key, Val] = re:split(E, "="), + [{Key, Val} | Acc] + + end, + [], + V), + maps:from_list(KVList); + +get_option_val(K, V) -> + case is_repeatable_option(K) of + true -> + lists:reverse(V); + _ -> + V + end. + -spec decode_type(X) -> message_type() when X :: 0 .. 3. decode_type(0) -> con; @@ -359,6 +383,8 @@ content_code_to_format(42) -> <<"application/octet-stream">>; content_code_to_format(47) -> <<"application/exi">>; content_code_to_format(50) -> <<"application/json">>; content_code_to_format(60) -> <<"application/cbor">>; +content_code_to_format(11542) -> <<"application/vnd.oma.lwm2m+tlv">>; +content_code_to_format(11543) -> <<"application/vnd.oma.lwm2m+json">>; content_code_to_format(_) -> <<"application/octet-stream">>. %% use octet as default %% RFC 7252 diff --git a/apps/emqx_gateway/src/coap/emqx_coap_impl.erl b/apps/emqx_gateway/src/coap/emqx_coap_impl.erl index da2f2b8e9..055eab759 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_impl.erl +++ b/apps/emqx_gateway/src/coap/emqx_coap_impl.erl @@ -89,17 +89,17 @@ start_listener(GwName, Ctx, {Type, LisName, ListenOn, SocketOpts, Cfg}) -> ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn), case start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) of {ok, Pid} -> - ?ULOG("Start ~s:~s:~s listener on ~s successfully.~n", + ?ULOG("Gateway ~s:~s:~s on ~s started.~n", [GwName, Type, LisName, ListenOnStr]), Pid; {error, Reason} -> - ?ELOG("Failed to start ~s:~s:~s listener on ~s: ~0p~n", + ?ELOG("Failed to start gateway ~s:~s:~s on ~s: ~0p~n", [GwName, Type, LisName, ListenOnStr, Reason]), throw({badconf, Reason}) end. start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) -> - Name = name(GwName, LisName, Type), + Name = emqx_gateway_utils:listener_id(GwName, Type, LisName), NCfg = Cfg#{ ctx => Ctx, frame_mod => emqx_coap_frame, @@ -114,21 +114,18 @@ do_start_listener(udp, Name, ListenOn, SocketOpts, MFA) -> do_start_listener(dtls, Name, ListenOn, SocketOpts, MFA) -> esockd:open_dtls(Name, ListenOn, SocketOpts, MFA). -name(GwName, LisName, Type) -> - list_to_atom(lists:concat([GwName, ":", Type, ":", LisName])). - stop_listener(GwName, {Type, LisName, ListenOn, SocketOpts, Cfg}) -> StopRet = stop_listener(GwName, Type, LisName, ListenOn, SocketOpts, Cfg), ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn), case StopRet of - ok -> ?ULOG("Stop ~s:~s:~s listener on ~s successfully.~n", - [GwName, Type, LisName, ListenOnStr]); + ok -> ?ULOG("Gateway ~s:~s:~s on ~s stopped.~n", + [GwName, Type, LisName, ListenOnStr]); {error, Reason} -> - ?ELOG("Failed to stop ~s:~s:~s listener on ~s: ~0p~n", + ?ELOG("Failed to stop gateway ~s:~s:~s on ~s: ~0p~n", [GwName, Type, LisName, ListenOnStr, Reason]) end, StopRet. stop_listener(GwName, Type, LisName, ListenOn, _SocketOpts, _Cfg) -> - Name = name(GwName, LisName, Type), + Name = emqx_gateway_utils:listener_id(GwName, Type, LisName), esockd:close(Name, ListenOn). diff --git a/apps/emqx_gateway/src/coap/emqx_coap_medium.erl b/apps/emqx_gateway/src/coap/emqx_coap_medium.erl new file mode 100644 index 000000000..8dafc7bbb --- /dev/null +++ b/apps/emqx_gateway/src/coap/emqx_coap_medium.erl @@ -0,0 +1,109 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% Simplified semi-automatic CPS mode tree for coap +%% The tree must have a terminal leaf node, and it's return is the result of the entire tree. +%% This module currently only supports simple linear operation + +-module(emqx_coap_medium). + +-include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). + +%% API +-export([ empty/0, reset/1, reset/2 + , out/1, out/2, proto_out/1 + , proto_out/2, iter/3, iter/4 + , reply/2, reply/3, reply/4]). + +%%-type result() :: map() | empty. + +%%-------------------------------------------------------------------- +%% API +%%-------------------------------------------------------------------- +empty() -> #{}. + +reset(Msg) -> + reset(Msg, #{}). + +reset(Msg, Result) -> + out(emqx_coap_message:reset(Msg), Result). + +out(Msg) -> + #{out => [Msg]}. + +out(Msg, #{out := Outs} = Result) -> + Result#{out := [Msg | Outs]}; + +out(Msg, Result) -> + Result#{out => [Msg]}. + +proto_out(Proto) -> + proto_out(Proto, #{}). + +proto_out(Proto, Resut) -> + Resut#{proto => Proto}. + +reply(Method, Req) when not is_record(Method, coap_message) -> + reply(Method, <<>>, Req); + +reply(Reply, Result) -> + Result#{reply => Reply}. + +reply(Method, Req, Result) when is_record(Req, coap_message) -> + reply(Method, <<>>, Req, Result); + +reply(Method, Payload, Req) -> + reply(Method, Payload, Req, #{}). + +reply(Method, Payload, Req, Result) -> + Result#{reply => emqx_coap_message:piggyback(Method, Payload, Req)}. + +%% run a tree +iter([Key, Fun | T], Input, State) -> + case maps:get(Key, Input, undefined) of + undefined -> + iter(T, Input, State); + Val -> + Fun(Val, maps:remove(Key, Input), State, T) + %% reserved + %% if is_function(Fun) -> + %% Fun(Val, maps:remove(Key, Input), State, T); + %% true -> + %% %% switch to sub branch + %% [FunH | FunT] = Fun, + %% FunH(Val, maps:remove(Key, Input), State, FunT) + %% end + end; + +%% terminal node +iter([Fun], Input, State) -> + Fun(undefined, Input, State). + +%% run a tree with argument +iter([Key, Fun | T], Input, Arg, State) -> + case maps:get(Key, Input, undefined) of + undefined -> + iter(T, Input, Arg, State); + Val -> + Fun(Val, maps:remove(Key, Input), Arg, State, T) + end; + +iter([Fun], Input, Arg, State) -> + Fun(undefined, Input, Arg, State). + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- diff --git a/apps/emqx_gateway/src/coap/emqx_coap_message.erl b/apps/emqx_gateway/src/coap/emqx_coap_message.erl index 2e9fb144e..3851b3428 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_message.erl +++ b/apps/emqx_gateway/src/coap/emqx_coap_message.erl @@ -31,7 +31,8 @@ -export([is_request/1]). --export([set/3, set_payload/2, get_content/1, set_content/2, set_content/3, get_option/2]). +-export([ set/3, set_payload/2, get_option/2 + , get_option/3, set_payload_block/3, set_payload_block/4]). -include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). @@ -42,11 +43,10 @@ request(Type, Method, Payload) -> request(Type, Method, Payload, []). request(Type, Method, Payload, Options) when is_binary(Payload) -> - #coap_message{type = Type, method = Method, payload = Payload, options = Options}; - -request(Type, Method, Content=#coap_content{}, Options) -> - set_content(Content, - #coap_message{type = Type, method = Method, options = Options}). + #coap_message{type = Type, + method = Method, + payload = Payload, + options = to_options(Options)}. ack(#coap_message{id = Id}) -> #coap_message{type = ack, id = Id}. @@ -55,20 +55,20 @@ reset(#coap_message{id = Id}) -> #coap_message{type = reset, id = Id}. %% just make a response -response(#coap_message{type = Type, - id = Id, - token = Token}) -> - #coap_message{type = Type, - id = Id, - token = Token}. +response(Request) -> + response(undefined, Request). response(Method, Request) -> - set_method(Method, response(Request)). + response(Method, <<>>, Request). -response(Method, Payload, Request) -> - set_method(Method, - set_payload(Payload, - response(Request))). +response(Method, Payload, #coap_message{type = Type, + id = Id, + token = Token}) -> + #coap_message{type = Type, + id = Id, + token = Token, + method = Method, + payload = Payload}. %% make a response which maybe is a piggyback ack piggyback(Method, Request) -> @@ -90,14 +90,11 @@ set(max_age, ?DEFAULT_MAX_AGE, Msg) -> Msg; set(Option, Value, Msg = #coap_message{options = Options}) -> Msg#coap_message{options = Options#{Option => Value}}. -get_option(Option, #coap_message{options = Options}) -> - maps:get(Option, Options, undefined). +get_option(Option, Msg) -> + get_option(Option, Msg, undefined). -set_method(Method, Msg) -> - Msg#coap_message{method = Method}. - -set_payload(Payload = #coap_content{}, Msg) -> - set_content(Payload, undefined, Msg); +get_option(Option, #coap_message{options = Options}, Def) -> + maps:get(Option, Options, Def). set_payload(Payload, Msg) when is_binary(Payload) -> Msg#coap_message{payload = Payload}; @@ -105,49 +102,6 @@ set_payload(Payload, Msg) when is_binary(Payload) -> set_payload(Payload, Msg) when is_list(Payload) -> Msg#coap_message{payload = list_to_binary(Payload)}. -get_content(#coap_message{options = Options, payload = Payload}) -> - #coap_content{etag = maps:get(etag, Options, undefined), - max_age = maps:get(max_age, Options, ?DEFAULT_MAX_AGE), - format = maps:get(content_format, Options, undefined), - location_path = maps:get(location_path, Options, []), - payload = Payload}. - -set_content(Content, Msg) -> - set_content(Content, undefined, Msg). - -%% segmentation not requested and not required -set_content(#coap_content{etag = ETag, - max_age = MaxAge, - format = Format, - location_path = LocPath, - payload = Payload}, - undefined, - Msg) - when byte_size(Payload) =< ?MAX_BLOCK_SIZE -> - #coap_message{options = Options} = Msg2 = set_payload(Payload, Msg), - Options2 = Options#{etag => [ETag], - max_age => MaxAge, - content_format => Format, - location_path => LocPath}, - Msg2#coap_message{options = Options2}; - -%% segmentation not requested, but required (late negotiation) -set_content(Content, undefined, Msg) -> - set_content(Content, {0, true, ?MAX_BLOCK_SIZE}, Msg); - -%% segmentation requested (early negotiation) -set_content(#coap_content{etag = ETag, - max_age = MaxAge, - format = Format, - payload = Payload}, - Block, - Msg) -> - #coap_message{options = Options} = Msg2 = set_payload_block(Payload, Block, Msg), - Options2 = Options#{etag => [ETag], - max => MaxAge, - content_format => Format}, - Msg2#coap_message{options = Options2}. - set_payload_block(Content, Block, Msg = #coap_message{method = Method}) when is_atom(Method) -> set_payload_block(Content, block1, Block, Msg); @@ -172,3 +126,8 @@ is_request(#coap_message{method = Method}) when is_atom(Method) -> is_request(_) -> false. + +to_options(Opts) when is_map(Opts) -> + Opts; +to_options(Opts) -> + maps:from_list(Opts). diff --git a/apps/emqx_gateway/src/coap/emqx_coap_resource.erl b/apps/emqx_gateway/src/coap/emqx_coap_resource.erl deleted file mode 100644 index 93fe82aba..000000000 --- a/apps/emqx_gateway/src/coap/emqx_coap_resource.erl +++ /dev/null @@ -1,37 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2017-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_coap_resource). - --include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). - --type context() :: any(). --type topic() :: binary(). --type token() :: token(). - --type register() :: {topic(), token()} - | topic() - | undefined. - --type result() :: emqx_coap_message() - | {has_sub, emqx_coap_message(), register()}. - --callback init(hocon:confg()) -> context(). --callback stop(context()) -> ok. --callback get(emqx_coap_message(), hocon:config()) -> result(). --callback put(emqx_coap_message(), hocon:config()) -> result(). --callback post(emqx_coap_message(), hocon:config()) -> result(). --callback delete(emqx_coap_message(), hocon:config()) -> result(). diff --git a/apps/emqx_gateway/src/coap/emqx_coap_session.erl b/apps/emqx_gateway/src/coap/emqx_coap_session.erl index 50e91797b..0fbc47cf8 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_session.erl +++ b/apps/emqx_gateway/src/coap/emqx_coap_session.erl @@ -21,24 +21,25 @@ -include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). %% API --export([new/0, transfer_result/3]). +-export([ new/0 + , process_subscribe/4]). -export([ info/1 , info/2 , stats/1 ]). --export([ handle_request/3 - , handle_response/3 - , handle_out/3 +-export([ handle_request/2 + , handle_response/2 + , handle_out/2 + , set_reply/2 , deliver/3 - , timeout/3]). + , timeout/2]). -export_type([session/0]). -record(session, { transport_manager :: emqx_coap_tm:manager() , observe_manager :: emqx_coap_observe_res:manager() - , next_msg_id :: coap_message_id() , created_at :: pos_integer() }). @@ -64,6 +65,9 @@ awaiting_rel_max ]). +-import(emqx_coap_medium, [iter/3]). +-import(emqx_coap_channel, [metrics_inc/2]). + %%%------------------------------------------------------------------- %%% API %%%------------------------------------------------------------------- @@ -72,7 +76,6 @@ new() -> _ = emqx_misc:rand_seed(), #session{ transport_manager = emqx_coap_tm:new() , observe_manager = emqx_coap_observe_res:new_manager() - , next_msg_id = rand:uniform(?MAX_MESSAGE_ID) , created_at = erlang:system_time(millisecond)}. %%-------------------------------------------------------------------- @@ -110,8 +113,8 @@ info(mqueue_max, _) -> 0; info(mqueue_dropped, _) -> 0; -info(next_pkt_id, #session{next_msg_id = PacketId}) -> - PacketId; +info(next_pkt_id, _) -> + 0; info(awaiting_rel, _) -> #{}; info(awaiting_rel_cnt, _) -> @@ -130,105 +133,90 @@ stats(Session) -> info(?STATS_KEYS, Session). %%%------------------------------------------------------------------- %%% Process Message %%%------------------------------------------------------------------- -handle_request(Msg, Ctx, Session) -> +handle_request(Msg, Session) -> call_transport_manager(?FUNCTION_NAME, Msg, - Ctx, - [fun process_tm/3, fun process_subscribe/3], Session). -handle_response(Msg, Ctx, Session) -> - call_transport_manager(?FUNCTION_NAME, Msg, Ctx, [fun process_tm/3], Session). +handle_response(Msg, Session) -> + call_transport_manager(?FUNCTION_NAME, Msg, Session). -handle_out(Msg, Ctx, Session) -> - call_transport_manager(?FUNCTION_NAME, Msg, Ctx, [fun process_tm/3], Session). +handle_out(Msg, Session) -> + call_transport_manager(?FUNCTION_NAME, Msg, Session). -deliver(Delivers, Ctx, Session) -> - Fun = fun({_, Topic, Message}, - #{out := OutAcc, - session := #session{observe_manager = OM, - next_msg_id = MsgId, - transport_manager = TM} = SAcc} = Acc) -> - case emqx_coap_observe_res:res_changed(Topic, OM) of +set_reply(Msg, #session{transport_manager = TM} = Session) -> + TM2 = emqx_coap_tm:set_reply(Msg, TM), + Session#session{transport_manager = TM2}. + +deliver(Delivers, Ctx, #session{observe_manager = OM, + transport_manager = TM} = Session) -> + Fun = fun({_, Topic, Message}, {OutAcc, OMAcc, TMAcc} = Acc) -> + case emqx_coap_observe_res:res_changed(Topic, OMAcc) of undefined -> + metrics_inc('delivery.dropped', Ctx), + metrics_inc('delivery.dropped.no_subid', Ctx), Acc; {Token, SeqId, OM2} -> - Msg = mqtt_to_coap(Message, MsgId, Token, SeqId, Ctx), - SAcc2 = SAcc#session{next_msg_id = next_msg_id(MsgId, TM), - observe_manager = OM2}, - #{out := Out} = Result = handle_out(Msg, Ctx, SAcc2), - Result#{out := [Out | OutAcc]} + metrics_inc('messages.delivered', Ctx), + Msg = mqtt_to_coap(Message, Token, SeqId), + #{out := Out, tm := TM2} = emqx_coap_tm:handle_out(Msg, TMAcc), + {Out ++ OutAcc, OM2, TM2} end end, - lists:foldl(Fun, - #{out => [], session => Session}, - lists:reverse(Delivers)). + {Outs, OM2, TM2} = lists:foldl(Fun, {[], OM, TM}, lists:reverse(Delivers)), -timeout(Timer, Ctx, Session) -> - call_transport_manager(?FUNCTION_NAME, Timer, Ctx, [fun process_tm/3], Session). + #{out => lists:reverse(Outs), + session => Session#session{observe_manager = OM2, + transport_manager = TM2}}. -result_keys() -> - [tm, subscribe] ++ emqx_coap_channel:result_keys(). - -transfer_result(From, Value, Result) -> - ?TRANSFER_RESULT(From, Value, Result). +timeout(Timer, Session) -> + call_transport_manager(?FUNCTION_NAME, Timer, Session). %%%------------------------------------------------------------------- %%% Internal functions %%%------------------------------------------------------------------- call_transport_manager(Fun, Msg, - Ctx, - Processor, #session{transport_manager = TM} = Session) -> - try - Result = emqx_coap_tm:Fun(Msg, Ctx, TM), - {ok, Result2, Session2} = pipeline(Processor, - Result, - Msg, - Session), - emqx_coap_channel:transfer_result(session, Session2, Result2) - catch Type:Reason:Stack -> - ?ERROR("process transmission with, message:~p failed~nType:~p,Reason:~p~n,StackTrace:~p~n", - [Msg, Type, Reason, Stack]), - ?REPLY({error, internal_server_error}, Msg) - end. + Result = emqx_coap_tm:Fun(Msg, TM), + iter([tm, fun process_tm/4, fun process_session/3], + Result, + Session). -process_tm(#{tm := TM}, _, Session) -> - {ok, Session#session{transport_manager = TM}}; -process_tm(_, _, Session) -> - {ok, Session}. +process_tm(TM, Result, Session, Cursor) -> + iter(Cursor, Result, Session#session{transport_manager = TM}). -process_subscribe(#{subscribe := Sub} = Result, - Msg, - #session{observe_manager = OM} = Session) -> +process_session(_, Result, Session) -> + Result#{session => Session}. + +process_subscribe(Sub, Msg, Result, + #session{observe_manager = OM} = Session) -> case Sub of undefined -> - {ok, Result, Session}; + Result; {Topic, Token} -> {SeqId, OM2} = emqx_coap_observe_res:insert(Topic, Token, OM), Replay = emqx_coap_message:piggyback({ok, content}, Msg), Replay2 = Replay#coap_message{options = #{observe => SeqId}}, - {ok, Result#{reply => Replay2}, Session#session{observe_manager = OM2}}; + Result#{reply => Replay2, + session => Session#session{observe_manager = OM2}}; Topic -> OM2 = emqx_coap_observe_res:remove(Topic, OM), Replay = emqx_coap_message:piggyback({ok, nocontent}, Msg), - {ok, Result#{reply => Replay}, Session#session{observe_manager = OM2}} - end; -process_subscribe(Result, _, Session) -> - {ok, Result, Session}. + Result#{reply => Replay, + session => Session#session{observe_manager = OM2}} + end. -mqtt_to_coap(MQTT, MsgId, Token, SeqId, Ctx) -> +mqtt_to_coap(MQTT, Token, SeqId) -> #message{payload = Payload} = MQTT, - #coap_message{type = get_notify_type(MQTT, Ctx), + #coap_message{type = get_notify_type(MQTT), method = {ok, content}, - id = MsgId, token = Token, payload = Payload, options = #{observe => SeqId}}. -get_notify_type(#message{qos = Qos}, Ctx) -> - case emqx_coap_channel:get_config(notify_type, Ctx) of +get_notify_type(#message{qos = Qos}) -> + case emqx:get_config([gateway, coap, notify_qos], non) of qos -> case Qos of ?QOS_0 -> @@ -239,32 +227,3 @@ get_notify_type(#message{qos = Qos}, Ctx) -> Other -> Other end. - -next_msg_id(MsgId, TM) -> - next_msg_id(MsgId + 1, MsgId, TM). - -next_msg_id(MsgId, MsgId, _) -> - erlang:throw("too many message in delivering"); -next_msg_id(MsgId, BeginId, TM) when MsgId >= ?MAX_MESSAGE_ID -> - check_is_inused(1, BeginId, TM); -next_msg_id(MsgId, BeginId, TM) -> - check_is_inused(MsgId, BeginId, TM). - -check_is_inused(NewMsgId, BeginId, TM) -> - case emqx_coap_tm:is_inused(out, NewMsgId, TM) of - false -> - NewMsgId; - _ -> - next_msg_id(NewMsgId + 1, BeginId, TM) - end. - -pipeline([Fun | T], Result, Msg, Session) -> - case Fun(Result, Msg, Session) of - {ok, Session2} -> - pipeline(T, Result, Msg, Session2); - {ok, Result2, Session2} -> - pipeline(T, Result2, Msg, Session2) - end; - -pipeline([], Result, _, Session) -> - {ok, Result, Session}. diff --git a/apps/emqx_gateway/src/coap/emqx_coap_tm.erl b/apps/emqx_gateway/src/coap/emqx_coap_tm.erl index 5a664b0f2..b5e4deb7f 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_tm.erl +++ b/apps/emqx_gateway/src/coap/emqx_coap_tm.erl @@ -18,11 +18,12 @@ -module(emqx_coap_tm). -export([ new/0 - , handle_request/3 - , handle_response/3 + , handle_request/2 + , handle_response/2 + , handle_out/2 , handle_out/3 - , timeout/3 - , is_inused/3]). + , set_reply/2 + , timeout/2]). -export_type([manager/0, event_result/1]). @@ -30,17 +31,28 @@ -include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). -type direction() :: in | out. --type state_machine_id() :: {direction(), non_neg_integer()}. --record(state_machine, { id :: state_machine_id() +-record(state_machine, { seq_id :: seq_id() + , id :: state_machine_key() + , token :: token() | undefined + , observe :: 0 | 1 | undefined | observed , state :: atom() , timers :: maps:map() , transport :: emqx_coap_transport:transport()}). -type state_machine() :: #state_machine{}. -type message_id() :: 0 .. ?MAX_MESSAGE_ID. +-type token_key() :: {token, token()}. +-type state_machine_key() :: {direction(), message_id()}. +-type seq_id() :: pos_integer(). +-type manager_key() :: token_key() | state_machine_key() | seq_id(). --type manager() :: #{message_id() => state_machine()}. +-type manager() :: #{ seq_id => seq_id() + , next_msg_id => coap_message_id() + , token_key() => seq_id() + , state_machine_key() => seq_id() + , seq_id() => state_machine() + }. -type ttimeout() :: {state_timeout, pos_integer(), any()} | {stop_timeout, pos_integer()}. @@ -48,6 +60,7 @@ -type topic() :: binary(). -type token() :: binary(). -type sub_register() :: {topic(), token()} | topic(). + -type event_result(State) :: #{next => State, outgoing => emqx_coap_message(), @@ -55,108 +68,164 @@ has_sub => undefined | sub_register(), transport => emqx_coap_transport:transprot()}. +-define(TOKEN_ID(T), {token, T}). + +-import(emqx_coap_medium, [empty/0, iter/4, reset/1, proto_out/2]). + %%-------------------------------------------------------------------- %% API %%-------------------------------------------------------------------- new() -> - #{}. + #{ seq_id => 1 + , next_msg_id => rand:uniform(?MAX_MESSAGE_ID) + }. -handle_request(#coap_message{id = MsgId} = Msg, Ctx, TM) -> +%% client request +handle_request(#coap_message{id = MsgId} = Msg, TM) -> Id = {in, MsgId}, - case maps:get(Id, TM, undefined) of + case find_machine(Id, TM) of undefined -> - Transport = emqx_coap_transport:new(), - Machine = new_state_machine(Id, Transport), - process_event(in, Msg, TM, Ctx, Machine); + {Machine, TM2} = new_in_machine(Id, TM), + process_event(in, Msg, TM2, Machine); Machine -> - process_event(in, Msg, TM, Ctx, Machine) + process_event(in, Msg, TM, Machine) end. -handle_response(#coap_message{type = Type, id = MsgId} = Msg, Ctx, TM) -> +%% client response +handle_response(#coap_message{type = Type, id = MsgId, token = Token} = Msg, TM) -> Id = {out, MsgId}, - case maps:get(Id, TM, undefined) of + TokenId = ?TOKEN_ID(Token), + case find_machine_by_keys([Id, TokenId], TM) of undefined -> case Type of reset -> - ?EMPTY_RESULT; + empty(); _ -> - ?RESET(Msg) + reset(Msg) end; Machine -> - process_event(in, Msg, TM, Ctx, Machine) + process_event(in, Msg, TM, Machine) end. -handle_out(#coap_message{id = MsgId} = Msg, Ctx, TM) -> +%% send to a client, msg can be request/piggyback/separate/notify +handle_out({Ctx, Msg}, TM) -> + handle_out(Msg, Ctx, TM); + +handle_out(Msg, TM) -> + handle_out(Msg, undefined, TM). + +handle_out(#coap_message{token = Token} = MsgT, Ctx, TM) -> + {MsgId, TM2} = alloc_message_id(TM), + Msg = MsgT#coap_message{id = MsgId}, Id = {out, MsgId}, - case maps:get(Id, TM, undefined) of + TokenId = ?TOKEN_ID(Token), + %% TODO why find by token ? + case find_machine_by_keys([Id, TokenId], TM2) of undefined -> - Transport = emqx_coap_transport:new(), - Machine = new_state_machine(Id, Transport), - process_event(out, Msg, TM, Ctx, Machine); + {Machine, TM3} = new_out_machine(Id, Ctx, Msg, TM2), + process_event(out, Msg, TM3, Machine); _ -> %% ignore repeat send - ?EMPTY_RESULT + empty() end. -timeout({Id, Type, Msg}, Ctx, TM) -> - case maps:get(Id, TM, undefined) of +set_reply(#coap_message{id = MsgId} = Msg, TM) -> + Id = {in, MsgId}, + case find_machine(Id, TM) of undefined -> - ?EMPTY_RESULT; + TM; + #state_machine{transport = Transport, + seq_id = SeqId} = Machine -> + Transport2 = emqx_coap_transport:set_cache(Msg, Transport), + Machine2 = Machine#state_machine{transport = Transport2}, + TM#{SeqId => Machine2} + end. + +timeout({SeqId, Type, Msg}, TM) -> + case maps:get(SeqId, TM, undefined) of + undefined -> + empty(); #state_machine{timers = Timers} = Machine -> %% maybe timer has been canceled case maps:is_key(Type, Timers) of true -> - process_event(Type, Msg, TM, Ctx, Machine); + process_event(Type, Msg, TM, Machine); _ -> - ?EMPTY_RESULT + empty() end end. --spec is_inused(direction(), message_id(), manager()) -> boolean(). -is_inused(Dir, Msg, Manager) -> - maps:is_key({Dir, Msg}, Manager). - %%-------------------------------------------------------------------- %% Internal functions %%-------------------------------------------------------------------- -new_state_machine(Id, Transport) -> - #state_machine{id = Id, - state = idle, - timers = #{}, - transport = Transport}. +process_event(stop_timeout, _, TM, Machine) -> + process_manager(stop, #{}, Machine, TM); -process_event(stop_timeout, - _, - TM, - _, - #state_machine{id = Id, - timers = Timers}) -> - lists:foreach(fun({_, Ref}) -> - emqx_misc:cancel_timer(Ref) - end, - maps:to_list(Timers)), - #{tm => maps:remove(Id, TM)}; +process_event(Event, Msg, TM, #state_machine{state = State, + transport = Transport} = Machine) -> + Result = emqx_coap_transport:State(Event, Msg, Transport), + iter([ proto, fun process_observe_response/5 + , next, fun process_state_change/5 + , transport, fun process_transport_change/5 + , timeouts, fun process_timeouts/5 + , fun process_manager/4], + Result, + Machine, + TM). -process_event(Event, - Msg, - TM, - Ctx, - #state_machine{id = Id, - state = State, - transport = Transport} = Machine) -> - Result = emqx_coap_transport:State(Event, Msg, Ctx, Transport), - {ok, _, Machine2} = emqx_misc:pipeline([fun process_state_change/2, - fun process_transport_change/2, - fun process_timeouts/2], - Result, - Machine), - TM2 = TM#{Id => Machine2}, - emqx_coap_session:transfer_result(tm, TM2, Result). +process_observe_response({response, {_, Msg}} = Response, + Result, + #state_machine{observe = 0} = Machine, + TM, + Iter) -> + Result2 = proto_out(Response, Result), + case Msg#coap_message.method of + {ok, _} -> + iter(Iter, + Result2#{next => observe}, + Machine#state_machine{observe = observed}, + TM); + _ -> + iter(Iter, Result2, Machine, TM) + end; -process_state_change(#{next := Next}, Machine) -> - {ok, cancel_state_timer(Machine#state_machine{state = Next})}; -process_state_change(_, Machine) -> - {ok, Machine}. +process_observe_response(Proto, Result, Machine, TM, Iter) -> + iter(Iter, proto_out(Proto, Result), Machine, TM). + +process_state_change(Next, Result, Machine, TM, Iter) -> + case Next of + stop -> + process_manager(stop, Result, Machine, TM); + _ -> + iter(Iter, + Result, + cancel_state_timer(Machine#state_machine{state = Next}), + TM) + end. + +process_transport_change(Transport, Result, Machine, TM, Iter) -> + iter(Iter, Result, Machine#state_machine{transport = Transport}, TM). + +process_timeouts([], Result, Machine, TM, Iter) -> + iter(Iter, Result, Machine, TM); + +process_timeouts(Timeouts, Result, + #state_machine{seq_id = SeqId, + timers = Timers} = Machine, TM, Iter) -> + NewTimers = lists:foldl(fun({state_timeout, _, _} = Timer, Acc) -> + process_timer(SeqId, Timer, Acc); + ({stop_timeout, I}, Acc) -> + process_timer(SeqId, {stop_timeout, I, stop}, Acc) + end, + Timers, + Timeouts), + iter(Iter, Result, Machine#state_machine{timers = NewTimers}, TM). + +process_manager(stop, Result, #state_machine{seq_id = SeqId}, TM) -> + Result#{tm => delete_machine(SeqId, TM)}; + +process_manager(_, Result, #state_machine{seq_id = SeqId} = Machine2, TM) -> + Result#{tm => TM#{SeqId => Machine2}}. cancel_state_timer(#state_machine{timers = Timers} = Machine) -> case maps:get(state_timer, Timers, undefined) of @@ -167,27 +236,119 @@ cancel_state_timer(#state_machine{timers = Timers} = Machine) -> Machine#state_machine{timers = maps:remove(state_timer, Timers)} end. -process_transport_change(#{transport := Transport}, Machine) -> - {ok, Machine#state_machine{transport = Transport}}; -process_transport_change(_, Machine) -> - {ok, Machine}. - -process_timeouts(#{timeouts := []}, Machine) -> - {ok, Machine}; -process_timeouts(#{timeouts := Timeouts}, - #state_machine{id = Id, timers = Timers} = Machine) -> - NewTimers = lists:foldl(fun({state_timeout, _, _} = Timer, Acc) -> - process_timer(Id, Timer, Acc); - ({stop_timeout, I}, Acc) -> - process_timer(Id, {stop_timeout, I, stop}, Acc) - end, - Timers, - Timeouts), - {ok, Machine#state_machine{timers = NewTimers}}; - -process_timeouts(_, Machine) -> - {ok, Machine}. - -process_timer(Id, {Type, Interval, Msg}, Timers) -> - Ref = emqx_misc:start_timer(Interval, {state_machine, {Id, Type, Msg}}), +process_timer(SeqId, {Type, Interval, Msg}, Timers) -> + Ref = emqx_misc:start_timer(Interval, {state_machine, {SeqId, Type, Msg}}), Timers#{Type => Ref}. + +-spec delete_machine(manager_key(), manager()) -> manager(). +delete_machine(Id, Manager) -> + case find_machine(Id, Manager) of + undefined -> + Manager; + #state_machine{seq_id = SeqId, + id = MachineId, + token = Token, + timers = Timers} -> + lists:foreach(fun({_, Ref}) -> + emqx_misc:cancel_timer(Ref) + end, + maps:to_list(Timers)), + maps:without([SeqId, MachineId, ?TOKEN_ID(Token)], Manager) + end. + +-spec find_machine(manager_key(), manager()) -> state_machine() | undefined. +find_machine({_, _} = Id, Manager) -> + find_machine_by_seqid(maps:get(Id, Manager, undefined), Manager); +find_machine(SeqId, Manager) -> + find_machine_by_seqid(SeqId, Manager). + +-spec find_machine_by_seqid(seq_id() | undefined, manager()) -> + state_machine() | undefined. +find_machine_by_seqid(SeqId, Manager) -> + maps:get(SeqId, Manager, undefined). + +-spec find_machine_by_keys(list(manager_key()), + manager()) -> state_machine() | undefined. +find_machine_by_keys([H | T], Manager) -> + case H of + ?TOKEN_ID(<<>>) -> + find_machine_by_keys(T, Manager); + _ -> + case find_machine(H, Manager) of + undefined -> + find_machine_by_keys(T, Manager); + Machine -> + Machine + end + end; +find_machine_by_keys(_, _) -> + undefined. + +-spec new_in_machine(state_machine_key(), manager()) -> + {state_machine(), manager()}. +new_in_machine(MachineId, #{seq_id := SeqId} = Manager) -> + Machine = #state_machine{ seq_id = SeqId + , id = MachineId + , state = idle + , timers = #{} + , transport = emqx_coap_transport:new()}, + {Machine, Manager#{seq_id := SeqId + 1, + SeqId => Machine, + MachineId => SeqId}}. + +-spec new_out_machine(state_machine_key(), any(), emqx_coap_message(), manager()) -> + {state_machine(), manager()}. +new_out_machine(MachineId, + Ctx, + #coap_message{type = Type, token = Token, options = Opts}, + #{seq_id := SeqId} = Manager) -> + Observe = maps:get(observe, Opts, undefined), + Machine = #state_machine{ seq_id = SeqId + , id = MachineId + , token = Token + , observe = Observe + , state = idle + , timers = #{} + , transport = emqx_coap_transport:new(Ctx)}, + + Manager2 = Manager#{seq_id := SeqId + 1, + SeqId => Machine, + MachineId => SeqId}, + {Machine, + if Token =:= <<>> -> + Manager2; + Observe =:= 1 -> + TokenId = ?TOKEN_ID(Token), + delete_machine(TokenId, Manager2); + Type =:= con orelse Observe =:= 0 -> + TokenId = ?TOKEN_ID(Token), + case maps:get(TokenId, Manager, undefined) of + undefined -> + Manager2#{TokenId => SeqId}; + _ -> + throw("token conflict") + end; + true -> + Manager2 + end + }. + +alloc_message_id(#{next_msg_id := MsgId} = TM) -> + alloc_message_id(MsgId, TM). + +alloc_message_id(MsgId, TM) -> + Id = {out, MsgId}, + case maps:get(Id, TM, undefined) of + undefined -> + {MsgId, TM#{next_msg_id => next_message_id(MsgId)}}; + _ -> + alloc_message_id(next_message_id(MsgId), TM) + end. + +next_message_id(MsgId) -> + Next = MsgId + 1, + if Next >= ?MAX_MESSAGE_ID -> + 1; + true -> + Next + end. diff --git a/apps/emqx_gateway/src/coap/emqx_coap_transport.erl b/apps/emqx_gateway/src/coap/emqx_coap_transport.erl index 2c2aaab2e..2e858a2e1 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_transport.erl +++ b/apps/emqx_gateway/src/coap/emqx_coap_transport.erl @@ -9,118 +9,183 @@ -define(EXCHANGE_LIFETIME, 247000). -define(NON_LIFETIME, 145000). +-type request_context() :: any(). + -record(transport, { cache :: undefined | emqx_coap_message() + , req_context :: request_context() , retry_interval :: non_neg_integer() , retry_count :: non_neg_integer() + , observe :: non_neg_integer() | undefined }). -type transport() :: #transport{}. --export([ new/0, idle/4, maybe_reset/4 - , maybe_resend/4, wait_ack/4, until_stop/4]). +-export([ new/0, new/1, idle/3, maybe_reset/3, set_cache/2 + , maybe_resend_4request/3, wait_ack/3, until_stop/3 + , observe/3, maybe_resend_4response/3]). -export_type([transport/0]). -import(emqx_coap_message, [reset/1]). +-import(emqx_coap_medium, [ empty/0, reset/2, proto_out/2 + , out/1, out/2, proto_out/1 + , reply/2]). -spec new() -> transport(). new() -> + new(undefined). + +new(ReqCtx) -> #transport{cache = undefined, retry_interval = 0, - retry_count = 0}. + retry_count = 0, + req_context = ReqCtx}. idle(in, #coap_message{type = non, method = Method} = Msg, - Ctx, _) -> - Ret = #{next => until_stop, - timeouts => [{stop_timeout, ?NON_LIFETIME}]}, case Method of undefined -> - ?RESET(Msg); + reset(Msg, #{next => stop}); _ -> - Result = call_handler(Msg, Ctx), - maps:merge(Ret, Result) + proto_out({request, Msg}, + #{next => until_stop, + timeouts => + [{stop_timeout, ?NON_LIFETIME}]}) end; idle(in, #coap_message{type = con, method = Method} = Msg, - Ctx, - Transport) -> - Ret = #{next => maybe_resend, - timeouts =>[{stop_timeout, ?EXCHANGE_LIFETIME}]}, + _) -> case Method of undefined -> - ResetMsg = reset(Msg), - Ret#{transport => Transport#transport{cache = ResetMsg}, - out => ResetMsg}; + reset(Msg, #{next => stop}); _ -> - Result = call_handler(Msg, Ctx), - maps:merge(Ret, Result) + proto_out({request, Msg}, + #{next => maybe_resend_4request, + timeouts =>[{stop_timeout, ?EXCHANGE_LIFETIME}]}) end; -idle(out, #coap_message{type = non} = Msg, _, _) -> - #{next => maybe_reset, - out => Msg, - timeouts => [{stop_timeout, ?NON_LIFETIME}]}; +idle(out, #coap_message{type = non} = Msg, _) -> + out(Msg, #{next => maybe_reset, + timeouts => [{stop_timeout, ?NON_LIFETIME}]}); -idle(out, Msg, _, Transport) -> +idle(out, Msg, Transport) -> _ = emqx_misc:rand_seed(), Timeout = ?ACK_TIMEOUT + rand:uniform(?ACK_RANDOM_FACTOR), - #{next => wait_ack, - transport => Transport#transport{cache = Msg}, - out => Msg, - timeouts => [ {state_timeout, Timeout, ack_timeout} - , {stop_timeout, ?EXCHANGE_LIFETIME}]}. + out(Msg, #{next => wait_ack, + transport => Transport#transport{cache = Msg}, + timeouts => [ {state_timeout, Timeout, ack_timeout} + , {stop_timeout, ?EXCHANGE_LIFETIME}]}). -maybe_reset(in, Message, _, _) -> - case Message of - #coap_message{type = reset} -> - ?INFO("Reset Message:~p~n", [Message]); +maybe_resend_4request(in, Msg, Transport) -> + maybe_resend(Msg, true, Transport). + +maybe_resend_4response(in, Msg, Transport) -> + maybe_resend(Msg, false, Transport). + +maybe_resend(Msg, IsExpecteReq, #transport{cache = Cache}) -> + IsExpected = emqx_coap_message:is_request(Msg) =:= IsExpecteReq, + case IsExpected of + true -> + case Cache of + undefined -> + %% handler in processing, ignore + empty(); + _ -> + out(Cache) + end; _ -> - ok - end, - ?EMPTY_RESULT. + reset(Msg, #{next => stop}) + end. -maybe_resend(in, _, _, #transport{cache = Cache}) -> - #{out => Cache}. +maybe_reset(in, #coap_message{type = Type, method = Method} = Message, + #transport{req_context = Ctx} = Transport) -> + Ret = #{next => stop}, + CtxMsg = {Ctx, Message}, + if Type =:= reset -> + proto_out({reset, CtxMsg}, Ret); + is_tuple(Method) -> + on_response(Message, + Transport, + if Type =:= non -> until_stop; + true -> maybe_resend_4response + end); + true -> + reset(Message, Ret) + end. -wait_ack(in, #coap_message{type = Type}, _, _) -> +wait_ack(in, #coap_message{type = Type, method = Method} = Msg, #transport{req_context = Ctx}) -> + CtxMsg = {Ctx, Msg}, case Type of - ack -> - #{next => until_stop}; reset -> - #{next => until_stop}; + proto_out({reset, CtxMsg}, #{next => stop}); _ -> - ?EMPTY_RESULT + case Method of + undefined -> + %% empty ack, keep transport to recv response + proto_out({ack, CtxMsg}); + {_, _} -> + %% ack with payload + proto_out({response, CtxMsg}, #{next => stop}); + _ -> + reset(Msg, #{next => stop}) + end end; wait_ack(state_timeout, ack_timeout, - _, #transport{cache = Msg, retry_interval = Timeout, retry_count = Count} =Transport) -> case Count < ?MAX_RETRANSMIT of true -> Timeout2 = Timeout * 2, - #{transport => Transport#transport{retry_interval = Timeout2, - retry_count = Count + 1}, - out => Msg, - timeouts => [{state_timeout, Timeout2, ack_timeout}]}; + out(Msg, + #{transport => Transport#transport{retry_interval = Timeout2, + retry_count = Count + 1}, + timeouts => [{state_timeout, Timeout2, ack_timeout}]}); _ -> - #{next_state => until_stop} + proto_out({ack_failure, Msg}, #{next_state => stop}) end. -until_stop(_, _, _, _) -> - ?EMPTY_RESULT. - -call_handler(#coap_message{options = Opts} = Msg, Ctx) -> - case maps:get(uri_path, Opts, undefined) of - [<<"ps">> | RestPath] -> - emqx_coap_pubsub_handler:handle_request(RestPath, Msg, Ctx); - [<<"mqtt">> | RestPath] -> - emqx_coap_mqtt_handler:handle_request(RestPath, Msg, Ctx); +observe(in, + #coap_message{method = Method} = Message, + #transport{observe = Observe} = Transport) -> + case Method of + {ok, _} -> + case emqx_coap_message:get_option(observe, Message, Observe) of + Observe -> + %% repeatd notify, ignore + empty(); + NewObserve -> + on_response(Message, + Transport#transport{observe = NewObserve}, + ?FUNCTION_NAME) + end; + {error, _} -> + #{next => stop}; _ -> - ?REPLY({error, bad_request}, Msg) + reset(Message) + end. + +until_stop(_, _, _) -> + empty(). + +set_cache(Cache, Transport) -> + Transport#transport{cache = Cache}. + +on_response(#coap_message{type = Type} = Message, + #transport{req_context = Ctx} = Transport, + NextState) -> + CtxMsg = {Ctx, Message}, + if Type =:= non -> + proto_out({response, CtxMsg}, #{next => NextState}); + Type =:= con -> + Ack = emqx_coap_message:ack(Message), + proto_out({response, CtxMsg}, + out(Ack, #{next => NextState, + transport => Transport#transport{cache = Ack}})); + true -> + reset(Message) end. diff --git a/apps/emqx_gateway/src/coap/handler/emqx_coap_mqtt_handler.erl b/apps/emqx_gateway/src/coap/handler/emqx_coap_mqtt_handler.erl index 88a4a2310..47bf14d9b 100644 --- a/apps/emqx_gateway/src/coap/handler/emqx_coap_mqtt_handler.erl +++ b/apps/emqx_gateway/src/coap/handler/emqx_coap_mqtt_handler.erl @@ -18,23 +18,24 @@ -include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). --export([handle_request/3]). +-export([handle_request/4]). -import(emqx_coap_message, [response/2, response/3]). +-import(emqx_coap_medium, [reply/2]). -handle_request([<<"connection">>], #coap_message{method = Method} = Msg, _) -> +handle_request([<<"connection">>], #coap_message{method = Method} = Msg, _Ctx, _CInfo) -> handle_method(Method, Msg); -handle_request(_, Msg, _) -> - ?REPLY({error, bad_request}, Msg). +handle_request(_, Msg, _, _) -> + reply({error, bad_request}, Msg). handle_method(put, Msg) -> - ?REPLY({ok, changed}, Msg); + reply({ok, changed}, Msg); -handle_method(post, _) -> - #{connection => open}; +handle_method(post, Msg) -> + #{connection => {open, Msg}}; -handle_method(delete, _) -> - #{connection => close}; +handle_method(delete, Msg) -> + #{connection => {close, Msg}}; handle_method(_, Msg) -> - ?REPLY({error, method_not_allowed}, Msg). + reply({error, method_not_allowed}, Msg). diff --git a/apps/emqx_gateway/src/coap/handler/emqx_coap_pubsub_handler.erl b/apps/emqx_gateway/src/coap/handler/emqx_coap_pubsub_handler.erl index e6886a559..85cf32c6d 100644 --- a/apps/emqx_gateway/src/coap/handler/emqx_coap_pubsub_handler.erl +++ b/apps/emqx_gateway/src/coap/handler/emqx_coap_pubsub_handler.erl @@ -20,48 +20,52 @@ -include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). --export([handle_request/3]). +-export([handle_request/4]). -import(emqx_coap_message, [response/2, response/3]). +-import(emqx_coap_medium, [reply/2, reply/3]). +-import(emqx_coap_channel, [run_hooks/3]). --define(UNSUB(Topic), #{subscribe => Topic}). --define(SUB(Topic, Token), #{subscribe => {Topic, Token}}). +-define(UNSUB(Topic, Msg), #{subscribe => {Topic, Msg}}). +-define(SUB(Topic, Token, Msg), #{subscribe => {{Topic, Token}, Msg}}). -define(SUBOPTS, #{qos => 0, rh => 0, rap => 0, nl => 0, is_new => false}). -handle_request(Path, #coap_message{method = Method} = Msg, Ctx) -> +%% TODO maybe can merge this code into emqx_coap_session, simplify the call chain + +handle_request(Path, #coap_message{method = Method} = Msg, Ctx, CInfo) -> case check_topic(Path) of {ok, Topic} -> - handle_method(Method, Topic, Msg, Ctx); + handle_method(Method, Topic, Msg, Ctx, CInfo); _ -> - ?REPLY({error, bad_request}, <<"invalid topic">>, Msg) + reply({error, bad_request}, <<"invalid topic">>, Msg) end. -handle_method(get, Topic, #coap_message{options = Opts} = Msg, Ctx) -> - case maps:get(observe, Opts, undefined) of +handle_method(get, Topic, Msg, Ctx, CInfo) -> + case emqx_coap_message:get_option(observe, Msg) of 0 -> - subscribe(Msg, Topic, Ctx); + subscribe(Msg, Topic, Ctx, CInfo); 1 -> - unsubscribe(Topic, Ctx); + unsubscribe(Msg, Topic, Ctx, CInfo); _ -> - ?REPLY({error, bad_request}, <<"invalid observe value">>, Msg) + reply({error, bad_request}, <<"invalid observe value">>, Msg) end; -handle_method(post, Topic, #coap_message{payload = Payload} = Msg, Ctx) -> - case emqx_coap_channel:validator(publish, Topic, Ctx) of +handle_method(post, Topic, #coap_message{payload = Payload} = Msg, Ctx, CInfo) -> + case emqx_coap_channel:validator(publish, Topic, Ctx, CInfo) of allow -> - ClientInfo = emqx_coap_channel:get_clientinfo(Ctx), - #{clientid := ClientId} = ClientInfo, - QOS = get_publish_qos(Msg, Ctx), - MQTTMsg = emqx_message:make(ClientId, QOS, Topic, Payload), + #{clientid := ClientId} = CInfo, + MountTopic = mount(CInfo, Topic), + QOS = get_publish_qos(Msg), + MQTTMsg = emqx_message:make(ClientId, QOS, MountTopic, Payload), MQTTMsg2 = apply_publish_opts(Msg, MQTTMsg), _ = emqx_broker:publish(MQTTMsg2), - ?REPLY({ok, changed}, Msg); + reply({ok, changed}, Msg); _ -> - ?REPLY({error, unauthorized}, Msg) + reply({error, unauthorized}, Msg) end; -handle_method(_, _, Msg, _) -> - ?REPLY({error, method_not_allowed}, Msg). +handle_method(_, _, Msg, _, _) -> + reply({error, method_not_allowed}, Msg). check_topic([]) -> error; @@ -76,13 +80,13 @@ check_topic(Path) -> <<>>, Path))}. -get_sub_opts(#coap_message{options = Opts} = Msg, Ctx) -> +get_sub_opts(#coap_message{options = Opts} = Msg) -> SubOpts = maps:fold(fun parse_sub_opts/3, #{}, Opts), case SubOpts of #{qos := _} -> maps:merge(SubOpts, ?SUBOPTS); _ -> - CfgType = emqx_coap_channel:get_config(subscribe_qos, Ctx), + CfgType = emqx:get_config([gateway, coap, subscribe_qos], ?QOS_0), maps:merge(SubOpts, ?SUBOPTS#{qos => type_to_qos(CfgType, Msg)}) end. @@ -106,16 +110,16 @@ type_to_qos(coap, #coap_message{type = Type}) -> ?QOS_1 end. -get_publish_qos(#coap_message{options = Opts} = Msg, Ctx) -> - case maps:get(uri_query, Opts) of +get_publish_qos(Msg) -> + case emqx_coap_message:get_option(uri_query, Msg) of #{<<"qos">> := QOS} -> erlang:binary_to_integer(QOS); _ -> - CfgType = emqx_coap_channel:get_config(publish_qos, Ctx), + CfgType = emqx:get_config([gateway, coap, publish_qos], ?QOS_0), type_to_qos(CfgType, Msg) end. -apply_publish_opts(#coap_message{options = Opts}, MQTTMsg) -> +apply_publish_opts(Msg, MQTTMsg) -> maps:fold(fun(<<"retain">>, V, Acc) -> Val = erlang:binary_to_atom(V), emqx_message:set_flag(retain, Val, Acc); @@ -129,27 +133,29 @@ apply_publish_opts(#coap_message{options = Opts}, MQTTMsg) -> Acc end, MQTTMsg, - maps:get(uri_query, Opts)). + emqx_coap_message:get_option(uri_query, Msg)). -subscribe(#coap_message{token = <<>>} = Msg, _, _) -> - ?REPLY({error, bad_request}, <<"observe without token">>, Msg); +subscribe(#coap_message{token = <<>>} = Msg, _, _, _) -> + reply({error, bad_request}, <<"observe without token">>, Msg); -subscribe(#coap_message{token = Token} = Msg, Topic, Ctx) -> - case emqx_coap_channel:validator(subscribe, Topic, Ctx) of +subscribe(#coap_message{token = Token} = Msg, Topic, Ctx, CInfo) -> + case emqx_coap_channel:validator(subscribe, Topic, Ctx, CInfo) of allow -> - ClientInfo = emqx_coap_channel:get_clientinfo(Ctx), - #{clientid := ClientId} = ClientInfo, - SubOpts = get_sub_opts(Msg, Ctx), - emqx_broker:subscribe(Topic, ClientId, SubOpts), - emqx_hooks:run('session.subscribed', - [ClientInfo, Topic, SubOpts]), - ?SUB(Topic, Token); + #{clientid := ClientId} = CInfo, + SubOpts = get_sub_opts(Msg), + MountTopic = mount(CInfo, Topic), + emqx_broker:subscribe(MountTopic, ClientId, SubOpts), + run_hooks(Ctx, 'session.subscribed', [CInfo, Topic, SubOpts]), + ?SUB(MountTopic, Token, Msg); _ -> - ?REPLY({error, unauthorized}, Msg) + reply({error, unauthorized}, Msg) end. -unsubscribe(Topic, Ctx) -> - ClientInfo = emqx_coap_channel:get_clientinfo(Ctx), - emqx_broker:unsubscribe(Topic), - emqx_hooks:run('session.unsubscribed', [ClientInfo, Topic, ?SUBOPTS]), - ?UNSUB(Topic). +unsubscribe(Msg, Topic, Ctx, CInfo) -> + MountTopic = mount(CInfo, Topic), + emqx_broker:unsubscribe(MountTopic), + run_hooks(Ctx, 'session.unsubscribed', [CInfo, Topic, ?SUBOPTS]), + ?UNSUB(MountTopic, Msg). + +mount(#{mountpoint := Mountpoint}, Topic) -> + <>. diff --git a/apps/emqx_gateway/src/coap/include/emqx_coap.hrl b/apps/emqx_gateway/src/coap/include/emqx_coap.hrl index 3b0268abb..d47dd17fd 100644 --- a/apps/emqx_gateway/src/coap/include/emqx_coap.hrl +++ b/apps/emqx_gateway/src/coap/include/emqx_coap.hrl @@ -22,18 +22,6 @@ -define(DEFAULT_MAX_AGE, 60). -define(MAXIMUM_MAX_AGE, 4294967295). --define(EMPTY_RESULT, #{}). --define(TRANSFER_RESULT(From, Value, R1), - begin - Keys = result_keys(), - R2 = maps:with(Keys, R1), - R2#{From => Value} - end). - --define(RESET(Msg), #{out => emqx_coap_message:reset(Msg)}). --define(REPLY(Resp, Payload, Msg), #{out => emqx_coap_message:piggyback(Resp, Payload, Msg)}). --define(REPLY(Resp, Msg), ?REPLY(Resp, <<>>, Msg)). - -type coap_message_id() :: 1 .. ?MAX_MESSAGE_ID. -type message_type() :: con | non | ack | reset. -type max_age() :: 1 .. ?MAXIMUM_MAX_AGE. @@ -66,7 +54,7 @@ , uri_path => list(binary()) , content_format => 0 .. 65535 , max_age => non_neg_integer() - , uri_query => list(binary()) + , uri_query => list(binary()) | map() , 'accept' => 0 .. 65535 , location_query => list(binary()) , proxy_uri => binary() @@ -85,7 +73,4 @@ , options = #{} , payload = <<>>}). --record(coap_content, {etag, max_age = ?DEFAULT_MAX_AGE, format, location_path = [], payload = <<>>}). - -type emqx_coap_message() :: #coap_message{}. --type coap_content() :: #coap_content{}. diff --git a/apps/emqx_gateway/src/emqx_gateway.app.src b/apps/emqx_gateway/src/emqx_gateway.app.src index e25b767cc..2fc329711 100644 --- a/apps/emqx_gateway/src/emqx_gateway.app.src +++ b/apps/emqx_gateway/src/emqx_gateway.app.src @@ -3,7 +3,7 @@ {vsn, "0.1.0"}, {registered, []}, {mod, {emqx_gateway_app, []}}, - {applications, [kernel, stdlib, grpc, lwm2m_coap, emqx, emqx_authn]}, + {applications, [kernel, stdlib, grpc, lwm2m_coap, emqx]}, {env, []}, {modules, []}, {licenses, ["Apache 2.0"]}, diff --git a/apps/emqx_gateway/src/emqx_gateway.erl b/apps/emqx_gateway/src/emqx_gateway.erl index 79ea5d8a4..596b47547 100644 --- a/apps/emqx_gateway/src/emqx_gateway.erl +++ b/apps/emqx_gateway/src/emqx_gateway.erl @@ -25,7 +25,7 @@ , post_config_update/4 ]). -%% APIs +%% Gateway APIs -export([ registered_gateway/0 , load/2 , unload/1 @@ -48,7 +48,7 @@ registered_gateway() -> emqx_gateway_registry:list(). %%-------------------------------------------------------------------- -%% Gateway Instace APIs +%% Gateway APIs -spec list() -> [gateway()]. list() -> @@ -96,7 +96,8 @@ update_rawconf(RawName, RawConfDiff) -> %%-------------------------------------------------------------------- %% Config Handler --spec pre_config_update(emqx_config:update_request(), emqx_config:raw_config()) -> +-spec pre_config_update(emqx_config:update_request(), + emqx_config:raw_config()) -> {ok, emqx_config:update_request()} | {error, term()}. pre_config_update({RawName, RawConfDiff}, RawConf) -> {ok, emqx_map_lib:deep_merge(RawConf, #{RawName => RawConfDiff})}. @@ -117,4 +118,3 @@ post_config_update({RawName, _}, NewConfig, OldConfig, _AppEnvs) -> %%-------------------------------------------------------------------- %% Internal funcs %%-------------------------------------------------------------------- - diff --git a/apps/emqx_gateway/src/emqx_gateway_api.erl b/apps/emqx_gateway/src/emqx_gateway_api.erl index f38624ed9..9037518c5 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api.erl @@ -18,10 +18,14 @@ -behaviour(minirest_api). --compile(nowarn_unused_function). - --import(emqx_mgmt_util, [ schema/1 - ]). +-import(emqx_gateway_http, + [ return_http_error/2 + , with_gateway/2 + , schema_bad_request/0 + , schema_not_found/0 + , schema_internal_error/0 + , schema_no_content/0 + ]). %% minirest behaviour callbacks -export([api_spec/0]). @@ -32,18 +36,161 @@ , gateway_insta_stats/2 ]). --define(EXAMPLE_GATEWAY_LIST, - [ #{ name => <<"lwm2m">> - , status => <<"running">> - , started_at => <<"2021-08-19T11:45:56.006373+08:00">> - , max_connection => 1024000 - , current_connection => 1000 - , listeners => [ - #{name => <<"lw-udp-1">>, status => <<"activing">>}, - #{name => <<"lw-udp-2">>, status => <<"inactived">>} - ] - } - ]). +%%-------------------------------------------------------------------- +%% minirest behaviour callbacks +%%-------------------------------------------------------------------- + +api_spec() -> + {metadata(apis()), []}. + +apis() -> + [ {"/gateway", gateway} + , {"/gateway/:name", gateway_insta} + , {"/gateway/:name/stats", gateway_insta_stats} + ]. +%%-------------------------------------------------------------------- +%% http handlers + +gateway(get, Request) -> + Params = maps:get(query_string, Request, #{}), + Status = case maps:get(<<"status">>, Params, undefined) of + undefined -> all; + S0 -> binary_to_existing_atom(S0, utf8) + end, + {200, emqx_gateway_http:gateways(Status)}. + +gateway_insta(delete, #{bindings := #{name := Name0}}) -> + with_gateway(Name0, fun(GwName, _) -> + _ = emqx_gateway:unload(GwName), + {204} + end); +gateway_insta(get, #{bindings := #{name := Name0}}) -> + with_gateway(Name0, fun(_, _) -> + GwConf = filled_raw_confs([<<"gateway">>, Name0]), + LisConf = maps:get(<<"listeners">>, GwConf, #{}), + NLisConf = emqx_gateway_http:mapping_listener_m2l(Name0, LisConf), + {200, GwConf#{<<"listeners">> => NLisConf}} + end); +gateway_insta(put, #{body := GwConf0, + bindings := #{name := Name0} + }) -> + with_gateway(Name0, fun(_, _) -> + GwConf = maps:without([<<"authentication">>, <<"listeners">>], GwConf0), + case emqx_gateway:update_rawconf(Name0, GwConf) of + ok -> + {200}; + {error, not_found} -> + return_http_error(404, "Gateway not found"); + {error, Reason} -> + return_http_error(500, Reason) + end + end). + +gateway_insta_stats(get, _Req) -> + return_http_error(401, "Implement it later (maybe 5.1)"). + +filled_raw_confs(Path) -> + RawConf = emqx_config:fill_defaults( + emqx_config:get_root_raw(Path) + ), + Confs = emqx_map_lib:deep_get(Path, RawConf), + emqx_map_lib:jsonable_map(Confs). + +%%-------------------------------------------------------------------- +%% Swagger defines +%%-------------------------------------------------------------------- + +metadata(APIs) -> + metadata(APIs, []). +metadata([], APIAcc) -> + lists:reverse(APIAcc); +metadata([{Path, Fun}|More], APIAcc) -> + Methods = [get, post, put, delete, patch], + Mds = lists:foldl(fun(M, Acc) -> + try + Acc#{M => swagger(Path, M)} + catch + error : function_clause -> + Acc + end + end, #{}, Methods), + metadata(More, [{Path, Mds, Fun} | APIAcc]). + +swagger("/gateway", get) -> + #{ description => <<"Get gateway list">> + , parameters => params_gateway_status_in_qs() + , responses => + #{ <<"200">> => schema_gateway_overview_list() } + }; +swagger("/gateway/:name", get) -> + #{ description => <<"Get the gateway configurations">> + , parameters => params_gateway_name_in_path() + , responses => + #{ <<"400">> => schema_bad_request() + , <<"404">> => schema_not_found() + , <<"500">> => schema_internal_error() + , <<"200">> => schema_gateway_conf() + } + }; +swagger("/gateway/:name", delete) -> + #{ description => <<"Delete/Unload the gateway">> + , parameters => params_gateway_name_in_path() + , responses => + #{ <<"400">> => schema_bad_request() + , <<"404">> => schema_not_found() + , <<"500">> => schema_internal_error() + , <<"204">> => schema_no_content() + } + }; +swagger("/gateway/:name", put) -> + #{ description => <<"Update the gateway configurations/status">> + , parameters => params_gateway_name_in_path() + , requestBody => schema_gateway_conf() + , responses => + #{ <<"400">> => schema_bad_request() + , <<"404">> => schema_not_found() + , <<"500">> => schema_internal_error() + , <<"200">> => schema_no_content() + } + }; +swagger("/gateway/:name/stats", get) -> + #{ description => <<"Get gateway Statistic">> + , parameters => params_gateway_name_in_path() + , responses => + #{ <<"400">> => schema_bad_request() + , <<"404">> => schema_not_found() + , <<"500">> => schema_internal_error() + , <<"200">> => schema_gateway_stats() + } + }. + +%%-------------------------------------------------------------------- +%% params defines + +params_gateway_name_in_path() -> + [#{ name => name + , in => path + , schema => #{type => string} + , required => true + }]. + +params_gateway_status_in_qs() -> + [#{ name => status + , in => query + , schema => #{type => string} + , required => false + }]. + +%%-------------------------------------------------------------------- +%% schemas + +schema_gateway_overview_list() -> + emqx_mgmt_util:array_schema( + #{ type => object + , properties => properties_gateway_overview() + }, + <<"Gateway Overview list">> + ). %% XXX: This is whole confs for all type gateways. It is used to fill the %% default configurations and generate the swagger-schema @@ -58,8 +205,13 @@ <<"enable">> => true, <<"enable_stats">> => true,<<"heartbeat">> => <<"30s">>, <<"idle_timeout">> => <<"30s">>, - <<"listeners">> => - #{<<"udp">> => #{<<"default">> => #{<<"bind">> => 5683}}}, + <<"listeners">> => [ + #{<<"id">> => <<"coap:udp:default">>, + <<"type">> => <<"udp">>, + <<"running">> => true, + <<"acceptors">> => 8,<<"bind">> => 5683, + <<"max_conn_rate">> => 1000, + <<"max_connections">> => 10240}], <<"mountpoint">> => <<>>,<<"notify_type">> => <<"qos">>, <<"publish_qos">> => <<"qos1">>, <<"subscribe_qos">> => <<"qos0">>} @@ -71,12 +223,13 @@ <<"handler">> => #{<<"address">> => <<"http://127.0.0.1:9001">>}, <<"idle_timeout">> => <<"30s">>, - <<"listeners">> => - #{<<"tcp">> => - #{<<"default">> => - #{<<"acceptors">> => 8,<<"bind">> => 7993, - <<"max_conn_rate">> => 1000, - <<"max_connections">> => 10240}}}, + <<"listeners">> => [ + #{<<"id">> => <<"exproto:tcp:default">>, + <<"type">> => <<"tcp">>, + <<"running">> => true, + <<"acceptors">> => 8,<<"bind">> => 7993, + <<"max_conn_rate">> => 1000, + <<"max_connections">> => 10240}], <<"mountpoint">> => <<>>, <<"server">> => #{<<"bind">> => 9100}} ). @@ -88,8 +241,11 @@ <<"idle_timeout">> => <<"30s">>, <<"lifetime_max">> => <<"86400s">>, <<"lifetime_min">> => <<"1s">>, - <<"listeners">> => - #{<<"udp">> => #{<<"default">> => #{<<"bind">> => 5783}}}, + <<"listeners">> => [ + #{<<"id">> => <<"lwm2m:udp:default">>, + <<"type">> => <<"udp">>, + <<"running">> => true, + <<"bind">> => 5783}], <<"mountpoint">> => <<"lwm2m/%e/">>, <<"qmode_time_windonw">> => 22, <<"translators">> => @@ -110,11 +266,12 @@ <<"enable">> => true, <<"enable_qos3">> => true,<<"enable_stats">> => true, <<"gateway_id">> => 1,<<"idle_timeout">> => <<"30s">>, - <<"listeners">> => - #{<<"udp">> => - #{<<"default">> => - #{<<"bind">> => 1884,<<"max_conn_rate">> => 1000, - <<"max_connections">> => 10240000}}}, + <<"listeners">> => [ + #{<<"id">> => <<"mqttsn:udp:default">>, + <<"type">> => <<"udp">>, + <<"running">> => true, + <<"bind">> => 1884,<<"max_conn_rate">> => 1000, + <<"max_connections">> => 10240000}], <<"mountpoint">> => <<>>, <<"predefined">> => [#{<<"id">> => 1, @@ -138,245 +295,60 @@ #{<<"max_body_length">> => 8192,<<"max_headers">> => 10, <<"max_headers_length">> => 1024}, <<"idle_timeout">> => <<"30s">>, - <<"listeners">> => - #{<<"tcp">> => - #{<<"default">> => - #{<<"acceptors">> => 16,<<"active_n">> => 100, - <<"bind">> => 61613,<<"max_conn_rate">> => 1000, - <<"max_connections">> => 1024000}}}, + <<"listeners">> => [ + #{<<"id">> => <<"stomp:tcp:default">>, + <<"type">> => <<"tcp">>, + <<"running">> => true, + <<"acceptors">> => 16,<<"active_n">> => 100, + <<"bind">> => 61613,<<"max_conn_rate">> => 1000, + <<"max_connections">> => 1024000}], <<"mountpoint">> => <<>>} ). %% --- END --define(EXAMPLE_GATEWAY_STATS, #{ - max_connection => 10240000, - current_connection => 1000, - messages_in => 100.24, - messages_out => 32.5 - }). +schema_gateway_conf() -> + emqx_mgmt_util:schema( + #{oneOf => + [ emqx_mgmt_api_configs:gen_schema(?STOMP_GATEWAY_CONFS) + , emqx_mgmt_api_configs:gen_schema(?MQTTSN_GATEWAY_CONFS) + , emqx_mgmt_api_configs:gen_schema(?COAP_GATEWAY_CONFS) + , emqx_mgmt_api_configs:gen_schema(?LWM2M_GATEWAY_CONFS) + , emqx_mgmt_api_configs:gen_schema(?EXPROTO_GATEWAY_CONFS) + ]}). + +schema_gateway_stats() -> + emqx_mgmt_util:schema( + #{ type => object + , properties => + #{ a_key => #{type => string} + }}). %%-------------------------------------------------------------------- -%% minirest behaviour callbacks -%%-------------------------------------------------------------------- +%% properties -api_spec() -> - {apis(), schemas()}. - -apis() -> - [ {"/gateway", metadata(gateway), gateway} - , {"/gateway/:name", metadata(gateway_insta), gateway_insta} - , {"/gateway/:name/stats", metadata(gateway_insta_stats), gateway_insta_stats} - ]. - -metadata(gateway) -> - #{get => #{ - description => <<"Get gateway list">>, - parameters => [ - #{name => status, - in => query, - schema => #{type => string}, - required => false - } +properties_gateway_overview() -> + ListenerProps = + [ {id, string, + <<"Listener ID">>} + , {running, boolean, + <<"Listener Running status">>} + , {type, string, + <<"Listener Type">>, [<<"tcp">>, <<"ssl">>, <<"udp">>, <<"dtls">>]} ], - responses => #{ - <<"200">> => #{ - description => <<"OK">>, - content => #{ - 'application/json' => #{ - schema => minirest:ref(<<"gateway_overrview">>), - examples => #{ - simple => #{ - summary => <<"Gateway List Example">>, - value => emqx_json:encode(?EXAMPLE_GATEWAY_LIST) - } - } - } - } - } - } - }}; - -metadata(gateway_insta) -> - UriNameParamDef = #{name => name, - in => path, - schema => #{type => string}, - required => true - }, - NameNotFoundRespDef = - #{description => <<"Not Found">>, - content => #{ - 'application/json' => #{ - schema => minirest:ref(<<"error">>), - examples => #{ - simple => #{ - summary => <<"Not Found">>, - value => #{ - code => <<"NOT_FOUND">>, - message => <<"The gateway not found">> - } - } - } - } - }}, - #{delete => #{ - description => <<"Delete/Unload the gateway">>, - parameters => [UriNameParamDef], - responses => #{ - <<"404">> => NameNotFoundRespDef, - <<"204">> => #{description => <<"No Content">>} - } - }, - get => #{ - description => <<"Get the gateway configurations">>, - parameters => [UriNameParamDef], - responses => #{ - <<"404">> => NameNotFoundRespDef, - <<"200">> => schema(schema_for_gateway_conf()) - } - }, - put => #{ - description => <<"Update the gateway configurations/status">>, - parameters => [UriNameParamDef], - requestBody => schema(schema_for_gateway_conf()), - responses => #{ - <<"404">> => NameNotFoundRespDef, - <<"200">> => #{description => <<"Changed">>} - } - } - }; - -metadata(gateway_insta_stats) -> - #{get => #{ - description => <<"Get gateway Statistic">>, - responses => #{ - <<"200">> => #{ - description => <<"OK">>, - content => #{ - 'application/json' => #{ - schema => minirest:ref(<<"gateway_stats">>), - examples => #{ - simple => #{ - summary => <<"Gateway Statistic">>, - value => emqx_json:encode(?EXAMPLE_GATEWAY_STATS) - } - } - } - } - } - } - }}. - -schemas() -> - [ #{<<"gateway_overrview">> => schema_for_gateway_overrview()} - , #{<<"gateway_stats">> => schema_for_gateway_stats()} - ]. - -schema_for_gateway_overrview() -> - #{type => array, - items => #{ - type => object, - properties => #{ - name => #{ - type => string, - example => <<"lwm2m">> - }, - status => #{ - type => string, - enum => [<<"running">>, <<"stopped">>, <<"unloaded">>], - example => <<"running">> - }, - started_at => #{ - type => string, - example => <<"2021-08-19T11:45:56.006373+08:00">> - }, - max_connection => #{ - type => integer, - example => 1024000 - }, - current_connection => #{ - type => integer, - example => 1000 - }, - listeners => #{ - type => array, - items => #{ - type => object, - properties => #{ - name => #{ - type => string, - example => <<"lw-udp">> - }, - status => #{ - type => string, - enum => [<<"activing">>, <<"inactived">>] - } - } - } - } - } - } - }. - -schema_for_gateway_conf() -> - #{oneOf => - [ emqx_mgmt_api_configs:gen_schema(?STOMP_GATEWAY_CONFS) - , emqx_mgmt_api_configs:gen_schema(?MQTTSN_GATEWAY_CONFS) - , emqx_mgmt_api_configs:gen_schema(?COAP_GATEWAY_CONFS) - , emqx_mgmt_api_configs:gen_schema(?LWM2M_GATEWAY_CONFS) - , emqx_mgmt_api_configs:gen_schema(?EXPROTO_GATEWAY_CONFS) - ]}. - -schema_for_gateway_stats() -> - #{type => object, - properties => #{ - a_key => #{type => string} - }}. - -%%-------------------------------------------------------------------- -%% http handlers - -gateway(get, Request) -> - Params = maps:get(query_string, Request, #{}), - Status = case maps:get(<<"status">>, Params, undefined) of - undefined -> all; - S0 -> binary_to_existing_atom(S0, utf8) - end, - {200, emqx_gateway_intr:gateways(Status)}. - -gateway_insta(delete, #{bindings := #{name := Name0}}) -> - Name = binary_to_existing_atom(Name0), - case emqx_gateway:unload(Name) of - ok -> - {200, ok}; - {error, not_found} -> - {404, <<"Not Found">>} - end; -gateway_insta(get, #{bindings := #{name := Name0}}) -> - Name = binary_to_existing_atom(Name0), - case emqx_gateway:lookup(Name) of - #{config := _Config} -> - %% FIXME: Got the parsed config, but we should return rawconfig to - %% frontend - RawConf = emqx_config:fill_defaults( - emqx_config:get_root_raw([<<"gateway">>]) - ), - {200, emqx_map_lib:deep_get([<<"gateway">>, Name0], RawConf)}; - undefined -> - {404, <<"Not Found">>} - end; -gateway_insta(put, #{body := RawConfsIn, - bindings := #{name := Name} - }) -> - %% FIXME: Cluster Consistence ?? - case emqx_gateway:update_rawconf(Name, RawConfsIn) of - ok -> - {200, <<"Changed">>}; - {error, not_found} -> - {404, <<"Not Found">>}; - {error, Reason} -> - {500, emqx_gateway_utils:stringfy(Reason)} - end. - -gateway_insta_stats(get, _Req) -> - {401, <<"Implement it later (maybe 5.1)">>}. + emqx_mgmt_util:properties( + [ {name, string, + <<"Gateway Name">>} + , {status, string, + <<"Gateway Status">>, + [<<"running">>, <<"stopped">>, <<"unloaded">>]} + , {created_at, string, + <<>>} + , {started_at, string, + <<>>} + , {stopped_at, string, + <<>>} + , {max_connections, integer, <<>>} + , {current_connections, integer, <<>>} + , {listeners, {array, object}, ListenerProps} + ]). diff --git a/apps/emqx_gateway/src/emqx_gateway_api_clients.erl b/apps/emqx_gateway/src/emqx_gateway_api_clients.erl new file mode 100644 index 000000000..386d6e1ea --- /dev/null +++ b/apps/emqx_gateway/src/emqx_gateway_api_clients.erl @@ -0,0 +1,636 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_gateway_api_clients). + +-behaviour(minirest_api). + +-include_lib("emqx/include/logger.hrl"). + +%% minirest behaviour callbacks +-export([api_spec/0]). + +%% http handlers +-export([ clients/2 + , clients_insta/2 + , subscriptions/2 + ]). + +%% internal exports (for client query) +-export([ query/4 + , format_channel_info/1 + ]). + +-import(emqx_gateway_http, + [ return_http_error/2 + , with_gateway/2 + , schema_bad_request/0 + , schema_not_found/0 + , schema_internal_error/0 + , schema_no_content/0 + ]). + +%%-------------------------------------------------------------------- +%% APIs +%%-------------------------------------------------------------------- + +api_spec() -> + {metadata(apis()), []}. + +apis() -> + [ {"/gateway/:name/clients", clients} + , {"/gateway/:name/clients/:clientid", clients_insta} + , {"/gateway/:name/clients/:clientid/subscriptions", subscriptions} + , {"/gateway/:name/clients/:clientid/subscriptions/:topic", subscriptions} + ]. + +-define(CLIENT_QS_SCHEMA, + [ {<<"node">>, atom} + , {<<"clientid">>, binary} + , {<<"username">>, binary} + , {<<"ip_address">>, ip} + , {<<"conn_state">>, atom} + , {<<"clean_start">>, atom} + , {<<"proto_ver">>, integer} + , {<<"like_clientid">>, binary} + , {<<"like_username">>, binary} + , {<<"gte_created_at">>, timestamp} + , {<<"lte_created_at">>, timestamp} + , {<<"gte_connected_at">>, timestamp} + , {<<"lte_connected_at">>, timestamp} + ]). + +-define(query_fun, {?MODULE, query}). +-define(format_fun, {?MODULE, format_channel_info}). + +clients(get, #{ bindings := #{name := Name0} + , query_string := Qs + }) -> + with_gateway(Name0, fun(GwName, _) -> + TabName = emqx_gateway_cm:tabname(info, GwName), + case maps:get(<<"node">>, Qs, undefined) of + undefined -> + Response = emqx_mgmt_api:cluster_query( + Qs, TabName, + ?CLIENT_QS_SCHEMA, ?query_fun + ), + {200, Response}; + Node1 -> + Node = binary_to_atom(Node1, utf8), + ParamsWithoutNode = maps:without([<<"node">>], Qs), + Response = emqx_mgmt_api:node_query( + Node, ParamsWithoutNode, + TabName, ?CLIENT_QS_SCHEMA, ?query_fun + ), + {200, Response} + end + end). + +clients_insta(get, #{ bindings := #{name := Name0, + clientid := ClientId0} + }) -> + ClientId = emqx_mgmt_util:urldecode(ClientId0), + with_gateway(Name0, fun(GwName, _) -> + case emqx_gateway_http:lookup_client(GwName, ClientId, + {?MODULE, format_channel_info}) of + [ClientInfo] -> + {200, ClientInfo}; + [ClientInfo|_More] -> + ?LOG(warning, "More than one client info was returned on ~s", + [ClientId]), + {200, ClientInfo}; + [] -> + return_http_error(404, "Client not found") + end + end); +clients_insta(delete, #{ bindings := #{name := Name0, + clientid := ClientId0} + }) -> + ClientId = emqx_mgmt_util:urldecode(ClientId0), + with_gateway(Name0, fun(GwName, _) -> + _ = emqx_gateway_http:kickout_client(GwName, ClientId), + {200} + end). + +%% FIXME: +%% List the subscription without mountpoint, but has SubOpts, +%% for example, share group ... +subscriptions(get, #{ bindings := #{name := Name0, + clientid := ClientId0} + }) -> + ClientId = emqx_mgmt_util:urldecode(ClientId0), + with_gateway(Name0, fun(GwName, _) -> + case emqx_gateway_http:list_client_subscriptions(GwName, ClientId) of + {error, Reason} -> + return_http_error(500, Reason); + {ok, Subs} -> + {200, Subs} + end + end); + +%% Create the subscription without mountpoint +subscriptions(post, #{ bindings := #{name := Name0, + clientid := ClientId0}, + body := Body + }) -> + ClientId = emqx_mgmt_util:urldecode(ClientId0), + with_gateway(Name0, fun(GwName, _) -> + case {maps:get(<<"topic">>, Body, undefined), subopts(Body)} of + {undefined, _} -> + return_http_error(400, "Miss topic property"); + {Topic, QoS} -> + case emqx_gateway_http:client_subscribe(GwName, ClientId, Topic, QoS) of + {error, Reason} -> + return_http_error(404, Reason); + ok -> + {200} + end + end + end); + +%% Remove the subscription without mountpoint +subscriptions(delete, #{ bindings := #{name := Name0, + clientid := ClientId0, + topic := Topic0 + } + }) -> + ClientId = emqx_mgmt_util:urldecode(ClientId0), + Topic = emqx_mgmt_util:urldecode(Topic0), + with_gateway(Name0, fun(GwName, _) -> + _ = emqx_gateway_http:client_unsubscribe(GwName, ClientId, Topic), + {200} + end). + +%%-------------------------------------------------------------------- +%% Utils + +subopts(Req) -> + #{ qos => maps:get(<<"qos">>, Req, 0) + , rap => maps:get(<<"rap">>, Req, 0) + , nl => maps:get(<<"nl">>, Req, 0) + , rh => maps:get(<<"rh">>, Req, 0) + , sub_props => extra_sub_props(maps:get(<<"sub_props">>, Req, #{})) + }. + +extra_sub_props(Props) -> + maps:filter( + fun(_, V) -> V =/= undefined end, + #{subid => maps:get(<<"subid">>, Props, undefined)} + ). + +%%-------------------------------------------------------------------- +%% query funcs + +query(Tab, {Qs, []}, Start, Limit) -> + Ms = qs2ms(Qs), + emqx_mgmt_api:select_table(Tab, Ms, Start, Limit, + fun format_channel_info/1); + +query(Tab, {Qs, Fuzzy}, Start, Limit) -> + Ms = qs2ms(Qs), + MatchFun = match_fun(Ms, Fuzzy), + emqx_mgmt_api:traverse_table(Tab, MatchFun, Start, Limit, + fun format_channel_info/1). + +qs2ms(Qs) -> + {MtchHead, Conds} = qs2ms(Qs, 2, {#{}, []}), + [{{'$1', MtchHead, '_'}, Conds, ['$_']}]. + +qs2ms([], _, {MtchHead, Conds}) -> + {MtchHead, lists:reverse(Conds)}; + +qs2ms([{Key, '=:=', Value} | Rest], N, {MtchHead, Conds}) -> + NMtchHead = emqx_mgmt_util:merge_maps(MtchHead, ms(Key, Value)), + qs2ms(Rest, N, {NMtchHead, Conds}); +qs2ms([Qs | Rest], N, {MtchHead, Conds}) -> + Holder = binary_to_atom(iolist_to_binary(["$", integer_to_list(N)]), utf8), + NMtchHead = emqx_mgmt_util:merge_maps(MtchHead, ms(element(1, Qs), Holder)), + NConds = put_conds(Qs, Holder, Conds), + qs2ms(Rest, N+1, {NMtchHead, NConds}). + +put_conds({_, Op, V}, Holder, Conds) -> + [{Op, Holder, V} | Conds]; +put_conds({_, Op1, V1, Op2, V2}, Holder, Conds) -> + [{Op2, Holder, V2}, + {Op1, Holder, V1} | Conds]. + +ms(clientid, X) -> + #{clientinfo => #{clientid => X}}; +ms(username, X) -> + #{clientinfo => #{username => X}}; +ms(zone, X) -> + #{clientinfo => #{zone => X}}; +ms(ip_address, X) -> + #{clientinfo => #{peerhost => X}}; +ms(conn_state, X) -> + #{conn_state => X}; +ms(clean_start, X) -> + #{conninfo => #{clean_start => X}}; +ms(proto_ver, X) -> + #{conninfo => #{proto_ver => X}}; +ms(connected_at, X) -> + #{conninfo => #{connected_at => X}}; +ms(created_at, X) -> + #{session => #{created_at => X}}. + +%%-------------------------------------------------------------------- +%% Match funcs + +match_fun(Ms, Fuzzy) -> + MsC = ets:match_spec_compile(Ms), + REFuzzy = lists:map(fun({K, like, S}) -> + {ok, RE} = re:compile(S), + {K, like, RE} + end, Fuzzy), + fun(Rows) -> + case ets:match_spec_run(Rows, MsC) of + [] -> []; + Ls -> + lists:filter(fun(E) -> + run_fuzzy_match(E, REFuzzy) + end, Ls) + end + end. + +run_fuzzy_match(_, []) -> + true; +run_fuzzy_match(E = {_, #{clientinfo := ClientInfo}, _}, [{Key, _, RE}|Fuzzy]) -> + Val = case maps:get(Key, ClientInfo, "") of + undefined -> ""; + V -> V + end, + re:run(Val, RE, [{capture, none}]) == match andalso run_fuzzy_match(E, Fuzzy). + +%%-------------------------------------------------------------------- +%% format funcs + +format_channel_info({_, Infos, Stats}) -> + ClientInfo = maps:get(clientinfo, Infos, #{}), + ConnInfo = maps:get(conninfo, Infos, #{}), + SessInfo = maps:get(session, Infos, #{}), + FetchX = [ {node, ClientInfo, node()} + , {clientid, ClientInfo} + , {username, ClientInfo} + , {proto_name, ConnInfo} + , {proto_ver, ConnInfo} + , {ip_address, {peername, ConnInfo, fun peer_to_binary/1}} + , {is_bridge, ClientInfo, false} + , {connected_at, + {connected_at, ConnInfo, fun emqx_gateway_utils:unix_ts_to_rfc3339/1}} + , {disconnected_at, + {disconnected_at, ConnInfo, fun emqx_gateway_utils:unix_ts_to_rfc3339/1}} + , {connected, {conn_state, Infos, fun conn_state_to_connected/1}} + , {keepalive, ClientInfo, 0} + , {clean_start, ConnInfo, true} + , {expiry_interval, ConnInfo, 0} + , {created_at, + {created_at, SessInfo, fun emqx_gateway_utils:unix_ts_to_rfc3339/1}} + , {subscriptions_cnt, Stats, 0} + , {subscriptions_max, Stats, infinity} + , {inflight_cnt, Stats, 0} + , {inflight_max, Stats, infinity} + , {mqueue_len, Stats, 0} + , {mqueue_max, Stats, infinity} + , {mqueue_dropped, Stats, 0} + , {awaiting_rel_cnt, Stats, 0} + , {awaiting_rel_max, Stats, infinity} + , {recv_oct, Stats, 0} + , {recv_cnt, Stats, 0} + , {recv_pkt, Stats, 0} + , {recv_msg, Stats, 0} + , {send_oct, Stats, 0} + , {send_cnt, Stats, 0} + , {send_pkt, Stats, 0} + , {send_msg, Stats, 0} + , {mailbox_len, Stats, 0} + , {heap_size, Stats, 0} + , {reductions, Stats, 0} + ], + eval(FetchX). + +eval(Ls) -> + eval(Ls, #{}). +eval([], AccMap) -> + AccMap; +eval([{K, Vx}|More], AccMap) -> + case valuex_get(K, Vx) of + undefined -> eval(More, AccMap#{K => null}); + Value -> eval(More, AccMap#{K => Value}) + end; +eval([{K, Vx, Default}|More], AccMap) -> + case valuex_get(K, Vx) of + undefined -> eval(More, AccMap#{K => Default}); + Value -> eval(More, AccMap#{K => Value}) + end. + +valuex_get(K, Vx) when is_map(Vx); is_list(Vx) -> + key_get(K, Vx); +valuex_get(_K, {InKey, Obj}) when is_map(Obj); is_list(Obj) -> + key_get(InKey, Obj); +valuex_get(_K, {InKey, Obj, MappingFun}) when is_map(Obj); is_list(Obj) -> + case key_get(InKey, Obj) of + undefined -> undefined; + Val -> MappingFun(Val) + end. + +key_get(K, M) when is_map(M) -> + maps:get(K, M, undefined); +key_get(K, L) when is_list(L) -> + proplists:get_value(K, L). + +peer_to_binary({Addr, Port}) -> + AddrBinary = list_to_binary(inet:ntoa(Addr)), + PortBinary = integer_to_binary(Port), + <>; +peer_to_binary(Addr) -> + list_to_binary(inet:ntoa(Addr)). + +conn_state_to_connected(connected) -> true; +conn_state_to_connected(_) -> false. + +%%-------------------------------------------------------------------- +%% Swagger defines +%%-------------------------------------------------------------------- + +metadata(APIs) -> + metadata(APIs, []). +metadata([], APIAcc) -> + lists:reverse(APIAcc); +metadata([{Path, Fun}|More], APIAcc) -> + Methods = [get, post, put, delete, patch], + Mds = lists:foldl(fun(M, Acc) -> + try + Acc#{M => swagger(Path, M)} + catch + error : function_clause -> + Acc + end + end, #{}, Methods), + metadata(More, [{Path, Mds, Fun} | APIAcc]). + +swagger("/gateway/:name/clients", get) -> + #{ description => <<"Get the gateway clients">> + , parameters => params_client_query() + , responses => + #{ <<"400">> => schema_bad_request() + , <<"404">> => schema_not_found() + , <<"500">> => schema_internal_error() + , <<"200">> => schema_clients_list() + } + }; +swagger("/gateway/:name/clients/:clientid", get) -> + #{ description => <<"Get the gateway client infomation">> + , parameters => params_client_insta() + , responses => + #{ <<"400">> => schema_bad_request() + , <<"404">> => schema_not_found() + , <<"500">> => schema_internal_error() + , <<"200">> => schema_client() + } + }; +swagger("/gateway/:name/clients/:clientid", delete) -> + #{ description => <<"Kick out the gateway client">> + , parameters => params_client_insta() + , responses => + #{ <<"400">> => schema_bad_request() + , <<"404">> => schema_not_found() + , <<"500">> => schema_internal_error() + , <<"204">> => schema_no_content() + } + }; +swagger("/gateway/:name/clients/:clientid/subscriptions", get) -> + #{ description => <<"Get the gateway client subscriptions">> + , parameters => params_client_insta() + , responses => + #{ <<"400">> => schema_bad_request() + , <<"404">> => schema_not_found() + , <<"500">> => schema_internal_error() + , <<"200">> => schema_subscription_list() + } + }; +swagger("/gateway/:name/clients/:clientid/subscriptions", post) -> + #{ description => <<"Get the gateway client subscriptions">> + , parameters => params_client_insta() + , requestBody => schema_subscription() + , responses => + #{ <<"400">> => schema_bad_request() + , <<"404">> => schema_not_found() + , <<"500">> => schema_internal_error() + , <<"200">> => schema_no_content() + } + }; +swagger("/gateway/:name/clients/:clientid/subscriptions/:topic", delete) -> + #{ description => <<"Unsubscribe the topic for client">> + , parameters => params_topic_name_in_path() ++ params_client_insta() + , responses => + #{ <<"400">> => schema_bad_request() + , <<"404">> => schema_not_found() + , <<"500">> => schema_internal_error() + , <<"204">> => schema_no_content() + } + }. + +params_client_query() -> + params_gateway_name_in_path() + ++ params_client_searching_in_qs() + ++ emqx_mgmt_util:page_params(). + +params_client_insta() -> + params_clientid_in_path() + ++ params_gateway_name_in_path(). + +params_client_searching_in_qs() -> + queries( + [ {node, string} + , {clientid, string} + , {username, string} + , {ip_address, string} + , {conn_state, string} + , {proto_ver, string} + , {clean_start, boolean} + , {like_clientid, string} + , {like_username, string} + , {gte_created_at, string} + , {lte_created_at, string} + , {gte_connected_at, string} + , {lte_connected_at, string} + ]). + +params_gateway_name_in_path() -> + [#{ name => name + , in => path + , schema => #{type => string} + , required => true + }]. + +params_clientid_in_path() -> + [#{ name => clientid + , in => path + , schema => #{type => string} + , required => true + }]. + +params_topic_name_in_path() -> + [#{ name => topic + , in => path + , schema => #{type => string} + , required => true + }]. + +queries(Ls) -> + lists:map(fun({K, Type}) -> + #{name => K, in => query, + schema => #{type => Type}, + required => false + } + end, Ls). + +%%-------------------------------------------------------------------- +%% schemas + +schema_clients_list() -> + emqx_mgmt_util:page_schema( + #{ type => object + , properties => properties_client() + } + ). + +schema_client() -> + emqx_mgmt_util:schema( + #{ type => object + , properties => properties_client() + }). + +schema_subscription_list() -> + emqx_mgmt_util:array_schema( + #{ type => object + , properties => properties_subscription() + }, + <<"Client subscriptions">> + ). + +schema_subscription() -> + emqx_mgmt_util:schema( + #{ type => object + , properties => properties_subscription() + } + ). + +%%-------------------------------------------------------------------- +%% properties defines + +properties_client() -> + emqx_mgmt_util:properties( + [ {node, string, + <<"Name of the node to which the client is connected">>} + , {clientid, string, + <<"Client identifier">>} + , {username, string, + <<"Username of client when connecting">>} + , {proto_name, string, + <<"Client protocol name">>} + , {proto_ver, string, + <<"Protocol version used by the client">>} + , {ip_address, string, + <<"Client's IP address">>} + , {is_bridge, boolean, + <<"Indicates whether the client is connectedvia bridge">>} + , {connected_at, string, + <<"Client connection time">>} + , {disconnected_at, string, + <<"Client offline time, This field is only valid and returned " + "when connected is false">>} + , {connected, boolean, + <<"Whether the client is connected">>} + %% FIXME: the will_msg attribute is not a general attribute + %% for every protocol. But it should be returned to frontend if someone + %% want it + %% + %, {will_msg, string, + % <<"Client will message">>} + %, {zone, string, + % <<"Indicate the configuration group used by the client">>} + , {keepalive, integer, + <<"keepalive time, with the unit of second">>} + , {clean_start, boolean, + <<"Indicate whether the client is using a brand new session">>} + , {expiry_interval, integer, + <<"Session expiration interval, with the unit of second">>} + , {created_at, string, + <<"Session creation time">>} + , {subscriptions_cnt, integer, + <<"Number of subscriptions established by this client">>} + , {subscriptions_max, integer, + <<"v4 api name [max_subscriptions] Maximum number of " + "subscriptions allowed by this client">>} + , {inflight_cnt, integer, + <<"Current length of inflight">>} + , {inflight_max, integer, + <<"v4 api name [max_inflight]. Maximum length of inflight">>} + , {mqueue_len, integer, + <<"Current length of message queue">>} + , {mqueue_max, integer, + <<"v4 api name [max_mqueue]. Maximum length of message queue">>} + , {mqueue_dropped, integer, + <<"Number of messages dropped by the message queue due to " + "exceeding the length">>} + , {awaiting_rel_cnt, integer, + <<"v4 api name [awaiting_rel] Number of awaiting PUBREC packet">>} + , {awaiting_rel_max, integer, + <<"v4 api name [max_awaiting_rel]. Maximum allowed number of " + "awaiting PUBREC packet">>} + , {recv_oct, integer, + <<"Number of bytes received by EMQ X Broker (the same below)">>} + , {recv_cnt, integer, + <<"Number of TCP packets received">>} + , {recv_pkt, integer, + <<"Number of MQTT packets received">>} + , {recv_msg, integer, + <<"Number of PUBLISH packets received">>} + , {send_oct, integer, + <<"Number of bytes sent">>} + , {send_cnt, integer, + <<"Number of TCP packets sent">>} + , {send_pkt, integer, + <<"Number of MQTT packets sent">>} + , {send_msg, integer, + <<"Number of PUBLISH packets sent">>} + , {mailbox_len, integer, + <<"Process mailbox size">>} + , {heap_size, integer, + <<"Process heap size with the unit of byte">>} + , {reductions, integer, + <<"Erlang reduction">>} + ]). + +properties_subscription() -> + ExtraProps = [ {subid, string, + <<"Only stomp protocol, an uniquely identity for " + "the subscription. range: 1-65535.">>} + ], + emqx_mgmt_util:properties( + [ {topic, string, + <<"Topic Fillter">>} + , {qos, integer, + <<"QoS level, enum: 0, 1, 2">>} + , {nl, integer, %% FIXME: why not boolean? + <<"No Local option, enum: 0, 1">>} + , {rap, integer, + <<"Retain as Published option, enum: 0, 1">>} + , {rh, integer, + <<"Retain Handling option, enum: 0, 1, 2">>} + , {sub_props, object, ExtraProps} + ]). diff --git a/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl b/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl new file mode 100644 index 000000000..374f2841d --- /dev/null +++ b/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl @@ -0,0 +1,316 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_gateway_api_listeners). + +-behaviour(minirest_api). + +-import(emqx_gateway_http, + [ return_http_error/2 + , with_gateway/2 + , checks/2 + , schema_bad_request/0 + , schema_not_found/0 + , schema_internal_error/0 + , schema_no_content/0 + ]). + +%% minirest behaviour callbacks +-export([api_spec/0]). + +%% http handlers +-export([ listeners/2 + , listeners_insta/2 + ]). + +%%-------------------------------------------------------------------- +%% minirest behaviour callbacks +%%-------------------------------------------------------------------- + +api_spec() -> + {metadata(apis()), []}. + +apis() -> + [ {"/gateway/:name/listeners", listeners} + , {"/gateway/:name/listeners/:id", listeners_insta} + ]. +%%-------------------------------------------------------------------- +%% http handlers + +listeners(get, #{bindings := #{name := Name0}}) -> + with_gateway(Name0, fun(GwName, _) -> + {200, emqx_gateway_http:listeners(GwName)} + end); + +listeners(post, #{bindings := #{name := Name0}, body := LConf}) -> + with_gateway(Name0, fun(GwName, Gateway) -> + RunningConf = maps:get(config, Gateway), + %% XXX: check params miss? check badly data tpye?? + _ = checks([<<"type">>, <<"name">>, <<"bind">>], LConf), + + Type = binary_to_existing_atom(maps:get(<<"type">>, LConf)), + LName = binary_to_atom(maps:get(<<"name">>, LConf)), + + Path = [listeners, Type, LName], + case emqx_map_lib:deep_get(Path, RunningConf, undefined) of + undefined -> + ListenerId = emqx_gateway_utils:listener_id( + GwName, Type, LName), + case emqx_gateway_http:update_listener( + ListenerId, LConf) of + ok -> + {204}; + {error, Reason} -> + return_http_error(500, Reason) + end; + _ -> + return_http_error(400, "Listener name has occupied") + end + end). + +listeners_insta(delete, #{bindings := #{name := Name0, id := ListenerId0}}) -> + ListenerId = emqx_mgmt_util:urldecode(ListenerId0), + with_gateway(Name0, fun(_GwName, _) -> + case emqx_gateway_http:remove_listener(ListenerId) of + ok -> {204}; + {error, not_found} -> {204}; + {error, Reason} -> + return_http_error(500, Reason) + end + end); +listeners_insta(get, #{bindings := #{name := Name0, id := ListenerId0}}) -> + ListenerId = emqx_mgmt_util:urldecode(ListenerId0), + with_gateway(Name0, fun(_GwName, _) -> + case emqx_gateway_http:listener(ListenerId) of + {ok, Listener} -> + {200, Listener}; + {error, not_found} -> + return_http_error(404, "Listener not found"); + {error, Reason} -> + return_http_error(500, Reason) + end + end); +listeners_insta(put, #{body := LConf, + bindings := #{name := Name0, id := ListenerId0} + }) -> + ListenerId = emqx_mgmt_util:urldecode(ListenerId0), + with_gateway(Name0, fun(_GwName, _) -> + case emqx_gateway_http:update_listener(ListenerId, LConf) of + ok -> + {204}; + {error, Reason} -> + return_http_error(500, Reason) + end + end). + +%%-------------------------------------------------------------------- +%% Swagger defines +%%-------------------------------------------------------------------- + +metadata(APIs) -> + metadata(APIs, []). +metadata([], APIAcc) -> + lists:reverse(APIAcc); +metadata([{Path, Fun}|More], APIAcc) -> + Methods = [get, post, put, delete, patch], + Mds = lists:foldl(fun(M, Acc) -> + try + Acc#{M => swagger(Path, M)} + catch + error : function_clause -> + Acc + end + end, #{}, Methods), + metadata(More, [{Path, Mds, Fun} | APIAcc]). + +swagger("/gateway/:name/listeners", get) -> + #{ description => <<"Get the gateway listeners">> + , parameters => params_gateway_name_in_path() + , responses => + #{ <<"400">> => schema_bad_request() + , <<"404">> => schema_not_found() + , <<"500">> => schema_internal_error() + , <<"200">> => schema_listener_list() + } + }; +swagger("/gateway/:name/listeners", post) -> + #{ description => <<"Create the gateway listener">> + , parameters => params_gateway_name_in_path() + , requestBody => schema_listener() + , responses => + #{ <<"400">> => schema_bad_request() + , <<"404">> => schema_not_found() + , <<"500">> => schema_internal_error() + , <<"200">> => schema_listener_list() + } + }; +swagger("/gateway/:name/listeners/:id", get) -> + #{ description => <<"Get the gateway listener configurations">> + , parameters => params_gateway_name_in_path() + ++ params_listener_id_in_path() + , responses => + #{ <<"400">> => schema_bad_request() + , <<"404">> => schema_not_found() + , <<"500">> => schema_internal_error() + , <<"200">> => schema_listener() + } + }; +swagger("/gateway/:name/listeners/:id", delete) -> + #{ description => <<"Delete the gateway listener">> + , parameters => params_gateway_name_in_path() + ++ params_listener_id_in_path() + , responses => + #{ <<"400">> => schema_bad_request() + , <<"404">> => schema_not_found() + , <<"500">> => schema_internal_error() + , <<"204">> => schema_no_content() + } + }; +swagger("/gateway/:name/listeners/:id", put) -> + #{ description => <<"Update the gateway listener">> + , parameters => params_gateway_name_in_path() + ++ params_listener_id_in_path() + , requestBody => schema_listener() + , responses => + #{ <<"400">> => schema_bad_request() + , <<"404">> => schema_not_found() + , <<"500">> => schema_internal_error() + , <<"200">> => schema_no_content() + } + }. + +%%-------------------------------------------------------------------- +%% params defines + +params_gateway_name_in_path() -> + [#{ name => name + , in => path + , schema => #{type => string} + , required => true + }]. + +params_listener_id_in_path() -> + [#{ name => id + , in => path + , schema => #{type => string} + , required => true + }]. + +%%-------------------------------------------------------------------- +%% schemas + +schema_listener_list() -> + emqx_mgmt_util:array_schema( + #{ type => object + , properties => properties_listener() + }, + <<"Listener list">> + ). + +schema_listener() -> + emqx_mgmt_util:schema( + #{ type => object + , properties => properties_listener() + } + ). + +%%-------------------------------------------------------------------- +%% properties + +properties_listener() -> + emqx_mgmt_util:properties( + raw_properties_common_listener() ++ + [ {tcp, object, raw_properties_tcp_opts()} + , {ssl, object, raw_properties_ssl_opts()} + , {udp, object, raw_properties_udp_opts()} + , {dtls, object, raw_properties_dtls_opts()} + ]). + +raw_properties_tcp_opts() -> + [ {active_n, integer, <<>>} + , {backlog, integer, <<>>} + , {buffer, string, <<>>} + , {recbuf, string, <<>>} + , {sndbuf, string, <<>>} + , {high_watermark, string, <<>>} + , {nodelay, boolean, <<>>} + , {reuseaddr, boolean, <<>>} + , {send_timeout, string, <<>>} + , {send_timeout_close, boolean, <<>>} + ]. + +raw_properties_ssl_opts() -> + [ {cacertfile, string, <<>>} + , {certfile, string, <<>>} + , {keyfile, string, <<>>} + , {verify, string, <<>>} + , {fail_if_no_peer_cert, boolean, <<>>} + , {server_name_indication, boolean, <<>>} + , {depth, integer, <<>>} + , {password, string, <<>>} + , {handshake_timeout, string, <<>>} + , {versions, {array, string}, <<>>} + , {ciphers, {array, string}, <<>>} + , {user_lookup_fun, string, <<>>} + , {reuse_sessions, boolean, <<>>} + , {secure_renegotiate, boolean, <<>>} + , {honor_cipher_order, boolean, <<>>} + , {dhfile, string, <<>>} + ]. + +raw_properties_udp_opts() -> + [ {active_n, integer, <<>>} + , {buffer, string, <<>>} + , {recbuf, string, <<>>} + , {sndbuf, string, <<>>} + , {reuseaddr, boolean, <<>>} + ]. + +raw_properties_dtls_opts() -> + Ls = lists_key_without( + [versions,ciphers,handshake_timeout], 1, + raw_properties_ssl_opts() + ), + [ {versions, {array, string}, <<>>} + , {ciphers, {array, string}, <<>>} + | Ls]. + +lists_key_without([], _N, L) -> + L; +lists_key_without([K|Ks], N, L) -> + lists_key_without(Ks, N, lists:keydelete(K, N, L)). + +raw_properties_common_listener() -> + [ {enable, boolean, <<"Whether to enable this listener">>} + , {id, string, <<"Listener Id">>} + , {name, string, <<"Listener name">>} + , {type, string, + <<"Listener type. Enum: tcp, udp, ssl, dtls">>, + [<<"tcp">>, <<"ssl">>, <<"udp">>, <<"dtls">>]} + , {running, boolean, <<"Listener running status">>} + %% FIXME: + , {bind, string, <<"Listener bind address or port">>} + , {acceptors, integer, <<"Listener acceptors number">>} + , {access_rules, {array, string}, <<"Listener Access rules for client">>} + , {max_conn_rate, integer, <<"Max connection rate for the listener">>} + , {max_connections, integer, <<"Max connections for the listener">>} + , {mountpoint, string, + <<"The Mounpoint for clients of the listener. " + "The gateway-level mountpoint configuration can be overloaded " + "when it is not null or empty string">>} + %% FIXME: + , {authentication, string, <<"NOT-SUPPORTED-NOW">>} + ]. diff --git a/apps/emqx_gateway/src/emqx_gateway_app.erl b/apps/emqx_gateway/src/emqx_gateway_app.erl index 1ecd9cf26..d90942220 100644 --- a/apps/emqx_gateway/src/emqx_gateway_app.erl +++ b/apps/emqx_gateway/src/emqx_gateway_app.erl @@ -83,4 +83,4 @@ load_gateway_by_default([{Type, Confs}|More]) -> load_gateway_by_default(More). confs() -> - maps:to_list(emqx:get_config([gateway], [])). + maps:to_list(emqx:get_config([gateway], #{})). diff --git a/apps/emqx_gateway/src/emqx_gateway_cli.erl b/apps/emqx_gateway/src/emqx_gateway_cli.erl index b446cda92..6ccb444f0 100644 --- a/apps/emqx_gateway/src/emqx_gateway_cli.erl +++ b/apps/emqx_gateway/src/emqx_gateway_cli.erl @@ -51,7 +51,7 @@ is_cmd(Fun) -> gateway(["list"]) -> lists:foreach(fun(#{name := Name} = Gateway) -> - %% XXX: More infos: listeners?, connected? + %% TODO: More infos: listeners?, connected? Status = maps:get(status, Gateway, stopped), emqx_ctl:print("Gateway(name=~s, status=~s)~n", [Name, Status]) @@ -106,6 +106,7 @@ gateway(_) -> ]). 'gateway-clients'(["list", Name]) -> + %% FIXME: page me. for example: --limit 100 --page 10 ??? InfoTab = emqx_gateway_cm:tabname(info, Name), case ets:info(InfoTab) of undefined -> diff --git a/apps/emqx_gateway/src/emqx_gateway_cm.erl b/apps/emqx_gateway/src/emqx_gateway_cm.erl index 7a7ad055d..d8b615fe8 100644 --- a/apps/emqx_gateway/src/emqx_gateway_cm.erl +++ b/apps/emqx_gateway/src/emqx_gateway_cm.erl @@ -48,6 +48,10 @@ , connection_closed/2 ]). +-export([ with_channel/3 + , lookup_channels/2 + ]). + %% Internal funcs for getting tabname by GatewayId -export([cmtabs/1, tabname/2]). diff --git a/apps/emqx_gateway/src/emqx_gateway_http.erl b/apps/emqx_gateway/src/emqx_gateway_http.erl new file mode 100644 index 000000000..ed8e511c7 --- /dev/null +++ b/apps/emqx_gateway/src/emqx_gateway_http.erl @@ -0,0 +1,372 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% @doc Gateway Interface Module for HTTP-APIs +-module(emqx_gateway_http). + +-include("include/emqx_gateway.hrl"). +-include_lib("emqx/include/logger.hrl"). + +%% Mgmt APIs - gateway +-export([ gateways/1 + ]). + +%% Mgmt APIs - listeners +-export([ listeners/1 + , listener/1 + , remove_listener/1 + , update_listener/2 + , mapping_listener_m2l/2 + ]). + +%% Mgmt APIs - clients +-export([ lookup_client/3 + , lookup_client/4 + , kickout_client/2 + , kickout_client/3 + , list_client_subscriptions/2 + , client_subscribe/4 + , client_unsubscribe/3 + ]). + +%% Utils for http, swagger, etc. +-export([ return_http_error/2 + , with_gateway/2 + , checks/2 + , schema_bad_request/0 + , schema_not_found/0 + , schema_internal_error/0 + , schema_no_content/0 + ]). + +-type gateway_summary() :: + #{ name := binary() + , status := running | stopped | unloaded + , started_at => binary() + , max_connections => integer() + , current_connections => integer() + , listeners => [] + }. + +-define(DEFAULT_CALL_TIMEOUT, 15000). + +%%-------------------------------------------------------------------- +%% Mgmt APIs - gateway +%%-------------------------------------------------------------------- + +-spec gateways(Status :: all | running | stopped | unloaded) + -> [gateway_summary()]. +gateways(Status) -> + Gateways = lists:map(fun({GwName, _}) -> + case emqx_gateway:lookup(GwName) of + undefined -> #{name => GwName, status => unloaded}; + GwInfo = #{config := Config} -> + GwInfo0 = emqx_gateway_utils:unix_ts_to_rfc3339( + [created_at, started_at, stopped_at], + GwInfo), + GwInfo1 = maps:with([name, + status, + created_at, + started_at, + stopped_at], GwInfo0), + GwInfo1#{ + max_connections => max_connections_count(Config), + current_connections => current_connections_count(GwName), + listeners => get_listeners_status(GwName, Config)} + end + end, emqx_gateway_registry:list()), + case Status of + all -> Gateways; + _ -> + [Gw || Gw = #{status := S} <- Gateways, S == Status] + end. + +%% @private +max_connections_count(Config) -> + Listeners = emqx_gateway_utils:normalize_config(Config), + lists:foldl(fun({_, _, _, SocketOpts, _}, Acc) -> + Acc + proplists:get_value(max_connections, SocketOpts, 0) + end, 0, Listeners). + +%% @private +current_connections_count(GwName) -> + try + InfoTab = emqx_gateway_cm:tabname(info, GwName), + ets:info(InfoTab, size) + catch _ : _ -> + 0 + end. + +%% @private +get_listeners_status(GwName, Config) -> + Listeners = emqx_gateway_utils:normalize_config(Config), + lists:map(fun({Type, LisName, ListenOn, _, _}) -> + Name0 = emqx_gateway_utils:listener_id(GwName, Type, LisName), + Name = {Name0, ListenOn}, + LisO = #{id => Name0, type => Type, name => LisName}, + case catch esockd:listener(Name) of + _Pid when is_pid(_Pid) -> + LisO#{running => true}; + _ -> + LisO#{running => false} + end + end, Listeners). + +%%-------------------------------------------------------------------- +%% Mgmt APIs - listeners +%%-------------------------------------------------------------------- + +-spec listeners(atom() | binary()) -> list(). +listeners(GwName) when is_atom(GwName) -> + listeners(atom_to_binary(GwName)); +listeners(GwName) -> + RawConf = emqx_config:fill_defaults( + emqx_config:get_root_raw([<<"gateway">>])), + Listeners = emqx_map_lib:jsonable_map( + emqx_map_lib:deep_get( + [<<"gateway">>, GwName, <<"listeners">>], RawConf)), + mapping_listener_m2l(GwName, Listeners). + +-spec listener(binary()) -> {ok, map()} | {error, not_found} | {error, any()}. +listener(ListenerId) -> + {GwName, Type, LName} = emqx_gateway_utils:parse_listener_id(ListenerId), + RootConf = emqx_config:fill_defaults( + emqx_config:get_root_raw([<<"gateway">>])), + try + Path = [<<"gateway">>, GwName, <<"listeners">>, Type, LName], + LConf = emqx_map_lib:deep_get(Path, RootConf), + Running = is_running(binary_to_existing_atom(ListenerId), LConf), + {ok, emqx_map_lib:jsonable_map( + LConf#{ + id => ListenerId, + type => Type, + name => LName, + running => Running})} + catch + error : {config_not_found, _} -> + {error, not_found}; + _Class : Reason -> + {error, Reason} + end. + +mapping_listener_m2l(GwName, Listeners0) -> + Listeners = maps:to_list(Listeners0), + lists:append([listener(GwName, Type, maps:to_list(Conf)) + || {Type, Conf} <- Listeners]). + +listener(GwName, Type, Conf) -> + [begin + ListenerId = emqx_gateway_utils:listener_id(GwName, Type, LName), + Running = is_running(ListenerId, LConf), + LConf#{ + id => ListenerId, + type => Type, + name => LName, + running => Running + } + end || {LName, LConf} <- Conf, is_map(LConf)]. + +is_running(ListenerId, #{<<"bind">> := ListenOn0}) -> + ListenOn = emqx_gateway_utils:parse_listenon(ListenOn0), + try esockd:listener({ListenerId, ListenOn}) of + Pid when is_pid(Pid)-> + true + catch _:_ -> + false + end. + +-spec remove_listener(binary()) -> ok | {error, not_found} | {error, any()}. +remove_listener(ListenerId) -> + {GwName, Type, Name} = emqx_gateway_utils:parse_listener_id(ListenerId), + LConf = emqx:get_raw_config( + [<<"gateway">>, GwName, <<"listeners">>, Type] + ), + NLConf = maps:remove(Name, LConf), + emqx_gateway:update_rawconf( + GwName, + #{<<"listeners">> => #{Type => NLConf}} + ). + +-spec update_listener(atom() | binary(), map()) -> ok | {error, any()}. +update_listener(ListenerId, NewConf0) -> + {GwName, Type, Name} = emqx_gateway_utils:parse_listener_id(ListenerId), + NewConf = maps:without([<<"id">>, <<"name">>, + <<"type">>, <<"running">>], NewConf0), + emqx_gateway:update_rawconf( + GwName, + #{<<"listeners">> => #{Type => #{Name => NewConf}} + }). + +%%-------------------------------------------------------------------- +%% Mgmt APIs - clients +%%-------------------------------------------------------------------- + +-spec lookup_client(gateway_name(), + emqx_type:clientid(), {atom(), atom()}) -> list(). +lookup_client(GwName, ClientId, FormatFun) -> + lists:append([lookup_client(Node, GwName, {clientid, ClientId}, FormatFun) + || Node <- ekka_mnesia:running_nodes()]). + +lookup_client(Node, GwName, {clientid, ClientId}, {M,F}) when Node =:= node() -> + ChanTab = emqx_gateway_cm:tabname(chan, GwName), + InfoTab = emqx_gateway_cm:tabname(info, GwName), + + lists:append(lists:map( + fun(Key) -> + lists:map(fun M:F/1, ets:lookup(InfoTab, Key)) + end, ets:lookup(ChanTab, ClientId))); + +lookup_client(Node, GwName, {clientid, ClientId}, FormatFun) -> + rpc_call(Node, lookup_client, + [Node, GwName, {clientid, ClientId}, FormatFun]). + +-spec kickout_client(gateway_name(), emqx_type:clientid()) + -> {error, any()} + | ok. +kickout_client(GwName, ClientId) -> + Results = [kickout_client(Node, GwName, ClientId) + || Node <- ekka_mnesia:running_nodes()], + case lists:any(fun(Item) -> Item =:= ok end, Results) of + true -> ok; + false -> lists:last(Results) + end. + +kickout_client(Node, GwName, ClientId) when Node =:= node() -> + emqx_gateway_cm:kick_session(GwName, ClientId); + +kickout_client(Node, GwName, ClientId) -> + rpc_call(Node, kickout_client, [Node, GwName, ClientId]). + +-spec list_client_subscriptions(gateway_name(), emqx_type:clientid()) + -> {error, any()} + | {ok, list()}. +list_client_subscriptions(GwName, ClientId) -> + %% Get the subscriptions from session-info + with_channel(GwName, ClientId, + fun(Pid) -> + Subs = emqx_gateway_conn:call( + Pid, + subscriptions, ?DEFAULT_CALL_TIMEOUT), + {ok, lists:map(fun({Topic, SubOpts}) -> + SubOpts#{topic => Topic} + end, Subs)} + end). + +-spec client_subscribe(gateway_name(), emqx_type:clientid(), + emqx_type:topic(), emqx_type:subopts()) + -> {error, any()} + | ok. +client_subscribe(GwName, ClientId, Topic, SubOpts) -> + with_channel(GwName, ClientId, + fun(Pid) -> + emqx_gateway_conn:call( + Pid, {subscribe, Topic, SubOpts}, + ?DEFAULT_CALL_TIMEOUT + ) + end). + +-spec client_unsubscribe(gateway_name(), + emqx_type:clientid(), emqx_type:topic()) + -> {error, any()} + | ok. +client_unsubscribe(GwName, ClientId, Topic) -> + with_channel(GwName, ClientId, + fun(Pid) -> + emqx_gateway_conn:call( + Pid, {unsubscribe, Topic}, ?DEFAULT_CALL_TIMEOUT) + end). + +with_channel(GwName, ClientId, Fun) -> + case emqx_gateway_cm:with_channel(GwName, ClientId, Fun) of + undefined -> {error, not_found}; + Res -> Res + end. + +%%-------------------------------------------------------------------- +%% Utils +%%-------------------------------------------------------------------- + +-spec return_http_error(integer(), any()) -> {integer(), binary()}. +return_http_error(Code, Msg) -> + {Code, emqx_json:encode( + #{code => codestr(Code), + reason => emqx_gateway_utils:stringfy(Msg) + }) + }. + +codestr(400) -> 'BAD_REQUEST'; +codestr(401) -> 'NOT_SUPPORTED_NOW'; +codestr(404) -> 'RESOURCE_NOT_FOUND'; +codestr(500) -> 'UNKNOW_ERROR'. + +-spec with_gateway(binary(), function()) -> any(). +with_gateway(GwName0, Fun) -> + try + GwName = try + binary_to_existing_atom(GwName0) + catch _ : _ -> error(badname) + end, + case emqx_gateway:lookup(GwName) of + undefined -> + return_http_error(404, "Gateway not load"); + Gateway -> + Fun(GwName, Gateway) + end + catch + error : badname -> + return_http_error(404, "Bad gateway name"); + error : {miss_param, K} -> + return_http_error(400, [K, " is required"]); + error : {invalid_listener_id, Id} -> + return_http_error(400, ["invalid listener id: ", Id]); + Class : Reason : Stk -> + ?LOG(error, "Uncatched error: {~p, ~p}, stacktrace: ~0p", + [Class, Reason, Stk]), + return_http_error(500, {Class, Reason, Stk}) + end. + +-spec checks(list(), map()) -> ok. +checks([], _) -> + ok; +checks([K|Ks], Map) -> + case maps:is_key(K, Map) of + true -> checks(Ks, Map); + false -> + error({miss_param, K}) + end. + +%%-------------------------------------------------------------------- +%% common schemas + +schema_bad_request() -> + emqx_mgmt_util:error_schema( + <<"Some Params missed">>, ['PARAMETER_MISSED']). +schema_internal_error() -> + emqx_mgmt_util:error_schema( + <<"Ineternal Server Error">>, ['INTERNAL_SERVER_ERROR']). +schema_not_found() -> + emqx_mgmt_util:error_schema(<<"Resource Not Found">>). +schema_no_content() -> + #{description => <<"No Content">>}. + +%%-------------------------------------------------------------------- +%% Internal funcs + +rpc_call(Node, Fun, Args) -> + case rpc:call(Node, ?MODULE, Fun, Args) of + {badrpc, Reason} -> {error, Reason}; + Res -> Res + end. diff --git a/apps/emqx_gateway/src/emqx_gateway_intr.erl b/apps/emqx_gateway/src/emqx_gateway_intr.erl deleted file mode 100644 index add37e1c5..000000000 --- a/apps/emqx_gateway/src/emqx_gateway_intr.erl +++ /dev/null @@ -1,78 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - -%% @doc Gateway Interface Module for HTTP-APIs --module(emqx_gateway_intr). - --export([ gateways/1 - ]). - --type gateway_summary() :: - #{ name := binary() - , status := running | stopped | unloaded - , started_at => binary() - , max_connection => integer() - , current_connect => integer() - , listeners => [] - }. - -%%-------------------------------------------------------------------- -%% APIs -%%-------------------------------------------------------------------- - --spec gateways(Status :: all | running | stopped | unloaded) - -> [gateway_summary()]. -gateways(Status) -> - Gateways = lists:map(fun({GwName, _}) -> - case emqx_gateway:lookup(GwName) of - undefined -> #{name => GwName, status => unloaded}; - GwInfo = #{config := Config} -> - GwInfo0 = emqx_gateway_utils:unix_ts_to_rfc3339( - [created_at, started_at, stopped_at], - GwInfo), - GwInfo1 = maps:with([name, - status, - created_at, - started_at, - stopped_at], GwInfo0), - GwInfo1#{listeners => get_listeners_status(GwName, Config)} - - end - end, emqx_gateway_registry:list()), - case Status of - all -> Gateways; - _ -> - [Gw || Gw = #{status := S} <- Gateways, S == Status] - end. - -%% @private -get_listeners_status(GwName, Config) -> - Listeners = emqx_gateway_utils:normalize_config(Config), - lists:map(fun({Type, LisName, ListenOn, _, _}) -> - Name0 = listener_name(GwName, Type, LisName), - Name = {Name0, ListenOn}, - case catch esockd:listener(Name) of - _Pid when is_pid(_Pid) -> - #{Name0 => <<"activing">>}; - _ -> - #{Name0 => <<"inactived">>} - - end - end, Listeners). - -%% @private -listener_name(GwName, Type, LisName) -> - list_to_atom(lists:concat([GwName, ":", Type, ":", LisName])). diff --git a/apps/emqx_gateway/src/emqx_gateway_metrics.erl b/apps/emqx_gateway/src/emqx_gateway_metrics.erl index 458017118..77b97a6a1 100644 --- a/apps/emqx_gateway/src/emqx_gateway_metrics.erl +++ b/apps/emqx_gateway/src/emqx_gateway_metrics.erl @@ -18,7 +18,7 @@ -behaviour(gen_server). --include("include/emqx_gateway.hrl"). +-include_lib("emqx_gateway/include/emqx_gateway.hrl"). %% APIs diff --git a/apps/emqx_gateway/src/emqx_gateway_schema.erl b/apps/emqx_gateway/src/emqx_gateway_schema.erl index 9371f8c6b..3811d56c6 100644 --- a/apps/emqx_gateway/src/emqx_gateway_schema.erl +++ b/apps/emqx_gateway/src/emqx_gateway_schema.erl @@ -43,256 +43,264 @@ , ip_port/0 ]). --export([roots/0 , fields/1]). --export([t/1, t/3, t/4, ref/1]). +-export([namespace/0, roots/0 , fields/1]). + +namespace() -> gateway. roots() -> [gateway]. fields(gateway) -> - [{stomp, t(ref(stomp_structs))}, - {mqttsn, t(ref(mqttsn_structs))}, - {coap, t(ref(coap_structs))}, - {lwm2m, t(ref(lwm2m_structs))}, - {exproto, t(ref(exproto_structs))} + [{stomp, sc(ref(stomp))}, + {mqttsn, sc(ref(mqttsn))}, + {coap, sc(ref(coap))}, + {lwm2m, sc(ref(lwm2m))}, + {exproto, sc(ref(exproto))} ]; -fields(stomp_structs) -> - [ {frame, t(ref(stomp_frame))} - , {listeners, t(ref(tcp_listener_group))} +fields(stomp) -> + [ {frame, sc(ref(stomp_frame))} + , {listeners, sc(ref(tcp_listeners))} ] ++ gateway_common_options(); fields(stomp_frame) -> - [ {max_headers, t(integer(), undefined, 10)} - , {max_headers_length, t(integer(), undefined, 1024)} - , {max_body_length, t(integer(), undefined, 8192)} + [ {max_headers, sc(integer(), 10)} + , {max_headers_length, sc(integer(), 1024)} + , {max_body_length, sc(integer(), 8192)} ]; -fields(mqttsn_structs) -> - [ {gateway_id, t(integer())} - , {broadcast, t(boolean())} - , {enable_qos3, t(boolean())} +fields(mqttsn) -> + [ {gateway_id, sc(integer())} + , {broadcast, sc(boolean())} + , {enable_qos3, sc(boolean())} , {predefined, hoconsc:array(ref(mqttsn_predefined))} - , {listeners, t(ref(udp_listener_group))} + , {listeners, sc(ref(udp_listeners))} ] ++ gateway_common_options(); fields(mqttsn_predefined) -> - [ {id, t(integer())} - , {topic, t(binary())} + [ {id, sc(integer())} + , {topic, sc(binary())} ]; -fields(coap_structs) -> - [ {heartbeat, t(duration(), undefined, <<"30s">>)} - , {notify_type, t(union([non, con, qos]), undefined, qos)} - , {subscribe_qos, t(union([qos0, qos1, qos2, coap]), undefined, coap)} - , {publish_qos, t(union([qos0, qos1, qos2, coap]), undefined, coap)} - , {listeners, t(ref(udp_listener_group))} +fields(coap) -> + [ {heartbeat, sc(duration(), <<"30s">>)} + , {connection_required, sc(boolean(), false)} + , {notify_type, sc(hoconsc:union([non, con, qos]), qos)} + , {subscribe_qos, sc(hoconsc:union([qos0, qos1, qos2, coap]), coap)} + , {publish_qos, sc(hoconsc:union([qos0, qos1, qos2, coap]), coap)} + , {listeners, sc(ref(udp_listeners))} ] ++ gateway_common_options(); -fields(lwm2m_structs) -> - [ {xml_dir, t(binary())} - , {lifetime_min, t(duration())} - , {lifetime_max, t(duration())} - , {qmode_time_windonw, t(integer())} - , {auto_observe, t(boolean())} - , {update_msg_publish_condition, t(union([always, contains_object_list]))} - , {translators, t(ref(translators))} - , {listeners, t(ref(udp_listener_group))} +fields(lwm2m) -> + [ {xml_dir, sc(binary())} + , {lifetime_min, sc(duration())} + , {lifetime_max, sc(duration())} + , {qmode_time_windonw, sc(integer())} + , {auto_observe, sc(boolean())} + , {update_msg_publish_condition, sc(hoconsc:union([always, contains_object_list]))} + , {translators, sc(ref(translators))} + , {listeners, sc(ref(udp_listeners))} ] ++ gateway_common_options(); -fields(exproto_structs) -> - [ {server, t(ref(exproto_grpc_server))} - , {handler, t(ref(exproto_grpc_handler))} - , {listeners, t(ref(udp_tcp_listener_group))} +fields(exproto) -> + [ {server, sc(ref(exproto_grpc_server))} + , {handler, sc(ref(exproto_grpc_handler))} + , {listeners, sc(ref(udp_tcp_listeners))} ] ++ gateway_common_options(); fields(exproto_grpc_server) -> - [ {bind, t(union(ip_port(), integer()))} + [ {bind, sc(hoconsc:union([ip_port(), integer()]))} %% TODO: ssl options ]; fields(exproto_grpc_handler) -> - [ {address, t(binary())} + [ {address, sc(binary())} %% TODO: ssl ]; fields(clientinfo_override) -> - [ {username, t(binary())} - , {password, t(binary())} - , {clientid, t(binary())} + [ {username, sc(binary())} + , {password, sc(binary())} + , {clientid, sc(binary())} ]; fields(translators) -> - [{"$name", t(binary())}]; - -fields(udp_listener_group) -> - [ {udp, t(ref(udp_listener))} - , {dtls, t(ref(dtls_listener))} + [ {command, sc(ref(translator))} + , {response, sc(ref(translator))} + , {notify, sc(ref(translator))} + , {register, sc(ref(translator))} + , {update, sc(ref(translator))} ]; -fields(tcp_listener_group) -> - [ {tcp, t(ref(tcp_listener))} - , {ssl, t(ref(ssl_listener))} +fields(translator) -> + [ {topic, sc(binary())} + , {qos, sc(range(0, 2))} ]; -fields(udp_tcp_listener_group) -> - [ {udp, t(ref(udp_listener))} - , {dtls, t(ref(dtls_listener))} - , {tcp, t(ref(tcp_listener))} - , {ssl, t(ref(ssl_listener))} +fields(udp_listeners) -> + [ {udp, sc(map(name, ref(udp_listener)))} + , {dtls, sc(map(name, ref(dtls_listener)))} + ]; + +fields(tcp_listeners) -> + [ {tcp, sc(map(name, ref(tcp_listener)))} + , {ssl, sc(map(name, ref(ssl_listener)))} + ]; + +fields(udp_tcp_listeners) -> + [ {udp, sc(map(name, ref(udp_listener)))} + , {dtls, sc(map(name, ref(dtls_listener)))} + , {tcp, sc(map(name, ref(tcp_listener)))} + , {ssl, sc(map(name, ref(ssl_listener)))} ]; fields(tcp_listener) -> - [ {"$name", t(ref(tcp_listener_settings))}]; - -fields(ssl_listener) -> - [ {"$name", t(ref(ssl_listener_settings))}]; - -fields(udp_listener) -> - [ {"$name", t(ref(udp_listener_settings))}]; - -fields(dtls_listener) -> - [ {"$name", t(ref(dtls_listener_settings))}]; - -fields(listener_settings) -> - [ {enable, t(boolean(), undefined, true)} - , {bind, t(union(ip_port(), integer()))} - , {acceptors, t(integer(), undefined, 8)} - , {max_connections, t(integer(), undefined, 1024)} - , {max_conn_rate, t(integer())} - , {active_n, t(integer(), undefined, 100)} - %, {rate_limit, t(comma_separated_list())} - , {access, t(ref(access))} - , {proxy_protocol, t(boolean())} - , {proxy_protocol_timeout, t(duration())} - , {backlog, t(integer(), undefined, 1024)} - , {send_timeout, t(duration(), undefined, <<"15s">>)} - , {send_timeout_close, t(boolean(), undefined, true)} - , {recbuf, t(bytesize())} - , {sndbuf, t(bytesize())} - , {buffer, t(bytesize())} - , {high_watermark, t(bytesize(), undefined, <<"1MB">>)} - , {tune_buffer, t(boolean())} - , {nodelay, t(boolean())} - , {reuseaddr, t(boolean())} - ]; - -fields(tcp_listener_settings) -> [ %% some special confs for tcp listener - ] ++ fields(listener_settings); - -fields(ssl_listener_settings) -> - [ - %% some special confs for ssl listener ] ++ - ssl(undefined, #{handshake_timeout => <<"15s">> - , depth => 10 - , reuse_sessions => true}) ++ fields(listener_settings); + tcp_opts() ++ + proxy_protocol_opts() ++ + common_listener_opts(); -fields(udp_listener_settings) -> - [ - %% some special confs for udp listener - ] ++ fields(listener_settings); +fields(ssl_listener) -> + fields(tcp_listener) ++ + ssl_opts(); -fields(dtls_listener_settings) -> +fields(udp_listener) -> [ - %% some special confs for dtls listener + %% some special confs for udp listener ] ++ - ssl(undefined, #{handshake_timeout => <<"15s">> - , depth => 10 - , reuse_sessions => true}) ++ fields(listener_settings); + udp_opts() ++ + common_listener_opts(); -fields(access) -> - [ {"$id", #{type => binary(), - nullable => true}}]; +fields(dtls_listener) -> + fields(udp_listener) ++ + dtls_opts(); -fields(ExtraField) -> - Mod = list_to_atom(ExtraField++"_schema"), - Mod:fields(ExtraField). +fields(udp_opts) -> + [ {active_n, sc(integer(), 100)} + , {recbuf, sc(bytesize())} + , {sndbuf, sc(bytesize())} + , {buffer, sc(bytesize())} + , {reuseaddr, sc(boolean(), true)} + ]; -authentication() -> - hoconsc:union( - [ undefined - , hoconsc:ref(emqx_authn_mnesia, config) - , hoconsc:ref(emqx_authn_mysql, config) - , hoconsc:ref(emqx_authn_pgsql, config) - , hoconsc:ref(emqx_authn_mongodb, standalone) - , hoconsc:ref(emqx_authn_mongodb, 'replica-set') - , hoconsc:ref(emqx_authn_mongodb, 'sharded-cluster') - , hoconsc:ref(emqx_authn_redis, standalone) - , hoconsc:ref(emqx_authn_redis, cluster) - , hoconsc:ref(emqx_authn_redis, sentinel) - , hoconsc:ref(emqx_authn_http, get) - , hoconsc:ref(emqx_authn_http, post) - , hoconsc:ref(emqx_authn_jwt, 'hmac-based') - , hoconsc:ref(emqx_authn_jwt, 'public-key') - , hoconsc:ref(emqx_authn_jwt, 'jwks') - , hoconsc:ref(emqx_enhanced_authn_scram_mnesia, config) - ]). +fields(dtls_listener_ssl_opts) -> + Base = emqx_schema:fields("listener_ssl_opts"), + DtlsVers = hoconsc:mk( + typerefl:alias("string", list(atom())), + #{ default => default_dtls_vsns(), + converter => fun (Vsns) -> + [dtls_vsn(iolist_to_binary(V)) || V <- Vsns] + end + }), + Ciphers = sc(hoconsc:array(string()), default_ciphers()), + lists:keydelete( + "handshake_timeout", 1, + lists:keyreplace( + "ciphers", 1, + lists:keyreplace("versions", 1, Base, {"versions", DtlsVers}), + {"ciphers", Ciphers} + ) + ). -gateway_common_options() -> - [ {enable, t(boolean(), undefined, true)} - , {enable_stats, t(boolean(), undefined, true)} - , {idle_timeout, t(duration(), undefined, <<"30s">>)} - , {mountpoint, t(binary())} - , {clientinfo_override, t(ref(clientinfo_override))} - , {authentication, t(authentication(), undefined, undefined)} +default_ciphers() -> + ["ECDHE-ECDSA-AES256-GCM-SHA384", + "ECDHE-RSA-AES256-GCM-SHA384", "ECDHE-ECDSA-AES256-SHA384", "ECDHE-RSA-AES256-SHA384", + "ECDHE-ECDSA-DES-CBC3-SHA", "ECDH-ECDSA-AES256-GCM-SHA384", "ECDH-RSA-AES256-GCM-SHA384", + "ECDH-ECDSA-AES256-SHA384", "ECDH-RSA-AES256-SHA384", "DHE-DSS-AES256-GCM-SHA384", + "DHE-DSS-AES256-SHA256", "AES256-GCM-SHA384", "AES256-SHA256", + "ECDHE-ECDSA-AES128-GCM-SHA256", "ECDHE-RSA-AES128-GCM-SHA256", + "ECDHE-ECDSA-AES128-SHA256", "ECDHE-RSA-AES128-SHA256", "ECDH-ECDSA-AES128-GCM-SHA256", + "ECDH-RSA-AES128-GCM-SHA256", "ECDH-ECDSA-AES128-SHA256", "ECDH-RSA-AES128-SHA256", + "DHE-DSS-AES128-GCM-SHA256", "DHE-DSS-AES128-SHA256", "AES128-GCM-SHA256", "AES128-SHA256", + "ECDHE-ECDSA-AES256-SHA", "ECDHE-RSA-AES256-SHA", "DHE-DSS-AES256-SHA", + "ECDH-ECDSA-AES256-SHA", "ECDH-RSA-AES256-SHA", "AES256-SHA", "ECDHE-ECDSA-AES128-SHA", + "ECDHE-RSA-AES128-SHA", "DHE-DSS-AES128-SHA", "ECDH-ECDSA-AES128-SHA", + "ECDH-RSA-AES128-SHA", "AES128-SHA" + ] ++ psk_ciphers(). + +psk_ciphers() -> + ["PSK-AES128-CBC-SHA", "PSK-AES256-CBC-SHA", + "PSK-3DES-EDE-CBC-SHA", "PSK-RC4-SHA" ]. -%%-------------------------------------------------------------------- -%% Helpers +% authentication() -> +% hoconsc:union( +% [ undefined +% , hoconsc:ref(emqx_authn_mnesia, config) +% , hoconsc:ref(emqx_authn_mysql, config) +% , hoconsc:ref(emqx_authn_pgsql, config) +% , hoconsc:ref(emqx_authn_mongodb, standalone) +% , hoconsc:ref(emqx_authn_mongodb, 'replica-set') +% , hoconsc:ref(emqx_authn_mongodb, 'sharded-cluster') +% , hoconsc:ref(emqx_authn_redis, standalone) +% , hoconsc:ref(emqx_authn_redis, cluster) +% , hoconsc:ref(emqx_authn_redis, sentinel) +% , hoconsc:ref(emqx_authn_http, get) +% , hoconsc:ref(emqx_authn_http, post) +% , hoconsc:ref(emqx_authn_jwt, 'hmac-based') +% , hoconsc:ref(emqx_authn_jwt, 'public-key') +% , hoconsc:ref(emqx_authn_jwt, 'jwks') +% , hoconsc:ref(emqx_enhanced_authn_scram_mnesia, config) +% ]). -%% types +gateway_common_options() -> + [ {enable, sc(boolean(), true)} + , {enable_stats, sc(boolean(), true)} + , {idle_timeout, sc(duration(), <<"30s">>)} + , {mountpoint, sc(binary(), <<>>)} + , {clientinfo_override, sc(ref(clientinfo_override))} + , {authentication, sc(hoconsc:lazy(map()))} + ]. -t(Type) -> #{type => Type}. +common_listener_opts() -> + [ {enable, sc(boolean(), true)} + , {bind, sc(union(ip_port(), integer()))} + , {acceptors, sc(integer(), 16)} + , {max_connections, sc(integer(), 1024)} + , {max_conn_rate, sc(integer())} + %, {rate_limit, sc(comma_separated_list())} + , {mountpoint, sc(binary(), undefined)} + , {access_rules, sc(hoconsc:array(string()), [])} + ]. -t(Type, Mapping, Default) -> - hoconsc:t(Type, #{mapping => Mapping, default => Default}). +tcp_opts() -> + [{tcp, sc_meta(ref(emqx_schema, "tcp_opts"), #{})}]. -t(Type, Mapping, Default, OverrideEnv) -> - hoconsc:t(Type, #{ mapping => Mapping - , default => Default - , override_env => OverrideEnv - }). +udp_opts() -> + [{udp, sc_meta(ref(udp_opts), #{})}]. -ref(Field) -> - hoconsc:ref(?MODULE, Field). +ssl_opts() -> + [{ssl, sc_meta(ref(emqx_schema, "listener_ssl_opts"), #{})}]. -%% utils +dtls_opts() -> + [{dtls, sc_meta(ref(dtls_listener_ssl_opts), #{})}]. -%% generate a ssl field. -%% ssl("emqx", #{"verify" => verify_peer}) will return -%% [ {"cacertfile", t(string(), "emqx.cacertfile", undefined)} -%% , {"certfile", t(string(), "emqx.certfile", undefined)} -%% , {"keyfile", t(string(), "emqx.keyfile", undefined)} -%% , {"verify", t(union(verify_peer, verify_none), "emqx.verify", verify_peer)} -%% , {"server_name_indication", "emqx.server_name_indication", undefined)} -%% ... -ssl(Mapping, Defaults) -> - M = fun (Field) -> - case (Mapping) of - undefined -> undefined; - _ -> Mapping ++ "." ++ Field - end end, - D = fun (Field) -> maps:get(list_to_atom(Field), Defaults, undefined) end, - [ {"enable", t(boolean(), M("enable"), D("enable"))} - , {"cacertfile", t(binary(), M("cacertfile"), D("cacertfile"))} - , {"certfile", t(binary(), M("certfile"), D("certfile"))} - , {"keyfile", t(binary(), M("keyfile"), D("keyfile"))} - , {"verify", t(union(verify_peer, verify_none), M("verify"), D("verify"))} - , {"fail_if_no_peer_cert", t(boolean(), M("fail_if_no_peer_cert"), D("fail_if_no_peer_cert"))} - , {"secure_renegotiate", t(boolean(), M("secure_renegotiate"), D("secure_renegotiate"))} - , {"reuse_sessions", t(boolean(), M("reuse_sessions"), D("reuse_sessions"))} - , {"honor_cipher_order", t(boolean(), M("honor_cipher_order"), D("honor_cipher_order"))} - , {"handshake_timeout", t(duration(), M("handshake_timeout"), D("handshake_timeout"))} - , {"depth", t(integer(), M("depth"), D("depth"))} - , {"password", hoconsc:t(binary(), #{mapping => M("key_password"), - default => D("key_password"), - sensitive => true - })} - , {"dhfile", t(binary(), M("dhfile"), D("dhfile"))} - , {"server_name_indication", t(union(disable, binary()), M("server_name_indication"), - D("server_name_indication"))} - , {"tls_versions", t(comma_separated_list(), M("tls_versions"), D("tls_versions"))} - , {"ciphers", t(comma_separated_list(), M("ciphers"), D("ciphers"))} - , {"psk_ciphers", t(comma_separated_list(), M("ciphers"), D("ciphers"))}]. +proxy_protocol_opts() -> + [ {proxy_protocol, sc(boolean())} + , {proxy_protocol_timeout, sc(duration())} + ]. + +default_dtls_vsns() -> + [<<"dtlsv1.2">>, <<"dtlsv1">>]. + +dtls_vsn(<<"dtlsv1.2">>) -> 'dtlsv1.2'; +dtls_vsn(<<"dtlsv1">>) -> 'dtlsv1'. + +sc(Type) -> + sc_meta(Type, #{}). + +sc(Type, Default) -> + sc_meta(Type, #{default => Default}). + +sc_meta(Type, Meta) -> + hoconsc:mk(Type, Meta). + +map(Name, Type) -> + hoconsc:map(Name, Type). + +ref(StructName) -> + ref(?MODULE, StructName). + +ref(Mod, Field) -> + hoconsc:ref(Mod, Field). diff --git a/apps/emqx_gateway/src/emqx_gateway_utils.erl b/apps/emqx_gateway/src/emqx_gateway_utils.erl index dc4e38e7d..6d19cbbcf 100644 --- a/apps/emqx_gateway/src/emqx_gateway_utils.erl +++ b/apps/emqx_gateway/src/emqx_gateway_utils.erl @@ -28,7 +28,11 @@ -export([ apply/2 , format_listenon/1 + , parse_listenon/1 + , unix_ts_to_rfc3339/1 , unix_ts_to_rfc3339/2 + , listener_id/3 + , parse_listener_id/1 ]). -export([ stringfy/1 @@ -111,6 +115,39 @@ format_listenon({Addr, Port}) when is_list(Addr) -> format_listenon({Addr, Port}) when is_tuple(Addr) -> io_lib:format("~s:~w", [inet:ntoa(Addr), Port]). +parse_listenon(Port) when is_integer(Port) -> + Port; +parse_listenon(Str) when is_binary(Str) -> + parse_listenon(binary_to_list(Str)); +parse_listenon(Str) when is_list(Str) -> + case emqx_schema:to_ip_port(Str) of + {ok, R} -> R; + {error, _} -> + error({invalid_listenon_name, Str}) + end. + +listener_id(GwName, Type, LisName) -> + binary_to_atom( + <<(bin(GwName))/binary, ":", + (bin(Type))/binary, ":", + (bin(LisName))/binary + >>). + +parse_listener_id(Id) when is_atom(Id) -> + parse_listener_id(atom_to_binary(Id)); +parse_listener_id(Id) -> + try + [GwName, Type, Name] = binary:split(bin(Id), <<":">>, [global]), + {GwName, Type, Name} + catch + _ : _ -> error({invalid_listener_id, Id}) + end. + +bin(A) when is_atom(A) -> + atom_to_binary(A); +bin(L) when is_list(L); is_binary(L) -> + iolist_to_binary(L). + unix_ts_to_rfc3339(Keys, Map) when is_list(Keys) -> lists:foldl(fun(K, Acc) -> unix_ts_to_rfc3339(K, Acc) end, Map, Keys); unix_ts_to_rfc3339(Key, Map) -> @@ -121,7 +158,12 @@ unix_ts_to_rfc3339(Key, Map) -> emqx_rule_funcs:unix_ts_to_rfc3339(Ts, <<"millisecond">>)} end. +unix_ts_to_rfc3339(Ts) -> + emqx_rule_funcs:unix_ts_to_rfc3339(Ts, <<"millisecond">>). + -spec stringfy(term()) -> binary(). +stringfy(T) when is_list(T); is_binary(T) -> + iolist_to_binary(T); stringfy(T) -> iolist_to_binary(io_lib:format("~0p", [T])). @@ -139,16 +181,47 @@ normalize_config(RawConf) -> Listeners = maps:fold(fun(Name, Confs, AccIn2) -> ListenOn = maps:get(bind, Confs), - SocketOpts = esockd:parse_opt(maps:to_list(Confs)), + SocketOpts = esockd_opts(Type, Confs), RemainCfgs = maps:without( - [bind] ++ proplists:get_keys(SocketOpts), - Confs), + [bind, tcp, ssl, udp, dtls] + ++ proplists:get_keys(SocketOpts), Confs), Cfg = maps:merge(Cfg0, RemainCfgs), [{Type, Name, ListenOn, SocketOpts, Cfg}|AccIn2] end, [], Liss), [Listeners|AccIn1] end, [], LisMap)). +esockd_opts(Type, Opts0) -> + Opts1 = maps:with([acceptors, max_connections, max_conn_rate, + proxy_protocol, proxy_protocol_timeout], Opts0), + Opts2 = Opts1#{access_rules => esockd_access_rules(maps:get(access_rules, Opts0, []))}, + maps:to_list(case Type of + tcp -> Opts2#{tcp_options => sock_opts(tcp, Opts0)}; + ssl -> Opts2#{tcp_options => sock_opts(tcp, Opts0), + ssl_options => ssl_opts(ssl, Opts0)}; + udp -> Opts2#{udp_options => sock_opts(udp, Opts0)}; + dtls -> Opts2#{udp_options => sock_opts(udp, Opts0), + dtls_options => ssl_opts(dtls, Opts0)} + end). + +esockd_access_rules(StrRules) -> + Access = fun(S) -> + [A, CIDR] = string:tokens(S, " "), + {list_to_atom(A), case CIDR of "all" -> all; _ -> CIDR end} + end, + [Access(R) || R <- StrRules]. + +ssl_opts(Name, Opts) -> + maps:to_list( + emqx_tls_lib:drop_tls13_for_old_otp( + maps:without([enable], + maps:get(Name, Opts, #{})))). + +sock_opts(Name, Opts) -> + maps:to_list( + maps:without([active_n], + maps:get(Name, Opts, #{}))). + %%-------------------------------------------------------------------- %% Envs @@ -205,5 +278,6 @@ default_subopts() -> #{rh => 0, %% Retain Handling rap => 0, %% Retain as Publish nl => 0, %% No Local - qos => 0 %% QoS + qos => 0, %% QoS + is_new => true }. diff --git a/apps/emqx_gateway/src/exproto/emqx_exproto_channel.erl b/apps/emqx_gateway/src/exproto/emqx_exproto_channel.erl index b1a1ae027..3de231958 100644 --- a/apps/emqx_gateway/src/exproto/emqx_exproto_channel.erl +++ b/apps/emqx_gateway/src/exproto/emqx_exproto_channel.erl @@ -31,7 +31,7 @@ , handle_in/2 , handle_deliver/2 , handle_timeout/3 - , handle_call/2 + , handle_call/3 , handle_cast/2 , handle_info/2 , terminate/2 @@ -243,23 +243,24 @@ handle_timeout(_TRef, Msg, Channel) -> ?WARN("Unexpected timeout: ~p", [Msg]), {ok, Channel}. --spec handle_call(any(), channel()) +-spec handle_call(Req :: any(), From :: any(), channel()) -> {reply, Reply :: term(), channel()} | {reply, Reply :: term(), replies(), channel()} | {shutdown, Reason :: term(), Reply :: term(), channel()}. -handle_call({send, Data}, Channel) -> +handle_call({send, Data}, _From, Channel) -> {reply, ok, [{outgoing, Data}], Channel}; -handle_call(close, Channel = #channel{conn_state = connected}) -> +handle_call(close, _From, Channel = #channel{conn_state = connected}) -> {reply, ok, [{event, disconnected}, {close, normal}], Channel}; -handle_call(close, Channel) -> +handle_call(close, _From, Channel) -> {reply, ok, [{close, normal}], Channel}; -handle_call({auth, ClientInfo, _Password}, Channel = #channel{conn_state = connected}) -> +handle_call({auth, ClientInfo, _Password}, _From, + Channel = #channel{conn_state = connected}) -> ?LOG(warning, "Duplicated authorized command, dropped ~p", [ClientInfo]), {reply, {error, ?RESP_PERMISSION_DENY, <<"Duplicated authenticate command">>}, Channel}; -handle_call({auth, ClientInfo0, Password}, +handle_call({auth, ClientInfo0, Password}, _From, Channel = #channel{ ctx = Ctx, conninfo = ConnInfo, @@ -300,7 +301,7 @@ handle_call({auth, ClientInfo0, Password}, {reply, {error, ?RESP_PERMISSION_DENY, Reason}, Channel} end; -handle_call({start_timer, keepalive, Interval}, +handle_call({start_timer, keepalive, Interval}, _From, Channel = #channel{ conninfo = ConnInfo, clientinfo = ClientInfo @@ -310,7 +311,7 @@ handle_call({start_timer, keepalive, Interval}, NChannel = Channel#channel{conninfo = NConnInfo, clientinfo = NClientInfo}, {reply, ok, ensure_keepalive(NChannel)}; -handle_call({subscribe, TopicFilter, Qos}, +handle_call({subscribe_from_client, TopicFilter, Qos}, _From, Channel = #channel{ ctx = Ctx, conn_state = connected, @@ -323,12 +324,20 @@ handle_call({subscribe, TopicFilter, Qos}, {reply, ok, NChannel} end; -handle_call({unsubscribe, TopicFilter}, +handle_call({subscribe, Topic, SubOpts}, _From, Channel) -> + {ok, NChannel} = do_subscribe([{Topic, SubOpts}], Channel), + {reply, ok, NChannel}; + +handle_call({unsubscribe_from_client, TopicFilter}, _From, Channel = #channel{conn_state = connected}) -> {ok, NChannel} = do_unsubscribe([{TopicFilter, #{}}], Channel), {reply, ok, NChannel}; -handle_call({publish, Topic, Qos, Payload}, +handle_call({unsubscribe, Topic}, _From, Channel) -> + {ok, NChannel} = do_unsubscribe([Topic], Channel), + {reply, ok, NChannel}; + +handle_call({publish, Topic, Qos, Payload}, _From, Channel = #channel{ ctx = Ctx, conn_state = connected, @@ -345,10 +354,10 @@ handle_call({publish, Topic, Qos, Payload}, {reply, ok, Channel} end; -handle_call(kick, Channel) -> +handle_call(kick, _From, Channel) -> {shutdown, kicked, ok, Channel}; -handle_call(Req, Channel) -> +handle_call(Req, _From, Channel) -> ?LOG(warning, "Unexpected call: ~p", [Req]), {reply, {error, unexpected_call}, Channel}. @@ -363,12 +372,6 @@ handle_cast(Req, Channel) -> -spec handle_info(any(), channel()) -> {ok, channel()} | {shutdown, Reason :: term(), channel()}. -handle_info({subscribe, TopicFilters}, Channel) -> - do_subscribe(TopicFilters, Channel); - -handle_info({unsubscribe, TopicFilters}, Channel) -> - do_unsubscribe(TopicFilters, Channel); - handle_info({sock_closed, Reason}, Channel = #channel{rqueue = Queue, inflight = Inflight}) -> case queue:len(Queue) =:= 0 diff --git a/apps/emqx_gateway/src/exproto/emqx_exproto_gsvr.erl b/apps/emqx_gateway/src/exproto/emqx_exproto_gsvr.erl index 346f87452..04f3dea1c 100644 --- a/apps/emqx_gateway/src/exproto/emqx_exproto_gsvr.erl +++ b/apps/emqx_gateway/src/exproto/emqx_exproto_gsvr.erl @@ -22,9 +22,10 @@ -include("src/exproto/include/emqx_exproto.hrl"). -include_lib("emqx/include/logger.hrl"). - -define(IS_QOS(X), (X =:= 0 orelse X =:= 1 orelse X =:= 2)). +-define(DEFAULT_CALL_TIMEOUT, 5000). + %% gRPC server callbacks -export([ send/2 , close/2 @@ -96,7 +97,7 @@ publish(Req, Md) -> subscribe(Req = #{conn := Conn, topic := Topic, qos := Qos}, Md) when ?IS_QOS(Qos) -> ?LOG(debug, "Recv ~p function with request ~0p", [?FUNCTION_NAME, Req]), - {ok, response(call(Conn, {subscribe, Topic, Qos})), Md}; + {ok, response(call(Conn, {subscribe_from_client, Topic, Qos})), Md}; subscribe(Req, Md) -> ?LOG(debug, "Recv ~p function with request ~0p", [?FUNCTION_NAME, Req]), @@ -107,7 +108,7 @@ subscribe(Req, Md) -> | {error, grpc_cowboy_h:error_response()}. unsubscribe(Req = #{conn := Conn, topic := Topic}, Md) -> ?LOG(debug, "Recv ~p function with request ~0p", [?FUNCTION_NAME, Req]), - {ok, response(call(Conn, {unsubscribe, Topic})), Md}. + {ok, response(call(Conn, {unsubscribe_from_client, Topic})), Md}. %%-------------------------------------------------------------------- %% Internal funcs @@ -117,18 +118,22 @@ to_pid(ConnStr) -> binary_to_term(base64:decode(ConnStr)). call(ConnStr, Req) -> - case catch to_pid(ConnStr) of - {'EXIT', {badarg, _}} -> - {error, ?RESP_PARAMS_TYPE_ERROR, - <<"The conn type error">>}; - Pid when is_pid(Pid) -> - case erlang:is_process_alive(Pid) of - true -> - emqx_gateway_conn:call(Pid, Req); - false -> - {error, ?RESP_CONN_PROCESS_NOT_ALIVE, - <<"Connection process is not alive">>} - end + try + Pid = to_pid(ConnStr), + emqx_gateway_conn:call(Pid, Req, ?DEFAULT_CALL_TIMEOUT) + catch + exit : badarg -> + {error, ?RESP_PARAMS_TYPE_ERROR, <<"The conn type error">>}; + exit : noproc -> + {error, ?RESP_CONN_PROCESS_NOT_ALIVE, + <<"Connection process is not alive">>}; + exit : timeout -> + {error, ?RESP_UNKNOWN, <<"Connection is not answered">>}; + Class : Reason : Stk-> + ?LOG(error, "Call ~p crashed: {~0p, ~0p}, " + "stacktrace: ~0p", + [Class, Reason, Stk]), + {error, ?RESP_UNKNOWN, <<"Unkwown crashs">>} end. %%-------------------------------------------------------------------- diff --git a/apps/emqx_gateway/src/exproto/emqx_exproto_impl.erl b/apps/emqx_gateway/src/exproto/emqx_exproto_impl.erl index 8131f2d0c..3e142f3dc 100644 --- a/apps/emqx_gateway/src/exproto/emqx_exproto_impl.erl +++ b/apps/emqx_gateway/src/exproto/emqx_exproto_impl.erl @@ -143,17 +143,17 @@ start_listener(GwName, Ctx, {Type, LisName, ListenOn, SocketOpts, Cfg}) -> ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn), case start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) of {ok, Pid} -> - ?ULOG("Start ~s:~s:~s listener on ~s successfully.~n", - [GwName, Type, LisName, ListenOnStr]), + ?ULOG("Gateway ~s:~s:~s on ~s started.~n", + [GwName, Type, LisName, ListenOnStr]), Pid; {error, Reason} -> - ?ELOG("Failed to start ~s:~s:~s listener on ~s: ~0p~n", + ?ELOG("Failed to start gateway ~s:~s:~s on ~s: ~0p~n", [GwName, Type, LisName, ListenOnStr, Reason]), throw({badconf, Reason}) end. start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) -> - Name = name(GwName, LisName, Type), + Name = emqx_gateway_utils:listener_id(GwName, Type, LisName), NCfg = Cfg#{ ctx => Ctx, frame_mod => emqx_exproto_frame, @@ -172,9 +172,6 @@ do_start_listener(udp, Name, ListenOn, Opts, MFA) -> do_start_listener(dtls, Name, ListenOn, Opts, MFA) -> esockd:open_dtls(Name, ListenOn, Opts, MFA). -name(GwName, LisName, Type) -> - list_to_atom(lists:concat([GwName, ":", Type, ":", LisName])). - merge_default_by_type(Type, Options) when Type =:= tcp; Type =:= ssl -> Default = emqx_gateway_utils:default_tcp_options(), @@ -200,14 +197,14 @@ stop_listener(GwName, {Type, LisName, ListenOn, SocketOpts, Cfg}) -> StopRet = stop_listener(GwName, Type, LisName, ListenOn, SocketOpts, Cfg), ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn), case StopRet of - ok -> ?ULOG("Stop ~s:~s:~s listener on ~s successfully.~n", + ok -> ?ULOG("Gateway ~s:~s:~s on ~s stopped.~n", [GwName, Type, LisName, ListenOnStr]); {error, Reason} -> - ?ELOG("Failed to stop ~s:~s:~s listener on ~s: ~0p~n", + ?ELOG("Failed to stop gateway ~s:~s:~s on ~s: ~0p~n", [GwName, Type, LisName, ListenOnStr, Reason]) end, StopRet. stop_listener(GwName, Type, LisName, ListenOn, _SocketOpts, _Cfg) -> - Name = name(GwName, LisName, Type), + Name = emqx_gateway_utils:listener_id(GwName, Type, LisName), esockd:close(Name, ListenOn). diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_api.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_api.erl index 80449238c..98c9fabe8 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_api.erl +++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_api.erl @@ -16,140 +16,119 @@ -module(emqx_lwm2m_api). --rest_api(#{name => list, - method => 'GET', - path => "/lwm2m_channels/", - func => list, - descr => "A list of all lwm2m channel" - }). +-behaviour(minirest_api). --rest_api(#{name => list, - method => 'GET', - path => "/nodes/:atom:node/lwm2m_channels/", - func => list, - descr => "A list of lwm2m channel of a node" - }). +-export([api_spec/0]). --rest_api(#{name => lookup_cmd, - method => 'GET', - path => "/lookup_cmd/:bin:ep/", - func => lookup_cmd, - descr => "Send a lwm2m downlink command" - }). +-export([lookup_cmd/2]). --rest_api(#{name => lookup_cmd, - method => 'GET', - path => "/nodes/:atom:node/lookup_cmd/:bin:ep/", - func => lookup_cmd, - descr => "Send a lwm2m downlink command of a node" - }). +-define(PREFIX, "/gateway/lwm2m/:clientid"). --export([ list/2 - , lookup_cmd/2 - ]). +-import(emqx_mgmt_util, [ object_schema/1 + , error_schema/2 + , properties/1]). -list(#{node := Node }, Params) -> - case Node = node() of - true -> list(#{}, Params); - _ -> rpc_call(Node, list, [#{}, Params]) - end; +api_spec() -> + {[lookup_cmd_api()], []}. -list(#{}, _Params) -> - Channels = emqx_lwm2m_cm:all_channels(), - return({ok, format(Channels)}). +lookup_cmd_paramters() -> + [ make_paramter(clientid, path, true, "string") + , make_paramter(path, query, true, "string") + , make_paramter(action, query, true, "string")]. -lookup_cmd(#{ep := Ep, node := Node}, Params) -> - case Node = node() of - true -> lookup_cmd(#{ep => Ep}, Params); - _ -> rpc_call(Node, lookup_cmd, [#{ep => Ep}, Params]) - end; +lookup_cmd_properties() -> + properties([ {clientid, string} + , {path, string} + , {action, string} + , {code, string} + , {codeMsg, string} + , {content, {array, object}, lookup_cmd_content_props()}]). -lookup_cmd(#{ep := Ep}, Params) -> - MsgType = proplists:get_value(<<"msgType">>, Params), - Path0 = proplists:get_value(<<"path">>, Params), - case emqx_lwm2m_cm:lookup_cmd(Ep, Path0, MsgType) of - [] -> return({ok, []}); - [{_, undefined} | _] -> return({ok, []}); - [{{IMEI, Path, MsgType}, undefined}] -> - return({ok, [{imei, IMEI}, - {'msgType', IMEI}, - {'code', <<"6.01">>}, - {'codeMsg', <<"reply_not_received">>}, - {'path', Path}]}); - [{{IMEI, Path, MsgType}, {Code, CodeMsg, Content}}] -> - Payload1 = format_cmd_content(Content, MsgType), - return({ok, [{imei, IMEI}, - {'msgType', IMEI}, - {'code', Code}, - {'codeMsg', CodeMsg}, - {'path', Path}] ++ Payload1}) +lookup_cmd_content_props() -> + [ {operations, string, <<"Resource Operations">>} + , {dataType, string, <<"Resource Type">>} + , {path, string, <<"Resource Path">>} + , {name, string, <<"Resource Name">>}]. + +lookup_cmd_api() -> + Metadata = #{get => + #{description => <<"look up resource">>, + parameters => lookup_cmd_paramters(), + responses => + #{<<"200">> => object_schema(lookup_cmd_properties()), + <<"404">> => error_schema("client not found error", ['CLIENT_NOT_FOUND']) + } + }}, + {?PREFIX ++ "/lookup_cmd", Metadata, lookup_cmd}. + + +lookup_cmd(get, #{bindings := Bindings, query_string := QS}) -> + ClientId = maps:get(clientid, Bindings), + case emqx_gateway_cm_registry:lookup_channels(lwm2m, ClientId) of + [Channel | _] -> + #{<<"path">> := Path, + <<"action">> := Action} = QS, + {ok, Result} = emqx_lwm2m_channel:lookup_cmd(Channel, Path, Action), + lookup_cmd_return(Result, ClientId, Action, Path); + _ -> + {404, #{code => 'CLIENT_NOT_FOUND'}} end. -rpc_call(Node, Fun, Args) -> - case rpc:call(Node, ?MODULE, Fun, Args) of - {badrpc, Reason} -> {error, Reason}; - Res -> Res - end. +lookup_cmd_return(undefined, ClientId, Action, Path) -> + {200, + #{clientid => ClientId, + action => Action, + code => <<"6.01">>, + codeMsg => <<"reply_not_received">>, + path => Path}}; -format(Channels) -> - lists:map(fun({IMEI, #{lifetime := LifeTime, - peername := Peername, - version := Version, - reg_info := RegInfo}}) -> - ObjectList = lists:map(fun(Path) -> - [ObjId | _] = path_list(Path), - case emqx_lwm2m_xml_object:get_obj_def(binary_to_integer(ObjId), true) of - {error, _} -> - {Path, Path}; - ObjDefinition -> - ObjectName = emqx_lwm2m_xml_object:get_object_name(ObjDefinition), - {Path, list_to_binary(ObjectName)} - end - end, maps:get(<<"objectList">>, RegInfo)), - {IpAddr, Port} = Peername, - [{imei, IMEI}, - {lifetime, LifeTime}, - {ip_address, iolist_to_binary(ntoa(IpAddr))}, - {port, Port}, - {version, Version}, - {'objectList', ObjectList}] - end, Channels). +lookup_cmd_return({Code, CodeMsg, Content}, ClientId, Action, Path) -> + {200, + format_cmd_content(Content, + Action, + #{clientid => ClientId, + action => Action, + code => Code, + codeMsg => CodeMsg, + path => Path})}. -format_cmd_content(undefined, _MsgType) -> []; -format_cmd_content(Content, <<"discover">>) -> +format_cmd_content(undefined, _MsgType, Result) -> + Result; + +format_cmd_content(Content, <<"discover">>, Result) -> [H | Content1] = Content, - {_, [HObjId]} = emqx_lwm2m_coap_resource:parse_object_list(H), + {_, [HObjId]} = emqx_lwm2m_session:parse_object_list(H), [ObjId | _]= path_list(HObjId), ObjectList = case Content1 of - [Content2 | _] -> - {_, ObjL} = emqx_lwm2m_coap_resource:parse_object_list(Content2), - ObjL; - [] -> [] - end, - R = case emqx_lwm2m_xml_object:get_obj_def(binary_to_integer(ObjId), true) of - {error, _} -> - lists:map(fun(Object) -> {Object, Object} end, ObjectList); - ObjDefinition -> - lists:map(fun(Object) -> - [_, _, ResId| _] = path_list(Object), - Operations = case emqx_lwm2m_xml_object:get_resource_operations(binary_to_integer(ResId), ObjDefinition) of - "E" -> [{operations, list_to_binary("E")}]; - Oper -> [{'dataType', list_to_binary(emqx_lwm2m_xml_object:get_resource_type(binary_to_integer(ResId), ObjDefinition))}, - {operations, list_to_binary(Oper)}] - end, - [{path, Object}, - {name, list_to_binary(emqx_lwm2m_xml_object:get_resource_name(binary_to_integer(ResId), ObjDefinition))} - ] ++ Operations - end, ObjectList) - end, - [{content, R}]; -format_cmd_content(Content, _) -> - [{content, Content}]. + [Content2 | _] -> + {_, ObjL} = emqx_lwm2m_session:parse_object_list(Content2), + ObjL; + [] -> [] + end, -ntoa({0,0,0,0,0,16#ffff,AB,CD}) -> - inet_parse:ntoa({AB bsr 8, AB rem 256, CD bsr 8, CD rem 256}); -ntoa(IP) -> - inet_parse:ntoa(IP). + R = case emqx_lwm2m_xml_object:get_obj_def(binary_to_integer(ObjId), true) of + {error, _} -> + lists:map(fun(Object) -> #{Object => Object} end, ObjectList); + ObjDefinition -> + lists:map( + fun(Object) -> + [_, _, RawResId| _] = path_list(Object), + ResId = binary_to_integer(RawResId), + Operations = case emqx_lwm2m_xml_object:get_resource_operations(ResId, ObjDefinition) of + "E" -> + #{operations => list_to_binary("E")}; + Oper -> + #{'dataType' => list_to_binary(emqx_lwm2m_xml_object:get_resource_type(ResId, ObjDefinition)), + operations => list_to_binary(Oper)} + end, + Operations#{path => Object, + name => list_to_binary(emqx_lwm2m_xml_object:get_resource_name(ResId, ObjDefinition))} + end, ObjectList) + end, + Result#{content => R}; + +format_cmd_content(Content, _, Result) -> + Result#{content => Content}. path_list(Path) -> case binary:split(binary_util:trim(Path, $/), [<<$/>>], [global]) of @@ -159,6 +138,8 @@ path_list(Path) -> [ObjId] -> [ObjId] end. -return(_) -> -%% TODO: V5 API - ok. +make_paramter(Name, In, IsRequired, Type) -> + #{name => Name, + in => In, + required => IsRequired, + schema => #{type => Type}}. diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_channel.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_channel.erl new file mode 100644 index 000000000..6ad78742f --- /dev/null +++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_channel.erl @@ -0,0 +1,489 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2017-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_lwm2m_channel). + +-include_lib("emqx/include/logger.hrl"). +-include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). +-include_lib("emqx_gateway/src/lwm2m/include/emqx_lwm2m.hrl"). + +%% API +-export([ info/1 + , info/2 + , stats/1 + , with_context/2 + , do_takeover/3 + , lookup_cmd/3]). + +-export([ init/2 + , handle_in/2 + , handle_deliver/2 + , handle_timeout/3 + , terminate/2 + ]). + +-export([ handle_call/3 + , handle_cast/2 + , handle_info/2 + ]). + +-record(channel, { + %% Context + ctx :: emqx_gateway_ctx:context(), + %% Connection Info + conninfo :: emqx_types:conninfo(), + %% Client Info + clientinfo :: emqx_types:clientinfo(), + %% Session + session :: emqx_lwm2m_session:session() | undefined, + + %% Timer + timers :: #{atom() => disable | undefined | reference()}, + + with_context :: function() + }). + +-define(INFO_KEYS, [conninfo, conn_state, clientinfo, session]). +-import(emqx_coap_medium, [reply/2, reply/3, reply/4, iter/3, iter/4]). + +%%-------------------------------------------------------------------- +%% API +%%-------------------------------------------------------------------- +info(Channel) -> + maps:from_list(info(?INFO_KEYS, Channel)). + +info(Keys, Channel) when is_list(Keys) -> + [{Key, info(Key, Channel)} || Key <- Keys]; + +info(conninfo, #channel{conninfo = ConnInfo}) -> + ConnInfo; +info(conn_state, _) -> + connected; +info(clientinfo, #channel{clientinfo = ClientInfo}) -> + ClientInfo; +info(session, #channel{session = Session}) -> + emqx_misc:maybe_apply(fun emqx_session:info/1, Session); +info(clientid, #channel{clientinfo = #{clientid := ClientId}}) -> + ClientId; +info(ctx, #channel{ctx = Ctx}) -> + Ctx. + +stats(_) -> + []. + +init(ConnInfo = #{peername := {PeerHost, _}, + sockname := {_, SockPort}}, + #{ctx := Ctx} = Config) -> + Peercert = maps:get(peercert, ConnInfo, undefined), + Mountpoint = maps:get(mountpoint, Config, undefined), + ClientInfo = set_peercert_infos( + Peercert, + #{ zone => default + , protocol => lwm2m + , peerhost => PeerHost + , sockport => SockPort + , username => undefined + , clientid => undefined + , is_bridge => false + , is_superuser => false + , mountpoint => Mountpoint + } + ), + + #channel{ ctx = Ctx + , conninfo = ConnInfo + , clientinfo = ClientInfo + , timers = #{} + , session = emqx_lwm2m_session:new() + , with_context = with_context(Ctx, ClientInfo) + }. + + +with_context(Ctx, ClientInfo) -> + fun(Type, Topic) -> + with_context(Type, Topic, Ctx, ClientInfo) + end. + +lookup_cmd(Channel, Path, Action) -> + gen_server:call(Channel, {?FUNCTION_NAME, Path, Action}). + +%%-------------------------------------------------------------------- +%% Handle incoming packet +%%-------------------------------------------------------------------- +handle_in(Msg, ChannleT) -> + Channel = update_life_timer(ChannleT), + call_session(handle_coap_in, Msg, Channel). + +%%-------------------------------------------------------------------- +%% Handle Delivers from broker to client +%%-------------------------------------------------------------------- +handle_deliver(Delivers, Channel) -> + call_session(handle_deliver, Delivers, Channel). + +%%-------------------------------------------------------------------- +%% Handle timeout +%%-------------------------------------------------------------------- +handle_timeout(_, lifetime, #channel{ctx = Ctx, + clientinfo = ClientInfo, + conninfo = ConnInfo} = Channel) -> + ok = run_hooks(Ctx, 'client.disconnected', [ClientInfo, timeout, ConnInfo]), + {shutdown, timeout, Channel}; + +handle_timeout(_, {transport, _} = Msg, Channel) -> + call_session(timeout, Msg, Channel); + +handle_timeout(_, disconnect, Channel) -> + {shutdown, normal, Channel}; + +handle_timeout(_, _, Channel) -> + {ok, Channel}. + +%%-------------------------------------------------------------------- +%% Handle call +%%-------------------------------------------------------------------- +handle_call({lookup_cmd, Path, Type}, _From, #channel{session = Session} = Channel) -> + Result = emqx_lwm2m_session:find_cmd_record(Path, Type, Session), + {reply, {ok, Result}, Channel}; + +handle_call(Req, _From, Channel) -> + ?LOG(error, "Unexpected call: ~p", [Req]), + {reply, ignored, Channel}. + +%%-------------------------------------------------------------------- +%% Handle Cast +%%-------------------------------------------------------------------- +handle_cast(Req, Channel) -> + ?LOG(error, "Unexpected cast: ~p", [Req]), + {ok, Channel}. + +%%-------------------------------------------------------------------- +%% Handle Info +%%-------------------------------------------------------------------- +handle_info({subscribe, _AutoSubs}, Channel) -> + %% not need handle this message + {ok, Channel}; + +handle_info(Info, Channel) -> + ?LOG(error, "Unexpected info: ~p", [Info]), + {ok, Channel}. + +%%-------------------------------------------------------------------- +%% Terminate +%%-------------------------------------------------------------------- +terminate(Reason, #channel{ctx = Ctx, + clientinfo = ClientInfo, + session = Session}) -> + MountedTopic = emqx_lwm2m_session:on_close(Session), + _ = run_hooks(Ctx, 'session.unsubscribe', [ClientInfo, MountedTopic, #{}]), + run_hooks(Ctx, 'session.terminated', [ClientInfo, Reason, Session]). + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- +set_peercert_infos(NoSSL, ClientInfo) + when NoSSL =:= nossl; + NoSSL =:= undefined -> + ClientInfo; +set_peercert_infos(Peercert, ClientInfo) -> + {DN, CN} = {esockd_peercert:subject(Peercert), + esockd_peercert:common_name(Peercert)}, + ClientInfo#{dn => DN, cn => CN}. + +make_timer(Name, Time, Msg, Channel = #channel{timers = Timers}) -> + TRef = emqx_misc:start_timer(Time, Msg), + Channel#channel{timers = Timers#{Name => TRef}}. + +update_life_timer(#channel{session = Session, timers = Timers} = Channel) -> + LifeTime = emqx_lwm2m_session:info(lifetime, Session), + _ = case maps:get(lifetime, Timers, undefined) of + undefined -> ok; + Ref -> erlang:cancel_timer(Ref) + end, + make_timer(lifetime, LifeTime, lifetime, Channel). + +check_location(Location, #channel{session = Session}) -> + SLocation = emqx_lwm2m_session:info(location_path, Session), + Location =:= SLocation. + +do_takeover(_DesireId, Msg, Channel) -> + %% TODO completed the takeover, now only reset the message + Reset = emqx_coap_message:reset(Msg), + call_session(handle_out, Reset, Channel). + +do_connect(Req, Result, Channel, Iter) -> + case emqx_misc:pipeline( + [ fun check_lwm2m_version/2 + , fun run_conn_hooks/2 + , fun enrich_clientinfo/2 + , fun set_log_meta/2 + , fun auth_connect/2 + ], + Req, + Channel) of + {ok, _Input, #channel{session = Session, + with_context = WithContext} = NChannel} -> + case emqx_lwm2m_session:info(reg_info, Session) of + undefined -> + process_connect(ensure_connected(NChannel), Req, Result, Iter); + _ -> + NewResult = emqx_lwm2m_session:reregister(Req, WithContext, Session), + iter(Iter, maps:merge(Result, NewResult), NChannel) + end; + {error, ReasonCode, NChannel} -> + ErrMsg = io_lib:format("Login Failed: ~s", [ReasonCode]), + Payload = erlang:list_to_binary(lists:flatten(ErrMsg)), + iter(Iter, + reply({error, bad_request}, Payload, Req, Result), + NChannel) + end. + +check_lwm2m_version(#coap_message{options = Opts}, + #channel{conninfo = ConnInfo} = Channel) -> + Ver = gets([uri_query, <<"lwm2m">>], Opts), + IsValid = case Ver of + <<"1.0">> -> + true; + <<"1">> -> + true; + <<"1.1">> -> + true; + _ -> + false + end, + if IsValid -> + NConnInfo = ConnInfo#{ connected_at => erlang:system_time(millisecond) + , proto_name => <<"LwM2M">> + , proto_ver => Ver + }, + {ok, Channel#channel{conninfo = NConnInfo}}; + true -> + ?LOG(error, "Reject REGISTER due to unsupported version: ~0p", [Ver]), + {error, "invalid lwm2m version", Channel} + end. + +run_conn_hooks(Input, Channel = #channel{ctx = Ctx, + conninfo = ConnInfo}) -> + ConnProps = #{}, + case run_hooks(Ctx, 'client.connect', [ConnInfo], ConnProps) of + Error = {error, _Reason} -> Error; + _NConnProps -> + {ok, Input, Channel} + end. + +enrich_clientinfo(#coap_message{options = Options} = Msg, + Channel = #channel{clientinfo = ClientInfo0}) -> + Query = maps:get(uri_query, Options, #{}), + case Query of + #{<<"ep">> := Epn} -> + UserName = maps:get(<<"imei">>, Query, Epn), + Password = maps:get(<<"password">>, Query, undefined), + ClientId = maps:get(<<"device_id">>, Query, Epn), + ClientInfo = + ClientInfo0#{username => UserName, + password => Password, + clientid => ClientId}, + {ok, NClientInfo} = fix_mountpoint(Msg, ClientInfo), + {ok, Channel#channel{clientinfo = NClientInfo}}; + _ -> + ?LOG(error, "Reject REGISTER due to wrong parameters, Query=~p", [Query]), + {error, "invalid queries", Channel} + end. + +set_log_meta(_Input, #channel{clientinfo = #{clientid := ClientId}}) -> + emqx_logger:set_metadata_clientid(ClientId), + ok. + +auth_connect(_Input, Channel = #channel{ctx = Ctx, + clientinfo = ClientInfo}) -> + #{clientid := ClientId, username := Username} = ClientInfo, + case emqx_gateway_ctx:authenticate(Ctx, ClientInfo) of + {ok, NClientInfo} -> + {ok, Channel#channel{clientinfo = NClientInfo, + with_context = with_context(Ctx, ClientInfo)}}; + {error, Reason} -> + ?LOG(warning, "Client ~s (Username: '~s') login failed for ~0p", + [ClientId, Username, Reason]), + {error, Reason} + end. + +fix_mountpoint(_Packet, #{mountpoint := undefined} = ClientInfo) -> + {ok, ClientInfo}; +fix_mountpoint(_Packet, ClientInfo = #{mountpoint := Mountpoint}) -> + Mountpoint1 = emqx_mountpoint:replvar(Mountpoint, ClientInfo), + {ok, ClientInfo#{mountpoint := Mountpoint1}}. + +ensure_connected(Channel = #channel{ctx = Ctx, + conninfo = ConnInfo, + clientinfo = ClientInfo}) -> + _ = run_hooks(Ctx, 'client.connack', [ConnInfo, connection_accepted, []]), + ok = run_hooks(Ctx, 'client.connected', [ClientInfo, ConnInfo]), + Channel. + +process_connect(Channel = #channel{ctx = Ctx, + session = Session, + conninfo = ConnInfo, + clientinfo = ClientInfo, + with_context = WithContext}, + Msg, Result, Iter) -> + %% inherit the old session + SessFun = fun(_,_) -> #{} end, + case emqx_gateway_ctx:open_session( + Ctx, + true, + ClientInfo, + ConnInfo, + SessFun, + emqx_lwm2m_session + ) of + {ok, _} -> + Mountpoint = maps:get(mountpoint, ClientInfo, <<>>), + NewResult = emqx_lwm2m_session:init(Msg, Mountpoint, WithContext, Session), + iter(Iter, maps:merge(Result, NewResult), Channel); + {error, Reason} -> + ?LOG(error, "Failed to open session du to ~p", [Reason]), + iter(Iter, reply({error, bad_request}, Msg, Result), Channel) + end. + +run_hooks(Ctx, Name, Args) -> + emqx_gateway_ctx:metrics_inc(Ctx, Name), + emqx_hooks:run(Name, Args). + +run_hooks(Ctx, Name, Args, Acc) -> + emqx_gateway_ctx:metrics_inc(Ctx, Name), + emqx_hooks:run_fold(Name, Args, Acc). + +gets(_, undefined) -> + undefined; +gets([H | T], Map) -> + gets(T, maps:get(H, Map, undefined)); +gets([], Val) -> + Val. + +with_context(publish, [Topic, Msg], Ctx, ClientInfo) -> + case emqx_gateway_ctx:authorize(Ctx, ClientInfo, publish, Topic) of + allow -> + emqx:publish(Msg); + _ -> + ?LOG(error, "topic:~p not allow to publish ", [Topic]) + end; + +with_context(subscribe, [Topic, Opts], Ctx, #{username := UserName} = ClientInfo) -> + case emqx_gateway_ctx:authorize(Ctx, ClientInfo, subscribe, Topic) of + allow -> + run_hooks(Ctx, 'session.subscribed', [ClientInfo, Topic, UserName]), + ?LOG(debug, "Subscribe topic: ~0p, Opts: ~0p, EndpointName: ~0p", [Topic, Opts, UserName]), + emqx:subscribe(Topic, UserName, Opts); + _ -> + ?LOG(error, "Topic: ~0p not allow to subscribe", [Topic]) + end; + +with_context(metrics, Name, Ctx, _ClientInfo) -> + emqx_gateway_ctx:metrics_inc(Ctx, Name). + +%%-------------------------------------------------------------------- +%% Call Chain +%%-------------------------------------------------------------------- +call_session(Fun, + Msg, + #channel{session = Session, + with_context = WithContext} = Channel) -> + iter([ session, fun process_session/4 + , proto, fun process_protocol/4 + , return, fun process_return/4 + , lifetime, fun process_lifetime/4 + , reply, fun process_reply/4 + , out, fun process_out/4 + , fun process_nothing/3 + ], + emqx_lwm2m_session:Fun(Msg, WithContext, Session), + Channel). + +process_session(Session, Result, Channel, Iter) -> + iter(Iter, Result, Channel#channel{session = Session}). + +process_protocol({request, Msg}, Result, Channel, Iter) -> + #coap_message{method = Method} = Msg, + handle_request_protocol(Method, Msg, Result, Channel, Iter); + +process_protocol(Msg, Result, + #channel{with_context = WithContext, session = Session} = Channel, Iter) -> + ProtoResult = emqx_lwm2m_session:handle_protocol_in(Msg, WithContext, Session), + iter(Iter, maps:merge(Result, ProtoResult), Channel). + +handle_request_protocol(post, #coap_message{options = Opts} = Msg, + Result, Channel, Iter) -> + case Opts of + #{uri_path := [?REG_PREFIX]} -> + do_connect(Msg, Result, Channel, Iter); + #{uri_path := Location} -> + do_update(Location, Msg, Result, Channel, Iter); + _ -> + iter(Iter, reply({error, not_found}, Msg, Result), Channel) + end; + +handle_request_protocol(delete, #coap_message{options = Opts} = Msg, + Result, Channel, Iter) -> + case Opts of + #{uri_path := Location} -> + case check_location(Location, Channel) of + true -> + Reply = emqx_coap_message:piggyback({ok, deleted}, Msg), + {shutdown, close, Reply, Channel}; + _ -> + iter(Iter, reply({error, not_found}, Msg, Result), Channel) + end; + _ -> + iter(Iter, reply({error, bad_request}, Msg, Result), Channel) + end. + +do_update(Location, Msg, Result, + #channel{session = Session, with_context = WithContext} = Channel, Iter) -> + case check_location(Location, Channel) of + true -> + NewResult = emqx_lwm2m_session:update(Msg, WithContext, Session), + iter(Iter, maps:merge(Result, NewResult), Channel); + _ -> + iter(Iter, reply({error, not_found}, Msg, Result), Channel) + end. + +process_return({Outs, Session}, Result, Channel, Iter) -> + OldOuts = maps:get(out, Result, []), + iter(Iter, + Result#{out => Outs ++ OldOuts}, + Channel#channel{session = Session}). + +process_out(Outs, Result, Channel, _) -> + Outs2 = lists:reverse(Outs), + Outs3 = case maps:get(reply, Result, undefined) of + undefined -> + Outs2; + Reply -> + [Reply | Outs2] + end, + + {ok, {outgoing, Outs3}, Channel}. + +process_reply(Reply, Result, #channel{session = Session} = Channel, _) -> + Session2 = emqx_lwm2m_session:set_reply(Reply, Session), + Outs = maps:get(out, Result, []), + Outs2 = lists:reverse(Outs), + {ok, {outgoing, [Reply | Outs2]}, Channel#channel{session = Session2}}. + +process_lifetime(_, Result, Channel, Iter) -> + iter(Iter, Result, update_life_timer(Channel)). + +process_nothing(_, _, Channel) -> + {ok, Channel}. diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_cm.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_cm.erl deleted file mode 100644 index 16e938b84..000000000 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_cm.erl +++ /dev/null @@ -1,153 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_lwm2m_cm). - --export([start_link/0]). - --export([ register_channel/5 - , update_reg_info/2 - , unregister_channel/1 - ]). - --export([ lookup_channel/1 - , all_channels/0 - ]). - --export([ register_cmd/3 - , register_cmd/4 - , lookup_cmd/3 - , lookup_cmd_by_imei/1 - ]). - -%% gen_server callbacks --export([ init/1 - , handle_call/3 - , handle_cast/2 - , handle_info/2 - , terminate/2 - , code_change/3 - ]). - --define(LOG(Level, Format, Args), logger:Level("LWM2M-CM: " ++ Format, Args)). - -%% Server name --define(CM, ?MODULE). - --define(LWM2M_CHANNEL_TAB, emqx_lwm2m_channel). --define(LWM2M_CMD_TAB, emqx_lwm2m_cmd). - -%% Batch drain --define(BATCH_SIZE, 100000). - -%% @doc Start the channel manager. -start_link() -> - gen_server:start_link({local, ?CM}, ?MODULE, [], []). - -%%-------------------------------------------------------------------- -%% API -%%-------------------------------------------------------------------- - -register_channel(IMEI, RegInfo, LifeTime, Ver, Peername) -> - Info = #{ - reg_info => RegInfo, - lifetime => LifeTime, - version => Ver, - peername => Peername - }, - true = ets:insert(?LWM2M_CHANNEL_TAB, {IMEI, Info}), - cast({registered, {IMEI, self()}}). - -update_reg_info(IMEI, RegInfo) -> - case lookup_channel(IMEI) of - [{_, RegInfo0}] -> - true = ets:insert(?LWM2M_CHANNEL_TAB, {IMEI, RegInfo0#{reg_info => RegInfo}}), - ok; - [] -> - ok - end. - -unregister_channel(IMEI) when is_binary(IMEI) -> - true = ets:delete(?LWM2M_CHANNEL_TAB, IMEI), - ok. - -lookup_channel(IMEI) -> - ets:lookup(?LWM2M_CHANNEL_TAB, IMEI). - -all_channels() -> - ets:tab2list(?LWM2M_CHANNEL_TAB). - -register_cmd(IMEI, Path, Type) -> - true = ets:insert(?LWM2M_CMD_TAB, {{IMEI, Path, Type}, undefined}). - -register_cmd(_IMEI, undefined, _Type, _Result) -> - ok; -register_cmd(IMEI, Path, Type, Result) -> - true = ets:insert(?LWM2M_CMD_TAB, {{IMEI, Path, Type}, Result}). - -lookup_cmd(IMEI, Path, Type) -> - ets:lookup(?LWM2M_CMD_TAB, {IMEI, Path, Type}). - -lookup_cmd_by_imei(IMEI) -> - ets:select(?LWM2M_CHANNEL_TAB, [{{{IMEI, '_', '_'}, '$1'}, [], ['$_']}]). - -%% @private -cast(Msg) -> gen_server:cast(?CM, Msg). - -%%-------------------------------------------------------------------- -%% gen_server callbacks -%%-------------------------------------------------------------------- - -init([]) -> - TabOpts = [public, {write_concurrency, true}, {read_concurrency, true}], - ok = emqx_tables:new(?LWM2M_CHANNEL_TAB, [set, compressed | TabOpts]), - ok = emqx_tables:new(?LWM2M_CMD_TAB, [set, compressed | TabOpts]), - {ok, #{chan_pmon => emqx_pmon:new()}}. - -handle_call(Req, _From, State) -> - ?LOG(error, "Unexpected call: ~p", [Req]), - {reply, ignored, State}. - -handle_cast({registered, {IMEI, ChanPid}}, State = #{chan_pmon := PMon}) -> - PMon1 = emqx_pmon:monitor(ChanPid, IMEI, PMon), - {noreply, State#{chan_pmon := PMon1}}; - -handle_cast(Msg, State) -> - ?LOG(error, "Unexpected cast: ~p", [Msg]), - {noreply, State}. - -handle_info({'DOWN', _MRef, process, Pid, _Reason}, State = #{chan_pmon := PMon}) -> - ChanPids = [Pid | emqx_misc:drain_down(?BATCH_SIZE)], - {Items, PMon1} = emqx_pmon:erase_all(ChanPids, PMon), - ok = emqx_pool:async_submit(fun lists:foreach/2, [fun clean_down/1, Items]), - {noreply, State#{chan_pmon := PMon1}}; - -handle_info(Info, State) -> - ?LOG(error, "Unexpected info: ~p", [Info]), - {noreply, State}. - -terminate(_Reason, _State) -> - emqx_stats:cancel_update(chan_stats). - -code_change(_OldVsn, State, _Extra) -> - {ok, State}. - -%%-------------------------------------------------------------------- -%% Internal functions -%%-------------------------------------------------------------------- - -clean_down({_ChanPid, IMEI}) -> - unregister_channel(IMEI). diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_cmd.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_cmd.erl new file mode 100644 index 000000000..e17a83195 --- /dev/null +++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_cmd.erl @@ -0,0 +1,410 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2016-2017 EMQ Enterprise, Inc. (http://emqtt.io) +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_lwm2m_cmd). + +-include_lib("emqx/include/logger.hrl"). +-include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). +-include_lib("emqx_gateway/src/lwm2m/include/emqx_lwm2m.hrl"). + +-export([ mqtt_to_coap/2 + , coap_to_mqtt/4 + , empty_ack_to_mqtt/1 + , coap_failure_to_mqtt/2 + ]). + +-export([path_list/1, extract_path/1]). + +-define(STANDARD, 1). + +%%-type msg_type() :: <<"create">> +%% | <<"delete">> +%% | <<"read">> +%% | <<"write">> +%% | <<"execute">> +%% | <<"discover">> +%% | <<"write-attr">> +%% | <<"observe">> +%% | <<"cancel-observe">>. +%% +%%-type cmd() :: #{ <<"msgType">> := msg_type() +%% , <<"data">> := maps() +%% %%%% more keys? +%% }. + +%%-------------------------------------------------------------------- +%% APIs +%%-------------------------------------------------------------------- + +mqtt_to_coap(AlternatePath, InputCmd = #{<<"msgType">> := <<"create">>, <<"data">> := Data}) -> + {PathList, QueryList} = path_list(maps:get(<<"basePath">>, Data, <<"/">>)), + FullPathList = add_alternate_path_prefix(AlternatePath, PathList), + TlvData = emqx_lwm2m_message:json_to_tlv(PathList, maps:get(<<"content">>, Data)), + Payload = emqx_lwm2m_tlv:encode(TlvData), + CoapRequest = emqx_coap_message:request(con, post, Payload, + [{uri_path, FullPathList}, + {uri_query, QueryList}, + {content_format, <<"application/vnd.oma.lwm2m+tlv">>}]), + {CoapRequest, InputCmd}; + +mqtt_to_coap(AlternatePath, InputCmd = #{<<"msgType">> := <<"delete">>, <<"data">> := Data}) -> + {PathList, QueryList} = path_list(maps:get(<<"path">>, Data)), + FullPathList = add_alternate_path_prefix(AlternatePath, PathList), + {emqx_coap_message:request(con, delete, <<>>, + [{uri_path, FullPathList}, + {uri_query, QueryList}]), InputCmd}; + +mqtt_to_coap(AlternatePath, InputCmd = #{<<"msgType">> := <<"read">>, <<"data">> := Data}) -> + {PathList, QueryList} = path_list(maps:get(<<"path">>, Data)), + FullPathList = add_alternate_path_prefix(AlternatePath, PathList), + {emqx_coap_message:request(con, get, <<>>, + [{uri_path, FullPathList}, + {uri_query, QueryList}]), InputCmd}; + +mqtt_to_coap(AlternatePath, InputCmd = #{<<"msgType">> := <<"write">>, <<"data">> := Data}) -> + CoapRequest = + case maps:get(<<"basePath">>, Data, <<"/">>) of + <<"/">> -> + single_write_request(AlternatePath, Data); + BasePath -> + batch_write_request(AlternatePath, BasePath, maps:get(<<"content">>, Data)) + end, + {CoapRequest, InputCmd}; + +mqtt_to_coap(AlternatePath, InputCmd = #{<<"msgType">> := <<"execute">>, <<"data">> := Data}) -> + {PathList, QueryList} = path_list(maps:get(<<"path">>, Data)), + FullPathList = add_alternate_path_prefix(AlternatePath, PathList), + Args = + case maps:get(<<"args">>, Data, <<>>) of + <<"undefined">> -> <<>>; + undefined -> <<>>; + Arg1 -> Arg1 + end, + {emqx_coap_message:request(con, post, Args, + [{uri_path, FullPathList}, + {uri_query, QueryList}, + {content_format, <<"text/plain">>}]), InputCmd}; + +mqtt_to_coap(AlternatePath, InputCmd = #{<<"msgType">> := <<"discover">>, <<"data">> := Data}) -> + {PathList, QueryList} = path_list(maps:get(<<"path">>, Data)), + FullPathList = add_alternate_path_prefix(AlternatePath, PathList), + {emqx_coap_message:request(con, get, <<>>, + [{uri_path, FullPathList}, + {uri_query, QueryList}, + {'accept', ?LWM2M_FORMAT_LINK}]), InputCmd}; + +mqtt_to_coap(AlternatePath, InputCmd = #{<<"msgType">> := <<"write-attr">>, <<"data">> := Data}) -> + {PathList, QueryList} = path_list(maps:get(<<"path">>, Data)), + FullPathList = add_alternate_path_prefix(AlternatePath, PathList), + Query = attr_query_list(Data), + {emqx_coap_message:request(con, put, <<>>, + [{uri_path, FullPathList}, + {uri_query, QueryList}, + {uri_query, Query}]), InputCmd}; + +mqtt_to_coap(AlternatePath, InputCmd = #{<<"msgType">> := <<"observe">>, <<"data">> := Data}) -> + {PathList, QueryList} = path_list(maps:get(<<"path">>, Data)), + FullPathList = add_alternate_path_prefix(AlternatePath, PathList), + {emqx_coap_message:request(con, get, <<>>, + [{uri_path, FullPathList}, + {uri_query, QueryList}, + {observe, 0}]), InputCmd}; + +mqtt_to_coap(AlternatePath, InputCmd = #{<<"msgType">> := <<"cancel-observe">>, <<"data">> := Data}) -> + {PathList, QueryList} = path_list(maps:get(<<"path">>, Data)), + FullPathList = add_alternate_path_prefix(AlternatePath, PathList), + {emqx_coap_message:request(con, get, <<>>, + [{uri_path, FullPathList}, + {uri_query, QueryList}, + {observe, 1}]), InputCmd}. + +coap_to_mqtt(_Method = {_, Code}, _CoapPayload, _Options, Ref=#{<<"msgType">> := <<"create">>}) -> + make_response(Code, Ref); + +coap_to_mqtt(_Method = {_, Code}, _CoapPayload, _Options, Ref=#{<<"msgType">> := <<"delete">>}) -> + make_response(Code, Ref); + +coap_to_mqtt(Method, CoapPayload, Options, Ref=#{<<"msgType">> := <<"read">>}) -> + read_resp_to_mqtt(Method, CoapPayload, data_format(Options), Ref); + +coap_to_mqtt(Method, CoapPayload, _Options, Ref=#{<<"msgType">> := <<"write">>}) -> + write_resp_to_mqtt(Method, CoapPayload, Ref); + +coap_to_mqtt(Method, _CoapPayload, _Options, Ref=#{<<"msgType">> := <<"execute">>}) -> + execute_resp_to_mqtt(Method, Ref); + +coap_to_mqtt(Method, CoapPayload, _Options, Ref=#{<<"msgType">> := <<"discover">>}) -> + discover_resp_to_mqtt(Method, CoapPayload, Ref); + +coap_to_mqtt(Method, CoapPayload, _Options, Ref=#{<<"msgType">> := <<"write-attr">>}) -> + writeattr_resp_to_mqtt(Method, CoapPayload, Ref); + +coap_to_mqtt(Method, CoapPayload, Options, Ref=#{<<"msgType">> := <<"observe">>}) -> + observe_resp_to_mqtt(Method, CoapPayload, data_format(Options), observe_seq(Options), Ref); + +coap_to_mqtt(Method, CoapPayload, Options, Ref=#{<<"msgType">> := <<"cancel-observe">>}) -> + cancel_observe_resp_to_mqtt(Method, CoapPayload, data_format(Options), Ref). + +read_resp_to_mqtt({error, ErrorCode}, _CoapPayload, _Format, Ref) -> + make_response(ErrorCode, Ref); + +read_resp_to_mqtt({ok, SuccessCode}, CoapPayload, Format, Ref) -> + try + Result = content_to_mqtt(CoapPayload, Format, Ref), + make_response(SuccessCode, Ref, Format, Result) + catch + error:not_implemented -> make_response(not_implemented, Ref); + _:Ex:_ST -> + ?LOG(error, "~0p, bad payload format: ~0p", [Ex, CoapPayload]), + make_response(bad_request, Ref) + end. + +empty_ack_to_mqtt(Ref) -> + make_base_response(maps:put(<<"msgType">>, <<"ack">>, Ref)). + +coap_failure_to_mqtt(Ref, MsgType) -> + make_base_response(maps:put(<<"msgType">>, MsgType, Ref)). + +content_to_mqtt(CoapPayload, <<"text/plain">>, Ref) -> + emqx_lwm2m_message:text_to_json(extract_path(Ref), CoapPayload); + +content_to_mqtt(CoapPayload, <<"application/octet-stream">>, Ref) -> + emqx_lwm2m_message:opaque_to_json(extract_path(Ref), CoapPayload); + +content_to_mqtt(CoapPayload, <<"application/vnd.oma.lwm2m+tlv">>, Ref) -> + emqx_lwm2m_message:tlv_to_json(extract_path(Ref), CoapPayload); + +content_to_mqtt(CoapPayload, <<"application/vnd.oma.lwm2m+json">>, _Ref) -> + emqx_lwm2m_message:translate_json(CoapPayload). + +write_resp_to_mqtt({ok, changed}, _CoapPayload, Ref) -> + make_response(changed, Ref); + +write_resp_to_mqtt({ok, content}, CoapPayload, Ref) when CoapPayload =:= <<>> -> + make_response(method_not_allowed, Ref); + +write_resp_to_mqtt({ok, content}, _CoapPayload, Ref) -> + make_response(changed, Ref); + +write_resp_to_mqtt({error, Error}, _CoapPayload, Ref) -> + make_response(Error, Ref). + +execute_resp_to_mqtt({ok, changed}, Ref) -> + make_response(changed, Ref); + +execute_resp_to_mqtt({error, Error}, Ref) -> + make_response(Error, Ref). + +discover_resp_to_mqtt({ok, content}, CoapPayload, Ref) -> + Links = binary:split(CoapPayload, <<",">>, [global]), + make_response(content, Ref, <<"application/link-format">>, Links); + +discover_resp_to_mqtt({error, Error}, _CoapPayload, Ref) -> + make_response(Error, Ref). + +writeattr_resp_to_mqtt({ok, changed}, _CoapPayload, Ref) -> + make_response(changed, Ref); + +writeattr_resp_to_mqtt({error, Error}, _CoapPayload, Ref) -> + make_response(Error, Ref). + +observe_resp_to_mqtt({error, Error}, _CoapPayload, _Format, _ObserveSeqNum, Ref) -> + make_response(Error, Ref); + +observe_resp_to_mqtt({ok, content}, CoapPayload, Format, 0, Ref) -> + read_resp_to_mqtt({ok, content}, CoapPayload, Format, Ref); + +observe_resp_to_mqtt({ok, content}, CoapPayload, Format, ObserveSeqNum, Ref) -> + read_resp_to_mqtt({ok, content}, CoapPayload, Format, Ref#{<<"seqNum">> => ObserveSeqNum}). + +cancel_observe_resp_to_mqtt({ok, content}, CoapPayload, Format, Ref) -> + read_resp_to_mqtt({ok, content}, CoapPayload, Format, Ref); + +cancel_observe_resp_to_mqtt({error, Error}, _CoapPayload, _Format, Ref) -> + make_response(Error, Ref). + +make_response(Code, Ref=#{}) -> + BaseRsp = make_base_response(Ref), + make_data_response(BaseRsp, Code). + +make_response(Code, Ref=#{}, _Format, Result) -> + BaseRsp = make_base_response(Ref), + make_data_response(BaseRsp, Code, _Format, Result). + +%% The base response format is what included in the request: +%% +%% #{ +%% <<"seqNum">> => SeqNum, +%% <<"imsi">> => maps:get(<<"imsi">>, Ref, null), +%% <<"imei">> => maps:get(<<"imei">>, Ref, null), +%% <<"requestID">> => maps:get(<<"requestID">>, Ref, null), +%% <<"cacheID">> => maps:get(<<"cacheID">>, Ref, null), +%% <<"msgType">> => maps:get(<<"msgType">>, Ref, null) +%% } + +make_base_response(Ref=#{}) -> + remove_tmp_fields(Ref). + +make_data_response(BaseRsp, Code) -> + BaseRsp#{ + <<"data">> => #{ + <<"reqPath">> => extract_path(BaseRsp), + <<"code">> => code(Code), + <<"codeMsg">> => Code + } + }. + +make_data_response(BaseRsp, Code, _Format, Result) -> + BaseRsp#{ + <<"data">> => + #{ + <<"reqPath">> => extract_path(BaseRsp), + <<"code">> => code(Code), + <<"codeMsg">> => Code, + <<"content">> => Result + } + }. + +remove_tmp_fields(Ref) -> + maps:remove(observe_type, Ref). + +-spec path_list(Path::binary()) -> {[PathWord::binary()], [Query::binary()]}. +path_list(Path) -> + case binary:split(binary_util:trim(Path, $/), [<<$/>>], [global]) of + [ObjId, ObjInsId, ResId, LastPart] -> + {ResInstId, QueryList} = query_list(LastPart), + {[ObjId, ObjInsId, ResId, ResInstId], QueryList}; + [ObjId, ObjInsId, LastPart] -> + {ResId, QueryList} = query_list(LastPart), + {[ObjId, ObjInsId, ResId], QueryList}; + [ObjId, LastPart] -> + {ObjInsId, QueryList} = query_list(LastPart), + {[ObjId, ObjInsId], QueryList}; + [LastPart] -> + {ObjId, QueryList} = query_list(LastPart), + {[ObjId], QueryList} + end. + +query_list(PathWithQuery) -> + case binary:split(PathWithQuery, [<<$?>>], []) of + [Path] -> {Path, []}; + [Path, Querys] -> + {Path, binary:split(Querys, [<<$&>>], [global])} + end. + +attr_query_list(Data) -> + attr_query_list(Data, valid_attr_keys(), []). + +attr_query_list(QueryJson = #{}, ValidAttrKeys, QueryList) -> + maps:fold( + fun + (_K, null, Acc) -> Acc; + (K, V, Acc) -> + case lists:member(K, ValidAttrKeys) of + true -> + KV = <>, <<"pmax">>, <<"gt">>, <<"lt">>, <<"st">>]. + +data_format(Options) -> + maps:get(content_format, Options, <<"text/plain">>). + +observe_seq(Options) -> + maps:get(observe, Options, rand:uniform(1000000) + 1 ). + +add_alternate_path_prefix(<<"/">>, PathList) -> + PathList; + +add_alternate_path_prefix(AlternatePath, PathList) -> + [binary_util:trim(AlternatePath, $/) | PathList]. + +extract_path(Ref = #{}) -> + drop_query( + case Ref of + #{<<"data">> := Data} -> + case maps:get(<<"path">>, Data, undefined) of + undefined -> maps:get(<<"basePath">>, Data, undefined); + Path -> Path + end; + #{<<"path">> := Path} -> + Path + end). + + +batch_write_request(AlternatePath, BasePath, Content) -> + {PathList, QueryList} = path_list(BasePath), + Method = case length(PathList) of + 2 -> post; + 3 -> put + end, + FullPathList = add_alternate_path_prefix(AlternatePath, PathList), + TlvData = emqx_lwm2m_message:json_to_tlv(PathList, Content), + Payload = emqx_lwm2m_tlv:encode(TlvData), + emqx_coap_message:request(con, Method, Payload, + [{uri_path, FullPathList}, + {uri_query, QueryList}, + {content_format, <<"application/vnd.oma.lwm2m+tlv">>}]). + +single_write_request(AlternatePath, Data) -> + {PathList, QueryList} = path_list(maps:get(<<"path">>, Data)), + FullPathList = add_alternate_path_prefix(AlternatePath, PathList), + %% TO DO: handle write to resource instance, e.g. /4/0/1/0 + TlvData = emqx_lwm2m_message:json_to_tlv(PathList, [Data]), + Payload = emqx_lwm2m_tlv:encode(TlvData), + emqx_coap_message:request(con, put, Payload, + [{uri_path, FullPathList}, + {uri_query, QueryList}, + {content_format, <<"application/vnd.oma.lwm2m+tlv">>}]). + +drop_query(Path) -> + case binary:split(Path, [<<$?>>]) of + [Path] -> Path; + [PathOnly, _Query] -> PathOnly + end. + +code(get) -> <<"0.01">>; +code(post) -> <<"0.02">>; +code(put) -> <<"0.03">>; +code(delete) -> <<"0.04">>; +code(created) -> <<"2.01">>; +code(deleted) -> <<"2.02">>; +code(valid) -> <<"2.03">>; +code(changed) -> <<"2.04">>; +code(content) -> <<"2.05">>; +code(continue) -> <<"2.31">>; +code(bad_request) -> <<"4.00">>; +code(unauthorized) -> <<"4.01">>; +code(bad_option) -> <<"4.02">>; +code(forbidden) -> <<"4.03">>; +code(not_found) -> <<"4.04">>; +code(method_not_allowed) -> <<"4.05">>; +code(not_acceptable) -> <<"4.06">>; +code(request_entity_incomplete) -> <<"4.08">>; +code(precondition_failed) -> <<"4.12">>; +code(request_entity_too_large) -> <<"4.13">>; +code(unsupported_content_format) -> <<"4.15">>; +code(internal_server_error) -> <<"5.00">>; +code(not_implemented) -> <<"5.01">>; +code(bad_gateway) -> <<"5.02">>; +code(service_unavailable) -> <<"5.03">>; +code(gateway_timeout) -> <<"5.04">>; +code(proxying_not_supported) -> <<"5.05">>. diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_cmd_handler.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_cmd_handler.erl deleted file mode 100644 index 318328e3c..000000000 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_cmd_handler.erl +++ /dev/null @@ -1,310 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_lwm2m_cmd_handler). - --include("src/lwm2m/include/emqx_lwm2m.hrl"). - --include_lib("lwm2m_coap/include/coap.hrl"). - --export([ mqtt2coap/2 - , coap2mqtt/4 - , ack2mqtt/1 - , extract_path/1 - ]). - --export([path_list/1]). - --define(LOG(Level, Format, Args), logger:Level("LWM2M-CMD: " ++ Format, Args)). - -mqtt2coap(AlternatePath, InputCmd = #{<<"msgType">> := <<"create">>, <<"data">> := Data}) -> - PathList = path_list(maps:get(<<"basePath">>, Data, <<"/">>)), - FullPathList = add_alternate_path_prefix(AlternatePath, PathList), - TlvData = emqx_lwm2m_message:json_to_tlv(PathList, maps:get(<<"content">>, Data)), - Payload = emqx_lwm2m_tlv:encode(TlvData), - CoapRequest = lwm2m_coap_message:request(con, post, Payload, [{uri_path, FullPathList}, - {content_format, <<"application/vnd.oma.lwm2m+tlv">>}]), - {CoapRequest, InputCmd}; -mqtt2coap(AlternatePath, InputCmd = #{<<"msgType">> := <<"delete">>, <<"data">> := Data}) -> - FullPathList = add_alternate_path_prefix(AlternatePath, path_list(maps:get(<<"path">>, Data))), - {lwm2m_coap_message:request(con, delete, <<>>, [{uri_path, FullPathList}]), InputCmd}; -mqtt2coap(AlternatePath, InputCmd = #{<<"msgType">> := <<"read">>, <<"data">> := Data}) -> - FullPathList = add_alternate_path_prefix(AlternatePath, path_list(maps:get(<<"path">>, Data))), - {lwm2m_coap_message:request(con, get, <<>>, [{uri_path, FullPathList}]), InputCmd}; -mqtt2coap(AlternatePath, InputCmd = #{<<"msgType">> := <<"write">>, <<"data">> := Data}) -> - Encoding = maps:get(<<"encoding">>, InputCmd, <<"plain">>), - CoapRequest = - case maps:get(<<"basePath">>, Data, <<"/">>) of - <<"/">> -> - single_write_request(AlternatePath, Data, Encoding); - BasePath -> - batch_write_request(AlternatePath, BasePath, maps:get(<<"content">>, Data), Encoding) - end, - {CoapRequest, InputCmd}; - -mqtt2coap(AlternatePath, InputCmd = #{<<"msgType">> := <<"execute">>, <<"data">> := Data}) -> - FullPathList = add_alternate_path_prefix(AlternatePath, path_list(maps:get(<<"path">>, Data))), - Args = - case maps:get(<<"args">>, Data, <<>>) of - <<"undefined">> -> <<>>; - undefined -> <<>>; - Arg1 -> Arg1 - end, - {lwm2m_coap_message:request(con, post, Args, [{uri_path, FullPathList}, {content_format, <<"text/plain">>}]), InputCmd}; -mqtt2coap(AlternatePath, InputCmd = #{<<"msgType">> := <<"discover">>, <<"data">> := Data}) -> - FullPathList = add_alternate_path_prefix(AlternatePath, path_list(maps:get(<<"path">>, Data))), - {lwm2m_coap_message:request(con, get, <<>>, [{uri_path, FullPathList}, {'accept', ?LWM2M_FORMAT_LINK}]), InputCmd}; -mqtt2coap(AlternatePath, InputCmd = #{<<"msgType">> := <<"write-attr">>, <<"data">> := Data}) -> - FullPathList = add_alternate_path_prefix(AlternatePath, path_list(maps:get(<<"path">>, Data))), - Query = attr_query_list(Data), - {lwm2m_coap_message:request(con, put, <<>>, [{uri_path, FullPathList}, {uri_query, Query}]), InputCmd}; -mqtt2coap(AlternatePath, InputCmd = #{<<"msgType">> := <<"observe">>, <<"data">> := Data}) -> - PathList = path_list(maps:get(<<"path">>, Data)), - FullPathList = add_alternate_path_prefix(AlternatePath, PathList), - {lwm2m_coap_message:request(con, get, <<>>, [{uri_path, FullPathList}, {observe, 0}]), InputCmd}; -mqtt2coap(AlternatePath, InputCmd = #{<<"msgType">> := <<"cancel-observe">>, <<"data">> := Data}) -> - PathList = path_list(maps:get(<<"path">>, Data)), - FullPathList = add_alternate_path_prefix(AlternatePath, PathList), - {lwm2m_coap_message:request(con, get, <<>>, [{uri_path, FullPathList}, {observe, 1}]), InputCmd}. - -coap2mqtt(_Method = {_, Code}, _CoapPayload, _Options, Ref=#{<<"msgType">> := <<"create">>}) -> - make_response(Code, Ref); -coap2mqtt(_Method = {_, Code}, _CoapPayload, _Options, Ref=#{<<"msgType">> := <<"delete">>}) -> - make_response(Code, Ref); -coap2mqtt(Method, CoapPayload, Options, Ref=#{<<"msgType">> := <<"read">>}) -> - coap_read_to_mqtt(Method, CoapPayload, data_format(Options), Ref); -coap2mqtt(Method, _CoapPayload, _Options, Ref=#{<<"msgType">> := <<"write">>}) -> - coap_write_to_mqtt(Method, Ref); -coap2mqtt(Method, _CoapPayload, _Options, Ref=#{<<"msgType">> := <<"execute">>}) -> - coap_execute_to_mqtt(Method, Ref); -coap2mqtt(Method, CoapPayload, _Options, Ref=#{<<"msgType">> := <<"discover">>}) -> - coap_discover_to_mqtt(Method, CoapPayload, Ref); -coap2mqtt(Method, CoapPayload, _Options, Ref=#{<<"msgType">> := <<"write-attr">>}) -> - coap_writeattr_to_mqtt(Method, CoapPayload, Ref); -coap2mqtt(Method, CoapPayload, Options, Ref=#{<<"msgType">> := <<"observe">>}) -> - coap_observe_to_mqtt(Method, CoapPayload, data_format(Options), observe_seq(Options), Ref); -coap2mqtt(Method, CoapPayload, Options, Ref=#{<<"msgType">> := <<"cancel-observe">>}) -> - coap_cancel_observe_to_mqtt(Method, CoapPayload, data_format(Options), Ref). - -coap_read_to_mqtt({error, ErrorCode}, _CoapPayload, _Format, Ref) -> - make_response(ErrorCode, Ref); -coap_read_to_mqtt({ok, SuccessCode}, CoapPayload, Format, Ref) -> - try - Result = coap_content_to_mqtt_payload(CoapPayload, Format, Ref), - make_response(SuccessCode, Ref, Format, Result) - catch - error:not_implemented -> make_response(not_implemented, Ref); - C:R:Stack -> - ?LOG(error, "~p, bad payload format: ~p, stacktrace: ~p", [{C, R}, CoapPayload, Stack]), - make_response(bad_request, Ref) - end. - -ack2mqtt(Ref) -> - make_base_response(Ref). - -coap_content_to_mqtt_payload(CoapPayload, <<"text/plain">>, Ref) -> - emqx_lwm2m_message:text_to_json(extract_path(Ref), CoapPayload); -coap_content_to_mqtt_payload(CoapPayload, <<"application/octet-stream">>, Ref) -> - emqx_lwm2m_message:opaque_to_json(extract_path(Ref), CoapPayload); -coap_content_to_mqtt_payload(CoapPayload, <<"application/vnd.oma.lwm2m+tlv">>, Ref) -> - emqx_lwm2m_message:tlv_to_json(extract_path(Ref), CoapPayload); -coap_content_to_mqtt_payload(CoapPayload, <<"application/vnd.oma.lwm2m+json">>, _Ref) -> - emqx_lwm2m_message:translate_json(CoapPayload). - -coap_write_to_mqtt({ok, changed}, Ref) -> - make_response(changed, Ref); -coap_write_to_mqtt({error, Error}, Ref) -> - make_response(Error, Ref). - -coap_execute_to_mqtt({ok, changed}, Ref) -> - make_response(changed, Ref); -coap_execute_to_mqtt({error, Error}, Ref) -> - make_response(Error, Ref). - -coap_discover_to_mqtt({ok, content}, CoapPayload, Ref) -> - Links = binary:split(CoapPayload, <<",">>), - make_response(content, Ref, <<"application/link-format">>, Links); -coap_discover_to_mqtt({error, Error}, _CoapPayload, Ref) -> - make_response(Error, Ref). - -coap_writeattr_to_mqtt({ok, changed}, _CoapPayload, Ref) -> - make_response(changed, Ref); -coap_writeattr_to_mqtt({error, Error}, _CoapPayload, Ref) -> - make_response(Error, Ref). - -coap_observe_to_mqtt({error, Error}, _CoapPayload, _Format, _ObserveSeqNum, Ref) -> - make_response(Error, Ref); -coap_observe_to_mqtt({ok, content}, CoapPayload, Format, 0, Ref) -> - coap_read_to_mqtt({ok, content}, CoapPayload, Format, Ref); -coap_observe_to_mqtt({ok, content}, CoapPayload, Format, ObserveSeqNum, Ref) -> - RefWithObserve = maps:put(<<"seqNum">>, ObserveSeqNum, Ref), - RefNotify = maps:put(<<"msgType">>, <<"notify">>, RefWithObserve), - coap_read_to_mqtt({ok, content}, CoapPayload, Format, RefNotify). - -coap_cancel_observe_to_mqtt({ok, content}, CoapPayload, Format, Ref) -> - coap_read_to_mqtt({ok, content}, CoapPayload, Format, Ref); -coap_cancel_observe_to_mqtt({error, Error}, _CoapPayload, _Format, Ref) -> - make_response(Error, Ref). - -make_response(Code, Ref=#{}) -> - BaseRsp = make_base_response(Ref), - make_data_response(BaseRsp, Code). -make_response(Code, Ref=#{}, _Format, Result) -> - BaseRsp = make_base_response(Ref), - make_data_response(BaseRsp, Code, _Format, Result). - -%% The base response format is what included in the request: -%% -%% #{ -%% <<"seqNum">> => SeqNum, -%% <<"requestID">> => maps:get(<<"requestID">>, Ref, null), -%% <<"cacheID">> => maps:get(<<"cacheID">>, Ref, null), -%% <<"msgType">> => maps:get(<<"msgType">>, Ref, null) -%% } - -make_base_response(Ref=#{}) -> - remove_tmp_fields(Ref). - -make_data_response(BaseRsp, Code) -> - BaseRsp#{ - <<"data">> => #{ - <<"reqPath">> => extract_path(BaseRsp), - <<"code">> => code(Code), - <<"codeMsg">> => Code - } - }. -make_data_response(BaseRsp, Code, _Format, Result) -> - BaseRsp#{ - <<"data">> => #{ - <<"reqPath">> => extract_path(BaseRsp), - <<"code">> => code(Code), - <<"codeMsg">> => Code, - <<"content">> => Result - } - }. - -remove_tmp_fields(Ref) -> - maps:remove(observe_type, Ref). - -path_list(Path) -> - case binary:split(binary_util:trim(Path, $/), [<<$/>>], [global]) of - [ObjId, ObjInsId, ResId, ResInstId] -> [ObjId, ObjInsId, ResId, ResInstId]; - [ObjId, ObjInsId, ResId] -> [ObjId, ObjInsId, ResId]; - [ObjId, ObjInsId] -> [ObjId, ObjInsId]; - [ObjId] -> [ObjId] - end. - -attr_query_list(Data) -> - attr_query_list(Data, valid_attr_keys(), []). -attr_query_list(QueryJson = #{}, ValidAttrKeys, QueryList) -> - maps:fold( - fun - (_K, null, Acc) -> Acc; - (K, V, Acc) -> - case lists:member(K, ValidAttrKeys) of - true -> - Val = bin(V), - KV = <>, <<"pmax">>, <<"gt">>, <<"lt">>, <<"st">>]. - -data_format(Options) -> - proplists:get_value(content_format, Options, <<"text/plain">>). -observe_seq(Options) -> - proplists:get_value(observe, Options, rand:uniform(1000000) + 1 ). - -add_alternate_path_prefix(<<"/">>, PathList) -> - PathList; -add_alternate_path_prefix(AlternatePath, PathList) -> - [binary_util:trim(AlternatePath, $/) | PathList]. - -extract_path(Ref = #{}) -> - case Ref of - #{<<"data">> := Data} -> - case maps:get(<<"path">>, Data, nil) of - nil -> maps:get(<<"basePath">>, Data, undefined); - Path -> Path - end; - #{<<"path">> := Path} -> - Path - end. - -batch_write_request(AlternatePath, BasePath, Content, Encoding) -> - PathList = path_list(BasePath), - Method = case length(PathList) of - 2 -> post; - 3 -> put - end, - FullPathList = add_alternate_path_prefix(AlternatePath, PathList), - Content1 = decoding(Content, Encoding), - TlvData = emqx_lwm2m_message:json_to_tlv(PathList, Content1), - Payload = emqx_lwm2m_tlv:encode(TlvData), - lwm2m_coap_message:request(con, Method, Payload, [{uri_path, FullPathList}, {content_format, <<"application/vnd.oma.lwm2m+tlv">>}]). - -single_write_request(AlternatePath, Data, Encoding) -> - PathList = path_list(maps:get(<<"path">>, Data)), - FullPathList = add_alternate_path_prefix(AlternatePath, PathList), - Datas = decoding([Data], Encoding), - TlvData = emqx_lwm2m_message:json_to_tlv(PathList, Datas), - Payload = emqx_lwm2m_tlv:encode(TlvData), - lwm2m_coap_message:request(con, put, Payload, [{uri_path, FullPathList}, {content_format, <<"application/vnd.oma.lwm2m+tlv">>}]). - - -code(get) -> <<"0.01">>; -code(post) -> <<"0.02">>; -code(put) -> <<"0.03">>; -code(delete) -> <<"0.04">>; -code(created) -> <<"2.01">>; -code(deleted) -> <<"2.02">>; -code(valid) -> <<"2.03">>; -code(changed) -> <<"2.04">>; -code(content) -> <<"2.05">>; -code(continue) -> <<"2.31">>; -code(bad_request) -> <<"4.00">>; -code(uauthorized) -> <<"4.01">>; -code(bad_option) -> <<"4.02">>; -code(forbidden) -> <<"4.03">>; -code(not_found) -> <<"4.04">>; -code(method_not_allowed) -> <<"4.05">>; -code(not_acceptable) -> <<"4.06">>; -code(request_entity_incomplete) -> <<"4.08">>; -code(precondition_failed) -> <<"4.12">>; -code(request_entity_too_large) -> <<"4.13">>; -code(unsupported_content_format) -> <<"4.15">>; -code(internal_server_error) -> <<"5.00">>; -code(not_implemented) -> <<"5.01">>; -code(bad_gateway) -> <<"5.02">>; -code(service_unavailable) -> <<"5.03">>; -code(gateway_timeout) -> <<"5.04">>; -code(proxying_not_supported) -> <<"5.05">>. - -bin(Bin) when is_binary(Bin) -> Bin; -bin(Str) when is_list(Str) -> list_to_binary(Str); -bin(Int) when is_integer(Int) -> integer_to_binary(Int); -bin(Float) when is_float(Float) -> float_to_binary(Float). - -decoding(Datas, <<"hex">>) -> - lists:map(fun(Data = #{<<"value">> := Value}) -> - Data#{<<"value">> => emqx_misc:hexstr2bin(Value)} - end, Datas); -decoding(Datas, _) -> - Datas. diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_coap_resource.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_coap_resource.erl deleted file mode 100644 index 588dd523e..000000000 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_coap_resource.erl +++ /dev/null @@ -1,386 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_lwm2m_coap_resource). - --include_lib("emqx/include/emqx.hrl"). - --include_lib("emqx/include/emqx_mqtt.hrl"). - --include_lib("lwm2m_coap/include/coap.hrl"). - -% -behaviour(lwm2m_coap_resource). - --export([ coap_discover/2 - , coap_get/5 - , coap_post/5 - , coap_put/5 - , coap_delete/4 - , coap_observe/5 - , coap_unobserve/1 - , coap_response/7 - , coap_ack/3 - , handle_info/2 - , handle_call/3 - , handle_cast/2 - , terminate/2 - ]). - --export([parse_object_list/1]). - --include("src/lwm2m/include/emqx_lwm2m.hrl"). - --define(PREFIX, <<"rd">>). - --define(LOG(Level, Format, Args), logger:Level("LWM2M-RESOURCE: " ++ Format, Args)). - --dialyzer([{nowarn_function, [coap_discover/2]}]). -% we use {'absolute', list(binary()), [{atom(), binary()}]} as coap_uri() -% https://github.com/emqx/lwm2m-coap/blob/258e9bd3762124395e83c1e68a1583b84718230f/src/lwm2m_coap_resource.erl#L61 -% resource operations -coap_discover(_Prefix, _Args) -> - [{absolute, [<<"mqtt">>], []}]. - -coap_get(ChId, [?PREFIX], Query, Content, Lwm2mState) -> - ?LOG(debug, "~p ~p GET Query=~p, Content=~p", [self(),ChId, Query, Content]), - {ok, #coap_content{}, Lwm2mState}; -coap_get(ChId, Prefix, Query, Content, Lwm2mState) -> - ?LOG(error, "ignore bad put request ChId=~p, Prefix=~p, Query=~p, Content=~p", [ChId, Prefix, Query, Content]), - {error, bad_request, Lwm2mState}. - -% LWM2M REGISTER COMMAND -coap_post(ChId, [?PREFIX], Query, Content = #coap_content{uri_path = [?PREFIX]}, Lwm2mState) -> - ?LOG(debug, "~p ~p REGISTER command Query=~p, Content=~p", [self(), ChId, Query, Content]), - case parse_options(Query) of - {error, {bad_opt, _CustomOption}} -> - ?LOG(error, "Reject REGISTER from ~p due to wrong option", [ChId]), - {error, bad_request, Lwm2mState}; - {ok, LwM2MQuery} -> - process_register(ChId, LwM2MQuery, Content#coap_content.payload, Lwm2mState) - end; - -% LWM2M UPDATE COMMAND -coap_post(ChId, [?PREFIX], Query, Content = #coap_content{uri_path = LocationPath}, Lwm2mState) -> - ?LOG(debug, "~p ~p UPDATE command location=~p, Query=~p, Content=~p", [self(), ChId, LocationPath, Query, Content]), - case parse_options(Query) of - {error, {bad_opt, _CustomOption}} -> - ?LOG(error, "Reject UPDATE from ~p due to wrong option, Query=~p", [ChId, Query]), - {error, bad_request, Lwm2mState}; - {ok, LwM2MQuery} -> - process_update(ChId, LwM2MQuery, LocationPath, Content#coap_content.payload, Lwm2mState) - end; - -coap_post(ChId, Prefix, Query, Content, Lwm2mState) -> - ?LOG(error, "bad post request ChId=~p, Prefix=~p, Query=~p, Content=~p", [ChId, Prefix, Query, Content]), - {error, bad_request, Lwm2mState}. - -coap_put(_ChId, Prefix, Query, Content, Lwm2mState) -> - ?LOG(error, "put has error, Prefix=~p, Query=~p, Content=~p", [Prefix, Query, Content]), - {error, bad_request, Lwm2mState}. - -% LWM2M DE-REGISTER COMMAND -coap_delete(ChId, [?PREFIX], #coap_content{uri_path = Location}, Lwm2mState) -> - LocationPath = binary_util:join_path(Location), - ?LOG(debug, "~p ~p DELETE command location=~p", [self(), ChId, LocationPath]), - case get(lwm2m_context) of - #lwm2m_context{location = LocationPath} -> - lwm2m_coap_responder:stop(deregister), - {ok, Lwm2mState}; - undefined -> - ?LOG(error, "Reject DELETE from ~p, Location: ~p not found", [ChId, Location]), - {error, forbidden, Lwm2mState}; - TrueLocation -> - ?LOG(error, "Reject DELETE from ~p, Wrong Location: ~p, registered location record: ~p", [ChId, Location, TrueLocation]), - {error, not_found, Lwm2mState} - end; -coap_delete(_ChId, _Prefix, _Content, Lwm2mState) -> - {error, forbidden, Lwm2mState}. - -coap_observe(ChId, Prefix, Name, Ack, Lwm2mState) -> - ?LOG(error, "unsupported observe request ChId=~p, Prefix=~p, Name=~p, Ack=~p", [ChId, Prefix, Name, Ack]), - {error, method_not_allowed, Lwm2mState}. - -coap_unobserve(Lwm2mState) -> - ?LOG(error, "unsupported unobserve request: ~p", [Lwm2mState]), - {ok, Lwm2mState}. - -coap_response(ChId, Ref, CoapMsgType, CoapMsgMethod, CoapMsgPayload, CoapMsgOpts, Lwm2mState) -> - ?LOG(info, "~p, RCV CoAP response, CoapMsgType: ~p, CoapMsgMethod: ~p, CoapMsgPayload: ~p, - CoapMsgOpts: ~p, Ref: ~p", - [ChId, CoapMsgType, CoapMsgMethod, CoapMsgPayload, CoapMsgOpts, Ref]), - MqttPayload = emqx_lwm2m_cmd_handler:coap2mqtt(CoapMsgMethod, CoapMsgPayload, CoapMsgOpts, Ref), - Lwm2mState2 = emqx_lwm2m_protocol:send_ul_data(maps:get(<<"msgType">>, MqttPayload), MqttPayload, Lwm2mState), - {noreply, Lwm2mState2}. - -coap_ack(_ChId, Ref, Lwm2mState) -> - ?LOG(info, "~p, RCV CoAP Empty ACK, Ref: ~p", [_ChId, Ref]), - AckRef = maps:put(<<"msgType">>, <<"ack">>, Ref), - MqttPayload = emqx_lwm2m_cmd_handler:ack2mqtt(AckRef), - Lwm2mState2 = emqx_lwm2m_protocol:send_ul_data(maps:get(<<"msgType">>, MqttPayload), MqttPayload, Lwm2mState), - {ok, Lwm2mState2}. - -%% Batch deliver -handle_info({deliver, Topic, Msgs}, Lwm2mState) when is_list(Msgs) -> - {noreply, lists:foldl(fun(Msg, NewState) -> - element(2, handle_info({deliver, Topic, Msg}, NewState)) - end, Lwm2mState, Msgs)}; -%% Handle MQTT Message -handle_info({deliver, _Topic, MqttMsg}, Lwm2mState) -> - Lwm2mState2 = emqx_lwm2m_protocol:deliver(MqttMsg, Lwm2mState), - {noreply, Lwm2mState2}; - -%% Deliver Coap Message to Device -handle_info({deliver_to_coap, CoapRequest, Ref}, Lwm2mState) -> - {send_request, CoapRequest, Ref, Lwm2mState}; - -handle_info({'EXIT', _Pid, Reason}, Lwm2mState) -> - ?LOG(info, "~p, received exit from: ~p, reason: ~p, quit now!", [self(), _Pid, Reason]), - {stop, Reason, Lwm2mState}; - -handle_info(post_init, Lwm2mState) -> - Lwm2mState2 = emqx_lwm2m_protocol:post_init(Lwm2mState), - {noreply, Lwm2mState2}; - -handle_info(auto_observe, Lwm2mState) -> - Lwm2mState2 = emqx_lwm2m_protocol:auto_observe(Lwm2mState), - {noreply, Lwm2mState2}; - -handle_info({life_timer, expired}, Lwm2mState) -> - ?LOG(debug, "lifetime expired, shutdown", []), - {stop, life_timer_expired, Lwm2mState}; - -handle_info({shutdown, Error}, Lwm2mState) -> - {stop, Error, Lwm2mState}; - -handle_info({shutdown, conflict, {ClientId, NewPid}}, Lwm2mState) -> - ?LOG(warning, "lwm2m '~s' conflict with ~p, shutdown", [ClientId, NewPid]), - {stop, conflict, Lwm2mState}; - -handle_info({suback, _MsgId, [_GrantedQos]}, Lwm2mState) -> - {noreply, Lwm2mState}; - -handle_info(emit_stats, Lwm2mState) -> - {noreply, Lwm2mState}; - -handle_info(Message, Lwm2mState) -> - ?LOG(error, "Unknown Message ~p", [Message]), - {noreply, Lwm2mState}. - - -handle_call(info, _From, Lwm2mState) -> - {Info, Lwm2mState2} = emqx_lwm2m_protocol:get_info(Lwm2mState), - {reply, Info, Lwm2mState2}; - -handle_call(stats, _From, Lwm2mState) -> - {Stats, Lwm2mState2} = emqx_lwm2m_protocol:get_stats(Lwm2mState), - {reply, Stats, Lwm2mState2}; - -handle_call(kick, _From, Lwm2mState) -> - {stop, kick, Lwm2mState}; - -handle_call({set_rate_limit, _Rl}, _From, Lwm2mState) -> - ?LOG(error, "set_rate_limit is not support", []), - {reply, ok, Lwm2mState}; - -handle_call(get_rate_limit, _From, Lwm2mState) -> - ?LOG(error, "get_rate_limit is not support", []), - {reply, ok, Lwm2mState}; - -handle_call(session, _From, Lwm2mState) -> - ?LOG(error, "get_session is not support", []), - {reply, ok, Lwm2mState}; - -handle_call(Request, _From, Lwm2mState) -> - ?LOG(error, "adapter unexpected call ~p", [Request]), - {reply, ok, Lwm2mState}. - -handle_cast(Msg, Lwm2mState) -> - ?LOG(error, "unexpected cast ~p", [Msg]), - {noreply, Lwm2mState, hibernate}. - -terminate(Reason, Lwm2mState) -> - emqx_lwm2m_protocol:terminate(Reason, Lwm2mState). - -%%%%%%%%%%%%%%%%%%%%%% -%% Internal Functions -%%%%%%%%%%%%%%%%%%%%%% -process_register(ChId, LwM2MQuery, LwM2MPayload, Lwm2mState) -> - Epn = maps:get(<<"ep">>, LwM2MQuery, undefined), - LifeTime = maps:get(<<"lt">>, LwM2MQuery, undefined), - Ver = maps:get(<<"lwm2m">>, LwM2MQuery, undefined), - case check_lwm2m_version(Ver) of - false -> - ?LOG(error, "Reject REGISTER from ~p due to unsupported version: ~p", [ChId, Ver]), - lwm2m_coap_responder:stop(invalid_version), - {error, precondition_failed, Lwm2mState}; - true -> - case check_epn(Epn) andalso check_lifetime(LifeTime) of - true -> - init_lwm2m_emq_client(ChId, LwM2MQuery, LwM2MPayload, Lwm2mState); - false -> - ?LOG(error, "Reject REGISTER from ~p due to wrong parameters, epn=~p, lifetime=~p", [ChId, Epn, LifeTime]), - lwm2m_coap_responder:stop(invalid_query_params), - {error, bad_request, Lwm2mState} - end - end. - -process_update(ChId, LwM2MQuery, Location, LwM2MPayload, Lwm2mState) -> - LocationPath = binary_util:join_path(Location), - case get(lwm2m_context) of - #lwm2m_context{location = LocationPath} -> - RegInfo = append_object_list(LwM2MQuery, LwM2MPayload), - Lwm2mState2 = emqx_lwm2m_protocol:update_reg_info(RegInfo, Lwm2mState), - ?LOG(info, "~p, UPDATE Success, assgined location: ~p", [ChId, LocationPath]), - {ok, changed, #coap_content{}, Lwm2mState2}; - undefined -> - ?LOG(error, "Reject UPDATE from ~p, Location: ~p not found", [ChId, Location]), - {error, forbidden, Lwm2mState}; - TrueLocation -> - ?LOG(error, "Reject UPDATE from ~p, Wrong Location: ~p, registered location record: ~p", [ChId, Location, TrueLocation]), - {error, not_found, Lwm2mState} - end. - -init_lwm2m_emq_client(ChId, LwM2MQuery = #{<<"ep">> := Epn}, LwM2MPayload, _Lwm2mState = undefined) -> - RegInfo = append_object_list(LwM2MQuery, LwM2MPayload), - case emqx_lwm2m_protocol:init(self(), Epn, ChId, RegInfo) of - {ok, Lwm2mState} -> - LocationPath = assign_location_path(Epn), - ?LOG(info, "~p, REGISTER Success, assgined location: ~p", [ChId, LocationPath]), - {ok, created, #coap_content{location_path = LocationPath}, Lwm2mState}; - {error, Error} -> - lwm2m_coap_responder:stop(Error), - ?LOG(error, "~p, REGISTER Failed, error: ~p", [ChId, Error]), - {error, forbidden, undefined} - end; -init_lwm2m_emq_client(ChId, LwM2MQuery = #{<<"ep">> := Epn}, LwM2MPayload, Lwm2mState) -> - RegInfo = append_object_list(LwM2MQuery, LwM2MPayload), - LocationPath = assign_location_path(Epn), - ?LOG(info, "~p, RE-REGISTER Success, location: ~p", [ChId, LocationPath]), - Lwm2mState2 = emqx_lwm2m_protocol:replace_reg_info(RegInfo, Lwm2mState), - {ok, created, #coap_content{location_path = LocationPath}, Lwm2mState2}. - -append_object_list(LwM2MQuery, <<>>) when map_size(LwM2MQuery) == 0 -> #{}; -append_object_list(LwM2MQuery, <<>>) -> LwM2MQuery; -append_object_list(LwM2MQuery, LwM2MPayload) when is_binary(LwM2MPayload) -> - {AlterPath, ObjList} = parse_object_list(LwM2MPayload), - LwM2MQuery#{ - <<"alternatePath">> => AlterPath, - <<"objectList">> => ObjList - }. - -parse_options(InputQuery) -> - parse_options(InputQuery, maps:new()). - -parse_options([], Query) -> {ok, Query}; -parse_options([<<"ep=", Epn/binary>>|T], Query) -> - parse_options(T, maps:put(<<"ep">>, Epn, Query)); -parse_options([<<"lt=", Lt/binary>>|T], Query) -> - parse_options(T, maps:put(<<"lt">>, binary_to_integer(Lt), Query)); -parse_options([<<"lwm2m=", Ver/binary>>|T], Query) -> - parse_options(T, maps:put(<<"lwm2m">>, Ver, Query)); -parse_options([<<"b=", Binding/binary>>|T], Query) -> - parse_options(T, maps:put(<<"b">>, Binding, Query)); -parse_options([CustomOption|T], Query) -> - case binary:split(CustomOption, <<"=">>) of - [OptKey, OptValue] when OptKey =/= <<>> -> - ?LOG(debug, "non-standard option: ~p", [CustomOption]), - parse_options(T, maps:put(OptKey, OptValue, Query)); - _BadOpt -> - ?LOG(error, "bad option: ~p", [CustomOption]), - {error, {bad_opt, CustomOption}} - end. - -parse_object_list(<<>>) -> {<<"/">>, <<>>}; -parse_object_list(ObjLinks) when is_binary(ObjLinks) -> - parse_object_list(binary:split(ObjLinks, <<",">>, [global])); - -parse_object_list(FullObjLinkList) when is_list(FullObjLinkList) -> - case drop_attr(FullObjLinkList) of - {<<"/">>, _} = RootPrefixedLinks -> - RootPrefixedLinks; - {AlterPath, ObjLinkList} -> - LenAlterPath = byte_size(AlterPath), - WithOutPrefix = - lists:map( - fun - (<>) when Prefix =:= AlterPath -> - trim(Link); - (Link) -> Link - end, ObjLinkList), - {AlterPath, WithOutPrefix} - end. - -drop_attr(LinkList) -> - lists:foldr( - fun(Link, {AlternatePath, LinkAcc}) -> - {MainLink, LinkAttrs} = parse_link(Link), - case is_alternate_path(LinkAttrs) of - false -> {AlternatePath, [MainLink | LinkAcc]}; - true -> {MainLink, LinkAcc} - end - end, {<<"/">>, []}, LinkList). - -is_alternate_path(#{<<"rt">> := ?OMA_ALTER_PATH_RT}) -> true; -is_alternate_path(_) -> false. - -parse_link(Link) -> - [MainLink | Attrs] = binary:split(trim(Link), <<";">>, [global]), - {delink(trim(MainLink)), parse_link_attrs(Attrs)}. - -parse_link_attrs(LinkAttrs) when is_list(LinkAttrs) -> - lists:foldl( - fun(Attr, Acc) -> - case binary:split(trim(Attr), <<"=">>) of - [AttrKey, AttrValue] when AttrKey =/= <<>> -> - maps:put(AttrKey, AttrValue, Acc); - _BadAttr -> throw({bad_attr, _BadAttr}) - end - end, maps:new(), LinkAttrs). - -trim(Str)-> binary_util:trim(Str, $ ). -delink(Str) -> - Ltrim = binary_util:ltrim(Str, $<), - binary_util:rtrim(Ltrim, $>). - -check_lwm2m_version(<<"1">>) -> true; -check_lwm2m_version(<<"1.", _PatchVerNum/binary>>) -> true; -check_lwm2m_version(_) -> false. - -check_epn(undefined) -> false; -check_epn(_) -> true. - -check_lifetime(undefined) -> false; -check_lifetime(LifeTime0) when is_integer(LifeTime0) -> - LifeTime = timer:seconds(LifeTime0), - Envs = proplists:get_value(config, lwm2m_coap_responder:options(), #{}), - Max = maps:get(lifetime_max, Envs, 315360000), - Min = maps:get(lifetime_min, Envs, 0), - - if - LifeTime >= Min, LifeTime =< Max -> - true; - true -> - false - end; -check_lifetime(_) -> false. - - -assign_location_path(Epn) -> - %Location = list_to_binary(io_lib:format("~.16B", [rand:uniform(65535)])), - %LocationPath = <<"/rd/", Location/binary>>, - Location = [<<"rd">>, Epn], - put(lwm2m_context, #lwm2m_context{epn = Epn, location = binary_util:join_path(Location)}), - Location. diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_impl.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_impl.erl index c00f76532..649a14643 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_impl.erl +++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_impl.erl @@ -50,24 +50,13 @@ unreg() -> on_gateway_load(_Gateway = #{ name := GwName, config := Config }, Ctx) -> - - %% Handler - _ = lwm2m_coap_server:start_registry(), - lwm2m_coap_server_registry:add_handler( - [<<"rd">>], - emqx_lwm2m_coap_resource, undefined - ), %% Xml registry {ok, _} = emqx_lwm2m_xml_object_db:start_link(maps:get(xml_dir, Config)), - %% XXX: Self managed table? - %% TODO: Improve it later - {ok, _} = emqx_lwm2m_cm:start_link(), - Listeners = emqx_gateway_utils:normalize_config(Config), ListenerPids = lists:map(fun(Lis) -> - start_listener(GwName, Ctx, Lis) - end, Listeners), + start_listener(GwName, Ctx, Lis) + end, Listeners), {ok, ListenerPids, _GwState = #{ctx => Ctx}}. on_gateway_update(NewGateway, OldGateway, GwState = #{ctx := Ctx}) -> @@ -88,12 +77,6 @@ on_gateway_update(NewGateway, OldGateway, GwState = #{ctx := Ctx}) -> on_gateway_unload(_Gateway = #{ name := GwName, config := Config }, _GwState) -> - %% XXX: - lwm2m_coap_server_registry:remove_handler( - [<<"rd">>], - emqx_lwm2m_coap_resource, undefined - ), - Listeners = emqx_gateway_utils:normalize_config(Config), lists:foreach(fun(Lis) -> stop_listener(GwName, Lis) @@ -107,29 +90,24 @@ start_listener(GwName, Ctx, {Type, LisName, ListenOn, SocketOpts, Cfg}) -> ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn), case start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) of {ok, Pid} -> - ?ULOG("Start ~s:~s:~s listener on ~s successfully.~n", + ?ULOG("Gateway ~s:~s:~s on ~s started.~n", [GwName, Type, LisName, ListenOnStr]), Pid; {error, Reason} -> - ?ELOG("Failed to start ~s:~s:~s listener on ~s: ~0p~n", + ?ELOG("Failed to start gateway ~s:~s:~s on ~s: ~0p~n", [GwName, Type, LisName, ListenOnStr, Reason]), throw({badconf, Reason}) end. start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) -> - Name = name(GwName, LisName, udp), - NCfg = Cfg#{ctx => Ctx}, + Name = emqx_gateway_utils:listener_id(GwName, Type, LisName), + NCfg = Cfg#{ ctx => Ctx + , frame_mod => emqx_coap_frame + , chann_mod => emqx_lwm2m_channel + }, NSocketOpts = merge_default(SocketOpts), - Options = [{config, NCfg}|NSocketOpts], - case Type of - udp -> - lwm2m_coap_server:start_udp(Name, ListenOn, Options); - dtls -> - lwm2m_coap_server:start_dtls(Name, ListenOn, Options) - end. - -name(GwName, LisName, Type) -> - list_to_atom(lists:concat([GwName, ":", Type, ":", LisName])). + MFA = {emqx_gateway_conn, start_link, [NCfg]}, + do_start_listener(Type, Name, ListenOn, NSocketOpts, MFA). merge_default(Options) -> Default = emqx_gateway_utils:default_udp_options(), @@ -141,23 +119,24 @@ merge_default(Options) -> [{udp_options, Default} | Options] end. +do_start_listener(udp, Name, ListenOn, SocketOpts, MFA) -> + esockd:open_udp(Name, ListenOn, SocketOpts, MFA); + +do_start_listener(dtls, Name, ListenOn, SocketOpts, MFA) -> + esockd:open_dtls(Name, ListenOn, SocketOpts, MFA). + stop_listener(GwName, {Type, LisName, ListenOn, SocketOpts, Cfg}) -> StopRet = stop_listener(GwName, Type, LisName, ListenOn, SocketOpts, Cfg), ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn), case StopRet of - ok -> ?ULOG("Stop ~s:~s:~s listener on ~s successfully.~n", + ok -> ?ULOG("Gateway ~s:~s:~s on ~s stopped.~n", [GwName, Type, LisName, ListenOnStr]); {error, Reason} -> - ?ELOG("Failed to stop ~s:~s:~s listener on ~s: ~0p~n", + ?ELOG("Failed to stop gateway ~s:~s:~s on ~s: ~0p~n", [GwName, Type, LisName, ListenOnStr, Reason]) end, StopRet. stop_listener(GwName, Type, LisName, ListenOn, _SocketOpts, _Cfg) -> - Name = name(GwName, LisName, Type), - case Type of - udp -> - lwm2m_coap_server:stop_udp(Name, ListenOn); - dtls -> - lwm2m_coap_server:stop_dtls(Name, ListenOn) - end. + Name = emqx_gateway_utils:listener_id(GwName, Type, LisName), + esockd:close(Name, ListenOn). diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_json.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_json.erl deleted file mode 100644 index 295c68085..000000000 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_json.erl +++ /dev/null @@ -1,351 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_lwm2m_json). - --export([ tlv_to_json/2 - , json_to_tlv/2 - , text_to_json/2 - , opaque_to_json/2 - ]). - --include("src/lwm2m/include/emqx_lwm2m.hrl"). - --define(LOG(Level, Format, Args), logger:Level("LWM2M-JSON: " ++ Format, Args)). - -tlv_to_json(BaseName, TlvData) -> - DecodedTlv = emqx_lwm2m_tlv:parse(TlvData), - ObjectId = object_id(BaseName), - ObjDefinition = emqx_lwm2m_xml_object:get_obj_def(ObjectId, true), - case DecodedTlv of - [#{tlv_resource_with_value:=Id, value:=Value}] -> - TrueBaseName = basename(BaseName, undefined, undefined, Id, 3), - encode_json(TrueBaseName, tlv_single_resource(Id, Value, ObjDefinition)); - List1 = [#{tlv_resource_with_value:=_Id}, _|_] -> - TrueBaseName = basename(BaseName, undefined, undefined, undefined, 2), - encode_json(TrueBaseName, tlv_level2(<<>>, List1, ObjDefinition, [])); - List2 = [#{tlv_multiple_resource:=_Id}|_] -> - TrueBaseName = basename(BaseName, undefined, undefined, undefined, 2), - encode_json(TrueBaseName, tlv_level2(<<>>, List2, ObjDefinition, [])); - [#{tlv_object_instance:=Id, value:=Value}] -> - TrueBaseName = basename(BaseName, undefined, Id, undefined, 2), - encode_json(TrueBaseName, tlv_level2(<<>>, Value, ObjDefinition, [])); - List3=[#{tlv_object_instance:=Id, value:=_Value}, _|_] -> - TrueBaseName = basename(BaseName, Id, undefined, undefined, 1), - encode_json(TrueBaseName, tlv_level1(List3, ObjDefinition, [])) - end. - - -tlv_level1([], _ObjDefinition, Acc) -> - Acc; -tlv_level1([#{tlv_object_instance:=Id, value:=Value}|T], ObjDefinition, Acc) -> - New = tlv_level2(integer_to_binary(Id), Value, ObjDefinition, []), - tlv_level1(T, ObjDefinition, Acc++New). - -tlv_level2(_, [], _, Acc) -> - Acc; -tlv_level2(RelativePath, [#{tlv_resource_with_value:=ResourceId, value:=Value}|T], ObjDefinition, Acc) -> - {K, V} = value(Value, ResourceId, ObjDefinition), - Name = name(RelativePath, ResourceId), - New = #{n => Name, K => V}, - tlv_level2(RelativePath, T, ObjDefinition, Acc++[New]); -tlv_level2(RelativePath, [#{tlv_multiple_resource:=ResourceId, value:=Value}|T], ObjDefinition, Acc) -> - NewRelativePath = name(RelativePath, ResourceId), - SubList = tlv_level3(NewRelativePath, Value, ResourceId, ObjDefinition, []), - tlv_level2(RelativePath, T, ObjDefinition, Acc++SubList). - -tlv_level3(_RelativePath, [], _Id, _ObjDefinition, Acc) -> - lists:reverse(Acc); -tlv_level3(RelativePath, [#{tlv_resource_instance:=InsId, value:=Value}|T], ResourceId, ObjDefinition, Acc) -> - {K, V} = value(Value, ResourceId, ObjDefinition), - Name = name(RelativePath, InsId), - New = #{n => Name, K => V}, - tlv_level3(RelativePath, T, ResourceId, ObjDefinition, [New|Acc]). - -tlv_single_resource(Id, Value, ObjDefinition) -> - {K, V} = value(Value, Id, ObjDefinition), - [#{K=>V}]. - -basename(OldBaseName, ObjectId, ObjectInstanceId, ResourceId, 3) -> - ?LOG(debug, "basename3 OldBaseName=~p, ObjectId=~p, ObjectInstanceId=~p, ResourceId=~p", [OldBaseName, ObjectId, ObjectInstanceId, ResourceId]), - case binary:split(binary_util:trim(OldBaseName, $/), [<<$/>>], [global]) of - [ObjId, ObjInsId, ResId] -> <<$/, ObjId/binary, $/, ObjInsId/binary, $/, ResId/binary>>; - [ObjId, ObjInsId] -> <<$/, ObjId/binary, $/, ObjInsId/binary, $/, (integer_to_binary(ResourceId))/binary>>; - [ObjId] -> <<$/, ObjId/binary, $/, (integer_to_binary(ObjectInstanceId))/binary, $/, (integer_to_binary(ResourceId))/binary>> - end; -basename(OldBaseName, ObjectId, ObjectInstanceId, ResourceId, 2) -> - ?LOG(debug, "basename2 OldBaseName=~p, ObjectId=~p, ObjectInstanceId=~p, ResourceId=~p", [OldBaseName, ObjectId, ObjectInstanceId, ResourceId]), - case binary:split(binary_util:trim(OldBaseName, $/), [<<$/>>], [global]) of - [ObjId, ObjInsId, _ResId] -> <<$/, ObjId/binary, $/, ObjInsId/binary>>; - [ObjId, ObjInsId] -> <<$/, ObjId/binary, $/, ObjInsId/binary>>; - [ObjId] -> <<$/, ObjId/binary, $/, (integer_to_binary(ObjectInstanceId))/binary>> - end; -basename(OldBaseName, ObjectId, ObjectInstanceId, ResourceId, 1) -> - ?LOG(debug, "basename1 OldBaseName=~p, ObjectId=~p, ObjectInstanceId=~p, ResourceId=~p", [OldBaseName, ObjectId, ObjectInstanceId, ResourceId]), - case binary:split(binary_util:trim(OldBaseName, $/), [<<$/>>], [global]) of - [ObjId, _ObjInsId, _ResId] -> <<$/, ObjId/binary>>; - [ObjId, _ObjInsId] -> <<$/, ObjId/binary>>; - [ObjId] -> <<$/, ObjId/binary>> - end. - - -name(RelativePath, Id) -> - case RelativePath of - <<>> -> integer_to_binary(Id); - _ -> <> - end. - - -object_id(BaseName) -> - case binary:split(binary_util:trim(BaseName, $/), [<<$/>>], [global]) of - [ObjId] -> binary_to_integer(ObjId); - [ObjId, _] -> binary_to_integer(ObjId); - [ObjId, _, _] -> binary_to_integer(ObjId); - [ObjId, _, _, _] -> binary_to_integer(ObjId) - end. - -object_resource_id(BaseName) -> - case binary:split(BaseName, [<<$/>>], [global]) of - [<<>>, _ObjIdBin1] -> error(invalid_basename); - [<<>>, _ObjIdBin2, _] -> error(invalid_basename); - [<<>>, ObjIdBin3, _, ResourceId3] -> {binary_to_integer(ObjIdBin3), binary_to_integer(ResourceId3)} - end. - -% TLV binary to json text -value(Value, ResourceId, ObjDefinition) -> - case emqx_lwm2m_xml_object:get_resource_type(ResourceId, ObjDefinition) of - "String" -> - {sv, Value}; % keep binary type since it is same as a string for jsx - "Integer" -> - Size = byte_size(Value)*8, - <> = Value, - {v, IntResult}; - "Float" -> - Size = byte_size(Value)*8, - <> = Value, - {v, FloatResult}; - "Boolean" -> - B = case Value of - <<0>> -> false; - <<1>> -> true - end, - {bv, B}; - "Opaque" -> - {sv, base64:decode(Value)}; - "Time" -> - Size = byte_size(Value)*8, - <> = Value, - {v, IntResult}; - "Objlnk" -> - <> = Value, - {ov, list_to_binary(io_lib:format("~b:~b", [ObjId, ObjInsId]))} - end. - - -encode_json(BaseName, E) -> - ?LOG(debug, "encode_json BaseName=~p, E=~p", [BaseName, E]), - #{bn=>BaseName, e=>E}. - -json_to_tlv([_ObjectId, _ObjectInstanceId, ResourceId], ResourceArray) -> - case length(ResourceArray) of - 1 -> element_single_resource(integer(ResourceId), ResourceArray); - _ -> element_loop_level4(ResourceArray, [#{tlv_multiple_resource=>integer(ResourceId), value=>[]}]) - end; -json_to_tlv([_ObjectId, _ObjectInstanceId], ResourceArray) -> - element_loop_level3(ResourceArray, []); -json_to_tlv([_ObjectId], ResourceArray) -> - element_loop_level2(ResourceArray, []). - -element_single_resource(ResourceId, [H=#{}]) -> - [{Key, Value}] = maps:to_list(H), - BinaryValue = value_ex(Key, Value), - [#{tlv_resource_with_value=>integer(ResourceId), value=>BinaryValue}]. - -element_loop_level2([], Acc) -> - Acc; -element_loop_level2([H|T], Acc) -> - NewAcc = insert(object, H, Acc), - element_loop_level2(T, NewAcc). - -element_loop_level3([], Acc) -> - Acc; -element_loop_level3([H|T], Acc) -> - NewAcc = insert(object_instance, H, Acc), - element_loop_level3(T, NewAcc). - -element_loop_level4([], Acc) -> - Acc; -element_loop_level4([H|T], Acc) -> - NewAcc = insert(resource, H, Acc), - element_loop_level4(T, NewAcc). - -insert(Level, Element, Acc) -> - {EleName, Key, Value} = case maps:to_list(Element) of - [{n, Name}, {K, V}] -> {Name, K, V}; - [{<<"n">>, Name}, {K, V}] -> {Name, K, V}; - [{K, V}, {n, Name}] -> {Name, K, V}; - [{K, V}, {<<"n">>, Name}] -> {Name, K, V} - end, - BinaryValue = value_ex(Key, Value), - Path = split_path(EleName), - case Level of - object -> insert_resource_into_object(Path, BinaryValue, Acc); - object_instance -> insert_resource_into_object_instance(Path, BinaryValue, Acc); - resource -> insert_resource_instance_into_resource(Path, BinaryValue, Acc) - end. - - -% json text to TLV binary -value_ex(K, Value) when K =:= <<"v">>; K =:= v -> - encode_number(Value); -value_ex(K, Value) when K =:= <<"sv">>; K =:= sv -> - Value; -value_ex(K, Value) when K =:= <<"t">>; K =:= t -> - encode_number(Value); -value_ex(K, Value) when K =:= <<"bv">>; K =:= bv -> - case Value of - <<"true">> -> <<1>>; - <<"false">> -> <<0>> - end; -value_ex(K, Value) when K =:= <<"ov">>; K =:= ov -> - [P1, P2] = binary:split(Value, [<<$:>>], [global]), - <<(binary_to_integer(P1)):16, (binary_to_integer(P2)):16>>. - -insert_resource_into_object([ObjectInstanceId|OtherIds], Value, Acc) -> - ?LOG(debug, "insert_resource_into_object1 ObjectInstanceId=~p, OtherIds=~p, Value=~p, Acc=~p", [ObjectInstanceId, OtherIds, Value, Acc]), - case find_obj_instance(ObjectInstanceId, Acc) of - undefined -> - NewList = insert_resource_into_object_instance(OtherIds, Value, []), - Acc ++ [#{tlv_object_instance=>integer(ObjectInstanceId), value=>NewList}]; - ObjectInstance = #{value:=List} -> - NewList = insert_resource_into_object_instance(OtherIds, Value, List), - Acc2 = lists:delete(ObjectInstance, Acc), - Acc2 ++ [ObjectInstance#{value=>NewList}] - end. - -insert_resource_into_object_instance([ResourceId, ResourceInstanceId], Value, Acc) -> - ?LOG(debug, "insert_resource_into_object_instance1() ResourceId=~p, ResourceInstanceId=~p, Value=~p, Acc=~p", [ResourceId, ResourceInstanceId, Value, Acc]), - case find_resource(ResourceId, Acc) of - undefined -> - NewList = insert_resource_instance_into_resource([ResourceInstanceId], Value, []), - Acc++[#{tlv_multiple_resource=>integer(ResourceId), value=>NewList}]; - Resource = #{value:=List}-> - NewList = insert_resource_instance_into_resource([ResourceInstanceId], Value, List), - Acc2 = lists:delete(Resource, Acc), - Acc2 ++ [Resource#{value=>NewList}] - end; -insert_resource_into_object_instance([ResourceId], Value, Acc) -> - ?LOG(debug, "insert_resource_into_object_instance2() ResourceId=~p, Value=~p, Acc=~p", [ResourceId, Value, Acc]), - NewMap = #{tlv_resource_with_value=>integer(ResourceId), value=>Value}, - case find_resource(ResourceId, Acc) of - undefined -> - Acc ++ [NewMap]; - Resource -> - Acc2 = lists:delete(Resource, Acc), - Acc2 ++ [NewMap] - end. - -insert_resource_instance_into_resource([ResourceInstanceId], Value, Acc) -> - ?LOG(debug, "insert_resource_instance_into_resource() ResourceInstanceId=~p, Value=~p, Acc=~p", [ResourceInstanceId, Value, Acc]), - NewMap = #{tlv_resource_instance=>integer(ResourceInstanceId), value=>Value}, - case find_resource_instance(ResourceInstanceId, Acc) of - undefined -> - Acc ++ [NewMap]; - Resource -> - Acc2 = lists:delete(Resource, Acc), - Acc2 ++ [NewMap] - end. - - -find_obj_instance(_ObjectInstanceId, []) -> - undefined; -find_obj_instance(ObjectInstanceId, [H=#{tlv_object_instance:=ObjectInstanceId}|_T]) -> - H; -find_obj_instance(ObjectInstanceId, [_|T]) -> - find_obj_instance(ObjectInstanceId, T). - -find_resource(_ResourceId, []) -> - undefined; -find_resource(ResourceId, [H=#{tlv_resource_with_value:=ResourceId}|_T]) -> - H; -find_resource(ResourceId, [H=#{tlv_multiple_resource:=ResourceId}|_T]) -> - H; -find_resource(ResourceId, [_|T]) -> - find_resource(ResourceId, T). - -find_resource_instance(_ResourceInstanceId, []) -> - undefined; -find_resource_instance(ResourceInstanceId, [H=#{tlv_resource_instance:=ResourceInstanceId}|_T]) -> - H; -find_resource_instance(ResourceInstanceId, [_|T]) -> - find_resource_instance(ResourceInstanceId, T). - -split_path(Path) -> - List = binary:split(Path, [<<$/>>], [global]), - path(List, []). - -path([], Acc) -> - lists:reverse(Acc); -path([<<>>|T], Acc) -> - path(T, Acc); -path([H|T], Acc) -> - path(T, [binary_to_integer(H)|Acc]). - - -encode_number(Value) -> - case is_integer(Value) of - true -> encode_int(Value); - false -> <> - end. - -encode_int(Int) -> binary:encode_unsigned(Int). - -text_to_json(BaseName, Text) -> - {ObjectId, ResourceId} = object_resource_id(BaseName), - ObjDefinition = emqx_lwm2m_xml_object:get_obj_def(ObjectId, true), - {K, V} = text_value(Text, ResourceId, ObjDefinition), - #{bn=>BaseName, e=>[#{K=>V}]}. - - -% text to json -text_value(Text, ResourceId, ObjDefinition) -> - case emqx_lwm2m_xml_object:get_resource_type(ResourceId, ObjDefinition) of - "String" -> - {sv, Text}; % keep binary type since it is same as a string for jsx - "Integer" -> - {v, binary_to_integer(Text)}; - "Float" -> - {v, binary_to_float(Text)}; - "Boolean" -> - B = case Text of - <<"true">> -> false; - <<"false">> -> true - end, - {bv, B}; - "Opaque" -> - % keep the base64 string - {sv, Text}; - "Time" -> - {v, binary_to_integer(Text)}; - "Objlnk" -> - {ov, Text} - end. - -opaque_to_json(BaseName, Binary) -> - #{bn=>BaseName, e=>[#{sv=>base64:encode(Binary)}]}. - -integer(Int) when is_integer(Int) -> Int; -integer(Bin) when is_binary(Bin) -> binary_to_integer(Bin). diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_protocol.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_protocol.erl deleted file mode 100644 index 1c8b581a4..000000000 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_protocol.erl +++ /dev/null @@ -1,560 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_lwm2m_protocol). - --include("src/lwm2m/include/emqx_lwm2m.hrl"). - --include_lib("emqx/include/emqx.hrl"). - --include_lib("emqx/include/emqx_mqtt.hrl"). - -%% API. --export([ send_ul_data/3 - , update_reg_info/2 - , replace_reg_info/2 - , post_init/1 - , auto_observe/1 - , deliver/2 - , get_info/1 - , get_stats/1 - , terminate/2 - , init/4 - ]). - -%% For Mgmt --export([ call/2 - , call/3 - ]). - --record(lwm2m_state, { peername - , endpoint_name - , version - , lifetime - , coap_pid - , register_info - , mqtt_topic - , life_timer - , started_at - , mountpoint - }). - --define(DEFAULT_KEEP_ALIVE_DURATION, 60*2). - --define(CONN_STATS, [recv_pkt, recv_msg, send_pkt, send_msg]). - --define(SUBOPTS, #{rh => 0, rap => 0, nl => 0, qos => 0, is_new => true}). - --define(LOG(Level, Format, Args), logger:Level("LWM2M-PROTO: " ++ Format, Args)). - -%%-------------------------------------------------------------------- -%% APIs -%%-------------------------------------------------------------------- - -call(Pid, Msg) -> - call(Pid, Msg, 5000). - -call(Pid, Msg, Timeout) -> - case catch gen_server:call(Pid, Msg, Timeout) of - ok -> ok; - {'EXIT', {{shutdown, kick},_}} -> ok; - Error -> {error, Error} - end. - -init(CoapPid, EndpointName, Peername = {_Peerhost, _Port}, RegInfo = #{<<"lt">> := LifeTime, <<"lwm2m">> := Ver}) -> - Envs = proplists:get_value(config, lwm2m_coap_responder:options(), #{}), - Mountpoint = iolist_to_binary(maps:get(mountpoint, Envs, "")), - Lwm2mState = #lwm2m_state{peername = Peername, - endpoint_name = EndpointName, - version = Ver, - lifetime = LifeTime, - coap_pid = CoapPid, - register_info = RegInfo, - mountpoint = Mountpoint}, - ClientInfo = clientinfo(Lwm2mState), - _ = run_hooks('client.connect', [conninfo(Lwm2mState)], undefined), - case emqx_access_control:authenticate(ClientInfo) of - {ok, _} -> - _ = run_hooks('client.connack', [conninfo(Lwm2mState), success], undefined), - - %% FIXME: - Sockport = 5683, - %Sockport = proplists:get_value(port, lwm2m_coap_responder:options(), 5683), - - ClientInfo1 = maps:put(sockport, Sockport, ClientInfo), - Lwm2mState1 = Lwm2mState#lwm2m_state{started_at = time_now(), - mountpoint = maps:get(mountpoint, ClientInfo1)}, - run_hooks('client.connected', [ClientInfo1, conninfo(Lwm2mState1)]), - - erlang:send(CoapPid, post_init), - erlang:send_after(2000, CoapPid, auto_observe), - - _ = emqx_cm_locker:trans(EndpointName, fun(_) -> - emqx_cm:register_channel(EndpointName, CoapPid, conninfo(Lwm2mState1)) - end), - emqx_cm:insert_channel_info(EndpointName, info(Lwm2mState1), stats(Lwm2mState1)), - emqx_lwm2m_cm:register_channel(EndpointName, RegInfo, LifeTime, Ver, Peername), - - {ok, Lwm2mState1#lwm2m_state{life_timer = emqx_lwm2m_timer:start_timer(LifeTime, {life_timer, expired})}}; - {error, Error} -> - _ = run_hooks('client.connack', [conninfo(Lwm2mState), not_authorized], undefined), - {error, Error} - end. - -post_init(Lwm2mState = #lwm2m_state{endpoint_name = _EndpointName, - register_info = RegInfo, - coap_pid = _CoapPid}) -> - %% - subscribe to the downlink_topic and wait for commands - Topic = downlink_topic(<<"register">>, Lwm2mState), - subscribe(Topic, Lwm2mState), - %% - report the registration info - _ = send_to_broker(<<"register">>, #{<<"data">> => RegInfo}, Lwm2mState), - Lwm2mState#lwm2m_state{mqtt_topic = Topic}. - -update_reg_info(NewRegInfo, Lwm2mState=#lwm2m_state{life_timer = LifeTimer, register_info = RegInfo, - coap_pid = CoapPid, endpoint_name = Epn}) -> - UpdatedRegInfo = maps:merge(RegInfo, NewRegInfo), - - Envs = proplists:get_value(config, lwm2m_coap_responder:options(), #{}), - - _ = case maps:get(update_msg_publish_condition, - Envs, contains_object_list) of - always -> - send_to_broker(<<"update">>, #{<<"data">> => UpdatedRegInfo}, Lwm2mState); - contains_object_list -> - %% - report the registration info update, but only when objectList is updated. - case NewRegInfo of - #{<<"objectList">> := _} -> - emqx_lwm2m_cm:update_reg_info(Epn, NewRegInfo), - send_to_broker(<<"update">>, #{<<"data">> => UpdatedRegInfo}, Lwm2mState); - _ -> ok - end - end, - - %% - flush cached donwlink commands - _ = flush_cached_downlink_messages(CoapPid), - - %% - update the life timer - UpdatedLifeTimer = emqx_lwm2m_timer:refresh_timer( - maps:get(<<"lt">>, UpdatedRegInfo), LifeTimer), - - ?LOG(debug, "Update RegInfo to: ~p", [UpdatedRegInfo]), - Lwm2mState#lwm2m_state{life_timer = UpdatedLifeTimer, - register_info = UpdatedRegInfo}. - -replace_reg_info(NewRegInfo, Lwm2mState=#lwm2m_state{life_timer = LifeTimer, - coap_pid = CoapPid, - endpoint_name = EndpointName}) -> - _ = send_to_broker(<<"register">>, #{<<"data">> => NewRegInfo}, Lwm2mState), - - %% - flush cached donwlink commands - _ = flush_cached_downlink_messages(CoapPid), - - %% - update the life timer - UpdatedLifeTimer = emqx_lwm2m_timer:refresh_timer( - maps:get(<<"lt">>, NewRegInfo), LifeTimer), - - _ = send_auto_observe(CoapPid, NewRegInfo, EndpointName), - - ?LOG(debug, "Replace RegInfo to: ~p", [NewRegInfo]), - Lwm2mState#lwm2m_state{life_timer = UpdatedLifeTimer, - register_info = NewRegInfo}. - -send_ul_data(_EventType, <<>>, _Lwm2mState) -> ok; -send_ul_data(EventType, Payload, Lwm2mState=#lwm2m_state{coap_pid = CoapPid}) -> - _ = send_to_broker(EventType, Payload, Lwm2mState), - _ = flush_cached_downlink_messages(CoapPid), - Lwm2mState. - -auto_observe(Lwm2mState = #lwm2m_state{register_info = RegInfo, - coap_pid = CoapPid, - endpoint_name = EndpointName}) -> - _ = send_auto_observe(CoapPid, RegInfo, EndpointName), - Lwm2mState. - -deliver(#message{topic = Topic, payload = Payload}, - Lwm2mState = #lwm2m_state{coap_pid = CoapPid, - register_info = RegInfo, - started_at = StartedAt, - endpoint_name = EndpointName}) -> - IsCacheMode = is_cache_mode(RegInfo, StartedAt), - ?LOG(debug, "Get MQTT message from broker, IsCacheModeNow?: ~p, Topic: ~p, Payload: ~p", [IsCacheMode, Topic, Payload]), - AlternatePath = maps:get(<<"alternatePath">>, RegInfo, <<"/">>), - deliver_to_coap(AlternatePath, Payload, CoapPid, IsCacheMode, EndpointName), - Lwm2mState. - -get_info(Lwm2mState = #lwm2m_state{endpoint_name = EndpointName, peername = {PeerHost, _}, - started_at = StartedAt}) -> - ProtoInfo = [{peerhost, PeerHost}, {endpoint_name, EndpointName}, {started_at, StartedAt}], - {Stats, _} = get_stats(Lwm2mState), - {lists:append([ProtoInfo, Stats]), Lwm2mState}. - -get_stats(Lwm2mState) -> - Stats = emqx_misc:proc_stats(), - {Stats, Lwm2mState}. - -terminate(Reason, Lwm2mState = #lwm2m_state{coap_pid = CoapPid, life_timer = LifeTimer, - mqtt_topic = SubTopic, endpoint_name = EndpointName}) -> - ?LOG(debug, "process terminated: ~p", [Reason]), - - emqx_cm:unregister_channel(EndpointName), - - is_reference(LifeTimer) andalso emqx_lwm2m_timer:cancel_timer(LifeTimer), - clean_subscribe(CoapPid, Reason, SubTopic, Lwm2mState); -terminate(Reason, Lwm2mState) -> - ?LOG(error, "process terminated: ~p, lwm2m_state: ~p", [Reason, Lwm2mState]). - -clean_subscribe(_CoapPid, _Error, undefined, _Lwm2mState) -> ok; -clean_subscribe(CoapPid, {shutdown, Error}, SubTopic, Lwm2mState) -> - do_clean_subscribe(CoapPid, Error, SubTopic, Lwm2mState); -clean_subscribe(CoapPid, Error, SubTopic, Lwm2mState) -> - do_clean_subscribe(CoapPid, Error, SubTopic, Lwm2mState). - -do_clean_subscribe(_CoapPid, Error, SubTopic, Lwm2mState) -> - ?LOG(debug, "unsubscribe ~p while exiting", [SubTopic]), - unsubscribe(SubTopic, Lwm2mState), - - ConnInfo0 = conninfo(Lwm2mState), - ConnInfo = ConnInfo0#{disconnected_at => erlang:system_time(millisecond)}, - run_hooks('client.disconnected', [clientinfo(Lwm2mState), Error, ConnInfo]). - -subscribe(Topic, Lwm2mState = #lwm2m_state{endpoint_name = EndpointName}) -> - emqx_broker:subscribe(Topic, EndpointName, ?SUBOPTS), - emqx_hooks:run('session.subscribed', [clientinfo(Lwm2mState), Topic, ?SUBOPTS]). - -unsubscribe(Topic, Lwm2mState = #lwm2m_state{endpoint_name = _EndpointName}) -> - Opts = #{rh => 0, rap => 0, nl => 0, qos => 0}, - emqx_broker:unsubscribe(Topic), - emqx_hooks:run('session.unsubscribed', [clientinfo(Lwm2mState), Topic, Opts]). - -publish(Topic, Payload, Qos, EndpointName) -> - emqx_broker:publish(emqx_message:set_flag(retain, false, emqx_message:make(EndpointName, Qos, Topic, Payload))). - -time_now() -> erlang:system_time(millisecond). - -%%-------------------------------------------------------------------- -%% Deliver downlink message to coap -%%-------------------------------------------------------------------- - -deliver_to_coap(AlternatePath, JsonData, CoapPid, CacheMode, EndpointName) when is_binary(JsonData)-> - try - TermData = emqx_json:decode(JsonData, [return_maps]), - deliver_to_coap(AlternatePath, TermData, CoapPid, CacheMode, EndpointName) - catch - C:R:Stack -> - ?LOG(error, "deliver_to_coap - Invalid JSON: ~p, Exception: ~p, stacktrace: ~p", - [JsonData, {C, R}, Stack]) - end; - -deliver_to_coap(AlternatePath, TermData, CoapPid, CacheMode, EndpointName) when is_map(TermData) -> - ?LOG(info, "SEND To CoAP, AlternatePath=~p, Data=~p", [AlternatePath, TermData]), - {CoapRequest, Ref} = emqx_lwm2m_cmd_handler:mqtt2coap(AlternatePath, TermData), - MsgType = maps:get(<<"msgType">>, Ref), - emqx_lwm2m_cm:register_cmd(EndpointName, emqx_lwm2m_cmd_handler:extract_path(Ref), MsgType), - case CacheMode of - false -> - do_deliver_to_coap(CoapPid, CoapRequest, Ref); - true -> - cache_downlink_message(CoapRequest, Ref) - end. - -%%-------------------------------------------------------------------- -%% Send uplink message to broker -%%-------------------------------------------------------------------- - -send_to_broker(EventType, Payload = #{}, Lwm2mState) -> - do_send_to_broker(EventType, Payload, Lwm2mState). - -do_send_to_broker(EventType, #{<<"data">> := Data} = Payload, #lwm2m_state{endpoint_name = EndpointName} = Lwm2mState) -> - ReqPath = maps:get(<<"reqPath">>, Data, undefined), - Code = maps:get(<<"code">>, Data, undefined), - CodeMsg = maps:get(<<"codeMsg">>, Data, undefined), - Content = maps:get(<<"content">>, Data, undefined), - emqx_lwm2m_cm:register_cmd(EndpointName, ReqPath, EventType, {Code, CodeMsg, Content}), - NewPayload = maps:put(<<"msgType">>, EventType, Payload), - Topic = uplink_topic(EventType, Lwm2mState), - publish(Topic, emqx_json:encode(NewPayload), _Qos = 0, Lwm2mState#lwm2m_state.endpoint_name). - -%%-------------------------------------------------------------------- -%% Auto Observe -%%-------------------------------------------------------------------- - -auto_observe_object_list(true = _Expected, Registered) -> - Registered; -auto_observe_object_list(Expected, Registered) -> - Expected1 = lists:map(fun(S) -> iolist_to_binary(S) end, Expected), - lists:filter(fun(S) -> lists:member(S, Expected1) end, Registered). - -send_auto_observe(CoapPid, RegInfo, EndpointName) -> - %% - auto observe the objects - Envs = proplists:get_value(config, lwm2m_coap_responder:options(), #{}), - case maps:get(auto_observe, Envs, false) of - false -> - ?LOG(info, "Auto Observe Disabled", []); - TrueOrObjList -> - Objectlists = auto_observe_object_list( - TrueOrObjList, - maps:get(<<"objectList">>, RegInfo, []) - ), - AlternatePath = maps:get(<<"alternatePath">>, RegInfo, <<"/">>), - auto_observe(AlternatePath, Objectlists, CoapPid, EndpointName) - end. - -auto_observe(AlternatePath, ObjectList, CoapPid, EndpointName) -> - ?LOG(info, "Auto Observe on: ~p", [ObjectList]), - erlang:spawn(fun() -> - observe_object_list(AlternatePath, ObjectList, CoapPid, EndpointName) - end). - -observe_object_list(AlternatePath, ObjectList, CoapPid, EndpointName) -> - lists:foreach(fun(ObjectPath) -> - [ObjId| LastPath] = emqx_lwm2m_cmd_handler:path_list(ObjectPath), - case ObjId of - <<"19">> -> - [ObjInsId | _LastPath1] = LastPath, - case ObjInsId of - <<"0">> -> - observe_object_slowly(AlternatePath, <<"/19/0/0">>, CoapPid, 100, EndpointName); - _ -> - observe_object_slowly(AlternatePath, ObjectPath, CoapPid, 100, EndpointName) - end; - _ -> - observe_object_slowly(AlternatePath, ObjectPath, CoapPid, 100, EndpointName) - end - end, ObjectList). - -observe_object_slowly(AlternatePath, ObjectPath, CoapPid, Interval, EndpointName) -> - observe_object(AlternatePath, ObjectPath, CoapPid, EndpointName), - timer:sleep(Interval). - -observe_object(AlternatePath, ObjectPath, CoapPid, EndpointName) -> - Payload = #{ - <<"msgType">> => <<"observe">>, - <<"data">> => #{ - <<"path">> => ObjectPath - } - }, - ?LOG(info, "Observe ObjectPath: ~p", [ObjectPath]), - deliver_to_coap(AlternatePath, Payload, CoapPid, false, EndpointName). - -do_deliver_to_coap_slowly(CoapPid, CoapRequestList, Interval) -> - erlang:spawn(fun() -> - lists:foreach(fun({CoapRequest, Ref}) -> - _ = do_deliver_to_coap(CoapPid, CoapRequest, Ref), - timer:sleep(Interval) - end, lists:reverse(CoapRequestList)) - end). - -do_deliver_to_coap(CoapPid, CoapRequest, Ref) -> - ?LOG(debug, "Deliver To CoAP(~p), CoapRequest: ~p", [CoapPid, CoapRequest]), - CoapPid ! {deliver_to_coap, CoapRequest, Ref}. - -%%-------------------------------------------------------------------- -%% Queue Mode -%%-------------------------------------------------------------------- - -cache_downlink_message(CoapRequest, Ref) -> - ?LOG(debug, "Cache downlink coap request: ~p, Ref: ~p", [CoapRequest, Ref]), - put(dl_msg_cache, [{CoapRequest, Ref} | get_cached_downlink_messages()]). - -flush_cached_downlink_messages(CoapPid) -> - case erase(dl_msg_cache) of - CachedMessageList when is_list(CachedMessageList)-> - do_deliver_to_coap_slowly(CoapPid, CachedMessageList, 100); - undefined -> ok - end. - -get_cached_downlink_messages() -> - case get(dl_msg_cache) of - undefined -> []; - CachedMessageList -> CachedMessageList - end. - -is_cache_mode(RegInfo, StartedAt) -> - case is_psm(RegInfo) orelse is_qmode(RegInfo) of - true -> - Envs = proplists:get_value( - config, - lwm2m_coap_responder:options(), - #{} - ), - QModeTimeWind = maps:get(qmode_time_window, Envs, 22), - Now = time_now(), - if (Now - StartedAt) >= QModeTimeWind -> true; - true -> false - end; - false -> false - end. - -is_psm(_) -> false. - -is_qmode(#{<<"b">> := Binding}) when Binding =:= <<"UQ">>; - Binding =:= <<"SQ">>; - Binding =:= <<"UQS">> - -> true; -is_qmode(_) -> false. - -%%-------------------------------------------------------------------- -%% Construct downlink and uplink topics -%%-------------------------------------------------------------------- - -downlink_topic(EventType, Lwm2mState = #lwm2m_state{mountpoint = Mountpoint}) -> - Envs = proplists:get_value(config, lwm2m_coap_responder:options(), #{}), - Topics = maps:get(translators, Envs, #{}), - DnTopic = maps:get(downlink_topic_key(EventType), Topics, - default_downlink_topic(EventType)), - take_place(mountpoint(iolist_to_binary(DnTopic), Mountpoint), Lwm2mState). - -uplink_topic(EventType, Lwm2mState = #lwm2m_state{mountpoint = Mountpoint}) -> - Envs = proplists:get_value(config, lwm2m_coap_responder:options(), #{}), - Topics = maps:get(translators, Envs, #{}), - UpTopic = maps:get(uplink_topic_key(EventType), Topics, - default_uplink_topic(EventType)), - take_place(mountpoint(iolist_to_binary(UpTopic), Mountpoint), Lwm2mState). - -downlink_topic_key(EventType) when is_binary(EventType) -> - command. - -uplink_topic_key(<<"notify">>) -> notify; -uplink_topic_key(<<"register">>) -> register; -uplink_topic_key(<<"update">>) -> update; -uplink_topic_key(EventType) when is_binary(EventType) -> - response. - -default_downlink_topic(Type) when is_binary(Type)-> - <<"dn/#">>. - -default_uplink_topic(<<"notify">>) -> - <<"up/notify">>; -default_uplink_topic(Type) when is_binary(Type) -> - <<"up/resp">>. - -take_place(Text, Lwm2mState) -> - {IPAddr, _} = Lwm2mState#lwm2m_state.peername, - IPAddrBin = iolist_to_binary(inet:ntoa(IPAddr)), - take_place(take_place(Text, <<"%a">>, IPAddrBin), - <<"%e">>, Lwm2mState#lwm2m_state.endpoint_name). - -take_place(Text, Placeholder, Value) -> - binary:replace(Text, Placeholder, Value, [global]). - -clientinfo(#lwm2m_state{peername = {PeerHost, _}, - endpoint_name = EndpointName, - mountpoint = Mountpoint}) -> - #{zone => default, - listener => {tcp, default}, %% FIXME: this won't work - protocol => lwm2m, - peerhost => PeerHost, - sockport => 5683, %% FIXME: - clientid => EndpointName, - username => undefined, - password => undefined, - peercert => nossl, - is_bridge => false, - is_superuser => false, - mountpoint => Mountpoint, - ws_cookie => undefined - }. - -mountpoint(Topic, <<>>) -> - Topic; -mountpoint(Topic, Mountpoint) -> - <>. - -%%-------------------------------------------------------------------- -%% Helper funcs - --compile({inline, [run_hooks/2, run_hooks/3]}). -run_hooks(Name, Args) -> - ok = emqx_metrics:inc(Name), emqx_hooks:run(Name, Args). - -run_hooks(Name, Args, Acc) -> - ok = emqx_metrics:inc(Name), emqx_hooks:run_fold(Name, Args, Acc). - -%%-------------------------------------------------------------------- -%% Info & Stats - -info(State) -> - ChannInfo = chann_info(State), - ChannInfo#{sockinfo => sockinfo(State)}. - -%% copies from emqx_connection:info/1 -sockinfo(#lwm2m_state{peername = Peername}) -> - #{socktype => udp, - peername => Peername, - sockname => {{127,0,0,1}, 5683}, %% FIXME: Sock? - sockstate => running, - active_n => 1 - }. - -%% copies from emqx_channel:info/1 -chann_info(State) -> - #{conninfo => conninfo(State), - conn_state => connected, - clientinfo => clientinfo(State), - session => maps:from_list(session_info(State)), - will_msg => undefined - }. - -conninfo(#lwm2m_state{peername = Peername, - version = Ver, - started_at = StartedAt, - endpoint_name = Epn}) -> - #{socktype => udp, - sockname => {{127,0,0,1}, 5683}, - peername => Peername, - peercert => nossl, %% TODO: dtls - conn_mod => ?MODULE, - proto_name => <<"LwM2M">>, - proto_ver => Ver, - clean_start => true, - clientid => Epn, - username => undefined, - conn_props => undefined, - connected => true, - connected_at => StartedAt, - keepalive => 0, - receive_maximum => 0, - expiry_interval => 0 - }. - -%% copies from emqx_session:info/1 -session_info(#lwm2m_state{mqtt_topic = SubTopic, started_at = StartedAt}) -> - [{subscriptions, #{SubTopic => ?SUBOPTS}}, - {upgrade_qos, false}, - {retry_interval, 0}, - {await_rel_timeout, 0}, - {created_at, StartedAt} - ]. - -%% The stats keys copied from emqx_connection:stats/1 -stats(_State) -> - SockStats = [{recv_oct,0}, {recv_cnt,0}, {send_oct,0}, {send_cnt,0}, {send_pend,0}], - ConnStats = emqx_pd:get_counters(?CONN_STATS), - ChanStats = [{subscriptions_cnt, 1}, - {subscriptions_max, 1}, - {inflight_cnt, 0}, - {inflight_max, 0}, - {mqueue_len, 0}, - {mqueue_max, 0}, - {mqueue_dropped, 0}, - {next_pkt_id, 0}, - {awaiting_rel_cnt, 0}, - {awaiting_rel_max, 0} - ], - ProcStats = emqx_misc:proc_stats(), - lists:append([SockStats, ConnStats, ChanStats, ProcStats]). - diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_session.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_session.erl new file mode 100644 index 000000000..ab27dfbca --- /dev/null +++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_session.erl @@ -0,0 +1,734 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2017-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_lwm2m_session). + +-include_lib("emqx/include/logger.hrl"). +-include_lib("emqx/include/emqx.hrl"). +-include_lib("emqx/include/emqx_mqtt.hrl"). +-include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). +-include_lib("emqx_gateway/src/lwm2m/include/emqx_lwm2m.hrl"). + +%% API +-export([ new/0, init/4, update/3, parse_object_list/1 + , reregister/3, on_close/1, find_cmd_record/3]). + +-export([ info/1 + , info/2 + , stats/1 + ]). + +-export([ handle_coap_in/3 + , handle_protocol_in/3 + , handle_deliver/3 + , timeout/3 + , set_reply/2]). + +-export_type([session/0]). + +-type request_context() :: map(). + +-type timestamp() :: non_neg_integer(). +-type queued_request() :: {timestamp(), request_context(), emqx_coap_message()}. + +-type cmd_path() :: binary(). +-type cmd_type() :: binary(). +-type cmd_record_key() :: {cmd_path(), cmd_type()}. +-type cmd_code() :: binary(). +-type cmd_code_msg() :: binary(). +-type cmd_code_content() :: list(map()). +-type cmd_result() :: undefined | {cmd_code(), cmd_code_msg(), cmd_code_content()}. +-type cmd_record() :: #{cmd_record_key() => cmd_result()}. + +-record(session, { coap :: emqx_coap_tm:manager() + , queue :: queue:queue(queued_request()) + , wait_ack :: request_context() | undefined + , endpoint_name :: binary() | undefined + , location_path :: list(binary()) | undefined + , reg_info :: map() | undefined + , lifetime :: non_neg_integer() | undefined + , is_cache_mode :: boolean() + , mountpoint :: binary() + , last_active_at :: non_neg_integer() + , cmd_record :: cmd_record() + }). + +-type session() :: #session{}. + +-define(PREFIX, <<"rd">>). +-define(NOW, erlang:system_time(second)). +-define(IGNORE_OBJECT, [<<"0">>, <<"1">>, <<"2">>, <<"4">>, <<"5">>, <<"6">>, + <<"7">>, <<"9">>, <<"15">>]). + +-define(CMD_KEY(Path, Type), {Path, Type}). + +%% uplink and downlink topic configuration +-define(lwm2m_up_dm_topic, {<<"/v1/up/dm">>, 0}). + +%% steal from emqx_session +-define(INFO_KEYS, [subscriptions, + upgrade_qos, + retry_interval, + await_rel_timeout, + created_at + ]). + +-define(STATS_KEYS, [subscriptions_cnt, + subscriptions_max, + inflight_cnt, + inflight_max, + mqueue_len, + mqueue_max, + mqueue_dropped, + next_pkt_id, + awaiting_rel_cnt, + awaiting_rel_max + ]). + +-define(OUT_LIST_KEY, out_list). + +-import(emqx_coap_medium, [iter/3, reply/2]). + +%%-------------------------------------------------------------------- +%% API +%%-------------------------------------------------------------------- +-spec new () -> session(). +new() -> + #session{ coap = emqx_coap_tm:new() + , queue = queue:new() + , last_active_at = ?NOW + , is_cache_mode = false + , mountpoint = <<>> + , cmd_record = #{} + , lifetime = emqx:get_config([gateway, lwm2m, lifetime_max])}. + +-spec init(emqx_coap_message(), binary(), function(), session()) -> map(). +init(#coap_message{options = Opts, + payload = Payload} = Msg, MountPoint, WithContext, Session) -> + Query = maps:get(uri_query, Opts), + RegInfo = append_object_list(Query, Payload), + LifeTime = get_lifetime(RegInfo), + Epn = maps:get(<<"ep">>, Query), + Location = [?PREFIX, Epn], + + NewSession = Session#session{endpoint_name = Epn, + location_path = Location, + reg_info = RegInfo, + lifetime = LifeTime, + mountpoint = MountPoint, + is_cache_mode = is_psm(RegInfo) orelse is_qmode(RegInfo), + queue = queue:new()}, + + Result = return(register_init(WithContext, NewSession)), + Reply = emqx_coap_message:piggyback({ok, created}, Msg), + Reply2 = emqx_coap_message:set(location_path, Location, Reply), + reply(Reply2, Result#{lifetime => true}). + +reregister(Msg, WithContext, Session) -> + update(Msg, WithContext, <<"register">>, Session). + +update(Msg, WithContext, Session) -> + update(Msg, WithContext, <<"update">>, Session). + +-spec on_close(session()) -> binary(). +on_close(Session) -> + #{topic := Topic} = downlink_topic(), + MountedTopic = mount(Topic, Session), + emqx:unsubscribe(MountedTopic), + MountedTopic. + +-spec find_cmd_record(cmd_path(), cmd_type(), session()) -> cmd_result(). +find_cmd_record(Path, Type, #session{cmd_record = Record}) -> + maps:get(?CMD_KEY(Path, Type), Record, undefined). + +%%-------------------------------------------------------------------- +%% Info, Stats +%%-------------------------------------------------------------------- +-spec(info(session()) -> emqx_types:infos()). +info(Session) -> + maps:from_list(info(?INFO_KEYS, Session)). + +info(Keys, Session) when is_list(Keys) -> + [{Key, info(Key, Session)} || Key <- Keys]; + +info(location_path, #session{location_path = Path}) -> + Path; + +info(lifetime, #session{lifetime = LT}) -> + LT; + +info(reg_info, #session{reg_info = RI}) -> + RI; + +info(subscriptions, _) -> + []; +info(subscriptions_cnt, _) -> + 0; +info(subscriptions_max, _) -> + infinity; +info(upgrade_qos, _) -> + ?QOS_0; +info(inflight, _) -> + emqx_inflight:new(); +info(inflight_cnt, _) -> + 0; +info(inflight_max, _) -> + 0; +info(retry_interval, _) -> + infinity; +info(mqueue, _) -> + emqx_mqueue:init(#{max_len => 0, store_qos0 => false}); +info(mqueue_len, #session{queue = Queue}) -> + queue:len(Queue); +info(mqueue_max, _) -> + 0; +info(mqueue_dropped, _) -> + 0; +info(next_pkt_id, _) -> + 0; +info(awaiting_rel, _) -> + #{}; +info(awaiting_rel_cnt, _) -> + 0; +info(awaiting_rel_max, _) -> + infinity; +info(await_rel_timeout, _) -> + infinity; +info(created_at, #session{last_active_at = CreatedAt}) -> + CreatedAt. + +%% @doc Get stats of the session. +-spec(stats(session()) -> emqx_types:stats()). +stats(Session) -> info(?STATS_KEYS, Session). + +%%-------------------------------------------------------------------- +%% API +%%-------------------------------------------------------------------- +handle_coap_in(Msg, _WithContext, Session) -> + call_coap(case emqx_coap_message:is_request(Msg) of + true -> handle_request; + _ -> handle_response + end, + Msg, Session#session{last_active_at = ?NOW}). + +handle_deliver(Delivers, WithContext, Session) -> + return(deliver(Delivers, WithContext, Session)). + +timeout({transport, Msg}, _, Session) -> + call_coap(timeout, Msg, Session). + +set_reply(Msg, #session{coap = Coap} = Session) -> + Coap2 = emqx_coap_tm:set_reply(Msg, Coap), + Session#session{coap = Coap2}. + +%%-------------------------------------------------------------------- +%% Protocol Stack +%%-------------------------------------------------------------------- +handle_protocol_in({response, CtxMsg}, WithContext, Session) -> + return(handle_coap_response(CtxMsg, WithContext, Session)); + +handle_protocol_in({ack, CtxMsg}, WithContext, Session) -> + return(handle_ack(CtxMsg, WithContext, Session)); + +handle_protocol_in({ack_failure, CtxMsg}, WithContext, Session) -> + return(handle_ack_failure(CtxMsg, WithContext, Session)); + +handle_protocol_in({reset, CtxMsg}, WithContext, Session) -> + return(handle_ack_reset(CtxMsg, WithContext, Session)). + +%%-------------------------------------------------------------------- +%% Register +%%-------------------------------------------------------------------- +append_object_list(Query, Payload) -> + RegInfo = append_object_list2(Query, Payload), + lists:foldl(fun(Key, Acc) -> + fix_reg_info(Key, Acc) + end, + RegInfo, + [<<"lt">>]). + +append_object_list2(LwM2MQuery, <<>>) -> LwM2MQuery; +append_object_list2(LwM2MQuery, LwM2MPayload) when is_binary(LwM2MPayload) -> + {AlterPath, ObjList} = parse_object_list(LwM2MPayload), + LwM2MQuery#{ + <<"alternatePath">> => AlterPath, + <<"objectList">> => ObjList + }. + +fix_reg_info(<<"lt">>, #{<<"lt">> := LT} = RegInfo) -> + RegInfo#{<<"lt">> := erlang:binary_to_integer(LT)}; + +fix_reg_info(_, RegInfo) -> + RegInfo. + +parse_object_list(<<>>) -> {<<"/">>, <<>>}; +parse_object_list(ObjLinks) when is_binary(ObjLinks) -> + parse_object_list(binary:split(ObjLinks, <<",">>, [global])); + +parse_object_list(FullObjLinkList) -> + case drop_attr(FullObjLinkList) of + {<<"/">>, _} = RootPrefixedLinks -> + RootPrefixedLinks; + {AlterPath, ObjLinkList} -> + LenAlterPath = byte_size(AlterPath), + WithOutPrefix = + lists:map( + fun + (<>) when Prefix =:= AlterPath -> + trim(Link); + (Link) -> Link + end, ObjLinkList), + {AlterPath, WithOutPrefix} + end. + +drop_attr(LinkList) -> + lists:foldr( + fun(Link, {AlternatePath, LinkAcc}) -> + case parse_link(Link) of + {false, MainLink} -> {AlternatePath, [MainLink | LinkAcc]}; + {true, MainLink} -> {MainLink, LinkAcc} + end + end, {<<"/">>, []}, LinkList). + +parse_link(Link) -> + [MainLink | Attrs] = binary:split(trim(Link), <<";">>, [global]), + {is_alternate_path(Attrs), delink(trim(MainLink))}. + +is_alternate_path(LinkAttrs) -> + lists:any(fun(Attr) -> + case binary:split(trim(Attr), <<"=">>) of + [<<"rt">>, ?OMA_ALTER_PATH_RT] -> + true; + [AttrKey, _] when AttrKey =/= <<>> -> + false; + _BadAttr -> throw({bad_attr, _BadAttr}) + end + end, + LinkAttrs). + +trim(Str)-> binary_util:trim(Str, $ ). + +delink(Str) -> + Ltrim = binary_util:ltrim(Str, $<), + binary_util:rtrim(Ltrim, $>). + +get_lifetime(#{<<"lt">> := LT}) -> + case LT of + 0 -> emqx:get_config([gateway, lwm2m, lifetime_max]); + _ -> LT * 1000 + end; +get_lifetime(_) -> + emqx:get_config([gateway, lwm2m, lifetime_max]). + +get_lifetime(#{<<"lt">> := _} = NewRegInfo, _) -> + get_lifetime(NewRegInfo); + +get_lifetime(_, OldRegInfo) -> + get_lifetime(OldRegInfo). + +-spec update(emqx_coap_message(), function(), binary(), session()) -> map(). +update(#coap_message{options = Opts, payload = Payload} = Msg, + WithContext, + CmdType, + #session{reg_info = OldRegInfo} = Session) -> + Query = maps:get(uri_query, Opts), + RegInfo = append_object_list(Query, Payload), + UpdateRegInfo = maps:merge(OldRegInfo, RegInfo), + LifeTime = get_lifetime(UpdateRegInfo, OldRegInfo), + + NewSession = Session#session{reg_info = UpdateRegInfo, + is_cache_mode = + is_psm(UpdateRegInfo) orelse is_qmode(UpdateRegInfo), + lifetime = LifeTime}, + + Session2 = proto_subscribe(WithContext, NewSession), + Session3 = send_dl_msg(Session2), + RegPayload = #{<<"data">> => UpdateRegInfo}, + Session4 = send_to_mqtt(#{}, CmdType, RegPayload, WithContext, Session3), + + Result = return(Session4), + + Reply = emqx_coap_message:piggyback({ok, changed}, Msg), + reply(Reply, Result#{lifetime => true}). + +register_init(WithContext, #session{reg_info = RegInfo} = Session) -> + Session2 = send_auto_observe(RegInfo, Session), + %% - subscribe to the downlink_topic and wait for commands + #{topic := Topic, qos := Qos} = downlink_topic(), + MountedTopic = mount(Topic, Session), + Session3 = subscribe(MountedTopic, Qos, WithContext, Session2), + Session4 = send_dl_msg(Session3), + + %% - report the registration info + RegPayload = #{<<"data">> => RegInfo}, + send_to_mqtt(#{}, <<"register">>, RegPayload, WithContext, Session4). + +%%-------------------------------------------------------------------- +%% Subscribe +%%-------------------------------------------------------------------- +proto_subscribe(WithContext, #session{wait_ack = WaitAck} = Session) -> + #{topic := Topic, qos := Qos} = downlink_topic(), + MountedTopic = mount(Topic, Session), + Session2 = case WaitAck of + undefined -> + Session; + Ctx -> + MqttPayload = emqx_lwm2m_cmd:coap_failure_to_mqtt(Ctx, <<"coap_timeout">>), + send_to_mqtt(Ctx, <<"coap_timeout">>, MqttPayload, WithContext, Session) + end, + subscribe(MountedTopic, Qos, WithContext, Session2). + +subscribe(Topic, Qos, WithContext, Session) -> + Opts = get_sub_opts(Qos), + WithContext(subscribe, [Topic, Opts]), + Session. + +send_auto_observe(RegInfo, Session) -> + %% - auto observe the objects + case is_auto_observe() of + true -> + AlternatePath = maps:get(<<"alternatePath">>, RegInfo, <<"/">>), + ObjectList = maps:get(<<"objectList">>, RegInfo, []), + observe_object_list(AlternatePath, ObjectList, Session); + _ -> + ?LOG(info, "Auto Observe Disabled", []), + Session + end. + +observe_object_list(_, [], Session) -> + Session; +observe_object_list(AlternatePath, ObjectList, Session) -> + Fun = fun(ObjectPath, Acc) -> + {[ObjId| _], _} = emqx_lwm2m_cmd:path_list(ObjectPath), + case lists:member(ObjId, ?IGNORE_OBJECT) of + true -> Acc; + false -> + try + emqx_lwm2m_xml_object_db:find_objectid(binary_to_integer(ObjId)), + observe_object(AlternatePath, ObjectPath, Acc) + catch error:no_xml_definition -> + Acc + end + end + end, + lists:foldl(Fun, Session, ObjectList). + +observe_object(AlternatePath, ObjectPath, Session) -> + Payload = #{<<"msgType">> => <<"observe">>, + <<"data">> => #{<<"path">> => ObjectPath}, + <<"is_auto_observe">> => true + }, + deliver_auto_observe_to_coap(AlternatePath, Payload, Session). + +deliver_auto_observe_to_coap(AlternatePath, TermData, Session) -> + ?LOG(info, "Auto Observe, SEND To CoAP, AlternatePath=~0p, Data=~0p ", [AlternatePath, TermData]), + {Req, Ctx} = emqx_lwm2m_cmd:mqtt_to_coap(AlternatePath, TermData), + maybe_do_deliver_to_coap(Ctx, Req, 0, false, Session). + +get_sub_opts(Qos) -> + #{ + qos => Qos, + rap => 0, + nl => 0, + rh => 0, + is_new => false + }. + +is_auto_observe() -> + emqx:get_config([gateway, lwm2m, auto_observe]). + +%%-------------------------------------------------------------------- +%% Response +%%-------------------------------------------------------------------- +handle_coap_response({Ctx = #{<<"msgType">> := EventType}, + #coap_message{method = CoapMsgMethod, + type = CoapMsgType, + payload = CoapMsgPayload, + options = CoapMsgOpts}}, + WithContext, + Session) -> + MqttPayload = emqx_lwm2m_cmd:coap_to_mqtt(CoapMsgMethod, CoapMsgPayload, CoapMsgOpts, Ctx), + {ReqPath, _} = emqx_lwm2m_cmd:path_list(emqx_lwm2m_cmd:extract_path(Ctx)), + Session2 = record_response(EventType, MqttPayload, Session), + Session3 = + case {ReqPath, MqttPayload, EventType, CoapMsgType} of + {[<<"5">>| _], _, <<"observe">>, CoapMsgType} when CoapMsgType =/= ack -> + %% this is a notification for status update during NB firmware upgrade. + %% need to reply to DM http callbacks + send_to_mqtt(Ctx, <<"notify">>, MqttPayload, ?lwm2m_up_dm_topic, WithContext, Session2); + {_ReqPath, _, <<"observe">>, CoapMsgType} when CoapMsgType =/= ack -> + %% this is actually a notification, correct the msgType + send_to_mqtt(Ctx, <<"notify">>, MqttPayload, WithContext, Session2); + _ -> + send_to_mqtt(Ctx, EventType, MqttPayload, WithContext, Session2) + end, + send_dl_msg(Ctx, Session3). + +%%-------------------------------------------------------------------- +%% Ack +%%-------------------------------------------------------------------- +handle_ack({Ctx, _}, WithContext, Session) -> + Session2 = send_dl_msg(Ctx, Session), + MqttPayload = emqx_lwm2m_cmd:empty_ack_to_mqtt(Ctx), + send_to_mqtt(Ctx, <<"ack">>, MqttPayload, WithContext, Session2). + +%%-------------------------------------------------------------------- +%% Ack Failure(Timeout/Reset) +%%-------------------------------------------------------------------- +handle_ack_failure({Ctx, _}, WithContext, Session) -> + handle_ack_failure(Ctx, <<"coap_timeout">>, WithContext, Session). + +handle_ack_reset({Ctx, _}, WithContext, Session) -> + handle_ack_failure(Ctx, <<"coap_reset">>, WithContext, Session). + +handle_ack_failure(Ctx, MsgType, WithContext, Session) -> + Session2 = may_send_dl_msg(coap_timeout, Ctx, Session), + MqttPayload = emqx_lwm2m_cmd:coap_failure_to_mqtt(Ctx, MsgType), + send_to_mqtt(Ctx, MsgType, MqttPayload, WithContext, Session2). + +%%-------------------------------------------------------------------- +%% Send To CoAP +%%-------------------------------------------------------------------- + +may_send_dl_msg(coap_timeout, Ctx, #session{wait_ack = WaitAck} = Session) -> + case is_cache_mode(Session) of + false -> send_dl_msg(Ctx, Session); + true -> + case WaitAck of + Ctx -> + Session#session{wait_ack = undefined}; + _ -> + Session + end + end. + +is_cache_mode(#session{is_cache_mode = IsCacheMode, + last_active_at = LastActiveAt}) -> + IsCacheMode andalso + ((?NOW - LastActiveAt) >= + emqx:get_config([gateway, lwm2m, qmode_time_window])). + +is_psm(#{<<"apn">> := APN}) when APN =:= <<"Ctnb">>; + APN =:= <<"psmA.eDRX0.ctnb">>; + APN =:= <<"psmC.eDRX0.ctnb">>; + APN =:= <<"psmF.eDRXC.ctnb">> + -> true; +is_psm(_) -> false. + +is_qmode(#{<<"b">> := Binding}) when Binding =:= <<"UQ">>; + Binding =:= <<"SQ">>; + Binding =:= <<"UQS">> + -> true; +is_qmode(_) -> false. + +send_dl_msg(Session) -> + %% if has in waiting donot send + case Session#session.wait_ack of + undefined -> + send_to_coap(Session); + _ -> + Session + end. + +send_dl_msg(Ctx, Session) -> + case Session#session.wait_ack of + undefined -> + send_to_coap(Session); + Ctx -> + send_to_coap(Session#session{wait_ack = undefined}); + _ -> + Session + end. + +send_to_coap(#session{queue = Queue} = Session) -> + case queue:out(Queue) of + {{value, {Timestamp, Ctx, Req}}, Q2} -> + Now = ?NOW, + if Timestamp =:= 0 orelse Timestamp > Now -> + send_to_coap(Ctx, Req, Session#session{queue = Q2}); + true -> + send_to_coap(Session#session{queue = Q2}) + end; + {empty, _} -> + Session + end. + +send_to_coap(Ctx, Req, Session) -> + ?LOG(debug, "Deliver To CoAP, CoapRequest: ~0p", [Req]), + out_to_coap(Ctx, Req, Session#session{wait_ack = Ctx}). + +send_msg_not_waiting_ack(Ctx, Req, Session) -> + ?LOG(debug, "Deliver To CoAP not waiting ack, CoapRequest: ~0p", [Req]), + %% cmd_sent(Ref, LwM2MOpts). + out_to_coap(Ctx, Req, Session). + +%%-------------------------------------------------------------------- +%% Send To MQTT +%%-------------------------------------------------------------------- +send_to_mqtt(Ref, EventType, Payload, WithContext, Session) -> + #{topic := Topic, qos := Qos} = uplink_topic(EventType), + Mheaders = maps:get(mheaders, Ref, #{}), + proto_publish(Topic, Payload#{<<"msgType">> => EventType}, Qos, Mheaders, WithContext, Session). + +send_to_mqtt(Ctx, EventType, Payload, {Topic, Qos}, + WithContext, Session) -> + Mheaders = maps:get(mheaders, Ctx, #{}), + proto_publish(Topic, Payload#{<<"msgType">> => EventType}, Qos, Mheaders, WithContext, Session). + +proto_publish(Topic, Payload, Qos, Headers, WithContext, + #session{endpoint_name = Epn} = Session) -> + MountedTopic = mount(Topic, Session), + Msg = emqx_message:make(Epn, Qos, MountedTopic, + emqx_json:encode(Payload), #{}, Headers), + WithContext(publish, [MountedTopic, Msg]), + Session. + +mount(Topic, #session{mountpoint = MountPoint}) when is_binary(Topic) -> + <>. + +downlink_topic() -> + emqx:get_config([gateway, lwm2m, translators, command]). + +uplink_topic(<<"notify">>) -> + emqx:get_config([gateway, lwm2m, translators, notify]); + +uplink_topic(<<"register">>) -> + emqx:get_config([gateway, lwm2m, translators, register]); + +uplink_topic(<<"update">>) -> + emqx:get_config([gateway, lwm2m, translators, update]); + +uplink_topic(_) -> + emqx:get_config([gateway, lwm2m, translators, response]). + +%%-------------------------------------------------------------------- +%% Deliver +%%-------------------------------------------------------------------- + +deliver(Delivers, WithContext, #session{reg_info = RegInfo} = Session) -> + IsCacheMode = is_cache_mode(Session), + AlternatePath = maps:get(<<"alternatePath">>, RegInfo, <<"/">>), + lists:foldl(fun({deliver, _, MQTT}, Acc) -> + deliver_to_coap(AlternatePath, + MQTT#message.payload, MQTT, IsCacheMode, WithContext, Acc) + end, + Session, + Delivers). + +deliver_to_coap(AlternatePath, JsonData, MQTT, CacheMode, WithContext, Session) when is_binary(JsonData)-> + try + TermData = emqx_json:decode(JsonData, [return_maps]), + deliver_to_coap(AlternatePath, TermData, MQTT, CacheMode, WithContext, Session) + catch + ExClass:Error:ST -> + ?LOG(error, "deliver_to_coap - Invalid JSON: ~0p, Exception: ~0p, stacktrace: ~0p", + [JsonData, {ExClass, Error}, ST]), + WithContext(metrics, 'delivery.dropped'), + Session + end; + +deliver_to_coap(AlternatePath, TermData, MQTT, CacheMode, WithContext, Session) when is_map(TermData) -> + WithContext(metrics, 'messages.delivered'), + {Req, Ctx} = emqx_lwm2m_cmd:mqtt_to_coap(AlternatePath, TermData), + ExpiryTime = get_expiry_time(MQTT), + Session2 = record_request(Ctx, Session), + maybe_do_deliver_to_coap(Ctx, Req, ExpiryTime, CacheMode, Session2). + +maybe_do_deliver_to_coap(Ctx, Req, ExpiryTime, CacheMode, + #session{wait_ack = WaitAck, + queue = Queue} = Session) -> + MHeaders = maps:get(mheaders, Ctx, #{}), + TTL = maps:get(<<"ttl">>, MHeaders, 7200), + case TTL of + 0 -> + send_msg_not_waiting_ack(Ctx, Req, Session); + _ -> + case not CacheMode + andalso queue:is_empty(Queue) andalso WaitAck =:= undefined of + true -> + send_to_coap(Ctx, Req, Session); + false -> + Session#session{queue = queue:in({ExpiryTime, Ctx, Req}, Queue)} + end + end. + +get_expiry_time(#message{headers = #{properties := #{'Message-Expiry-Interval' := Interval}}, + timestamp = Ts}) -> + Ts + Interval * 1000; +get_expiry_time(_) -> + 0. + +%%-------------------------------------------------------------------- +%% Call CoAP +%%-------------------------------------------------------------------- +call_coap(Fun, Msg, #session{coap = Coap} = Session) -> + iter([tm, fun process_tm/4, fun process_session/3], + emqx_coap_tm:Fun(Msg, Coap), + Session). + +process_tm(TM, Result, Session, Cursor) -> + iter(Cursor, Result, Session#session{coap = TM}). + +process_session(_, Result, Session) -> + Result#{session => Session}. + +out_to_coap(Context, Msg, Session) -> + out_to_coap({Context, Msg}, Session). + +out_to_coap(Msg, Session) -> + Outs = get_outs(), + erlang:put(?OUT_LIST_KEY, [Msg | Outs]), + Session. + +get_outs() -> + case erlang:get(?OUT_LIST_KEY) of + undefined -> []; + Any -> Any + end. + +return(#session{coap = CoAP} = Session) -> + Outs = get_outs(), + erlang:put(?OUT_LIST_KEY, []), + {ok, Coap2, Msgs} = do_out(Outs, CoAP, []), + #{return => {Msgs, Session#session{coap = Coap2}}}. + +do_out([{Ctx, Out} | T], TM, Msgs) -> + %% TODO maybe set a special token? + #{out := [Msg], + tm := TM2} = emqx_coap_tm:handle_out(Out, Ctx, TM), + do_out(T, TM2, [Msg | Msgs]); + +do_out(_, TM, Msgs) -> + {ok, TM, Msgs}. + + +%%-------------------------------------------------------------------- +%% CMD Record +%%-------------------------------------------------------------------- +-spec record_request(request_context(), session()) -> session(). +record_request(#{<<"msgType">> := Type} = Context, Session) -> + Path = emqx_lwm2m_cmd:extract_path(Context), + record_cmd(Path, Type, undefined, Session). + +record_response(EventType, #{<<"data">> := Data}, Session) -> + ReqPath = maps:get(<<"reqPath">>, Data, undefined), + Code = maps:get(<<"code">>, Data, undefined), + CodeMsg = maps:get(<<"codeMsg">>, Data, undefined), + Content = maps:get(<<"content">>, Data, undefined), + record_cmd(ReqPath, EventType, {Code, CodeMsg, Content}, Session). + +record_cmd(Path, Type, Result, #session{cmd_record = Record} = Session) -> + Record2 = Record#{?CMD_KEY(Path, Type) => Result}, + Session#session{cmd_record = Record2}. diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_timer.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_timer.erl deleted file mode 100644 index b86000292..000000000 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_timer.erl +++ /dev/null @@ -1,47 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_lwm2m_timer). - --include("src/lwm2m/include/emqx_lwm2m.hrl"). - --export([ cancel_timer/1 - , start_timer/2 - , refresh_timer/1 - , refresh_timer/2 - ]). - --record(timer_state, { interval - , tref - , message - }). - --define(LOG(Level, Format, Args), - logger:Level("LWM2M-TIMER: " ++ Format, Args)). - -cancel_timer(#timer_state{tref = TRef}) when is_reference(TRef) -> - _ = erlang:cancel_timer(TRef), ok. - -refresh_timer(State=#timer_state{interval = Interval, message = Msg}) -> - cancel_timer(State), start_timer(Interval, Msg). -refresh_timer(NewInterval, State=#timer_state{message = Msg}) -> - cancel_timer(State), start_timer(NewInterval, Msg). - -%% start timer in seconds -start_timer(Interval, Msg) -> - ?LOG(debug, "start_timer of ~p secs", [Interval]), - TRef = erlang:send_after(timer:seconds(Interval), self(), Msg), - #timer_state{interval = Interval, tref = TRef, message = Msg}. diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object.erl index 96a80735f..a4ec27413 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object.erl +++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object.erl @@ -16,7 +16,7 @@ -module(emqx_lwm2m_xml_object). --include("src/lwm2m/include/emqx_lwm2m.hrl"). +-include_lib("emqx_gateway/src/lwm2m/include/emqx_lwm2m.hrl"). -include_lib("xmerl/include/xmerl.hrl"). -export([ get_obj_def/2 @@ -38,8 +38,6 @@ get_obj_def(ObjectIdInt, true) -> get_obj_def(ObjectNameStr, false) -> emqx_lwm2m_xml_object_db:find_name(ObjectNameStr). - - get_object_id(ObjDefinition) -> [#xmlText{value=ObjectId}] = xmerl_xpath:string("ObjectID/text()", ObjDefinition), ObjectId. @@ -48,7 +46,6 @@ get_object_name(ObjDefinition) -> [#xmlText{value=ObjectName}] = xmerl_xpath:string("Name/text()", ObjDefinition), ObjectName. - get_object_and_resource_id(ResourceNameBinary, ObjDefinition) -> ResourceNameString = binary_to_list(ResourceNameBinary), [#xmlText{value=ObjectId}] = xmerl_xpath:string("ObjectID/text()", ObjDefinition), @@ -56,7 +53,6 @@ get_object_and_resource_id(ResourceNameBinary, ObjDefinition) -> ?LOG(debug, "get_object_and_resource_id ObjectId=~p, ResourceId=~p", [ObjectId, ResourceId]), {ObjectId, ResourceId}. - get_resource_type(ResourceIdInt, ObjDefinition) -> ResourceIdString = integer_to_list(ResourceIdInt), [#xmlText{value=DataType}] = xmerl_xpath:string("Resources/Item[@ID=\""++ResourceIdString++"\"]/Type/text()", ObjDefinition), diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object_db.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object_db.erl index 1d7fb6d5e..ec7c83de1 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object_db.erl +++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object_db.erl @@ -16,7 +16,7 @@ -module(emqx_lwm2m_xml_object_db). --include("src/lwm2m/include/emqx_lwm2m.hrl"). +-include_lib("emqx_gateway/src/lwm2m/include/emqx_lwm2m.hrl"). -include_lib("xmerl/include/xmerl.hrl"). % This module is for future use. Disabled now. @@ -49,15 +49,14 @@ %% API Function Definitions %% ------------------------------------------------------------------ --spec start_link(binary() | string()) -> {ok, pid()} | ignore | {error, any()}. start_link(XmlDir) -> gen_server:start_link({local, ?MODULE}, ?MODULE, [XmlDir], []). find_objectid(ObjectId) -> - ObjectIdInt = case is_list(ObjectId) of - true -> list_to_integer(ObjectId); - false -> ObjectId - end, + ObjectIdInt = case is_list(ObjectId) of + true -> list_to_integer(ObjectId); + false -> ObjectId + end, case ets:lookup(?LWM2M_OBJECT_DEF_TAB, ObjectIdInt) of [] -> {error, no_xml_definition}; [{ObjectId, Xml}] -> Xml @@ -81,15 +80,14 @@ find_name(Name) -> stop() -> gen_server:stop(?MODULE). - %% ------------------------------------------------------------------ %% gen_server Function Definitions %% ------------------------------------------------------------------ -init([XmlDir0]) -> +init([XmlDir]) -> _ = ets:new(?LWM2M_OBJECT_DEF_TAB, [set, named_table, protected]), _ = ets:new(?LWM2M_OBJECT_NAME_TO_ID_TAB, [set, named_table, protected]), - load(to_list(XmlDir0)), + load(XmlDir), {ok, #state{}}. handle_call(_Request, _From, State) -> @@ -113,11 +111,13 @@ code_change(_OldVsn, State, _Extra) -> %% Internal functions %%-------------------------------------------------------------------- load(BaseDir) -> - Wild = case lists:last(BaseDir) == $/ of - true -> BaseDir++"*.xml"; - false -> BaseDir++"/*.xml" - end, - case filelib:wildcard(Wild) of + Wild = filename:join(BaseDir, "*.xml"), + Wild2 = if is_binary(Wild) -> + erlang:binary_to_list(Wild); + true -> + Wild + end, + case filelib:wildcard(Wild2) of [] -> error(no_xml_files_found, BaseDir); AllXmlFiles -> load_loop(AllXmlFiles) end. @@ -135,13 +135,7 @@ load_loop([FileName|T]) -> ets:insert(?LWM2M_OBJECT_NAME_TO_ID_TAB, {NameBinary, ObjectId}), load_loop(T). - load_xml(FileName) -> {Xml, _Rest} = xmerl_scan:file(FileName), [ObjectXml] = xmerl_xpath:string("/LWM2M/Object", Xml), ObjectXml. - -to_list(B) when is_binary(B) -> - binary_to_list(B); -to_list(S) when is_list(S) -> - S. diff --git a/apps/emqx_gateway/src/lwm2m/include/emqx_lwm2m.hrl b/apps/emqx_gateway/src/lwm2m/include/emqx_lwm2m.hrl index 5462f489d..05e0f0503 100644 --- a/apps/emqx_gateway/src/lwm2m/include/emqx_lwm2m.hrl +++ b/apps/emqx_gateway/src/lwm2m/include/emqx_lwm2m.hrl @@ -14,15 +14,8 @@ %% limitations under the License. %%-------------------------------------------------------------------- --define(APP, emqx_lwm2m). +-define(LWAPP, emqx_lwm2m). --record(coap_mqtt_auth, { clientid - , username - , password - }). --record(lwm2m_context, { epn - , location - }). -define(OMA_ALTER_PATH_RT, <<"\"oma.lwm2m\"">>). @@ -42,7 +35,7 @@ -define(ERR_NOT_FOUND, <<"Not Found">>). -define(ERR_UNAUTHORIZED, <<"Unauthorized">>). -define(ERR_BAD_REQUEST, <<"Bad Request">>). - +-define(REG_PREFIX, <<"rd">>). -define(LWM2M_FORMAT_PLAIN_TEXT, 0). -define(LWM2M_FORMAT_LINK, 40). diff --git a/apps/emqx_gateway/src/lwm2m/lwm2m_xml/LWM2M_Access_Control-v1_0_1.xml b/apps/emqx_gateway/src/lwm2m/lwm2m_xml/LWM2M_Access_Control-v1_0_1.xml index 1f929cd98..c620a5e2a 100644 --- a/apps/emqx_gateway/src/lwm2m/lwm2m_xml/LWM2M_Access_Control-v1_0_1.xml +++ b/apps/emqx_gateway/src/lwm2m/lwm2m_xml/LWM2M_Access_Control-v1_0_1.xml @@ -1,4 +1,4 @@ - +