diff --git a/.ci/build_packages/tests.sh b/.ci/build_packages/tests.sh index 87c19621a..53ab9ac57 100755 --- a/.ci/build_packages/tests.sh +++ b/.ci/build_packages/tests.sh @@ -91,6 +91,12 @@ emqx_test(){ ;; "rpm") packagename=$(basename "${PACKAGE_PATH}/${EMQX_NAME}"-*.rpm) + + if [[ "${ARCH}" == "amd64" && $(rpm -E '%{rhel}') == 7 ]] ; + then + # EMQX OTP requires openssl11 to have TLS1.3 support + yum install -y openssl11; + fi rpm -ivh "${PACKAGE_PATH}/${packagename}" if ! rpm -q emqx | grep -q emqx; then echo "package install error" @@ -126,7 +132,7 @@ export EMQX_LOG__FILE_HANDLERS__DEFAULT__LEVEL=debug EOF ## for ARM, due to CI env issue, skip start of quic listener for the moment [[ $(arch) == *arm* || $(arch) == aarch64 ]] && tee -a "$emqx_env_vars" < $HOME/.git-credentials + git config --global credential.helper store + make emqx-ee-zip + - uses: actions/upload-artifact@v2 + with: + name: emqx-broker + path: _packages/**/*.zip + api-test: + needs: build + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + script_name: + - api_metrics + - api_subscriptions + steps: + - uses: actions/checkout@v2 + with: + repository: emqx/emqx-fvt + path: . + - uses: actions/setup-java@v1 + with: + java-version: '8.0.282' # The JDK version to make available on the path. + java-package: jdk # (jre, jdk, or jdk+fx) - defaults to jdk + architecture: x64 # (x64 or x86) - defaults to x64 + - uses: actions/download-artifact@v2 + with: + name: emqx-broker + path: . + - name: start emqx-broker + env: + EMQX_LISTENERS__WSS__DEFAULT__BIND: "0.0.0.0:8085" + run: | + unzip ./emqx/*.zip + ./emqx/bin/emqx start + - name: install jmeter + timeout-minutes: 10 + env: + JMETER_VERSION: 5.3 + run: | + wget --no-verbose --no-check-certificate -O /tmp/apache-jmeter.tgz https://downloads.apache.org/jmeter/binaries/apache-jmeter-$JMETER_VERSION.tgz + cd /tmp && tar -xvf apache-jmeter.tgz + echo "jmeter.save.saveservice.output_format=xml" >> /tmp/apache-jmeter-$JMETER_VERSION/user.properties + echo "jmeter.save.saveservice.response_data.on_error=true" >> /tmp/apache-jmeter-$JMETER_VERSION/user.properties + wget --no-verbose -O /tmp/apache-jmeter-$JMETER_VERSION/lib/ext/mqtt-xmeter-2.0.2-jar-with-dependencies.jar https://raw.githubusercontent.com/xmeter-net/mqtt-jmeter/master/Download/v2.0.2/mqtt-xmeter-2.0.2-jar-with-dependencies.jar + ln -s /tmp/apache-jmeter-$JMETER_VERSION /opt/jmeter + - name: run ${{ matrix.script_name }} + run: | + /opt/jmeter/bin/jmeter.sh \ + -Jjmeter.save.saveservice.output_format=xml -n \ + -t .ci/api-test-suite/${{ matrix.script_name }}.jmx \ + -Demqx_ip="127.0.0.1" \ + -l jmeter_logs/${{ matrix.script_name }}.jtl \ + -j jmeter_logs/logs/${{ matrix.script_name }}.log + - name: check test logs + run: | + if cat jmeter_logs/${{ matrix.script_name }}.jtl | grep -e 'true' > /dev/null 2>&1; then + grep -A 5 -B 3 'true' jmeter_logs/${{ matrix.script_name }}.jtl > jmeter_logs/${{ matrix.script_name }}_err_api.txt + echo "check logs failed" + exit 1 + fi + - uses: actions/upload-artifact@v1 + if: failure() + with: + name: jmeter_logs + path: ./jmeter_logs + - uses: actions/upload-artifact@v1 + if: failure() + with: + name: jmeter_logs + path: emqx/log + delete-package: + runs-on: ubuntu-20.04 + needs: api-test + if: always() + steps: + - uses: geekyeggo/delete-artifact@v1 + with: + name: emqx-broker diff --git a/Makefile b/Makefile index 52b0ea209..e646f5b7a 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ BUILD = $(CURDIR)/build SCRIPTS = $(CURDIR)/scripts export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh) export EMQX_DESC ?= EMQ X -export EMQX_DASHBOARD_VERSION ?= v5.0.0-beta.9 +export EMQX_DASHBOARD_VERSION ?= v5.0.0-beta.11 ifeq ($(OS),Windows_NT) export REBAR_COLOR=none endif diff --git a/README-CN.md b/README-CN.md index b430d4b5f..80e926199 100644 --- a/README-CN.md +++ b/README-CN.md @@ -4,7 +4,7 @@ [![Build Status](https://travis-ci.org/emqx/emqx.svg)](https://travis-ci.org/emqx/emqx) [![Coverage Status](https://coveralls.io/repos/github/emqx/emqx/badge.svg)](https://coveralls.io/github/emqx/emqx) [![Docker Pulls](https://img.shields.io/docker/pulls/emqx/emqx)](https://hub.docker.com/r/emqx/emqx) -[![Slack Invite]()](https://slack-invite.emqx.io) +[![Slack](https://img.shields.io/badge/Slack-EMQ%20X-39AE85?logo=slack)](https://slack-invite.emqx.io/) [![Twitter](https://img.shields.io/badge/Twitter-EMQ-1DA1F2?logo=twitter)](https://twitter.com/EMQTech) [![Community](https://img.shields.io/badge/Community-EMQ%20X-yellow)](https://askemq.com) [![YouTube](https://img.shields.io/badge/Subscribe-EMQ%20中文-FF0000?logo=youtube)](https://www.youtube.com/channel/UCir_r04HIsLjf2qqyZ4A8Cg) @@ -90,7 +90,7 @@ make eunit ct ### 执行部分应用的 common tests ```bash -make apps/emqx_bridge_mqtt-ct +make apps/emqx_retainer-ct ``` ### 静态分析(Dialyzer) diff --git a/README-JP.md b/README-JP.md index 6e1c62f2f..57c9a1809 100644 --- a/README-JP.md +++ b/README-JP.md @@ -4,7 +4,7 @@ [![Build Status](https://travis-ci.org/emqx/emqx.svg)](https://travis-ci.org/emqx/emqx) [![Coverage Status](https://coveralls.io/repos/github/emqx/emqx/badge.svg)](https://coveralls.io/github/emqx/emqx) [![Docker Pulls](https://img.shields.io/docker/pulls/emqx/emqx)](https://hub.docker.com/r/emqx/emqx) -[![Slack Invite]()](https://slack-invite.emqx.io) +[![Slack](https://img.shields.io/badge/Slack-EMQ%20X-39AE85?logo=slack)](https://slack-invite.emqx.io/) [![Twitter](https://img.shields.io/badge/Twitter-EMQ-1DA1F2?logo=twitter)](https://twitter.com/EMQTech) [![YouTube](https://img.shields.io/badge/Subscribe-EMQ-FF0000?logo=youtube)](https://www.youtube.com/channel/UC5FjR77ErAxvZENEWzQaO5Q) @@ -84,7 +84,7 @@ make eunit ct ### common test の一部を実行する ```bash -make apps/emqx_bridge_mqtt-ct +make apps/emqx_retainer-ct ``` ### Dialyzer diff --git a/README-RU.md b/README-RU.md index 2a06dac71..e02f47aa4 100644 --- a/README-RU.md +++ b/README-RU.md @@ -4,7 +4,7 @@ [![Build Status](https://travis-ci.org/emqx/emqx.svg)](https://travis-ci.org/emqx/emqx) [![Coverage Status](https://coveralls.io/repos/github/emqx/emqx/badge.svg?branch=master)](https://coveralls.io/github/emqx/emqx?branch=master) [![Docker Pulls](https://img.shields.io/docker/pulls/emqx/emqx)](https://hub.docker.com/r/emqx/emqx) -[![Slack Invite]()](https://slack-invite.emqx.io) +[![Slack](https://img.shields.io/badge/Slack-EMQ%20X-39AE85?logo=slack)](https://slack-invite.emqx.io/) [![Twitter](https://img.shields.io/badge/Follow-EMQ-1DA1F2?logo=twitter)](https://twitter.com/EMQTech) [![Community](https://img.shields.io/badge/Community-EMQ%20X-yellow?logo=github)](https://github.com/emqx/emqx/discussions) [![YouTube](https://img.shields.io/badge/Subscribe-EMQ-FF0000?logo=youtube)](https://www.youtube.com/channel/UC5FjR77ErAxvZENEWzQaO5Q) @@ -93,7 +93,7 @@ make eunit ct Пример: ```bash -make apps/emqx_bridge_mqtt-ct +make apps/emqx_retainer-ct ``` ### Dialyzer diff --git a/README.md b/README.md index 1726d426b..f60ed3cd9 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ [![Build Status](https://travis-ci.org/emqx/emqx.svg)](https://travis-ci.org/emqx/emqx) [![Coverage Status](https://coveralls.io/repos/github/emqx/emqx/badge.svg?branch=master)](https://coveralls.io/github/emqx/emqx?branch=master) [![Docker Pulls](https://img.shields.io/docker/pulls/emqx/emqx)](https://hub.docker.com/r/emqx/emqx) -[![Slack Invite]()](https://slack-invite.emqx.io) +[![Slack](https://img.shields.io/badge/Slack-EMQ%20X-39AE85?logo=slack)](https://slack-invite.emqx.io/) [![Twitter](https://img.shields.io/badge/Follow-EMQ-1DA1F2?logo=twitter)](https://twitter.com/EMQTech) [![YouTube](https://img.shields.io/badge/Subscribe-EMQ-FF0000?logo=youtube)](https://www.youtube.com/channel/UC5FjR77ErAxvZENEWzQaO5Q) @@ -92,7 +92,7 @@ make eunit ct Examples ```bash -make apps/emqx_bridge_mqtt-ct +make apps/emqx_retainer-ct ``` ### Dialyzer diff --git a/apps/emqx/etc/emqx.conf b/apps/emqx/etc/emqx.conf index 7c404ff2d..6834d2a6e 100644 --- a/apps/emqx/etc/emqx.conf +++ b/apps/emqx/etc/emqx.conf @@ -64,7 +64,7 @@ listeners.tcp.default { proxy_protocol = false ## Sets the timeout for proxy protocol. EMQ X will close the TCP connection - ## if no proxy protocol packet recevied within the timeout. + ## if no proxy protocol packet received within the timeout. ## ## @doc listeners.tcp..proxy_protocol_timeout ## ValueType: Duration @@ -163,7 +163,7 @@ listeners.ssl.default { proxy_protocol = false ## Sets the timeout for proxy protocol. EMQ X will close the TCP connection - ## if no proxy protocol packet recevied within the timeout. + ## if no proxy protocol packet received within the timeout. ## ## @doc listeners.ssl..proxy_protocol_timeout ## ValueType: Duration @@ -345,7 +345,7 @@ listeners.ws.default { proxy_protocol = false ## Sets the timeout for proxy protocol. EMQ X will close the TCP connection - ## if no proxy protocol packet recevied within the timeout. + ## if no proxy protocol packet received within the timeout. ## ## @doc listeners.ws..proxy_protocol_timeout ## ValueType: Duration @@ -448,7 +448,7 @@ listeners.wss.default { proxy_protocol = false ## Sets the timeout for proxy protocol. EMQ X will close the TCP connection - ## if no proxy protocol packet recevied within the timeout. + ## if no proxy protocol packet received within the timeout. ## ## @doc listeners.wss..proxy_protocol_timeout ## ValueType: Duration diff --git a/apps/emqx/include/emqx.hrl b/apps/emqx/include/emqx.hrl index 633527b57..550e650a2 100644 --- a/apps/emqx/include/emqx.hrl +++ b/apps/emqx/include/emqx.hrl @@ -134,3 +134,19 @@ }). -endif. + +%%-------------------------------------------------------------------- +%% Authentication +%%-------------------------------------------------------------------- + +-record(authenticator, + { id :: binary() + , provider :: module() + , enable :: boolean() + , state :: map() + }). + +-record(chain, + { name :: atom() + , authenticators :: [#authenticator{}] + }). \ No newline at end of file diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index 271558f6d..bb3a588a9 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -15,7 +15,7 @@ , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.2"}}} , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.10.8"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.1"}}} - , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.14.0"}}} + , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.15.0"}}} , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}} , {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.14.1"}}} diff --git a/apps/emqx/src/emqx_access_control.erl b/apps/emqx/src/emqx_access_control.erl index 7d5b009ba..914651535 100644 --- a/apps/emqx/src/emqx_access_control.erl +++ b/apps/emqx/src/emqx_access_control.erl @@ -29,9 +29,9 @@ -spec(authenticate(emqx_types:clientinfo()) -> {ok, map()} | {ok, map(), binary()} | {continue, map()} | {continue, binary(), map()} | {error, term()}). authenticate(Credential) -> - case run_hooks('client.authenticate', [Credential], {ok, #{superuser => false}}) of + case run_hooks('client.authenticate', [Credential], {ok, #{is_superuser => false}}) of ok -> - {ok, #{superuser => false}}; + {ok, #{is_superuser => false}}; Other -> Other end. diff --git a/apps/emqx/src/emqx_alarm.erl b/apps/emqx/src/emqx_alarm.erl index 10bb6facc..6bd2d5d49 100644 --- a/apps/emqx/src/emqx_alarm.erl +++ b/apps/emqx/src/emqx_alarm.erl @@ -159,6 +159,7 @@ format(#activated_alarm{name = Name, message = Message, activate_at = At, detail name => Name, message => Message, duration => (Now - At) div 1000, %% to millisecond + activate_at => to_rfc3339(At), details => Details }; format(#deactivated_alarm{name = Name, message = Message, activate_at = At, details = Details, @@ -168,11 +169,16 @@ format(#deactivated_alarm{name = Name, message = Message, activate_at = At, deta name => Name, message => Message, duration => DAt - At, + activate_at => to_rfc3339(At), + deactivate_at => to_rfc3339(DAt), details => Details }; format(_) -> {error, unknow_alarm}. +to_rfc3339(Timestamp) -> + list_to_binary(calendar:system_time_to_rfc3339(Timestamp div 1000, [{unit, millisecond}])). + %%-------------------------------------------------------------------- %% gen_server callbacks %%-------------------------------------------------------------------- diff --git a/apps/emqx/src/emqx_authentication.erl b/apps/emqx/src/emqx_authentication.erl new file mode 100644 index 000000000..8cc8cf2df --- /dev/null +++ b/apps/emqx/src/emqx_authentication.erl @@ -0,0 +1,735 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_authentication). + +-behaviour(gen_server). +-behaviour(hocon_schema). +-behaviour(emqx_config_handler). + +-include("emqx.hrl"). +-include("logger.hrl"). + +-export([ roots/0 + , fields/1 + ]). + +-export([ pre_config_update/2 + , post_config_update/4 + ]). + +-export([ authenticate/2 + ]). + +-export([ initialize_authentication/2 ]). + +-export([ start_link/0 + , stop/0 + ]). + +-export([ add_provider/2 + , remove_provider/1 + , create_chain/1 + , delete_chain/1 + , lookup_chain/1 + , list_chains/0 + , create_authenticator/2 + , delete_authenticator/2 + , update_authenticator/3 + , lookup_authenticator/2 + , list_authenticators/1 + , move_authenticator/3 + ]). + +-export([ import_users/3 + , add_user/3 + , delete_user/3 + , update_user/4 + , lookup_user/3 + , list_users/2 + ]). + +-export([ generate_id/1 ]). + +%% gen_server callbacks +-export([ init/1 + , handle_call/3 + , handle_cast/2 + , handle_info/2 + , terminate/2 + , code_change/3 + ]). + +-define(CHAINS_TAB, emqx_authn_chains). + +-define(VER_1, <<"1">>). +-define(VER_2, <<"2">>). + +-type config() :: #{atom() => term()}. +-type state() :: #{atom() => term()}. +-type extra() :: #{is_superuser := boolean(), + atom() => term()}. +-type user_info() :: #{user_id := binary(), + atom() => term()}. + +-callback refs() -> [{ref, Module, Name}] when Module::module(), Name::atom(). + +-callback create(Config) + -> {ok, State} + | {error, term()} + when Config::config(), State::state(). + +-callback update(Config, State) + -> {ok, NewState} + | {error, term()} + when Config::config(), State::state(), NewState::state(). + +-callback authenticate(Credential, State) + -> ignore + | {ok, Extra} + | {ok, Extra, AuthData} + | {continue, AuthCache} + | {continue, AuthData, AuthCache} + | {error, term()} + when Credential::map(), State::state(), Extra::extra(), AuthData::binary(), AuthCache::map(). + +-callback destroy(State) + -> ok + when State::state(). + +-callback import_users(Filename, State) + -> ok + | {error, term()} + when Filename::binary(), State::state(). + +-callback add_user(UserInfo, State) + -> {ok, User} + | {error, term()} + when UserInfo::user_info(), State::state(), User::user_info(). + +-callback delete_user(UserID, State) + -> ok + | {error, term()} + when UserID::binary(), State::state(). + +-callback update_user(UserID, UserInfo, State) + -> {ok, User} + | {error, term()} + when UserID::binary, UserInfo::map(), State::state(), User::user_info(). + +-callback list_users(State) + -> {ok, Users} + when State::state(), Users::[user_info()]. + +-optional_callbacks([ import_users/2 + , add_user/2 + , delete_user/2 + , update_user/3 + , list_users/1 + ]). + +%%------------------------------------------------------------------------------ +%% Hocon Schema +%%------------------------------------------------------------------------------ + +roots() -> [{authentication, fun authentication/1}]. + +fields(_) -> []. + +authentication(type) -> + {ok, Refs} = get_refs(), + hoconsc:union([hoconsc:array(hoconsc:union(Refs)) | Refs]); +authentication(default) -> []; +authentication(_) -> undefined. + +%%------------------------------------------------------------------------------ +%% Callbacks of config handler +%%------------------------------------------------------------------------------ + +pre_config_update(UpdateReq, OldConfig) -> + case do_pre_config_update(UpdateReq, to_list(OldConfig)) of + {error, Reason} -> {error, Reason}; + {ok, NewConfig} -> {ok, may_to_map(NewConfig)} + end. + +do_pre_config_update({create_authenticator, _ChainName, Config}, OldConfig) -> + {ok, OldConfig ++ [Config]}; +do_pre_config_update({delete_authenticator, _ChainName, AuthenticatorID}, OldConfig) -> + NewConfig = lists:filter(fun(OldConfig0) -> + AuthenticatorID =/= generate_id(OldConfig0) + end, OldConfig), + {ok, NewConfig}; +do_pre_config_update({update_authenticator, _ChainName, AuthenticatorID, Config}, OldConfig) -> + NewConfig = lists:map(fun(OldConfig0) -> + case AuthenticatorID =:= generate_id(OldConfig0) of + true -> maps:merge(OldConfig0, Config); + false -> OldConfig0 + end + end, OldConfig), + {ok, NewConfig}; +do_pre_config_update({move_authenticator, _ChainName, AuthenticatorID, Position}, OldConfig) -> + case split_by_id(AuthenticatorID, OldConfig) of + {error, Reason} -> {error, Reason}; + {ok, Part1, [Found | Part2]} -> + case Position of + <<"top">> -> + {ok, [Found | Part1] ++ Part2}; + <<"bottom">> -> + {ok, Part1 ++ Part2 ++ [Found]}; + <<"before:", Before/binary>> -> + case split_by_id(Before, Part1 ++ Part2) of + {error, Reason} -> + {error, Reason}; + {ok, NPart1, [NFound | NPart2]} -> + {ok, NPart1 ++ [Found, NFound | NPart2]} + end; + _ -> + {error, {invalid_parameter, position}} + end + end. + +post_config_update(UpdateReq, NewConfig, OldConfig, AppEnvs) -> + do_post_config_update(UpdateReq, check_config(to_list(NewConfig)), OldConfig, AppEnvs). + +do_post_config_update({create_authenticator, ChainName, Config}, _NewConfig, _OldConfig, _AppEnvs) -> + NConfig = check_config(Config), + _ = create_chain(ChainName), + create_authenticator(ChainName, NConfig); + +do_post_config_update({delete_authenticator, ChainName, AuthenticatorID}, _NewConfig, _OldConfig, _AppEnvs) -> + delete_authenticator(ChainName, AuthenticatorID); + +do_post_config_update({update_authenticator, ChainName, AuthenticatorID, _Config}, NewConfig, _OldConfig, _AppEnvs) -> + [Config] = lists:filter(fun(NewConfig0) -> + AuthenticatorID =:= generate_id(NewConfig0) + end, NewConfig), + NConfig = check_config(Config), + update_authenticator(ChainName, AuthenticatorID, NConfig); + +do_post_config_update({move_authenticator, ChainName, AuthenticatorID, Position}, _NewConfig, _OldConfig, _AppEnvs) -> + NPosition = case Position of + <<"top">> -> top; + <<"bottom">> -> bottom; + <<"before:", Before/binary>> -> + {before, Before} + end, + move_authenticator(ChainName, AuthenticatorID, NPosition). + +check_config(Config) -> + #{authentication := CheckedConfig} = hocon_schema:check_plain(emqx_authentication, + #{<<"authentication">> => Config}, #{nullable => true, atom_key => true}), + CheckedConfig. + +%%------------------------------------------------------------------------------ +%% Authenticate +%%------------------------------------------------------------------------------ + +authenticate(#{listener := Listener, protocol := Protocol} = Credential, _AuthResult) -> + case ets:lookup(?CHAINS_TAB, Listener) of + [#chain{authenticators = Authenticators}] when Authenticators =/= [] -> + do_authenticate(Authenticators, Credential); + _ -> + case ets:lookup(?CHAINS_TAB, global_chain(Protocol)) of + [#chain{authenticators = Authenticators}] when Authenticators =/= [] -> + do_authenticate(Authenticators, Credential); + _ -> + ignore + end + end. + +do_authenticate([], _) -> + {stop, {error, not_authorized}}; +do_authenticate([#authenticator{provider = Provider, state = State} | More], Credential) -> + case Provider:authenticate(Credential, State) of + ignore -> + do_authenticate(More, Credential); + Result -> + %% {ok, Extra} + %% {ok, Extra, AuthData} + %% {continue, AuthCache} + %% {continue, AuthData, AuthCache} + %% {error, Reason} + {stop, Result} + end. + +%%------------------------------------------------------------------------------ +%% APIs +%%------------------------------------------------------------------------------ + +initialize_authentication(_, []) -> + ok; +initialize_authentication(ChainName, AuthenticatorsConfig) -> + _ = create_chain(ChainName), + CheckedConfig = check_config(to_list(AuthenticatorsConfig)), + lists:foreach(fun(AuthenticatorConfig) -> + case create_authenticator(ChainName, AuthenticatorConfig) of + {ok, _} -> + ok; + {error, Reason} -> + ?LOG(error, "Failed to create authenticator '~s': ~p", [generate_id(AuthenticatorConfig), Reason]) + end + end, CheckedConfig). + +start_link() -> + gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). + +stop() -> + gen_server:stop(?MODULE). + +get_refs() -> + gen_server:call(?MODULE, get_refs). + +add_provider(AuthNType, Provider) -> + gen_server:call(?MODULE, {add_provider, AuthNType, Provider}). + +remove_provider(AuthNType) -> + gen_server:call(?MODULE, {remove_provider, AuthNType}). + +create_chain(Name) -> + gen_server:call(?MODULE, {create_chain, Name}). + +delete_chain(Name) -> + gen_server:call(?MODULE, {delete_chain, Name}). + +lookup_chain(Name) -> + gen_server:call(?MODULE, {lookup_chain, Name}). + +list_chains() -> + Chains = ets:tab2list(?CHAINS_TAB), + {ok, [serialize_chain(Chain) || Chain <- Chains]}. + +create_authenticator(ChainName, Config) -> + gen_server:call(?MODULE, {create_authenticator, ChainName, Config}). + +delete_authenticator(ChainName, AuthenticatorID) -> + gen_server:call(?MODULE, {delete_authenticator, ChainName, AuthenticatorID}). + +update_authenticator(ChainName, AuthenticatorID, Config) -> + gen_server:call(?MODULE, {update_authenticator, ChainName, AuthenticatorID, Config}). + +lookup_authenticator(ChainName, AuthenticatorID) -> + case ets:lookup(?CHAINS_TAB, ChainName) of + [] -> + {error, {not_found, {chain, ChainName}}}; + [#chain{authenticators = Authenticators}] -> + case lists:keyfind(AuthenticatorID, #authenticator.id, Authenticators) of + false -> + {error, {not_found, {authenticator, AuthenticatorID}}}; + Authenticator -> + {ok, serialize_authenticator(Authenticator)} + end + end. + +list_authenticators(ChainName) -> + case ets:lookup(?CHAINS_TAB, ChainName) of + [] -> + {error, {not_found, {chain, ChainName}}}; + [#chain{authenticators = Authenticators}] -> + {ok, serialize_authenticators(Authenticators)} + end. + +move_authenticator(ChainName, AuthenticatorID, Position) -> + gen_server:call(?MODULE, {move_authenticator, ChainName, AuthenticatorID, Position}). + +import_users(ChainName, AuthenticatorID, Filename) -> + gen_server:call(?MODULE, {import_users, ChainName, AuthenticatorID, Filename}). + +add_user(ChainName, AuthenticatorID, UserInfo) -> + gen_server:call(?MODULE, {add_user, ChainName, AuthenticatorID, UserInfo}). + +delete_user(ChainName, AuthenticatorID, UserID) -> + gen_server:call(?MODULE, {delete_user, ChainName, AuthenticatorID, UserID}). + +update_user(ChainName, AuthenticatorID, UserID, NewUserInfo) -> + gen_server:call(?MODULE, {update_user, ChainName, AuthenticatorID, UserID, NewUserInfo}). + +lookup_user(ChainName, AuthenticatorID, UserID) -> + gen_server:call(?MODULE, {lookup_user, ChainName, AuthenticatorID, UserID}). + +%% TODO: Support pagination +list_users(ChainName, AuthenticatorID) -> + gen_server:call(?MODULE, {list_users, ChainName, AuthenticatorID}). + +generate_id(#{mechanism := Mechanism0, backend := Backend0}) -> + Mechanism = atom_to_binary(Mechanism0), + Backend = atom_to_binary(Backend0), + <>; +generate_id(#{mechanism := Mechanism}) -> + atom_to_binary(Mechanism); +generate_id(#{<<"mechanism">> := Mechanism, <<"backend">> := Backend}) -> + <>; +generate_id(#{<<"mechanism">> := Mechanism}) -> + Mechanism. + +%%-------------------------------------------------------------------- +%% gen_server callbacks +%%-------------------------------------------------------------------- + +init(_Opts) -> + _ = ets:new(?CHAINS_TAB, [ named_table, set, public + , {keypos, #chain.name} + , {read_concurrency, true}]), + ok = emqx_config_handler:add_handler([authentication], ?MODULE), + ok = emqx_config_handler:add_handler([listeners, '?', '?', authentication], ?MODULE), + {ok, #{hooked => false, providers => #{}}}. + +handle_call({add_provider, AuthNType, Provider}, _From, #{providers := Providers} = State) -> + reply(ok, State#{providers := Providers#{AuthNType => Provider}}); + +handle_call({remove_provider, AuthNType}, _From, #{providers := Providers} = State) -> + reply(ok, State#{providers := maps:remove(AuthNType, Providers)}); + +handle_call(get_refs, _From, #{providers := Providers} = State) -> + Refs = lists:foldl(fun({_, Provider}, Acc) -> + Acc ++ Provider:refs() + end, [], maps:to_list(Providers)), + reply({ok, Refs}, State); + +handle_call({create_chain, Name}, _From, State) -> + case ets:member(?CHAINS_TAB, Name) of + true -> + reply({error, {already_exists, {chain, Name}}}, State); + false -> + Chain = #chain{name = Name, + authenticators = []}, + true = ets:insert(?CHAINS_TAB, Chain), + reply({ok, serialize_chain(Chain)}, State) + end; + +handle_call({delete_chain, Name}, _From, State) -> + case ets:lookup(?CHAINS_TAB, Name) of + [] -> + reply({error, {not_found, {chain, Name}}}, State); + [#chain{authenticators = Authenticators}] -> + _ = [do_delete_authenticator(Authenticator) || Authenticator <- Authenticators], + true = ets:delete(?CHAINS_TAB, Name), + reply(ok, may_unhook(State)) + end; + +handle_call({lookup_chain, Name}, _From, State) -> + case ets:lookup(?CHAINS_TAB, Name) of + [] -> + reply({error, {not_found, {chain, Name}}}, State); + [Chain] -> + reply({ok, serialize_chain(Chain)}, State) + end; + +handle_call({create_authenticator, ChainName, Config}, _From, #{providers := Providers} = State) -> + UpdateFun = + fun(#chain{authenticators = Authenticators} = Chain) -> + AuthenticatorID = generate_id(Config), + case lists:keymember(AuthenticatorID, #authenticator.id, Authenticators) of + true -> + {error, {already_exists, {authenticator, AuthenticatorID}}}; + false -> + case do_create_authenticator(ChainName, AuthenticatorID, Config, Providers) of + {ok, Authenticator} -> + NAuthenticators = Authenticators ++ [Authenticator], + true = ets:insert(?CHAINS_TAB, Chain#chain{authenticators = NAuthenticators}), + {ok, serialize_authenticator(Authenticator)}; + {error, Reason} -> + {error, Reason} + end + end + end, + Reply = update_chain(ChainName, UpdateFun), + reply(Reply, may_hook(State)); + +handle_call({delete_authenticator, ChainName, AuthenticatorID}, _From, State) -> + UpdateFun = + fun(#chain{authenticators = Authenticators} = Chain) -> + case lists:keytake(AuthenticatorID, #authenticator.id, Authenticators) of + false -> + {error, {not_found, {authenticator, AuthenticatorID}}}; + {value, Authenticator, NAuthenticators} -> + _ = do_delete_authenticator(Authenticator), + true = ets:insert(?CHAINS_TAB, Chain#chain{authenticators = NAuthenticators}), + ok + end + end, + Reply = update_chain(ChainName, UpdateFun), + reply(Reply, may_unhook(State)); + +handle_call({update_authenticator, ChainName, AuthenticatorID, Config}, _From, State) -> + UpdateFun = + fun(#chain{authenticators = Authenticators} = Chain) -> + case lists:keyfind(AuthenticatorID, #authenticator.id, Authenticators) of + false -> + {error, {not_found, {authenticator, AuthenticatorID}}}; + #authenticator{provider = Provider, + state = #{version := Version} = ST} = Authenticator -> + case AuthenticatorID =:= generate_id(Config) of + true -> + Unique = unique(ChainName, AuthenticatorID, Version), + case Provider:update(Config#{'_unique' => Unique}, ST) of + {ok, NewST} -> + NewAuthenticator = Authenticator#authenticator{state = switch_version(NewST)}, + NewAuthenticators = replace_authenticator(AuthenticatorID, NewAuthenticator, Authenticators), + true = ets:insert(?CHAINS_TAB, Chain#chain{authenticators = NewAuthenticators}), + {ok, serialize_authenticator(NewAuthenticator)}; + {error, Reason} -> + {error, Reason} + end; + false -> + {error, mechanism_or_backend_change_is_not_alloed} + end + end + end, + Reply = update_chain(ChainName, UpdateFun), + reply(Reply, State); + +handle_call({move_authenticator, ChainName, AuthenticatorID, Position}, _From, State) -> + UpdateFun = + fun(#chain{authenticators = Authenticators} = Chain) -> + case do_move_authenticator(AuthenticatorID, Authenticators, Position) of + {ok, NAuthenticators} -> + true = ets:insert(?CHAINS_TAB, Chain#chain{authenticators = NAuthenticators}), + ok; + {error, Reason} -> + {error, Reason} + end + end, + Reply = update_chain(ChainName, UpdateFun), + reply(Reply, State); + +handle_call({import_users, ChainName, AuthenticatorID, Filename}, _From, State) -> + Reply = call_authenticator(ChainName, AuthenticatorID, import_users, [Filename]), + reply(Reply, State); + +handle_call({add_user, ChainName, AuthenticatorID, UserInfo}, _From, State) -> + Reply = call_authenticator(ChainName, AuthenticatorID, add_user, [UserInfo]), + reply(Reply, State); + +handle_call({delete_user, ChainName, AuthenticatorID, UserID}, _From, State) -> + Reply = call_authenticator(ChainName, AuthenticatorID, delete_user, [UserID]), + reply(Reply, State); + +handle_call({update_user, ChainName, AuthenticatorID, UserID, NewUserInfo}, _From, State) -> + Reply = call_authenticator(ChainName, AuthenticatorID, update_user, [UserID, NewUserInfo]), + reply(Reply, State); + +handle_call({lookup_user, ChainName, AuthenticatorID, UserID}, _From, State) -> + Reply = call_authenticator(ChainName, AuthenticatorID, lookup_user, [UserID]), + reply(Reply, State); + +handle_call({list_users, ChainName, AuthenticatorID}, _From, State) -> + Reply = call_authenticator(ChainName, AuthenticatorID, list_users, []), + reply(Reply, State); + +handle_call(Req, _From, State) -> + ?LOG(error, "Unexpected call: ~p", [Req]), + {reply, ignored, State}. + +handle_cast(Req, State) -> + ?LOG(error, "Unexpected case: ~p", [Req]), + {noreply, State}. + +handle_info(Info, State) -> + ?LOG(error, "Unexpected info: ~p", [Info]), + {noreply, State}. + +terminate(_Reason, _State) -> + emqx_config_handler:remove_handler([authentication]), + emqx_config_handler:remove_handler([listeners, '?', '?', authentication]), + ok. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +reply(Reply, State) -> + {reply, Reply, State}. + +%%------------------------------------------------------------------------------ +%% Internal functions +%%------------------------------------------------------------------------------ + +split_by_id(ID, AuthenticatorsConfig) -> + case lists:foldl( + fun(C, {P1, P2, F0}) -> + F = case ID =:= generate_id(C) of + true -> true; + false -> F0 + end, + case F of + false -> {[C | P1], P2, F}; + true -> {P1, [C | P2], F} + end + end, {[], [], false}, AuthenticatorsConfig) of + {_, _, false} -> + {error, {not_found, {authenticator, ID}}}; + {Part1, Part2, true} -> + {ok, lists:reverse(Part1), lists:reverse(Part2)} + end. + +global_chain(mqtt) -> + 'mqtt:global'; +global_chain('mqtt-sn') -> + 'mqtt-sn:global'; +global_chain(coap) -> + 'coap:global'; +global_chain(lwm2m) -> + 'lwm2m:global'; +global_chain(stomp) -> + 'stomp:global'; +global_chain(_) -> + 'unknown:global'. + +may_hook(#{hooked := false} = State) -> + case lists:any(fun(#chain{authenticators = []}) -> false; + (_) -> true + end, ets:tab2list(?CHAINS_TAB)) of + true -> + _ = emqx:hook('client.authenticate', {emqx_authentication, authenticate, []}), + State#{hooked => true}; + false -> + State + end; +may_hook(State) -> + State. + +may_unhook(#{hooked := true} = State) -> + case lists:all(fun(#chain{authenticators = []}) -> true; + (_) -> false + end, ets:tab2list(?CHAINS_TAB)) of + true -> + _ = emqx:unhook('client.authenticate', {emqx_authentication, authenticate, []}), + State#{hooked => false}; + false -> + State + end; +may_unhook(State) -> + State. + +do_create_authenticator(ChainName, AuthenticatorID, #{enable := Enable} = Config, Providers) -> + case maps:get(authn_type(Config), Providers, undefined) of + undefined -> + {error, no_available_provider}; + Provider -> + Unique = unique(ChainName, AuthenticatorID, ?VER_1), + case Provider:create(Config#{'_unique' => Unique}) of + {ok, State} -> + Authenticator = #authenticator{id = AuthenticatorID, + provider = Provider, + enable = Enable, + state = switch_version(State)}, + {ok, Authenticator}; + {error, Reason} -> + {error, Reason} + end + end. + +do_delete_authenticator(#authenticator{provider = Provider, state = State}) -> + _ = Provider:destroy(State), + ok. + +replace_authenticator(ID, Authenticator, Authenticators) -> + lists:keyreplace(ID, #authenticator.id, Authenticators, Authenticator). + +do_move_authenticator(ID, Authenticators, Position) -> + case lists:keytake(ID, #authenticator.id, Authenticators) of + false -> + {error, {not_found, {authenticator, ID}}}; + {value, Authenticator, NAuthenticators} -> + case Position of + top -> + {ok, [Authenticator | NAuthenticators]}; + bottom -> + {ok, NAuthenticators ++ [Authenticator]}; + {before, ID0} -> + insert(Authenticator, NAuthenticators, ID0, []) + end + end. + +insert(_, [], ID, _) -> + {error, {not_found, {authenticator, ID}}}; +insert(Authenticator, [#authenticator{id = ID} | _] = Authenticators, ID, Acc) -> + {ok, lists:reverse(Acc) ++ [Authenticator | Authenticators]}; +insert(Authenticator, [Authenticator0 | More], ID, Acc) -> + insert(Authenticator, More, ID, [Authenticator0 | Acc]). + +update_chain(ChainName, UpdateFun) -> + case ets:lookup(?CHAINS_TAB, ChainName) of + [] -> + {error, {not_found, {chain, ChainName}}}; + [Chain] -> + UpdateFun(Chain) + end. + +call_authenticator(ChainName, AuthenticatorID, Func, Args) -> + UpdateFun = + fun(#chain{authenticators = Authenticators}) -> + case lists:keyfind(AuthenticatorID, #authenticator.id, Authenticators) of + false -> + {error, {not_found, {authenticator, AuthenticatorID}}}; + #authenticator{provider = Provider, state = State} -> + case erlang:function_exported(Provider, Func, length(Args) + 1) of + true -> + erlang:apply(Provider, Func, Args ++ [State]); + false -> + {error, unsupported_feature} + end + end + end, + update_chain(ChainName, UpdateFun). + +serialize_chain(#chain{name = Name, + authenticators = Authenticators}) -> + #{ name => Name + , authenticators => serialize_authenticators(Authenticators) + }. + +serialize_authenticators(Authenticators) -> + [serialize_authenticator(Authenticator) || Authenticator <- Authenticators]. + +serialize_authenticator(#authenticator{id = ID, + provider = Provider, + enable = Enable, + state = State}) -> + #{ id => ID + , provider => Provider + , enable => Enable + , state => State + }. + +unique(ChainName, AuthenticatorID, Version) -> + NChainName = atom_to_binary(ChainName), + <>. + +switch_version(State = #{version := ?VER_1}) -> + State#{version := ?VER_2}; +switch_version(State = #{version := ?VER_2}) -> + State#{version := ?VER_1}; +switch_version(State) -> + State#{version => ?VER_1}. + +authn_type(#{mechanism := Mechanism, backend := Backend}) -> + {Mechanism, Backend}; +authn_type(#{mechanism := Mechanism}) -> + Mechanism. + +may_to_map([L]) -> + L; +may_to_map(L) -> + L. + +to_list(undefined) -> + []; +to_list(M) when M =:= #{} -> + []; +to_list(M) when is_map(M) -> + [M]; +to_list(L) when is_list(L) -> + L. diff --git a/apps/emqx/src/emqx_banned.erl b/apps/emqx/src/emqx_banned.erl index 715548d41..608734363 100644 --- a/apps/emqx/src/emqx_banned.erl +++ b/apps/emqx/src/emqx_banned.erl @@ -37,6 +37,7 @@ , delete/1 , info/1 , format/1 + , parse/1 ]). %% gen_server callbacks @@ -107,6 +108,33 @@ format(#banned{who = Who0, until => to_rfc3339(Until) }. +parse(Params) -> + Who = pares_who(Params), + By = maps:get(<<"by">>, Params, <<"mgmt_api">>), + Reason = maps:get(<<"reason">>, Params, <<"">>), + At = pares_time(maps:get(<<"at">>, Params, undefined), erlang:system_time(second)), + Until = pares_time(maps:get(<<"until">>, Params, undefined), At + 5 * 60), + #banned{ + who = Who, + by = By, + reason = Reason, + at = At, + until = Until + }. + +pares_who(#{as := As, who := Who}) -> + pares_who(#{<<"as">> => As, <<"who">> => Who}); +pares_who(#{<<"as">> := <<"peerhost">>, <<"who">> := Peerhost0}) -> + {ok, Peerhost} = inet:parse_address(binary_to_list(Peerhost0)), + {peerhost, Peerhost}; +pares_who(#{<<"as">> := As, <<"who">> := Who}) -> + {binary_to_atom(As, utf8), Who}. + +pares_time(undefined, Default) -> + Default; +pares_time(Rfc3339, _Default) -> + to_timestamp(Rfc3339). + maybe_format_host({peerhost, Host}) -> AddrBinary = list_to_binary(inet:ntoa(Host)), {peerhost, AddrBinary}; @@ -116,6 +144,11 @@ maybe_format_host({As, Who}) -> to_rfc3339(Timestamp) -> list_to_binary(calendar:system_time_to_rfc3339(Timestamp, [{unit, second}])). +to_timestamp(Rfc3339) when is_binary(Rfc3339) -> + to_timestamp(binary_to_list(Rfc3339)); +to_timestamp(Rfc3339) -> + calendar:rfc3339_to_system_time(Rfc3339, [{unit, second}]). + -spec(create(emqx_types:banned() | map()) -> ok). create(#{who := Who, by := By, @@ -130,12 +163,16 @@ create(#{who := Who, create(Banned) when is_record(Banned, banned) -> ekka_mnesia:dirty_write(?BANNED_TAB, Banned). +look_up(Who) when is_map(Who) -> + look_up(pares_who(Who)); look_up(Who) -> mnesia:dirty_read(?BANNED_TAB, Who). -spec(delete({clientid, emqx_types:clientid()} | {username, emqx_types:username()} | {peerhost, emqx_types:peerhost()}) -> ok). +delete(Who) when is_map(Who)-> + delete(pares_who(Who)); delete(Who) -> ekka_mnesia:dirty_delete(?BANNED_TAB, Who). diff --git a/apps/emqx/src/emqx_broker_sup.erl b/apps/emqx/src/emqx_broker_sup.erl index 69df72408..a479e9ff1 100644 --- a/apps/emqx/src/emqx_broker_sup.erl +++ b/apps/emqx/src/emqx_broker_sup.erl @@ -43,6 +43,14 @@ init([]) -> type => worker, modules => [emqx_shared_sub]}, + %% Authentication + AuthN = #{id => authn, + start => {emqx_authentication, start_link, []}, + restart => permanent, + shutdown => 2000, + type => worker, + modules => [emqx_authentication]}, + %% Broker helper Helper = #{id => helper, start => {emqx_broker_helper, start_link, []}, @@ -51,5 +59,5 @@ init([]) -> type => worker, modules => [emqx_broker_helper]}, - {ok, {{one_for_all, 0, 1}, [BrokerPool, SharedSub, Helper]}}. + {ok, {{one_for_all, 0, 1}, [BrokerPool, SharedSub, AuthN, Helper]}}. diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl index e25a9c8d6..0b1ff7e25 100644 --- a/apps/emqx/src/emqx_channel.erl +++ b/apps/emqx/src/emqx_channel.erl @@ -214,7 +214,7 @@ init(ConnInfo = #{peername := {PeerHost, _Port}, ClientInfo = set_peercert_infos( Peercert, #{zone => Zone, - listener => Listener, + listener => emqx_listeners:listener_id(Type, Listener), protocol => Protocol, peerhost => PeerHost, sockport => SockPort, @@ -223,7 +223,7 @@ init(ConnInfo = #{peername := {PeerHost, _Port}, mountpoint => MountPoint, is_bridge => false, is_superuser => false - }, Zone, Listener), + }, Zone), {NClientInfo, NConnInfo} = take_ws_cookie(ClientInfo, ConnInfo), #channel{conninfo = NConnInfo, clientinfo = NClientInfo, @@ -244,12 +244,12 @@ quota_policy(RawPolicy) -> erlang:trunc(hocon_postprocess:duration(StrWind) / 1000)}} || {Name, [StrCount, StrWind]} <- maps:to_list(RawPolicy)]. -set_peercert_infos(NoSSL, ClientInfo, _, _) +set_peercert_infos(NoSSL, ClientInfo, _) when NoSSL =:= nossl; NoSSL =:= undefined -> ClientInfo#{username => undefined}; -set_peercert_infos(Peercert, ClientInfo, Zone, _Listener) -> +set_peercert_infos(Peercert, ClientInfo, Zone) -> {DN, CN} = {esockd_peercert:subject(Peercert), esockd_peercert:common_name(Peercert)}, PeercetAs = fun(Key) -> @@ -1303,11 +1303,11 @@ do_authenticate(#{auth_method := AuthMethod} = Credential, #channel{clientinfo = case emqx_access_control:authenticate(Credential) of {ok, Result} -> {ok, Properties, - Channel#channel{clientinfo = ClientInfo#{is_superuser => maps:get(superuser, Result, false)}, + Channel#channel{clientinfo = ClientInfo#{is_superuser => maps:get(is_superuser, Result, false)}, auth_cache = #{}}}; {ok, Result, AuthData} -> {ok, Properties#{'Authentication-Data' => AuthData}, - Channel#channel{clientinfo = ClientInfo#{is_superuser => maps:get(superuser, Result, false)}, + Channel#channel{clientinfo = ClientInfo#{is_superuser => maps:get(is_superuser, Result, false)}, auth_cache = #{}}}; {continue, AuthCache} -> {continue, Properties, Channel#channel{auth_cache = AuthCache}}; @@ -1320,8 +1320,8 @@ do_authenticate(#{auth_method := AuthMethod} = Credential, #channel{clientinfo = do_authenticate(Credential, #channel{clientinfo = ClientInfo} = Channel) -> case emqx_access_control:authenticate(Credential) of - {ok, #{superuser := Superuser}} -> - {ok, #{}, Channel#channel{clientinfo = ClientInfo#{is_superuser => Superuser}}}; + {ok, #{is_superuser := IsSuperuser}} -> + {ok, #{}, Channel#channel{clientinfo = ClientInfo#{is_superuser => IsSuperuser}}}; {error, Reason} -> {error, emqx_reason_codes:connack_error(Reason)} end. diff --git a/apps/emqx/src/emqx_cm_sup.erl b/apps/emqx/src/emqx_cm_sup.erl index f332a0868..cddd8aa5e 100644 --- a/apps/emqx/src/emqx_cm_sup.erl +++ b/apps/emqx/src/emqx_cm_sup.erl @@ -22,49 +22,38 @@ -export([init/1]). +%%-------------------------------------------------------------------- +%% API +%%-------------------------------------------------------------------- + start_link() -> supervisor:start_link({local, ?MODULE}, ?MODULE, []). +%%-------------------------------------------------------------------- +%% Supervisor callbacks +%%-------------------------------------------------------------------- + init([]) -> - Banned = #{id => banned, - start => {emqx_banned, start_link, []}, - restart => permanent, - shutdown => 1000, - type => worker, - modules => [emqx_banned]}, - Flapping = #{id => flapping, - start => {emqx_flapping, start_link, []}, - restart => permanent, - shutdown => 1000, - type => worker, - modules => [emqx_flapping]}, - %% Channel locker - Locker = #{id => locker, - start => {emqx_cm_locker, start_link, []}, - restart => permanent, - shutdown => 5000, - type => worker, - modules => [emqx_cm_locker] - }, - %% Channel registry - Registry = #{id => registry, - start => {emqx_cm_registry, start_link, []}, - restart => permanent, - shutdown => 5000, - type => worker, - modules => [emqx_cm_registry] - }, - %% Channel Manager - Manager = #{id => manager, - start => {emqx_cm, start_link, []}, - restart => permanent, - shutdown => 5000, - type => worker, - modules => [emqx_cm] - }, SupFlags = #{strategy => one_for_one, intensity => 100, period => 10 }, + Banned = child_spec(emqx_banned, 1000, worker), + Flapping = child_spec(emqx_flapping, 1000, worker), + Locker = child_spec(emqx_cm_locker, 5000, worker), + Registry = child_spec(emqx_cm_registry, 5000, worker), + Manager = child_spec(emqx_cm, 5000, worker), {ok, {SupFlags, [Banned, Flapping, Locker, Registry, Manager]}}. +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- + +child_spec(Mod, Shutdown, Type) -> + #{id => Mod, + start => {Mod, start_link, []}, + restart => permanent, + shutdown => Shutdown, + type => Type, + modules => [Mod] + }. diff --git a/apps/emqx/src/emqx_config.erl b/apps/emqx/src/emqx_config.erl index 2f5bc9551..bd6e14e8e 100644 --- a/apps/emqx/src/emqx_config.erl +++ b/apps/emqx/src/emqx_config.erl @@ -87,15 +87,21 @@ -type update_request() :: term(). -type update_cmd() :: {update, update_request()} | remove. -type update_opts() :: #{ - %% fill the default values into the rawconf map - rawconf_with_defaults => boolean() + %% rawconf_with_defaults: + %% fill the default values into the `raw_config` field of the return value + %% defaults to `false` + rawconf_with_defaults => boolean(), + %% persistent: + %% save the updated config to the emqx_override.conf file + %% defaults to `true` + persistent => boolean() }. -type update_args() :: {update_cmd(), Opts :: update_opts()}. -type update_stage() :: pre_config_update | post_config_update. -type update_error() :: {update_stage(), module(), term()} | {save_configs, term()} | term(). -type update_result() :: #{ - config := emqx_config:config(), - raw_config := emqx_config:raw_config(), + config => emqx_config:config(), + raw_config => emqx_config:raw_config(), post_config_update => #{module() => any()} }. @@ -235,7 +241,7 @@ put_raw(KeyPath, Config) -> do_put(?RAW_CONF, KeyPath, Config). %% in the rear of the list overrides prior values. -spec init_load(module(), [string()] | binary() | hocon:config()) -> ok. init_load(SchemaMod, Conf) when is_list(Conf) orelse is_binary(Conf) -> - ParseOptions = #{format => richmap}, + ParseOptions = #{format => map}, Parser = case is_binary(Conf) of true -> fun hocon:binary/2; false -> fun hocon:files/2 @@ -249,19 +255,17 @@ init_load(SchemaMod, Conf) when is_list(Conf) orelse is_binary(Conf) -> }), error(failed_to_load_hocon_conf) end; -init_load(SchemaMod, RawRichConf) when is_map(RawRichConf) -> - %% check with richmap for line numbers in error reports (future enhancement) - Opts = #{return_plain => true, - nullable => true - }, - %% this call throws exception in case of check failure - {_AppEnvs, CheckedConf} = hocon_schema:map_translate(SchemaMod, RawRichConf, Opts), +init_load(SchemaMod, RawConf0) when is_map(RawConf0) -> ok = save_schema_mod_and_names(SchemaMod), - ok = save_to_config_map(emqx_map_lib:unsafe_atom_key_map(normalize_conf(CheckedConf)), - normalize_conf(hocon_schema:richmap_to_map(RawRichConf))). + %% override part of the input conf using emqx_override.conf + RawConf = merge_with_override_conf(RawConf0), + %% check and save configs + {_AppEnvs, CheckedConf} = check_config(SchemaMod, RawConf), + ok = save_to_config_map(maps:with(get_atom_root_names(), CheckedConf), + maps:with(get_root_names(), RawConf)). -normalize_conf(Conf) -> - maps:with(get_root_names(bin), Conf). +merge_with_override_conf(RawConf) -> + maps:merge(RawConf, maps:with(maps:keys(RawConf), read_override_conf())). -spec check_config(module(), raw_config()) -> {AppEnvs, CheckedConf} when AppEnvs :: app_envs(), CheckedConf :: config(). @@ -277,7 +281,7 @@ check_config(SchemaMod, RawConf) -> -spec fill_defaults(raw_config()) -> map(). fill_defaults(RawConf) -> - RootNames = get_root_names(bin), + RootNames = get_root_names(), maps:fold(fun(Key, Conf, Acc) -> SubMap = #{Key => Conf}, WithDefaults = case lists:member(Key, RootNames) of @@ -320,8 +324,8 @@ get_schema_mod(RootName) -> get_root_names() -> maps:get(names, persistent_term:get(?PERSIS_SCHEMA_MODS, #{names => []})). -get_root_names(bin) -> - maps:keys(get_schema_mod()). +get_atom_root_names() -> + [atom(N) || N <- get_root_names()]. -spec save_configs(app_envs(), config(), raw_config(), raw_config()) -> ok | {error, term()}. save_configs(_AppEnvs, Conf, RawConf, OverrideConf) -> @@ -344,14 +348,19 @@ save_to_config_map(Conf, RawConf) -> ?MODULE:put_raw(RawConf). -spec save_to_override_conf(raw_config()) -> ok | {error, term()}. +save_to_override_conf(undefined) -> + ok; save_to_override_conf(RawConf) -> - FileName = emqx_override_conf_name(), - ok = filelib:ensure_dir(FileName), - case file:write_file(FileName, jsx:prettify(jsx:encode(RawConf))) of - ok -> ok; - {error, Reason} -> - logger:error("write to ~s failed, ~p", [FileName, Reason]), - {error, Reason} + case emqx_override_conf_name() of + undefined -> ok; + FileName -> + ok = filelib:ensure_dir(FileName), + case file:write_file(FileName, jsx:prettify(jsx:encode(RawConf))) of + ok -> ok; + {error, Reason} -> + logger:error("write to ~s failed, ~p", [FileName, Reason]), + {error, Reason} + end end. load_hocon_file(FileName, LoadType) -> @@ -363,7 +372,7 @@ load_hocon_file(FileName, LoadType) -> end. emqx_override_conf_name() -> - application:get_env(emqx, override_conf_file, "emqx_override.conf"). + application:get_env(emqx, override_conf_file, undefined). do_get(Type, KeyPath) -> Ref = make_ref(), @@ -412,14 +421,7 @@ do_deep_put(?RAW_CONF, KeyPath, Map, Value) -> root_names_from_conf(RawConf) -> Keys = maps:keys(RawConf), - StrNames = [str(K) || K <- Keys], - AtomNames = lists:foldl(fun(K, Acc) -> - try [atom(K) | Acc] - catch error:badarg -> Acc - end - end, [], Keys), - PossibleNames = StrNames ++ AtomNames, - [Name || Name <- get_root_names(), lists:member(Name, PossibleNames)]. + [Name || Name <- get_root_names(), lists:member(Name, Keys)]. atom(Bin) when is_binary(Bin) -> binary_to_existing_atom(Bin, latin1); @@ -428,13 +430,6 @@ atom(Str) when is_list(Str) -> atom(Atom) when is_atom(Atom) -> Atom. -str(Bin) when is_binary(Bin) -> - binary_to_list(Bin); -str(Str) when is_list(Str) -> - Str; -str(Atom) when is_atom(Atom) -> - atom_to_list(Atom). - bin(Bin) when is_binary(Bin) -> Bin; bin(Str) when is_list(Str) -> list_to_binary(Str); bin(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8). diff --git a/apps/emqx/src/emqx_config_handler.erl b/apps/emqx/src/emqx_config_handler.erl index b45f89538..d92f1d35a 100644 --- a/apps/emqx/src/emqx_config_handler.erl +++ b/apps/emqx/src/emqx_config_handler.erl @@ -23,6 +23,7 @@ %% API functions -export([ start_link/0 + , stop/0 , add_handler/2 , remove_handler/1 , update_config/3 @@ -38,6 +39,7 @@ code_change/3]). -define(MOD, {mod}). +-define(WKEY, '?'). -define(ATOM_CONF_PATH(PATH, EXP, EXP_ON_FAIL), try [safe_atom(Key) || Key <- PATH] of @@ -68,6 +70,9 @@ start_link() -> gen_server:start_link({local, ?MODULE}, ?MODULE, {}, []). +stop() -> + gen_server:stop(?MODULE). + -spec update_config(module(), emqx_config:config_key_path(), emqx_config:update_args()) -> {ok, emqx_config:update_result()} | {error, emqx_config:update_error()}. update_config(SchemaModule, ConfKeyPath, UpdateArgs) -> @@ -76,11 +81,11 @@ update_config(SchemaModule, ConfKeyPath, UpdateArgs) -> -spec add_handler(emqx_config:config_key_path(), handler_name()) -> ok. add_handler(ConfKeyPath, HandlerName) -> - gen_server:call(?MODULE, {add_child, ConfKeyPath, HandlerName}). + gen_server:call(?MODULE, {add_handler, ConfKeyPath, HandlerName}). -spec remove_handler(emqx_config:config_key_path()) -> ok. remove_handler(ConfKeyPath) -> - gen_server:call(?MODULE, {remove_child, ConfKeyPath}). + gen_server:call(?MODULE, {remove_handler, ConfKeyPath}). %%============================================================================ @@ -88,15 +93,18 @@ remove_handler(ConfKeyPath) -> init(_) -> {ok, #{handlers => #{?MOD => ?MODULE}}}. -handle_call({add_child, ConfKeyPath, HandlerName}, _From, - State = #{handlers := Handlers}) -> - {reply, ok, State#{handlers => - emqx_map_lib:deep_put(ConfKeyPath, Handlers, #{?MOD => HandlerName})}}; +handle_call({add_handler, ConfKeyPath, HandlerName}, _From, State = #{handlers := Handlers}) -> + case deep_put_handler(ConfKeyPath, Handlers, HandlerName) of + {ok, NewHandlers} -> + {reply, ok, State#{handlers => NewHandlers}}; + Error -> + {reply, Error, State} + end; -handle_call({remove_child, ConfKeyPath}, _From, +handle_call({remove_handler, ConfKeyPath}, _From, State = #{handlers := Handlers}) -> {reply, ok, State#{handlers => - emqx_map_lib:deep_remove(ConfKeyPath, Handlers)}}; + emqx_map_lib:deep_remove(ConfKeyPath ++ [?MOD], Handlers)}}; handle_call({change_config, SchemaModule, ConfKeyPath, UpdateArgs}, _From, #{handlers := Handlers} = State) -> @@ -130,17 +138,40 @@ terminate(_Reason, _State) -> code_change(_OldVsn, State, _Extra) -> {ok, State}. -process_update_request(ConfKeyPath, _Handlers, {remove, _Opts}) -> +deep_put_handler([], Handlers, Mod) when is_map(Handlers) -> + {ok, Handlers#{?MOD => Mod}}; +deep_put_handler([], _Handlers, Mod) -> + {ok, #{?MOD => Mod}}; +deep_put_handler([?WKEY | KeyPath], Handlers, Mod) -> + deep_put_handler2(?WKEY, KeyPath, Handlers, Mod); +deep_put_handler([Key | KeyPath], Handlers, Mod) -> + case maps:find(?WKEY, Handlers) of + error -> + deep_put_handler2(Key, KeyPath, Handlers, Mod); + {ok, _SubHandlers} -> + {error, {cannot_override_a_wildcard_path, [?WKEY | KeyPath]}} + end. + +deep_put_handler2(Key, KeyPath, Handlers, Mod) -> + SubHandlers = maps:get(Key, Handlers, #{}), + case deep_put_handler(KeyPath, SubHandlers, Mod) of + {ok, SubHandlers1} -> + {ok, Handlers#{Key => SubHandlers1}}; + Error -> + Error + end. + +process_update_request(ConfKeyPath, _Handlers, {remove, Opts}) -> OldRawConf = emqx_config:get_root_raw(ConfKeyPath), BinKeyPath = bin_path(ConfKeyPath), NewRawConf = emqx_map_lib:deep_remove(BinKeyPath, OldRawConf), - OverrideConf = emqx_map_lib:deep_remove(BinKeyPath, emqx_config:read_override_conf()), + OverrideConf = remove_from_override_config(BinKeyPath, Opts), {ok, NewRawConf, OverrideConf}; -process_update_request(ConfKeyPath, Handlers, {{update, UpdateReq}, _Opts}) -> +process_update_request(ConfKeyPath, Handlers, {{update, UpdateReq}, Opts}) -> OldRawConf = emqx_config:get_root_raw(ConfKeyPath), case do_update_config(ConfKeyPath, Handlers, OldRawConf, UpdateReq) of {ok, NewRawConf} -> - OverrideConf = update_override_config(NewRawConf), + OverrideConf = update_override_config(NewRawConf, Opts), {ok, NewRawConf, OverrideConf}; Error -> Error end. @@ -149,7 +180,7 @@ do_update_config([], Handlers, OldRawConf, UpdateReq) -> call_pre_config_update(Handlers, OldRawConf, UpdateReq); do_update_config([ConfKey | ConfKeyPath], Handlers, OldRawConf, UpdateReq) -> SubOldRawConf = get_sub_config(bin(ConfKey), OldRawConf), - SubHandlers = maps:get(ConfKey, Handlers, #{}), + SubHandlers = get_sub_handlers(ConfKey, Handlers), case do_update_config(ConfKeyPath, SubHandlers, SubOldRawConf, UpdateReq) of {ok, NewUpdateReq} -> call_pre_config_update(Handlers, OldRawConf, #{bin(ConfKey) => NewUpdateReq}); @@ -180,7 +211,7 @@ do_post_config_update([ConfKey | ConfKeyPath], Handlers, OldConf, NewConf, AppEn Result) -> SubOldConf = get_sub_config(ConfKey, OldConf), SubNewConf = get_sub_config(ConfKey, NewConf), - SubHandlers = maps:get(ConfKey, Handlers, #{}), + SubHandlers = get_sub_handlers(ConfKey, Handlers), case do_post_config_update(ConfKeyPath, SubHandlers, SubOldConf, SubNewConf, AppEnvs, UpdateArgs, Result) of {ok, Result1} -> @@ -189,6 +220,12 @@ do_post_config_update([ConfKey | ConfKeyPath], Handlers, OldConf, NewConf, AppEn Error -> Error end. +get_sub_handlers(ConfKey, Handlers) -> + case maps:find(ConfKey, Handlers) of + error -> maps:get(?WKEY, Handlers, #{}); + {ok, SubHandlers} -> SubHandlers + end. + get_sub_config(ConfKey, Conf) when is_map(Conf) -> maps:get(ConfKey, Conf, undefined); get_sub_config(_, _Conf) -> %% the Conf is a primitive @@ -217,10 +254,9 @@ call_post_config_update(Handlers, OldConf, NewConf, AppEnvs, UpdateReq, Result) false -> {ok, Result} end. -save_configs(ConfKeyPath, AppEnvs, CheckedConf, NewRawConf, OverrideConf, {_Cmd, Opts}) -> +save_configs(ConfKeyPath, AppEnvs, CheckedConf, NewRawConf, OverrideConf, UpdateArgs) -> case emqx_config:save_configs(AppEnvs, CheckedConf, NewRawConf, OverrideConf) of - ok -> {ok, #{config => emqx_config:get(ConfKeyPath), - raw_config => return_rawconf(ConfKeyPath, Opts)}}; + ok -> {ok, return_change_result(ConfKeyPath, UpdateArgs)}; {error, Reason} -> {error, {save_configs, Reason}} end. @@ -234,13 +270,27 @@ merge_to_old_config(UpdateReq, RawConf) when is_map(UpdateReq), is_map(RawConf) merge_to_old_config(UpdateReq, _RawConf) -> {ok, UpdateReq}. -update_override_config(RawConf) -> +remove_from_override_config(_BinKeyPath, #{persistent := false}) -> + undefined; +remove_from_override_config(BinKeyPath, _Opts) -> + OldConf = emqx_config:read_override_conf(), + emqx_map_lib:deep_remove(BinKeyPath, OldConf). + +update_override_config(_RawConf, #{persistent := false}) -> + undefined; +update_override_config(RawConf, _Opts) -> OldConf = emqx_config:read_override_conf(), maps:merge(OldConf, RawConf). up_req({remove, _Opts}) -> '$remove'; up_req({{update, Req}, _Opts}) -> Req. +return_change_result(ConfKeyPath, {{update, _Req}, Opts}) -> + #{config => emqx_config:get(ConfKeyPath), + raw_config => return_rawconf(ConfKeyPath, Opts)}; +return_change_result(_ConfKeyPath, {remove, _Opts}) -> + #{}. + return_rawconf(ConfKeyPath, #{rawconf_with_defaults := true}) -> FullRawConf = emqx_config:fill_defaults(emqx_config:get_raw([])), emqx_map_lib:deep_get(bin_path(ConfKeyPath), FullRawConf); diff --git a/apps/emqx/src/emqx_frame.erl b/apps/emqx/src/emqx_frame.erl index 082801bad..79a740bed 100644 --- a/apps/emqx/src/emqx_frame.erl +++ b/apps/emqx/src/emqx_frame.erl @@ -100,14 +100,10 @@ parse(<>, StrictMode andalso validate_header(Type, Dup, QoS, Retain), Header = #mqtt_packet_header{type = Type, dup = bool(Dup), - qos = QoS, + qos = fixqos(Type, QoS), retain = bool(Retain) }, - Header1 = case fixqos(Type, QoS) of - QoS -> Header; - FixedQoS -> Header#mqtt_packet_header{qos = FixedQoS} - end, - parse_remaining_len(Rest, Header1, Options); + parse_remaining_len(Rest, Header, Options); parse(Bin, {{len, #{hdr := Header, len := {Multiplier, Length}} diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl index 2d3357f37..06d900ed5 100644 --- a/apps/emqx/src/emqx_listeners.erl +++ b/apps/emqx/src/emqx_listeners.erl @@ -26,6 +26,8 @@ , restart/0 , stop/0 , is_running/1 + , current_conns/2 + , max_conns/2 ]). -export([ start_listener/1 @@ -41,10 +43,14 @@ , parse_listener_id/1 ]). +-export([post_config_update/4]). + +-define(CONF_KEY_PATH, [listeners]). + %% @doc List configured listeners. -spec(list() -> [{ListenerId :: atom(), ListenerConf :: map()}]). list() -> - [{listener_id(ZoneName, LName), LConf} || {ZoneName, LName, LConf} <- do_list()]. + [{listener_id(Type, LName), LConf} || {Type, LName, LConf} <- do_list()]. do_list() -> Listeners = maps:to_list(emqx:get_config([listeners], #{})), @@ -58,7 +64,7 @@ list(Type, Conf) -> -spec is_running(ListenerId :: atom()) -> boolean() | {error, no_found}. is_running(ListenerId) -> - case lists:filtermap(fun({_Zone, Id, #{running := IsRunning}}) -> + case lists:filtermap(fun({_Type, Id, #{running := IsRunning}}) -> Id =:= ListenerId andalso {true, IsRunning} end, do_list()) of [IsRunning] -> IsRunning; @@ -85,9 +91,34 @@ is_running(quic, _ListenerId, _Conf)-> %% TODO: quic support {error, no_found}. +current_conns(ID, ListenOn) -> + {Type, Name} = parse_listener_id(ID), + current_conns(Type, Name, ListenOn). + +current_conns(Type, Name, ListenOn) when Type == tcl; Type == ssl -> + esockd:get_current_connections({listener_id(Type, Name), ListenOn}); +current_conns(Type, Name, _ListenOn) when Type =:= ws; Type =:= wss -> + proplists:get_value(all_connections, ranch:info(listener_id(Type, Name))); +current_conns(_, _, _) -> + {error, not_support}. + +max_conns(ID, ListenOn) -> + {Type, Name} = parse_listener_id(ID), + max_conns(Type, Name, ListenOn). + +max_conns(Type, Name, ListenOn) when Type == tcl; Type == ssl -> + esockd:get_max_connections({listener_id(Type, Name), ListenOn}); +max_conns(Type, Name, _ListenOn) when Type =:= ws; Type =:= wss -> + proplists:get_value(max_connections, ranch:info(listener_id(Type, Name))); +max_conns(_, _, _) -> + {error, not_support}. + %% @doc Start all listeners. -spec(start() -> ok). start() -> + %% The ?MODULE:start/0 will be called by emqx_app when emqx get started, + %% so we install the config handler here. + ok = emqx_config_handler:add_handler(?CONF_KEY_PATH, ?MODULE), foreach_listeners(fun start_listener/3). -spec start_listener(atom()) -> ok | {error, term()}. @@ -102,7 +133,7 @@ start_listener(Type, ListenerName, #{bind := Bind} = Conf) -> console_print("- Skip - starting listener ~s on ~s ~n due to ~p", [listener_id(Type, ListenerName), format_addr(Bind), Reason]); {ok, _} -> - console_print("Start listener ~s on ~s successfully.~n", + console_print("Listener ~s on ~s started.~n", [listener_id(Type, ListenerName), format_addr(Bind)]); {error, {already_started, Pid}} -> {error, {already_started, Pid}}; @@ -122,27 +153,47 @@ restart_listener(ListenerId) -> apply_on_listener(ListenerId, fun restart_listener/3). -spec(restart_listener(atom(), atom(), map()) -> ok | {error, term()}). +restart_listener(Type, ListenerName, {OldConf, NewConf}) -> + restart_listener(Type, ListenerName, OldConf, NewConf); restart_listener(Type, ListenerName, Conf) -> - case stop_listener(Type, ListenerName, Conf) of - ok -> start_listener(Type, ListenerName, Conf); + restart_listener(Type, ListenerName, Conf, Conf). + +restart_listener(Type, ListenerName, OldConf, NewConf) -> + case stop_listener(Type, ListenerName, OldConf) of + ok -> start_listener(Type, ListenerName, NewConf); Error -> Error end. %% @doc Stop all listeners. -spec(stop() -> ok). stop() -> + %% The ?MODULE:stop/0 will be called by emqx_app when emqx is going to shutdown, + %% so we uninstall the config handler here. + _ = emqx_config_handler:remove_handler(?CONF_KEY_PATH), foreach_listeners(fun stop_listener/3). -spec(stop_listener(atom()) -> ok | {error, term()}). stop_listener(ListenerId) -> apply_on_listener(ListenerId, fun stop_listener/3). --spec(stop_listener(atom(), atom(), map()) -> ok | {error, term()}). -stop_listener(Type, ListenerName, #{bind := ListenOn}) when Type == tcp; Type == ssl -> +stop_listener(Type, ListenerName, #{bind := Bind} = Conf) -> + case do_stop_listener(Type, ListenerName, Conf) of + ok -> + console_print("Listener ~s on ~s stopped.~n", + [listener_id(Type, ListenerName), format_addr(Bind)]), + ok; + {error, Reason} -> + ?ELOG("Failed to stop listener ~s on ~s: ~0p~n", + [listener_id(Type, ListenerName), format_addr(Bind), Reason]), + {error, Reason} + end. + +-spec(do_stop_listener(atom(), atom(), map()) -> ok | {error, term()}). +do_stop_listener(Type, ListenerName, #{bind := ListenOn}) when Type == tcp; Type == ssl -> esockd:close(listener_id(Type, ListenerName), ListenOn); -stop_listener(Type, ListenerName, _Conf) when Type == ws; Type == wss -> +do_stop_listener(Type, ListenerName, _Conf) when Type == ws; Type == wss -> cowboy:stop_listener(listener_id(Type, ListenerName)); -stop_listener(quic, ListenerName, _Conf) -> +do_stop_listener(quic, ListenerName, _Conf) -> quicer:stop_listener(listener_id(quic, ListenerName)). -ifndef(TEST). @@ -201,6 +252,36 @@ do_start_listener(quic, ListenerName, #{bind := ListenOn} = Opts) -> {ok, {skipped, quic_app_missing}} end. +delete_authentication(Type, ListenerName, _Conf) -> + emqx_authentication:delete_chain(atom_to_binary(listener_id(Type, ListenerName))). + +%% Update the listeners at runtime +post_config_update(_Req, NewListeners, OldListeners, _AppEnvs) -> + #{added := Added, removed := Removed, changed := Updated} + = diff_listeners(NewListeners, OldListeners), + perform_listener_changes(fun stop_listener/3, Removed), + perform_listener_changes(fun delete_authentication/3, Removed), + perform_listener_changes(fun start_listener/3, Added), + perform_listener_changes(fun restart_listener/3, Updated). + +perform_listener_changes(Action, MapConfs) -> + lists:foreach(fun + ({Id, Conf}) -> + {Type, Name} = parse_listener_id(Id), + Action(Type, Name, Conf) + end, maps:to_list(MapConfs)). + +diff_listeners(NewListeners, OldListeners) -> + emqx_map_lib:diff_maps(flatten_listeners(NewListeners), flatten_listeners(OldListeners)). + +flatten_listeners(Conf0) -> + maps:from_list( + lists:append([do_flatten_listeners(Type, Conf) + || {Type, Conf} <- maps:to_list(Conf0)])). + +do_flatten_listeners(Type, Conf0) -> + [{listener_id(Type, Name), maps:remove(authentication, Conf)} || {Name, Conf} <- maps:to_list(Conf0)]. + esockd_opts(Type, Opts0) -> Opts1 = maps:with([acceptors, max_connections, proxy_protocol, proxy_protocol_timeout], Opts0), Opts2 = case emqx_config:get_zone_conf(zone(Opts0), [rate_limit, max_conn_rate]) of @@ -265,12 +346,12 @@ format_addr({Addr, Port}) when is_tuple(Addr) -> io_lib:format("~s:~w", [inet:ntoa(Addr), Port]). listener_id(Type, ListenerName) -> - list_to_atom(lists:append([atom_to_list(Type), ":", atom_to_list(ListenerName)])). + list_to_atom(lists:append([str(Type), ":", str(ListenerName)])). parse_listener_id(Id) -> try - [Zone, Listen] = string:split(atom_to_list(Id), ":", leading), - {list_to_existing_atom(Zone), list_to_existing_atom(Listen)} + [Type, Name] = string:split(str(Id), ":", leading), + {list_to_existing_atom(Type), list_to_atom(Name)} catch _ : _ -> error({invalid_listener_id, Id}) end. @@ -291,8 +372,8 @@ tcp_opts(Opts) -> foreach_listeners(Do) -> lists:foreach( - fun({ZoneName, LName, LConf}) -> - Do(ZoneName, LName, LConf) + fun({Type, LName, LConf}) -> + Do(Type, LName, LConf) end, do_list()). has_enabled_listener_conf_by_type(Type) -> @@ -307,3 +388,10 @@ apply_on_listener(ListenerId, Do) -> {not_found, _, _} -> error({listener_config_not_found, Type, ListenerName}); {ok, Conf} -> Do(Type, ListenerName, Conf) end. + +str(A) when is_atom(A) -> + atom_to_list(A); +str(B) when is_binary(B) -> + binary_to_list(B); +str(S) when is_list(S) -> + S. diff --git a/apps/emqx/src/emqx_map_lib.erl b/apps/emqx/src/emqx_map_lib.erl index 0486c10da..6aa6606c0 100644 --- a/apps/emqx/src/emqx_map_lib.erl +++ b/apps/emqx/src/emqx_map_lib.erl @@ -24,13 +24,16 @@ , safe_atom_key_map/1 , unsafe_atom_key_map/1 , jsonable_map/1 - , jsonable_value/1 - , deep_convert/2 + , jsonable_map/2 + , binary_string/1 + , deep_convert/3 + , diff_maps/2 ]). -export_type([config_key/0, config_key_path/0]). -type config_key() :: atom() | binary(). -type config_key_path() :: [config_key()]. +-type convert_fun() :: fun((...) -> {K1::any(), V1::any()} | drop). %%----------------------------------------------------------------- -spec deep_get(config_key_path(), map()) -> term(). @@ -62,13 +65,11 @@ deep_find(_KeyPath, Data) -> {not_found, _KeyPath, Data}. -spec deep_put(config_key_path(), map(), term()) -> map(). -deep_put([], Map, Data) when is_map(Map) -> - Data; -deep_put([], _Map, Data) -> %% not map, replace it +deep_put([], _Map, Data) -> Data; deep_put([Key | KeyPath], Map, Data) -> - SubMap = deep_put(KeyPath, maps:get(Key, Map, #{}), Data), - Map#{Key => SubMap}. + SubMap = maps:get(Key, Map, #{}), + Map#{Key => deep_put(KeyPath, SubMap, Data)}. -spec deep_remove(config_key_path(), map()) -> map(). deep_remove([], Map) -> @@ -100,15 +101,17 @@ deep_merge(BaseMap, NewMap) -> end, #{}, BaseMap), maps:merge(MergedBase, maps:with(NewKeys, NewMap)). --spec deep_convert(map(), fun((K::any(), V::any()) -> {K1::any(), V1::any()})) -> map(). -deep_convert(Map, ConvFun) when is_map(Map) -> +-spec deep_convert(map(), convert_fun(), Args::list()) -> map(). +deep_convert(Map, ConvFun, Args) when is_map(Map) -> maps:fold(fun(K, V, Acc) -> - {K1, V1} = ConvFun(K, deep_convert(V, ConvFun)), - Acc#{K1 => V1} + case apply(ConvFun, [K, deep_convert(V, ConvFun, Args) | Args]) of + drop -> Acc; + {K1, V1} -> Acc#{K1 => V1} + end end, #{}, Map); -deep_convert(ListV, ConvFun) when is_list(ListV) -> - [deep_convert(V, ConvFun) || V <- ListV]; -deep_convert(Val, _) -> Val. +deep_convert(ListV, ConvFun, Args) when is_list(ListV) -> + [deep_convert(V, ConvFun, Args) || V <- ListV]; +deep_convert(Val, _, _Args) -> Val. -spec unsafe_atom_key_map(#{binary() | atom() => any()}) -> #{atom() => any()}. unsafe_atom_key_map(Map) -> @@ -120,17 +123,45 @@ safe_atom_key_map(Map) -> -spec jsonable_map(map() | list()) -> map() | list(). jsonable_map(Map) -> - deep_convert(Map, fun(K, V) -> - {jsonable_value(K), jsonable_value(V)} - end). + jsonable_map(Map, fun(K, V) -> {K, V} end). -jsonable_value([]) -> []; -jsonable_value(Val) when is_list(Val) -> +jsonable_map(Map, JsonableFun) -> + deep_convert(Map, fun binary_string_kv/3, [JsonableFun]). + +-spec diff_maps(map(), map()) -> + #{added := map(), identical := map(), removed := map(), + changed := #{any() => {OldValue::any(), NewValue::any()}}}. +diff_maps(NewMap, OldMap) -> + InitR = #{identical => #{}, changed => #{}, removed => #{}}, + {Result, RemInNew} = + lists:foldl(fun({OldK, OldV}, {Result0 = #{identical := I, changed := U, removed := D}, + RemNewMap}) -> + Result1 = case maps:find(OldK, NewMap) of + error -> + Result0#{removed => D#{OldK => OldV}}; + {ok, NewV} when NewV == OldV -> + Result0#{identical => I#{OldK => OldV}}; + {ok, NewV} -> + Result0#{changed => U#{OldK => {OldV, NewV}}} + end, + {Result1, maps:remove(OldK, RemNewMap)} + end, {InitR, NewMap}, maps:to_list(OldMap)), + Result#{added => RemInNew}. + + +binary_string_kv(K, V, JsonableFun) -> + case JsonableFun(K, V) of + drop -> drop; + {K1, V1} -> {binary_string(K1), binary_string(V1)} + end. + +binary_string([]) -> []; +binary_string(Val) when is_list(Val) -> case io_lib:printable_unicode_list(Val) of true -> unicode:characters_to_binary(Val); - false -> Val + false -> [binary_string(V) || V <- Val] end; -jsonable_value(Val) -> +binary_string(Val) -> Val. %%--------------------------------------------------------------------------- @@ -138,4 +169,4 @@ covert_keys_to_atom(BinKeyMap, Conv) -> deep_convert(BinKeyMap, fun (K, V) when is_atom(K) -> {K, V}; (K, V) when is_binary(K) -> {Conv(K), V} - end). + end, []). diff --git a/apps/emqx/src/emqx_metrics.erl b/apps/emqx/src/emqx_metrics.erl index 736bb05b0..282b8b5f3 100644 --- a/apps/emqx/src/emqx_metrics.erl +++ b/apps/emqx/src/emqx_metrics.erl @@ -22,8 +22,6 @@ -include("logger.hrl"). -include("types.hrl"). -include("emqx_mqtt.hrl"). --include("emqx.hrl"). - -export([ start_link/0 , stop/0 diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 7d1e39510..01989c5a1 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -49,6 +49,10 @@ -typerefl_from_string({cipher/0, emqx_schema, to_erl_cipher_suite}). -typerefl_from_string({comma_separated_atoms/0, emqx_schema, to_comma_separated_atoms}). +-export([ validate_heap_size/1 + , parse_user_lookup_fun/1 + ]). + % workaround: prevent being recognized as unused functions -export([to_duration/1, to_duration_s/1, to_duration_ms/1, to_bytesize/1, to_wordsize/1, @@ -65,204 +69,541 @@ cipher/0, comma_separated_atoms/0]). --export([roots/0, fields/1]). --export([t/1, t/3, t/4, ref/1]). +-export([namespace/0, roots/0, fields/1]). -export([conf_get/2, conf_get/3, keys/2, filter/1]). -export([ssl/1]). +namespace() -> undefined. + roots() -> - ["zones", "mqtt", "flapping_detect", "force_shutdown", "force_gc", - "conn_congestion", "rate_limit", "quota", "listeners", "broker", "plugins", - "stats", "sysmon", "alarm", "authorization"]. + ["zones", + "mqtt", + "flapping_detect", + "force_shutdown", + "force_gc", + "conn_congestion", + "rate_limit", + "quota", + {"listeners", + sc(ref("listeners"), + #{ desc => "MQTT listeners identified by their protocol type and assigned names. " + "The listeners enabled by default are named with 'default'"}) + }, + "broker", + "plugins", + "stats", + "sysmon", + "alarm", + "authorization", + {"authentication", sc(hoconsc:lazy(hoconsc:array(map())), #{})} + ]. fields("stats") -> - [ {"enable", t(boolean(), undefined, true)} + [ {"enable", + sc(boolean(), + #{ default => true + })} ]; fields("authorization") -> - [ {"no_match", t(union(allow, deny), undefined, allow)} - , {"deny_action", t(union(ignore, disconnect), undefined, ignore)} - , {"cache", ref("authorization_cache")} + [ {"no_match", + sc(union(allow, deny), + #{ default => allow + })} + , {"deny_action", + sc(union(ignore, disconnect), + #{ default => ignore + })} + , {"cache", + sc(ref(?MODULE, "cache"), + #{ + }) + } ]; -fields("authorization_cache") -> - [ {"enable", t(boolean(), undefined, true)} - , {"max_size", t(range(1, 1048576), undefined, 32)} - , {"ttl", t(duration(), undefined, "1m")} +fields("cache") -> + [ {"enable", + sc(boolean(), + #{ default => true + }) + } + , {"max_size", + sc(range(1, 1048576), + #{ default => 32 + }) + } + , {"ttl", + sc(duration(), + #{ default => "1m" + }) + } ]; fields("mqtt") -> - [ {"idle_timeout", maybe_infinity(duration(), "15s")} - , {"max_packet_size", t(bytesize(), undefined, "1MB")} - , {"max_clientid_len", t(range(23, 65535), undefined, 65535)} - , {"max_topic_levels", t(range(1, 65535), undefined, 65535)} - , {"max_qos_allowed", t(range(0, 2), undefined, 2)} - , {"max_topic_alias", t(range(0, 65535), undefined, 65535)} - , {"retain_available", t(boolean(), undefined, true)} - , {"wildcard_subscription", t(boolean(), undefined, true)} - , {"shared_subscription", t(boolean(), undefined, true)} - , {"ignore_loop_deliver", t(boolean(), undefined, false)} - , {"strict_mode", t(boolean(), undefined, false)} - , {"response_information", t(string(), undefined, "")} - , {"server_keepalive", maybe_disabled(integer())} - , {"keepalive_backoff", t(float(), undefined, 0.75)} - , {"max_subscriptions", maybe_infinity(range(1, inf))} - , {"upgrade_qos", t(boolean(), undefined, false)} - , {"max_inflight", t(range(1, 65535), undefined, 32)} - , {"retry_interval", t(duration(), undefined, "30s")} - , {"max_awaiting_rel", maybe_infinity(integer(), 100)} - , {"await_rel_timeout", t(duration(), undefined, "300s")} - , {"session_expiry_interval", t(duration(), undefined, "2h")} - , {"max_mqueue_len", maybe_infinity(range(0, inf), 1000)} - , {"mqueue_priorities", maybe_disabled(map())} - , {"mqueue_default_priority", t(union(highest, lowest), undefined, lowest)} - , {"mqueue_store_qos0", t(boolean(), undefined, true)} - , {"use_username_as_clientid", t(boolean(), undefined, false)} - , {"peer_cert_as_username", maybe_disabled(union([cn, dn, crt, pem, md5]))} - , {"peer_cert_as_clientid", maybe_disabled(union([cn, dn, crt, pem, md5]))} + [ {"idle_timeout", + sc(hoconsc:union([infinity, duration()]), + #{ default => "15s" + })} + , {"max_packet_size", + sc(bytesize(), + #{ default => "1MB" + })} + , {"max_clientid_len", + sc(range(23, 65535), + #{ default => 65535 + })} + , {"max_topic_levels", + sc(range(1, 65535), + #{ default => 65535 + })} + , {"max_qos_allowed", + sc(range(0, 2), + #{ default => 2 + })} + , {"max_topic_alias", + sc(range(0, 65535), + #{ default => 65535 + })} + , {"retain_available", + sc(boolean(), + #{ default => true + })} + , {"wildcard_subscription", + sc(boolean(), + #{ default => true + })} + , {"shared_subscription", + sc(boolean(), + #{ default => true + })} + , {"ignore_loop_deliver", + sc(boolean(), + #{ default => false + })} + , {"strict_mode", + sc(boolean(), + #{default => false + }) + } + , {"response_information", + sc(string(), + #{default => "" + }) + } + , {"server_keepalive", + sc(hoconsc:union([integer(), disabled]), + #{ default => disabled + }) + } + , {"keepalive_backoff", + sc(float(), + #{default => 0.75 + }) + } + , {"max_subscriptions", + sc(hoconsc:union([range(1, inf), infinity]), + #{ default => infinity + }) + } + , {"upgrade_qos", + sc(boolean(), + #{ default => false + }) + } + , {"max_inflight", + sc(range(1, 65535), + #{ default => 32 + }) + } + , {"retry_interval", + sc(duration(), + #{default => "30s" + }) + } + , {"max_awaiting_rel", + sc(hoconsc:union([integer(), infinity]), + #{ default => 100 + }) + } + , {"await_rel_timeout", + sc(duration(), + #{ default => "300s" + }) + } + , {"session_expiry_interval", + sc(duration(), + #{ default => "2h" + }) + } + , {"max_mqueue_len", + sc(hoconsc:union([range(0, inf), infinity]), + #{ default => 1000 + }) + } + , {"mqueue_priorities", + sc(hoconsc:union([map(), disabled]), + #{ default => disabled + }) + } + , {"mqueue_default_priority", + sc(union(highest, lowest), + #{ default => lowest + }) + } + , {"mqueue_store_qos0", + sc(boolean(), + #{ default => true + }) + } + , {"use_username_as_clientid", + sc(boolean(), + #{ default => false + }) + } + , {"peer_cert_as_username", + sc(hoconsc:union([disabled, cn, dn, crt, pem, md5]), + #{ default => disabled + })} + , {"peer_cert_as_clientid", + sc(hoconsc:union([disabled, cn, dn, crt, pem, md5]), + #{ default => disabled + })} ]; fields("zones") -> - [ {"$name", ref("zone_settings")}]; + [ {"$name", + sc(ref("zone_settings"), + #{ + } + )}]; fields("zone_settings") -> - Fields = ["mqtt", "stats", "authorization", "flapping_detect", "force_shutdown", + Fields = ["mqtt", "stats", "flapping_detect", "force_shutdown", "conn_congestion", "rate_limit", "quota", "force_gc"], - [{F, ref("strip_default:" ++ F)} || F <- Fields]; + [{F, ref(emqx_zone_schema, F)} || F <- Fields]; fields("rate_limit") -> - [ {"max_conn_rate", maybe_infinity(integer(), 1000)} - , {"conn_messages_in", maybe_infinity(comma_separated_list())} - , {"conn_bytes_in", maybe_infinity(comma_separated_list())} + [ {"max_conn_rate", + sc(hoconsc:union([infinity, integer()]), + #{ default => 1000 + }) + } + , {"conn_messages_in", + sc(hoconsc:union([infinity, comma_separated_list()]), + #{ default => infinity + }) + } + , {"conn_bytes_in", + sc(hoconsc:union([infinity, comma_separated_list()]), + #{ default => infinity + }) + } ]; fields("quota") -> - [ {"conn_messages_routing", maybe_infinity(comma_separated_list())} - , {"overall_messages_routing", maybe_infinity(comma_separated_list())} + [ {"conn_messages_routing", + sc(hoconsc:union([infinity, comma_separated_list()]), + #{ default => infinity + }) + } + , {"overall_messages_routing", + sc(hoconsc:union([infinity, comma_separated_list()]), + #{ default => infinity + }) + } ]; fields("flapping_detect") -> - [ {"enable", t(boolean(), undefined, false)} - , {"max_count", t(integer(), undefined, 15)} - , {"window_time", t(duration(), undefined, "1m")} - , {"ban_time", t(duration(), undefined, "5m")} + [ {"enable", + sc(boolean(), + #{ default => false + })} + , {"max_count", + sc(integer(), + #{ default => 15 + })} + , {"window_time", + sc(duration(), + #{ default => "1m" + })} + , {"ban_time", + sc(duration(), + #{ default => "5m" + })} ]; fields("force_shutdown") -> - [ {"enable", t(boolean(), undefined, true)} - , {"max_message_queue_len", t(range(0, inf), undefined, 1000)} - , {"max_heap_size", t(wordsize(), undefined, "32MB", undefined, - fun(Siz) -> - MaxSiz = case erlang:system_info(wordsize) of - 8 -> % arch_64 - (1 bsl 59) - 1; - 4 -> % arch_32 - (1 bsl 27) - 1 - end, - case Siz > MaxSiz of - true -> - error(io_lib:format("force_shutdown_policy: heap-size ~s is too large", [Siz])); - false -> - ok - end - end)} + [ {"enable", + sc(boolean(), + #{ default => true})} + , {"max_message_queue_len", + sc(range(0, inf), + #{ default => 1000 + })} + , {"max_heap_size", + sc(wordsize(), + #{ default => "32MB", + validator => fun ?MODULE:validate_heap_size/1 + })} ]; fields("conn_congestion") -> - [ {"enable_alarm", t(boolean(), undefined, false)} - , {"min_alarm_sustain_duration", t(duration(), undefined, "1m")} + [ {"enable_alarm", + sc(boolean(), + #{ default => false + })} + , {"min_alarm_sustain_duration", + sc(duration(), + #{ default => "1m" + })} ]; fields("force_gc") -> - [ {"enable", t(boolean(), undefined, true)} - , {"count", t(range(0, inf), undefined, 16000)} - , {"bytes", t(bytesize(), undefined, "16MB")} + [ {"enable", + sc(boolean(), + #{ default => true + })} + , {"count", + sc(range(0, inf), + #{ default => 16000 + })} + , {"bytes", + sc(bytesize(), + #{ default => "16MB" + })} ]; fields("listeners") -> - [ {"tcp", ref("t_tcp_listeners")} - , {"ssl", ref("t_ssl_listeners")} - , {"ws", ref("t_ws_listeners")} - , {"wss", ref("t_wss_listeners")} - , {"quic", ref("t_quic_listeners")} + [ {"tcp", + sc(ref("tcp_listeners"), + #{ desc => "TCP listeners" + }) + } + , {"ssl", + sc(ref("ssl_listeners"), + #{ desc => "SSL listeners" + }) + } + , {"ws", + sc(ref("ws_listeners"), + #{ desc => "HTTP websocket listeners" + }) + } + , {"wss", + sc(ref("wss_listeners"), + #{ desc => "HTTPS websocket listeners" + }) + } + , {"quic", + sc(ref("quic_listeners"), + #{ desc => "QUIC listeners" + }) + } ]; -fields("t_tcp_listeners") -> +fields("tcp_listeners") -> [ {"$name", ref("mqtt_tcp_listener")} ]; -fields("t_ssl_listeners") -> +fields("ssl_listeners") -> [ {"$name", ref("mqtt_ssl_listener")} ]; -fields("t_ws_listeners") -> +fields("ws_listeners") -> [ {"$name", ref("mqtt_ws_listener")} ]; -fields("t_wss_listeners") -> +fields("wss_listeners") -> [ {"$name", ref("mqtt_wss_listener")} ]; -fields("t_quic_listeners") -> +fields("quic_listeners") -> [ {"$name", ref("mqtt_quic_listener")} ]; fields("mqtt_tcp_listener") -> - [ {"tcp", ref("tcp_opts")} + [ {"tcp", + sc(ref("tcp_opts"), + #{ desc => "TCP listener options" + }) + } ] ++ mqtt_listener(); fields("mqtt_ssl_listener") -> - [ {"tcp", ref("tcp_opts")} - , {"ssl", ref("ssl_opts")} + [ {"tcp", + sc(ref("tcp_opts"), + #{}) + } + , {"ssl", + sc(ref("listener_ssl_opts"), + #{}) + } ] ++ mqtt_listener(); fields("mqtt_ws_listener") -> - [ {"tcp", ref("tcp_opts")} - , {"websocket", ref("ws_opts")} + [ {"tcp", + sc(ref("tcp_opts"), + #{}) + } + , {"websocket", + sc(ref("ws_opts"), + #{}) + } ] ++ mqtt_listener(); fields("mqtt_wss_listener") -> - [ {"tcp", ref("tcp_opts")} - , {"ssl", ref("ssl_opts")} - , {"websocket", ref("ws_opts")} + [ {"tcp", + sc(ref("tcp_opts"), + #{}) + } + , {"ssl", + sc(ref("listener_ssl_opts"), + #{}) + } + , {"websocket", + sc(ref("ws_opts"), + #{}) + } ] ++ mqtt_listener(); fields("mqtt_quic_listener") -> - [ {"enabled", t(boolean(), undefined, true)} - , {"certfile", t(string(), undefined, undefined)} - , {"keyfile", t(string(), undefined, undefined)} - , {"ciphers", t(comma_separated_list(), undefined, "TLS_AES_256_GCM_SHA384," - "TLS_AES_128_GCM_SHA256,TLS_CHACHA20_POLY1305_SHA256")} - , {"idle_timeout", t(duration(), undefined, "15s")} + [ {"enabled", + sc(boolean(), + #{ default => true + }) + } + , {"certfile", + sc(string(), + #{}) + } + , {"keyfile", + sc(string(), + #{}) + } + , {"ciphers", + sc(comma_separated_list(), + #{ default => "TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256," + "TLS_CHACHA20_POLY1305_SHA256" + })} + , {"idle_timeout", + sc(duration(), + #{ default => "15s" + }) + } ] ++ base_listener(); fields("ws_opts") -> - [ {"mqtt_path", t(string(), undefined, "/mqtt")} - , {"mqtt_piggyback", t(union(single, multiple), undefined, multiple)} - , {"compress", t(boolean(), undefined, false)} - , {"idle_timeout", t(duration(), undefined, "15s")} - , {"max_frame_size", maybe_infinity(integer())} - , {"fail_if_no_subprotocol", t(boolean(), undefined, true)} - , {"supported_subprotocols", t(comma_separated_list(), undefined, - "mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5")} - , {"check_origin_enable", t(boolean(), undefined, false)} - , {"allow_origin_absence", t(boolean(), undefined, true)} - , {"check_origins", t(hoconsc:array(binary()), undefined, [])} - , {"proxy_address_header", t(string(), undefined, "x-forwarded-for")} - , {"proxy_port_header", t(string(), undefined, "x-forwarded-port")} - , {"deflate_opts", ref("deflate_opts")} + [ {"mqtt_path", + sc(string(), + #{ default => "/mqtt" + }) + } + , {"mqtt_piggyback", + sc(hoconsc:union([single, multiple]), + #{ default => multiple + }) + } + , {"compress", + sc(boolean(), + #{ default => false + }) + } + , {"idle_timeout", + sc(duration(), + #{ default => "15s" + }) + } + , {"max_frame_size", + sc(hoconsc:union([infinity, integer()]), + #{ default => infinity + }) + } + , {"fail_if_no_subprotocol", + sc(boolean(), + #{ default => true + }) + } + , {"supported_subprotocols", + sc(comma_separated_list(), + #{ default => "mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5" + }) + } + , {"check_origin_enable", + sc(boolean(), + #{ default => false + }) + } + , {"allow_origin_absence", + sc(boolean(), + #{ default => true + }) + } + , {"check_origins", + sc(hoconsc:array(binary()), + #{ default => [] + }) + } + , {"proxy_address_header", + sc(string(), + #{ default => "x-forwarded-for" + }) + } + , {"proxy_port_header", + sc(string(), + #{ default => "x-forwarded-port" + }) + } + , {"deflate_opts", + sc(ref("deflate_opts"), + #{}) + } ]; fields("tcp_opts") -> - [ {"active_n", t(integer(), undefined, 100)} - , {"backlog", t(integer(), undefined, 1024)} - , {"send_timeout", t(duration(), undefined, "15s")} - , {"send_timeout_close", t(boolean(), undefined, true)} - , {"recbuf", t(bytesize())} - , {"sndbuf", t(bytesize())} - , {"buffer", t(bytesize())} - , {"high_watermark", t(bytesize(), undefined, "1MB")} - , {"nodelay", t(boolean(), undefined, false)} - , {"reuseaddr", t(boolean(), undefined, true)} + [ {"active_n", + sc(integer(), + #{ default => 100 + }) + } + , {"backlog", + sc(integer(), + #{ default => 1024 + }) + } + , {"send_timeout", + sc(duration(), + #{ default => "15s" + }) + } + , {"send_timeout_close", + sc(boolean(), + #{ default => true + }) + } + , {"recbuf", + sc(bytesize(), + #{}) + } + , {"sndbuf", + sc(bytesize(), + #{}) + } + , {"buffer", + sc(bytesize(), + #{}) + } + , {"high_watermark", + sc(bytesize(), + #{ default => "1MB"}) + } + , {"nodelay", + sc(boolean(), + #{ default => false}) + } + , {"reuseaddr", + sc(boolean(), + #{ default => true + }) + } ]; -fields("ssl_opts") -> +fields("listener_ssl_opts") -> ssl(#{handshake_timeout => "15s" , depth => 10 , reuse_sessions => true @@ -271,82 +612,241 @@ fields("ssl_opts") -> }); fields("deflate_opts") -> - [ {"level", t(union([none, default, best_compression, best_speed]))} - , {"mem_level", t(range(1, 9), undefined, 8)} - , {"strategy", t(union([default, filtered, huffman_only, rle]))} - , {"server_context_takeover", t(union(takeover, no_takeover))} - , {"client_context_takeover", t(union(takeover, no_takeover))} - , {"server_max_window_bits", t(range(8, 15), undefined, 15)} - , {"client_max_window_bits", t(range(8, 15), undefined, 15)} + [ {"level", + sc(hoconsc:union([none, default, best_compression, best_speed]), + #{}) + } + , {"mem_level", + sc(range(1, 9), + #{ default => 8 + }) + } + , {"strategy", + sc(hoconsc:union([default, filtered, huffman_only, rle]), + #{}) + } + , {"server_context_takeover", + sc(hoconsc:union([takeover, no_takeover]), + #{}) + } + , {"client_context_takeover", + sc(hoconsc:union([takeover, no_takeover]), + #{}) + } + , {"server_max_window_bits", + sc(range(8, 15), + #{ default => 15 + }) + } + , {"client_max_window_bits", + sc(range(8, 15), + #{ default => 15 + }) + } ]; fields("plugins") -> - [ {"expand_plugins_dir", t(string())} + [ {"expand_plugins_dir", + sc(string(), + #{}) + } ]; fields("broker") -> - [ {"sys_msg_interval", maybe_disabled(duration(), "1m")} - , {"sys_heartbeat_interval", maybe_disabled(duration(), "30s")} - , {"enable_session_registry", t(boolean(), undefined, true)} - , {"session_locking_strategy", t(union([local, leader, quorum, all]), undefined, quorum)} - , {"shared_subscription_strategy", t(union(random, round_robin), undefined, round_robin)} - , {"shared_dispatch_ack_enabled", t(boolean(), undefined, false)} - , {"route_batch_clean", t(boolean(), undefined, true)} - , {"perf", ref("perf")} + [ {"sys_msg_interval", + sc(hoconsc:union([disabled, duration()]), + #{ default => "1m" + }) + } + , {"sys_heartbeat_interval", + sc(hoconsc:union([disabled, duration()]), + #{ default => "30s" + }) + } + , {"enable_session_registry", + sc(boolean(), + #{ default => true + }) + } + , {"session_locking_strategy", + sc(hoconsc:union([local, leader, quorum, all]), + #{ default => quorum + }) + } + , {"shared_subscription_strategy", + sc(hoconsc:union([random, round_robin]), + #{ default => round_robin + }) + } + , {"shared_dispatch_ack_enabled", + sc(boolean(), + #{ default => false + }) + } + , {"route_batch_clean", + sc(boolean(), + #{ default => true + })} + , {"perf", + sc(ref("broker_perf"), + #{ desc => "Broker performance tuning pamaters" + }) + } ]; -fields("perf") -> - [ {"route_lock_type", t(union([key, tab, global]), undefined, key)} - , {"trie_compaction", t(boolean(), undefined, true)} +fields("broker_perf") -> + [ {"route_lock_type", + sc(hoconsc:union([key, tab, global]), + #{ default => key + })} + , {"trie_compaction", + sc(boolean(), + #{ default => true + })} ]; fields("sysmon") -> - [ {"vm", ref("sysmon_vm")} - , {"os", ref("sysmon_os")} + [ {"vm", + sc(ref("sysmon_vm"), + #{}) + } + , {"os", + sc(ref("sysmon_os"), + #{}) + } ]; fields("sysmon_vm") -> - [ {"process_check_interval", t(duration(), undefined, "30s")} - , {"process_high_watermark", t(percent(), undefined, "80%")} - , {"process_low_watermark", t(percent(), undefined, "60%")} - , {"long_gc", maybe_disabled(duration())} - , {"long_schedule", maybe_disabled(duration(), "240ms")} - , {"large_heap", maybe_disabled(bytesize(), "32MB")} - , {"busy_dist_port", t(boolean(), undefined, true)} - , {"busy_port", t(boolean(), undefined, true)} + [ {"process_check_interval", + sc(duration(), + #{ default => "30s" + }) + } + , {"process_high_watermark", + sc(percent(), + #{ default => "80%" + }) + } + , {"process_low_watermark", + sc(percent(), + #{ default => "60%" + }) + } + , {"long_gc", + sc(hoconsc:union([disabled, duration()]), + #{}) + } + , {"long_schedule", + sc(hoconsc:union([disabled, duration()]), + #{ default => "240ms" + }) + } + , {"large_heap", + sc(hoconsc:union([disabled, bytesize()]), + #{default => "32MB"}) + } + , {"busy_dist_port", + sc(boolean(), + #{ default => true + }) + } + , {"busy_port", + sc(boolean(), + #{ default => true + })} ]; fields("sysmon_os") -> - [ {"cpu_check_interval", t(duration(), undefined, "60s")} - , {"cpu_high_watermark", t(percent(), undefined, "80%")} - , {"cpu_low_watermark", t(percent(), undefined, "60%")} - , {"mem_check_interval", maybe_disabled(duration(), "60s")} - , {"sysmem_high_watermark", t(percent(), undefined, "70%")} - , {"procmem_high_watermark", t(percent(), undefined, "5%")} + [ {"cpu_check_interval", + sc(duration(), + #{ default => "60s"}) + } + , {"cpu_high_watermark", + sc(percent(), + #{ default => "80%" + }) + } + , {"cpu_low_watermark", + sc(percent(), + #{ default => "60%" + }) + } + , {"mem_check_interval", + sc(hoconsc:union([disabled, duration()]), + #{ default => "60s" + })} + , {"sysmem_high_watermark", + sc(percent(), + #{ default => "70%" + }) + } + , {"procmem_high_watermark", + sc(percent(), + #{ default => "5%" + }) + } ]; fields("alarm") -> - [ {"actions", t(hoconsc:array(atom()), undefined, [log, publish])} - , {"size_limit", t(integer(), undefined, 1000)} - , {"validity_period", t(duration(), undefined, "24h")} - ]; - -fields("strip_default:" ++ Name) -> - strip_default(fields(Name)). + [ {"actions", + sc(hoconsc:array(atom()), + #{ default => [log, publish] + }) + } + , {"size_limit", + sc(integer(), + #{ default => 1000 + }) + } + , {"validity_period", + sc(duration(), + #{ default => "24h" + }) + } + ]. mqtt_listener() -> base_listener() ++ - [ {"access_rules", t(hoconsc:array(string()))} - , {"proxy_protocol", t(boolean(), undefined, false)} - , {"proxy_protocol_timeout", t(duration())} + [ {"access_rules", + sc(hoconsc:array(string()), + #{}) + } + , {"proxy_protocol", + sc(boolean(), + #{ default => false + }) + } + , {"proxy_protocol_timeout", + sc(duration(), + #{}) + } + , {"authentication", + sc(hoconsc:lazy(hoconsc:array(map())), + #{}) + } ]. base_listener() -> - [ {"bind", t(union(ip_port(), integer()))} - , {"acceptors", t(integer(), undefined, 16)} - , {"max_connections", maybe_infinity(integer(), infinity)} - , {"mountpoint", t(binary(), undefined, <<>>)} - , {"zone", t(atom(), undefined, default)} + [ {"bind", + sc(hoconsc:union([ip_port(), integer()]), + #{ nullable => false + })} + , {"acceptors", + sc(integer(), + #{ default => 16 + })} + , {"max_connections", + sc(hoconsc:union([infinity, integer()]), + #{ default => infinity + })} + , {"mountpoint", + sc(binary(), + #{ default => <<>> + })} + , {"zone", + sc(atom(), + #{ default => 'default' + })} ]. %% utils @@ -372,43 +872,101 @@ conf_get(Key, Conf, Default) -> filter(Opts) -> [{K, V} || {K, V} <- Opts, V =/= undefined]. -%% generate a ssl field. -%% ssl(#{"verify" => verify_peer}) will return: -%% [ {"cacertfile", t(string(), undefined, undefined)} -%% , {"certfile", t(string(), undefined, undefined)} -%% , {"keyfile", t(string(), undefined, undefined)} -%% , {"verify", t(union(verify_peer, verify_none), undefined, verify_peer)} -%% , {"server_name_indication", undefined, undefined)} -%% ...] ssl(Defaults) -> D = fun (Field) -> maps:get(to_atom(Field), Defaults, undefined) end, - [ {"enable", t(boolean(), undefined, D("enable"))} - , {"cacertfile", t(string(), undefined, D("cacertfile"))} - , {"certfile", t(string(), undefined, D("certfile"))} - , {"keyfile", t(string(), undefined, D("keyfile"))} - , {"verify", t(union(verify_peer, verify_none), undefined, D("verify"))} - , {"fail_if_no_peer_cert", t(boolean(), undefined, D("fail_if_no_peer_cert"))} - , {"secure_renegotiate", t(boolean(), undefined, D("secure_renegotiate"))} - , {"reuse_sessions", t(boolean(), undefined, D("reuse_sessions"))} - , {"honor_cipher_order", t(boolean(), undefined, D("honor_cipher_order"))} - , {"handshake_timeout", t(duration(), undefined, D("handshake_timeout"))} - , {"depth", t(integer(), undefined, D("depth"))} - , {"password", hoconsc:t(string(), #{default => D("key_password"), - sensitive => true - })} - , {"dhfile", t(string(), undefined, D("dhfile"))} - , {"server_name_indication", t(union(disable, string()), undefined, - D("server_name_indication"))} - , {"versions", #{ type => list(atom()) - , default => maps:get(versions, Defaults, default_tls_vsns()) - , converter => fun (Vsns) -> [tls_vsn(V) || V <- Vsns] end - }} - , {"ciphers", t(hoconsc:array(string()), undefined, D("ciphers"))} - , {"user_lookup_fun", t(any(), undefined, {fun emqx_psk:lookup/3, <<>>})} + [ {"enable", + sc(boolean(), + #{ default => D("enable") + }) + } + , {"cacertfile", + sc(string(), + #{ default => D("cacertfile") + }) + } + , {"certfile", + sc(string(), + #{ default => D("certfile") + }) + } + , {"keyfile", + sc(string(), + #{ default => D("keyfile") + }) + } + , {"verify", + sc(hoconsc:union([verify_peer, verify_none]), + #{ default => D("verify") + }) + } + , {"fail_if_no_peer_cert", + sc(boolean(), + #{ default => D("fail_if_no_peer_cert") + }) + } + , {"secure_renegotiate", + sc(boolean(), + #{ default => D("secure_renegotiate") + }) + } + , {"reuse_sessions", + sc(boolean(), + #{ default => D("reuse_sessions") + }) + } + , {"honor_cipher_order", + sc(boolean(), + #{ default => D("honor_cipher_order") + }) + } + , {"handshake_timeout", + sc(duration(), + #{ default => D("handshake_timeout") + }) + } + , {"depth", + sc(integer(), + #{default => D("depth") + }) + } + , {"password", + sc(string(), + #{ default => D("key_password") + , sensitive => true + }) + } + , {"dhfile", + sc(string(), + #{ default => D("dhfile") + }) + } + , {"server_name_indication", + sc(hoconsc:union([disable, string()]), + #{ default => D("server_name_indication") + }) + } + , {"versions", + sc(typerefl:alias("string", list(atom())), + #{ default => maps:get(versions, Defaults, default_tls_vsns()) + , converter => fun (Vsns) -> [tls_vsn(iolist_to_binary(V)) || V <- Vsns] end + }) + } + , {"ciphers", + sc(hoconsc:array(string()), + #{ default => D("ciphers") + }) + } + , {"user_lookup_fun", + sc(typerefl:alias("string", any()), + #{ default => "emqx_psk:lookup" + , converter => fun ?MODULE:parse_user_lookup_fun/1 + }) + } ]. %% on erl23.2.7.2-emqx-2, sufficient_crypto_support('tlsv1.3') -> false default_tls_vsns() -> [<<"tlsv1.2">>, <<"tlsv1.1">>, <<"tlsv1">>]. + tls_vsn(<<"tlsv1.3">>) -> 'tlsv1.3'; tls_vsn(<<"tlsv1.2">>) -> 'tlsv1.2'; tls_vsn(<<"tlsv1.1">>) -> 'tlsv1.1'; @@ -451,40 +1009,11 @@ ceiling(X) -> %% types -t(Type) -> hoconsc:t(Type). +sc(Type, Meta) -> hoconsc:mk(Type, Meta). -t(Type, Mapping, Default) -> - hoconsc:t(Type, #{mapping => Mapping, default => Default}). +ref(Field) -> hoconsc:ref(?MODULE, Field). -t(Type, Mapping, Default, OverrideEnv) -> - hoconsc:t(Type, #{ mapping => Mapping - , default => Default - , override_env => OverrideEnv - }). - -t(Type, Mapping, Default, OverrideEnv, Validator) -> - hoconsc:t(Type, #{ mapping => Mapping - , default => Default - , override_env => OverrideEnv - , validator => Validator - }). - -ref(Field) -> hoconsc:t(hoconsc:ref(?MODULE, Field)). - -maybe_disabled(T) -> - maybe_sth(disabled, T, disabled). - -maybe_disabled(T, Default) -> - maybe_sth(disabled, T, Default). - -maybe_infinity(T) -> - maybe_sth(infinity, T, infinity). - -maybe_infinity(T, Default) -> - maybe_sth(infinity, T, Default). - -maybe_sth(What, Type, Default) -> - t(union([What, Type]), undefined, Default). +ref(Module, Field) -> hoconsc:ref(Module, Field). to_duration(Str) -> case hocon_postprocess:duration(Str) of @@ -545,22 +1074,26 @@ to_erl_cipher_suite(Str) -> Cipher -> Cipher end. -strip_default(Fields) -> - [do_strip_default(F) || F <- Fields]. - -do_strip_default({Name, #{type := {ref, Ref}}}) -> - {Name, nullable_no_def(ref("strip_default:" ++ Ref))}; -do_strip_default({Name, #{type := {ref, _Mod, Ref}}}) -> - {Name, nullable_no_def(ref("strip_default:" ++ Ref))}; -do_strip_default({Name, Type}) -> - {Name, nullable_no_def(Type)}. - -nullable_no_def(Type) when is_map(Type) -> - Type#{default => undefined, nullable => true}. - to_atom(Atom) when is_atom(Atom) -> Atom; to_atom(Str) when is_list(Str) -> list_to_atom(Str); to_atom(Bin) when is_binary(Bin) -> binary_to_atom(Bin, utf8). + +validate_heap_size(Siz) -> + MaxSiz = case erlang:system_info(wordsize) of + 8 -> % arch_64 + (1 bsl 59) - 1; + 4 -> % arch_32 + (1 bsl 27) - 1 + end, + case Siz > MaxSiz of + true -> error(io_lib:format("force_shutdown_policy: heap-size ~s is too large", [Siz])); + false -> ok + end. +parse_user_lookup_fun(StrConf) -> + [ModStr, FunStr] = string:tokens(StrConf, ":"), + Mod = list_to_atom(ModStr), + Fun = list_to_atom(FunStr), + {fun Mod:Fun/3, <<>>}. diff --git a/apps/emqx_gateway/src/emqx_gateway_api_client.erl b/apps/emqx/src/emqx_zone_schema.erl similarity index 63% rename from apps/emqx_gateway/src/emqx_gateway_api_client.erl rename to apps/emqx/src/emqx_zone_schema.erl index 03fb056ad..013ffb22f 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api_client.erl +++ b/apps/emqx/src/emqx_zone_schema.erl @@ -13,13 +13,22 @@ %% See the License for the specific language governing permissions and %% limitations under the License. %%-------------------------------------------------------------------- -%% --module(emqx_gateway_api_client). --behaviour(minirest_api). +-module(emqx_zone_schema). -%% minirest behaviour callbacks --export([api_spec/0]). +-export([namespace/0, roots/0, fields/1]). -api_spec() -> - {[], []}. +namespace() -> zone. + +roots() -> []. + +%% zone schemas are clones from the same name from root level +%% only not allowed to have default values. +fields(Name) -> + [{N, no_default(Sc)} || {N, Sc} <- emqx_schema:fields(Name)]. + +%% no default values for zone settings +no_default(Sc) -> + fun(default) -> undefined; + (Other) -> hocon_schema:field_schema(Sc, Other) + end. diff --git a/apps/emqx/test/emqx_authentication_SUITE.erl b/apps/emqx/test/emqx_authentication_SUITE.erl new file mode 100644 index 000000000..aa4d55fee --- /dev/null +++ b/apps/emqx/test/emqx_authentication_SUITE.erl @@ -0,0 +1,238 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_authentication_SUITE). + +-behaviour(hocon_schema). +-behaviour(emqx_authentication). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("typerefl/include/types.hrl"). + +-export([ fields/1 ]). + +-export([ refs/0 + , create/1 + , update/2 + , authenticate/2 + , destroy/1 + ]). + +-define(AUTHN, emqx_authentication). + +%%------------------------------------------------------------------------------ +%% Hocon Schema +%%------------------------------------------------------------------------------ + +fields(type1) -> + [ {mechanism, {enum, ['password-based']}} + , {backend, {enum, ['built-in-database']}} + , {enable, fun enable/1} + ]; + +fields(type2) -> + [ {mechanism, {enum, ['password-based']}} + , {backend, {enum, ['mysql']}} + , {enable, fun enable/1} + ]. + +enable(type) -> boolean(); +enable(default) -> true; +enable(_) -> undefined. + +%%------------------------------------------------------------------------------ +%% Callbacks +%%------------------------------------------------------------------------------ + +refs() -> + [ hoconsc:ref(?MODULE, type1) + , hoconsc:ref(?MODULE, type2) + ]. + +create(_Config) -> + {ok, #{mark => 1}}. + +update(_Config, _State) -> + {ok, #{mark => 2}}. + +authenticate(#{username := <<"good">>}, _State) -> + {ok, #{is_superuser => true}}; +authenticate(#{username := _}, _State) -> + {error, bad_username_or_password}. + +destroy(_State) -> + ok. + +all() -> + emqx_ct:all(?MODULE). + +init_per_suite(Config) -> + application:set_env(ekka, strict_mode, true), + emqx_ct_helpers:start_apps([]), + Config. + +end_per_suite(_) -> + emqx_ct_helpers:stop_apps([]), + ok. + +t_chain(_) -> + % CRUD of authentication chain + ChainName = 'test', + ?assertMatch({ok, []}, ?AUTHN:list_chains()), + ?assertMatch({ok, #{name := ChainName, authenticators := []}}, ?AUTHN:create_chain(ChainName)), + ?assertEqual({error, {already_exists, {chain, ChainName}}}, ?AUTHN:create_chain(ChainName)), + ?assertMatch({ok, #{name := ChainName, authenticators := []}}, ?AUTHN:lookup_chain(ChainName)), + ?assertMatch({ok, [#{name := ChainName}]}, ?AUTHN:list_chains()), + ?assertEqual(ok, ?AUTHN:delete_chain(ChainName)), + ?assertMatch({error, {not_found, {chain, ChainName}}}, ?AUTHN:lookup_chain(ChainName)), + ok. + +t_authenticator(_) -> + ChainName = 'test', + AuthenticatorConfig1 = #{mechanism => 'password-based', + backend => 'built-in-database', + enable => true}, + + % Create an authenticator when the authentication chain does not exist + ?assertEqual({error, {not_found, {chain, ChainName}}}, ?AUTHN:create_authenticator(ChainName, AuthenticatorConfig1)), + ?AUTHN:create_chain(ChainName), + % Create an authenticator when the provider does not exist + ?assertEqual({error, no_available_provider}, ?AUTHN:create_authenticator(ChainName, AuthenticatorConfig1)), + + AuthNType1 = {'password-based', 'built-in-database'}, + ?AUTHN:add_provider(AuthNType1, ?MODULE), + ID1 = <<"password-based:built-in-database">>, + + % CRUD of authencaticator + ?assertMatch({ok, #{id := ID1, state := #{mark := 1}}}, ?AUTHN:create_authenticator(ChainName, AuthenticatorConfig1)), + ?assertMatch({ok, #{id := ID1}}, ?AUTHN:lookup_authenticator(ChainName, ID1)), + ?assertMatch({ok, [#{id := ID1}]}, ?AUTHN:list_authenticators(ChainName)), + ?assertEqual({error, {already_exists, {authenticator, ID1}}}, ?AUTHN:create_authenticator(ChainName, AuthenticatorConfig1)), + ?assertMatch({ok, #{id := ID1, state := #{mark := 2}}}, ?AUTHN:update_authenticator(ChainName, ID1, AuthenticatorConfig1)), + ?assertEqual(ok, ?AUTHN:delete_authenticator(ChainName, ID1)), + ?assertEqual({error, {not_found, {authenticator, ID1}}}, ?AUTHN:update_authenticator(ChainName, ID1, AuthenticatorConfig1)), + ?assertMatch({ok, []}, ?AUTHN:list_authenticators(ChainName)), + + % Multiple authenticators exist at the same time + AuthNType2 = {'password-based', mysql}, + ?AUTHN:add_provider(AuthNType2, ?MODULE), + ID2 = <<"password-based:mysql">>, + AuthenticatorConfig2 = #{mechanism => 'password-based', + backend => mysql, + enable => true}, + ?assertMatch({ok, #{id := ID1}}, ?AUTHN:create_authenticator(ChainName, AuthenticatorConfig1)), + ?assertMatch({ok, #{id := ID2}}, ?AUTHN:create_authenticator(ChainName, AuthenticatorConfig2)), + + % Move authenticator + ?assertMatch({ok, [#{id := ID1}, #{id := ID2}]}, ?AUTHN:list_authenticators(ChainName)), + ?assertEqual(ok, ?AUTHN:move_authenticator(ChainName, ID2, top)), + ?assertMatch({ok, [#{id := ID2}, #{id := ID1}]}, ?AUTHN:list_authenticators(ChainName)), + ?assertEqual(ok, ?AUTHN:move_authenticator(ChainName, ID2, bottom)), + ?assertMatch({ok, [#{id := ID1}, #{id := ID2}]}, ?AUTHN:list_authenticators(ChainName)), + ?assertEqual(ok, ?AUTHN:move_authenticator(ChainName, ID2, {before, ID1})), + ?assertMatch({ok, [#{id := ID2}, #{id := ID1}]}, ?AUTHN:list_authenticators(ChainName)), + + ?AUTHN:delete_chain(ChainName), + ?AUTHN:remove_provider(AuthNType1), + ?AUTHN:remove_provider(AuthNType2), + ok. + +t_authenticate(_) -> + ListenerID = 'tcp:default', + ClientInfo = #{zone => default, + listener => ListenerID, + protocol => mqtt, + username => <<"good">>, + password => <<"any">>}, + ?assertEqual({ok, #{is_superuser => false}}, emqx_access_control:authenticate(ClientInfo)), + + AuthNType = {'password-based', 'built-in-database'}, + ?AUTHN:add_provider(AuthNType, ?MODULE), + + AuthenticatorConfig = #{mechanism => 'password-based', + backend => 'built-in-database', + enable => true}, + ?AUTHN:create_chain(ListenerID), + ?assertMatch({ok, _}, ?AUTHN:create_authenticator(ListenerID, AuthenticatorConfig)), + ?assertEqual({ok, #{is_superuser => true}}, emqx_access_control:authenticate(ClientInfo)), + ?assertEqual({error, bad_username_or_password}, emqx_access_control:authenticate(ClientInfo#{username => <<"bad">>})), + + ?AUTHN:delete_chain(ListenerID), + ?AUTHN:remove_provider(AuthNType), + ok. + +t_update_config(_) -> + emqx_config_handler:add_handler([authentication], emqx_authentication), + + AuthNType1 = {'password-based', 'built-in-database'}, + AuthNType2 = {'password-based', mysql}, + ?AUTHN:add_provider(AuthNType1, ?MODULE), + ?AUTHN:add_provider(AuthNType2, ?MODULE), + + Global = 'mqtt:global', + AuthenticatorConfig1 = #{mechanism => 'password-based', + backend => 'built-in-database', + enable => true}, + AuthenticatorConfig2 = #{mechanism => 'password-based', + backend => mysql, + enable => true}, + ID1 = <<"password-based:built-in-database">>, + ID2 = <<"password-based:mysql">>, + + ?assertMatch({ok, []}, ?AUTHN:list_chains()), + ?assertMatch({ok, _}, update_config([authentication], {create_authenticator, Global, AuthenticatorConfig1})), + ?assertMatch({ok, #{id := ID1, state := #{mark := 1}}}, ?AUTHN:lookup_authenticator(Global, ID1)), + + ?assertMatch({ok, _}, update_config([authentication], {create_authenticator, Global, AuthenticatorConfig2})), + ?assertMatch({ok, #{id := ID2, state := #{mark := 1}}}, ?AUTHN:lookup_authenticator(Global, ID2)), + + ?assertMatch({ok, _}, update_config([authentication], {update_authenticator, Global, ID1, #{}})), + ?assertMatch({ok, #{id := ID1, state := #{mark := 2}}}, ?AUTHN:lookup_authenticator(Global, ID1)), + + ?assertMatch({ok, _}, update_config([authentication], {move_authenticator, Global, ID2, <<"top">>})), + ?assertMatch({ok, [#{id := ID2}, #{id := ID1}]}, ?AUTHN:list_authenticators(Global)), + + ?assertMatch({ok, _}, update_config([authentication], {delete_authenticator, Global, ID1})), + ?assertEqual({error, {not_found, {authenticator, ID1}}}, ?AUTHN:lookup_authenticator(Global, ID1)), + + ListenerID = 'tcp:default', + ConfKeyPath = [listeners, tcp, default, authentication], + ?assertMatch({ok, _}, update_config(ConfKeyPath, {create_authenticator, ListenerID, AuthenticatorConfig1})), + ?assertMatch({ok, #{id := ID1, state := #{mark := 1}}}, ?AUTHN:lookup_authenticator(ListenerID, ID1)), + + ?assertMatch({ok, _}, update_config(ConfKeyPath, {create_authenticator, ListenerID, AuthenticatorConfig2})), + ?assertMatch({ok, #{id := ID2, state := #{mark := 1}}}, ?AUTHN:lookup_authenticator(ListenerID, ID2)), + + ?assertMatch({ok, _}, update_config(ConfKeyPath, {update_authenticator, ListenerID, ID1, #{}})), + ?assertMatch({ok, #{id := ID1, state := #{mark := 2}}}, ?AUTHN:lookup_authenticator(ListenerID, ID1)), + + ?assertMatch({ok, _}, update_config(ConfKeyPath, {move_authenticator, ListenerID, ID2, <<"top">>})), + ?assertMatch({ok, [#{id := ID2}, #{id := ID1}]}, ?AUTHN:list_authenticators(ListenerID)), + + ?assertMatch({ok, _}, update_config(ConfKeyPath, {delete_authenticator, ListenerID, ID1})), + ?assertEqual({error, {not_found, {authenticator, ID1}}}, ?AUTHN:lookup_authenticator(ListenerID, ID1)), + + ?AUTHN:delete_chain(Global), + ?AUTHN:remove_provider(AuthNType1), + ?AUTHN:remove_provider(AuthNType2), + ok. + +update_config(Path, ConfigRequest) -> + emqx:update_config(Path, ConfigRequest, #{rawconf_with_defaults => true}). diff --git a/apps/emqx/test/emqx_channel_SUITE.erl b/apps/emqx/test/emqx_channel_SUITE.erl index 031f89612..775b40ee8 100644 --- a/apps/emqx/test/emqx_channel_SUITE.erl +++ b/apps/emqx/test/emqx_channel_SUITE.erl @@ -144,7 +144,7 @@ init_per_suite(Config) -> %% Access Control Meck ok = meck:new(emqx_access_control, [passthrough, no_history, no_link]), ok = meck:expect(emqx_access_control, authenticate, - fun(_) -> {ok, #{superuser => false}} end), + fun(_) -> {ok, #{is_superuser => false}} end), ok = meck:expect(emqx_access_control, authorize, fun(_, _, _) -> allow end), %% Broker Meck ok = meck:new(emqx_broker, [passthrough, no_history, no_link]), diff --git a/apps/emqx/test/emqx_client_SUITE.erl b/apps/emqx/test/emqx_client_SUITE.erl index 117a0f5b9..0a3a050ac 100644 --- a/apps/emqx/test/emqx_client_SUITE.erl +++ b/apps/emqx/test/emqx_client_SUITE.erl @@ -114,8 +114,8 @@ t_cm(_) -> emqx_config:put_zone_conf(default, [mqtt, idle_timeout], 15000). t_cm_registry(_) -> - Info = supervisor:which_children(emqx_cm_sup), - {_, Pid, _, _} = lists:keyfind(registry, 1, Info), + Children = supervisor:which_children(emqx_cm_sup), + {_, Pid, _, _} = lists:keyfind(emqx_cm_registry, 1, Children), ignored = gen_server:call(Pid, <<"Unexpected call">>), gen_server:cast(Pid, <<"Unexpected cast">>), Pid ! <<"Unexpected info">>. diff --git a/apps/emqx/test/emqx_config_SUITE.erl b/apps/emqx/test/emqx_config_SUITE.erl new file mode 100644 index 000000000..50d575c0e --- /dev/null +++ b/apps/emqx/test/emqx_config_SUITE.erl @@ -0,0 +1,50 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_config_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). +-include_lib("eunit/include/eunit.hrl"). + +all() -> emqx_ct:all(?MODULE). + +init_per_suite(Config) -> + emqx_ct_helpers:boot_modules(all), + emqx_ct_helpers:start_apps([]), + Config. + +end_per_suite(_Config) -> + emqx_ct_helpers:stop_apps([]). + +t_fill_default_values(_) -> + Conf = #{ + <<"broker">> => #{ + <<"perf">> => #{}, + <<"route_batch_clean">> => false} + }, + ?assertMatch(#{<<"broker">> := + #{<<"enable_session_registry">> := true, + <<"perf">> := + #{<<"route_lock_type">> := key, + <<"trie_compaction">> := true}, + <<"route_batch_clean">> := false, + <<"session_locking_strategy">> := quorum, + <<"shared_dispatch_ack_enabled">> := false, + <<"shared_subscription_strategy">> := round_robin, + <<"sys_heartbeat_interval">> := "30s", + <<"sys_msg_interval">> := "1m"}}, + emqx_config:fill_defaults(Conf)). diff --git a/apps/emqx/test/emqx_flapping_SUITE.erl b/apps/emqx/test/emqx_flapping_SUITE.erl index 5ac6b9cdf..a8e783c49 100644 --- a/apps/emqx/test/emqx_flapping_SUITE.erl +++ b/apps/emqx/test/emqx_flapping_SUITE.erl @@ -55,8 +55,8 @@ t_detect_check(_) -> true = emqx_banned:check(ClientInfo), timer:sleep(3000), false = emqx_banned:check(ClientInfo), - Childrens = supervisor:which_children(emqx_cm_sup), - {flapping, Pid, _, _} = lists:keyfind(flapping, 1, Childrens), + Children = supervisor:which_children(emqx_cm_sup), + {emqx_flapping, Pid, _, _} = lists:keyfind(emqx_flapping, 1, Children), gen_server:call(Pid, unexpected_msg), gen_server:cast(Pid, unexpected_msg), Pid ! test, @@ -72,4 +72,4 @@ t_expired_detecting(_) -> (_) -> false end, ets:tab2list(emqx_flapping))), timer:sleep(200), ?assertEqual(true, lists:all(fun({flapping, <<"client008">>, _, _, _}) -> false; - (_) -> true end, ets:tab2list(emqx_flapping))). \ No newline at end of file + (_) -> true end, ets:tab2list(emqx_flapping))). diff --git a/apps/emqx/test/emqx_listeners_SUITE.erl b/apps/emqx/test/emqx_listeners_SUITE.erl index a8760c7e8..a3bfb2d47 100644 --- a/apps/emqx/test/emqx_listeners_SUITE.erl +++ b/apps/emqx/test/emqx_listeners_SUITE.erl @@ -37,6 +37,14 @@ end_per_suite(_Config) -> application:stop(esockd), application:stop(cowboy). +init_per_testcase(_, Config) -> + {ok, _} = emqx_config_handler:start_link(), + Config. + +end_per_testcase(_, _Config) -> + _ = emqx_config_handler:stop(), + ok. + t_start_stop_listeners(_) -> ok = emqx_listeners:start(), ?assertException(error, _, emqx_listeners:start_listener({ws,{"127.0.0.1", 8083}, []})), diff --git a/apps/emqx_authn/data/user-credentials.csv b/apps/emqx_authn/data/user-credentials.csv index 0548308b7..cbadaefbc 100644 --- a/apps/emqx_authn/data/user-credentials.csv +++ b/apps/emqx_authn/data/user-credentials.csv @@ -1,3 +1,3 @@ -user_id,password_hash,salt,superuser +user_id,password_hash,salt,is_superuser myuser3,b6c743545a7817ae8c8f624371d5f5f0373234bb0ff36b8ffbf19bce0e06ab75,de1024f462fb83910fd13151bd4bd235,true myuser4,ee68c985a69208b6eda8c6c9b4c7c2d2b15ee2352cdd64a903171710a99182e8,ad773b5be9dd0613fe6c2f4d8c403139,false diff --git a/apps/emqx_authn/data/user-credentials.json b/apps/emqx_authn/data/user-credentials.json index e54501233..94375df22 100644 --- a/apps/emqx_authn/data/user-credentials.json +++ b/apps/emqx_authn/data/user-credentials.json @@ -3,12 +3,12 @@ "user_id":"myuser1", "password_hash":"c5e46903df45e5dc096dc74657610dbee8deaacae656df88a1788f1847390242", "salt": "e378187547bf2d6f0545a3f441aa4d8a", - "superuser": true + "is_superuser": true }, { "user_id":"myuser2", "password_hash":"f4d17f300b11e522fd33f497c11b126ef1ea5149c74d2220f9a16dc876d4567b", "salt": "6d3f9bd5b54d94b98adbcfe10b6d181f", - "superuser": false + "is_superuser": false } ] diff --git a/apps/emqx_authn/etc/emqx_authn.conf b/apps/emqx_authn/etc/emqx_authn.conf index 59f4aa9ee..d1d3d16f8 100644 --- a/apps/emqx_authn/etc/emqx_authn.conf +++ b/apps/emqx_authn/etc/emqx_authn.conf @@ -1,37 +1,6 @@ -authentication { - enable = false - authenticators = [ - # { - # name: "authenticator1" - # mechanism: password-based - # server_type: built-in-database - # user_id_type: clientid - # }, - # { - # name: "authenticator2" - # mechanism: password-based - # server_type: mongodb - # server: "127.0.0.1:27017" - # database: mqtt - # collection: users - # selector: { - # username: "${mqtt-username}" - # } - # password_hash_field: password_hash - # salt_field: salt - # password_hash_algorithm: sha256 - # salt_position: prefix - # }, - # { - # name: "authenticator 3" - # mechanism: password-based - # server_type: redis - # server: "127.0.0.1:6379" - # password: "public" - # database: 0 - # query: "HMGET ${mqtt-username} password_hash salt" - # password_hash_algorithm: sha256 - # salt_position: prefix - # } - ] -} +# authentication: { +# mechanism: password-based +# backend: built-in-database +# user_id_type: clientid +# } + diff --git a/apps/emqx_authn/include/emqx_authn.hrl b/apps/emqx_authn/include/emqx_authn.hrl index c5a392fd0..5eef08012 100644 --- a/apps/emqx_authn/include/emqx_authn.hrl +++ b/apps/emqx_authn/include/emqx_authn.hrl @@ -15,24 +15,11 @@ %%-------------------------------------------------------------------- -define(APP, emqx_authn). --define(CHAIN, <<"mqtt">>). --define(VER_1, <<"1">>). --define(VER_2, <<"2">>). +-define(AUTHN, emqx_authentication). + +-define(GLOBAL, 'mqtt:global'). -define(RE_PLACEHOLDER, "\\$\\{[a-z0-9\\-]+\\}"). --record(authenticator, - { id :: binary() - , name :: binary() - , provider :: module() - , state :: map() - }). - --record(chain, - { id :: binary() - , authenticators :: [{binary(), binary(), #authenticator{}}] - , created_at :: integer() - }). - -define(AUTH_SHARD, emqx_authn_shard). diff --git a/apps/emqx_authn/rebar.config b/apps/emqx_authn/rebar.config index 32b5a43e0..73696b033 100644 --- a/apps/emqx_authn/rebar.config +++ b/apps/emqx_authn/rebar.config @@ -1,6 +1,4 @@ -{deps, [ - {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.1"}}} -]}. +{deps, []}. {edoc_opts, [{preprocess, true}]}. {erl_opts, [warn_unused_vars, diff --git a/apps/emqx_authn/src/emqx_authn.erl b/apps/emqx_authn/src/emqx_authn.erl index 1034682e5..3ab05e6b0 100644 --- a/apps/emqx_authn/src/emqx_authn.erl +++ b/apps/emqx_authn/src/emqx_authn.erl @@ -15,640 +15,3 @@ %%-------------------------------------------------------------------- -module(emqx_authn). - --behaviour(gen_server). - --behaviour(emqx_config_handler). - --include("emqx_authn.hrl"). --include_lib("emqx/include/logger.hrl"). - --export([ pre_config_update/2 - , post_config_update/4 - , update_config/2 - ]). - --export([ enable/0 - , disable/0 - , is_enabled/0 - ]). - --export([authenticate/2]). - --export([ start_link/0 - , stop/0 - ]). - --export([ create_chain/1 - , delete_chain/1 - , lookup_chain/1 - , list_chains/0 - , create_authenticator/2 - , delete_authenticator/2 - , update_authenticator/3 - , update_or_create_authenticator/3 - , lookup_authenticator/2 - , list_authenticators/1 - , move_authenticator/3 - ]). - --export([ import_users/3 - , add_user/3 - , delete_user/3 - , update_user/4 - , lookup_user/3 - , list_users/2 - ]). - -%% gen_server callbacks --export([ init/1 - , handle_call/3 - , handle_cast/2 - , handle_info/2 - , terminate/2 - , code_change/3 - ]). - --define(CHAIN_TAB, emqx_authn_chain). - -%%------------------------------------------------------------------------------ -%% APIs -%%------------------------------------------------------------------------------ - -pre_config_update({enable, Enable}, _OldConfig) -> - {ok, Enable}; -pre_config_update({create_authenticator, Config}, OldConfig) -> - {ok, OldConfig ++ [Config]}; -pre_config_update({delete_authenticator, ID}, OldConfig) -> - case lookup_authenticator(?CHAIN, ID) of - {error, Reason} -> {error, Reason}; - {ok, #{name := Name}} -> - NewConfig = lists:filter(fun(#{<<"name">> := N}) -> - N =/= Name - end, OldConfig), - {ok, NewConfig} - end; -pre_config_update({update_authenticator, ID, Config}, OldConfig) -> - case lookup_authenticator(?CHAIN, ID) of - {error, Reason} -> {error, Reason}; - {ok, #{name := Name}} -> - NewConfig = lists:map(fun(#{<<"name">> := N} = C) -> - case N =:= Name of - true -> Config; - false -> C - end - end, OldConfig), - {ok, NewConfig} - end; -pre_config_update({update_or_create_authenticator, ID, Config}, OldConfig) -> - case lookup_authenticator(?CHAIN, ID) of - {error, _Reason} -> OldConfig ++ [Config]; - {ok, #{name := Name}} -> - NewConfig = lists:map(fun(#{<<"name">> := N} = C) -> - case N =:= Name of - true -> Config; - false -> C - end - end, OldConfig), - {ok, NewConfig} - end; -pre_config_update({move_authenticator, ID, Position}, OldConfig) -> - case lookup_authenticator(?CHAIN, ID) of - {error, Reason} -> {error, Reason}; - {ok, #{name := Name}} -> - {ok, Found, Part1, Part2} = split_by_name(Name, OldConfig), - case Position of - <<"top">> -> - {ok, [Found | Part1] ++ Part2}; - <<"bottom">> -> - {ok, Part1 ++ Part2 ++ [Found]}; - Before -> - case binary:split(Before, <<":">>, [global]) of - [<<"before">>, ID0] -> - case lookup_authenticator(?CHAIN, ID0) of - {error, Reason} -> {error, Reason}; - {ok, #{name := Name1}} -> - {ok, NFound, NPart1, NPart2} = split_by_name(Name1, Part1 ++ Part2), - {ok, NPart1 ++ [Found, NFound | NPart2]} - end; - _ -> - {error, {invalid_parameter, position}} - end - end - end. - -post_config_update({enable, true}, _NewConfig, _OldConfig, _AppEnvs) -> - emqx_authn:enable(); -post_config_update({enable, false}, _NewConfig, _OldConfig, _AppEnvs) -> - emqx_authn:disable(); -post_config_update({create_authenticator, #{<<"name">> := Name}}, NewConfig, _OldConfig, _AppEnvs) -> - case lists:filter( - fun(#{name := N}) -> - N =:= Name - end, NewConfig) of - [Config] -> - create_authenticator(?CHAIN, Config); - [_Config | _] -> - {error, name_has_be_used} - end; -post_config_update({delete_authenticator, ID}, _NewConfig, _OldConfig, _AppEnvs) -> - case delete_authenticator(?CHAIN, ID) of - ok -> ok; - {error, Reason} -> throw(Reason) - end; -post_config_update({update_authenticator, ID, #{<<"name">> := Name}}, NewConfig, _OldConfig, _AppEnvs) -> - case lists:filter( - fun(#{name := N}) -> - N =:= Name - end, NewConfig) of - [Config] -> - update_authenticator(?CHAIN, ID, Config); - [_Config | _] -> - {error, name_has_be_used} - end; -post_config_update({update_or_create_authenticator, ID, #{<<"name">> := Name}}, NewConfig, _OldConfig, _AppEnvs) -> - case lists:filter( - fun(#{name := N}) -> - N =:= Name - end, NewConfig) of - [Config] -> - update_or_create_authenticator(?CHAIN, ID, Config); - [_Config | _] -> - {error, name_has_be_used} - end; -post_config_update({move_authenticator, ID, Position}, _NewConfig, _OldConfig, _AppEnvs) -> - NPosition = case Position of - <<"top">> -> top; - <<"bottom">> -> bottom; - Before -> - case binary:split(Before, <<":">>, [global]) of - [<<"before">>, ID0] -> - {before, ID0}; - _ -> - {error, {invalid_parameter, position}} - end - end, - move_authenticator(?CHAIN, ID, NPosition). - -update_config(Path, ConfigRequest) -> - emqx:update_config(Path, ConfigRequest, #{rawconf_with_defaults => true}). - -enable() -> - case emqx:hook('client.authenticate', {?MODULE, authenticate, []}) of - ok -> ok; - {error, already_exists} -> ok - end. - -disable() -> - emqx:unhook('client.authenticate', {?MODULE, authenticate, []}), - ok. - -is_enabled() -> - Callbacks = emqx_hooks:lookup('client.authenticate'), - lists:any(fun({callback, {?MODULE, authenticate, []}, _, _}) -> - true; - (_) -> - false - end, Callbacks). - -authenticate(Credential, _AuthResult) -> - case ets:lookup(?CHAIN_TAB, ?CHAIN) of - [#chain{authenticators = Authenticators}] -> - do_authenticate(Authenticators, Credential); - [] -> - {stop, {error, not_authorized}} - end. - -do_authenticate([], _) -> - {stop, {error, not_authorized}}; -do_authenticate([{_, _, #authenticator{provider = Provider, state = State}} | More], Credential) -> - case Provider:authenticate(Credential, State) of - ignore -> - do_authenticate(More, Credential); - Result -> - %% {ok, Extra} - %% {ok, Extra, AuthData} - %% {ok, MetaData} - %% {continue, AuthCache} - %% {continue, AuthData, AuthCache} - %% {error, Reason} - {stop, Result} - end. - -start_link() -> - gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). - -stop() -> - gen_server:stop(?MODULE). - -create_chain(#{id := ID}) -> - gen_server:call(?MODULE, {create_chain, ID}). - -delete_chain(ID) -> - gen_server:call(?MODULE, {delete_chain, ID}). - -lookup_chain(ID) -> - gen_server:call(?MODULE, {lookup_chain, ID}). - -list_chains() -> - Chains = ets:tab2list(?CHAIN_TAB), - {ok, [serialize_chain(Chain) || Chain <- Chains]}. - -create_authenticator(ChainID, Config) -> - gen_server:call(?MODULE, {create_authenticator, ChainID, Config}). - -delete_authenticator(ChainID, AuthenticatorID) -> - gen_server:call(?MODULE, {delete_authenticator, ChainID, AuthenticatorID}). - -update_authenticator(ChainID, AuthenticatorID, Config) -> - gen_server:call(?MODULE, {update_authenticator, ChainID, AuthenticatorID, Config}). - -update_or_create_authenticator(ChainID, AuthenticatorID, Config) -> - gen_server:call(?MODULE, {update_or_create_authenticator, ChainID, AuthenticatorID, Config}). - -lookup_authenticator(ChainID, AuthenticatorID) -> - case ets:lookup(?CHAIN_TAB, ChainID) of - [] -> - {error, {not_found, {chain, ChainID}}}; - [#chain{authenticators = Authenticators}] -> - case lists:keyfind(AuthenticatorID, 1, Authenticators) of - false -> - {error, {not_found, {authenticator, AuthenticatorID}}}; - {_, _, Authenticator} -> - {ok, serialize_authenticator(Authenticator)} - end - end. - -list_authenticators(ChainID) -> - case ets:lookup(?CHAIN_TAB, ChainID) of - [] -> - {error, {not_found, {chain, ChainID}}}; - [#chain{authenticators = Authenticators}] -> - {ok, serialize_authenticators(Authenticators)} - end. - -move_authenticator(ChainID, AuthenticatorID, Position) -> - gen_server:call(?MODULE, {move_authenticator, ChainID, AuthenticatorID, Position}). - -import_users(ChainID, AuthenticatorID, Filename) -> - gen_server:call(?MODULE, {import_users, ChainID, AuthenticatorID, Filename}). - -add_user(ChainID, AuthenticatorID, UserInfo) -> - gen_server:call(?MODULE, {add_user, ChainID, AuthenticatorID, UserInfo}). - -delete_user(ChainID, AuthenticatorID, UserID) -> - gen_server:call(?MODULE, {delete_user, ChainID, AuthenticatorID, UserID}). - -update_user(ChainID, AuthenticatorID, UserID, NewUserInfo) -> - gen_server:call(?MODULE, {update_user, ChainID, AuthenticatorID, UserID, NewUserInfo}). - -lookup_user(ChainID, AuthenticatorID, UserID) -> - gen_server:call(?MODULE, {lookup_user, ChainID, AuthenticatorID, UserID}). - -%% TODO: Support pagination -list_users(ChainID, AuthenticatorID) -> - gen_server:call(?MODULE, {list_users, ChainID, AuthenticatorID}). - -%%-------------------------------------------------------------------- -%% gen_server callbacks -%%-------------------------------------------------------------------- - -init(_Opts) -> - _ = ets:new(?CHAIN_TAB, [ named_table, set, public - , {keypos, #chain.id} - , {read_concurrency, true}]), - {ok, #{}}. - -handle_call({create_chain, ID}, _From, State) -> - case ets:member(?CHAIN_TAB, ID) of - true -> - reply({error, {already_exists, {chain, ID}}}, State); - false -> - Chain = #chain{id = ID, - authenticators = [], - created_at = erlang:system_time(millisecond)}, - true = ets:insert(?CHAIN_TAB, Chain), - reply({ok, serialize_chain(Chain)}, State) - end; - -handle_call({delete_chain, ID}, _From, State) -> - case ets:lookup(?CHAIN_TAB, ID) of - [] -> - reply({error, {not_found, {chain, ID}}}, State); - [#chain{authenticators = Authenticators}] -> - _ = [do_delete_authenticator(Authenticator) || {_, _, Authenticator} <- Authenticators], - true = ets:delete(?CHAIN_TAB, ID), - reply(ok, State) - end; - -handle_call({lookup_chain, ID}, _From, State) -> - case ets:lookup(?CHAIN_TAB, ID) of - [] -> - reply({error, {not_found, {chain, ID}}}, State); - [Chain] -> - reply({ok, serialize_chain(Chain)}, State) - end; - -handle_call({create_authenticator, ChainID, #{name := Name} = Config}, _From, State) -> - UpdateFun = - fun(#chain{authenticators = Authenticators} = Chain) -> - case lists:keymember(Name, 2, Authenticators) of - true -> - {error, name_has_be_used}; - false -> - AlreadyExist = fun(ID) -> - lists:keymember(ID, 1, Authenticators) - end, - AuthenticatorID = gen_id(AlreadyExist), - case do_create_authenticator(ChainID, AuthenticatorID, Config) of - {ok, Authenticator} -> - NAuthenticators = Authenticators ++ [{AuthenticatorID, Name, Authenticator}], - true = ets:insert(?CHAIN_TAB, Chain#chain{authenticators = NAuthenticators}), - {ok, serialize_authenticator(Authenticator)}; - {error, Reason} -> - {error, Reason} - end - end - end, - Reply = update_chain(ChainID, UpdateFun), - reply(Reply, State); - -handle_call({delete_authenticator, ChainID, AuthenticatorID}, _From, State) -> - UpdateFun = - fun(#chain{authenticators = Authenticators} = Chain) -> - case lists:keytake(AuthenticatorID, 1, Authenticators) of - false -> - {error, {not_found, {authenticator, AuthenticatorID}}}; - {value, {_, _, Authenticator}, NAuthenticators} -> - _ = do_delete_authenticator(Authenticator), - true = ets:insert(?CHAIN_TAB, Chain#chain{authenticators = NAuthenticators}), - ok - end - end, - Reply = update_chain(ChainID, UpdateFun), - reply(Reply, State); - -handle_call({update_authenticator, ChainID, AuthenticatorID, Config}, _From, State) -> - Reply = update_or_create_authenticator(ChainID, AuthenticatorID, Config, false), - reply(Reply, State); - -handle_call({update_or_create_authenticator, ChainID, AuthenticatorID, Config}, _From, State) -> - Reply = update_or_create_authenticator(ChainID, AuthenticatorID, Config, true), - reply(Reply, State); - -handle_call({move_authenticator, ChainID, AuthenticatorID, Position}, _From, State) -> - UpdateFun = - fun(#chain{authenticators = Authenticators} = Chain) -> - case do_move_authenticator(AuthenticatorID, Authenticators, Position) of - {ok, NAuthenticators} -> - true = ets:insert(?CHAIN_TAB, Chain#chain{authenticators = NAuthenticators}), - ok; - {error, Reason} -> - {error, Reason} - end - end, - Reply = update_chain(ChainID, UpdateFun), - reply(Reply, State); - -handle_call({import_users, ChainID, AuthenticatorID, Filename}, _From, State) -> - Reply = call_authenticator(ChainID, AuthenticatorID, import_users, [Filename]), - reply(Reply, State); - -handle_call({add_user, ChainID, AuthenticatorID, UserInfo}, _From, State) -> - Reply = call_authenticator(ChainID, AuthenticatorID, add_user, [UserInfo]), - reply(Reply, State); - -handle_call({delete_user, ChainID, AuthenticatorID, UserID}, _From, State) -> - Reply = call_authenticator(ChainID, AuthenticatorID, delete_user, [UserID]), - reply(Reply, State); - -handle_call({update_user, ChainID, AuthenticatorID, UserID, NewUserInfo}, _From, State) -> - Reply = call_authenticator(ChainID, AuthenticatorID, update_user, [UserID, NewUserInfo]), - reply(Reply, State); - -handle_call({lookup_user, ChainID, AuthenticatorID, UserID}, _From, State) -> - Reply = call_authenticator(ChainID, AuthenticatorID, lookup_user, [UserID]), - reply(Reply, State); - -handle_call({list_users, ChainID, AuthenticatorID}, _From, State) -> - Reply = call_authenticator(ChainID, AuthenticatorID, list_users, []), - reply(Reply, State); - -handle_call(Req, _From, State) -> - ?LOG(error, "Unexpected call: ~p", [Req]), - {reply, ignored, State}. - -handle_cast(Req, State) -> - ?LOG(error, "Unexpected case: ~p", [Req]), - {noreply, State}. - -handle_info(Info, State) -> - ?LOG(error, "Unexpected info: ~p", [Info]), - {noreply, State}. - -terminate(_Reason, _State) -> - ok. - -code_change(_OldVsn, State, _Extra) -> - {ok, State}. - -reply(Reply, State) -> - {reply, Reply, State}. - -%%------------------------------------------------------------------------------ -%% Internal functions -%%------------------------------------------------------------------------------ - -authenticator_provider(#{mechanism := 'password-based', server_type := 'built-in-database'}) -> - emqx_authn_mnesia; -authenticator_provider(#{mechanism := 'password-based', server_type := 'mysql'}) -> - emqx_authn_mysql; -authenticator_provider(#{mechanism := 'password-based', server_type := 'pgsql'}) -> - emqx_authn_pgsql; -authenticator_provider(#{mechanism := 'password-based', server_type := 'mongodb'}) -> - emqx_authn_mongodb; -authenticator_provider(#{mechanism := 'password-based', server_type := 'redis'}) -> - emqx_authn_redis; -authenticator_provider(#{mechanism := 'password-based', server_type := 'http-server'}) -> - emqx_authn_http; -authenticator_provider(#{mechanism := jwt}) -> - emqx_authn_jwt; -authenticator_provider(#{mechanism := scram, server_type := 'built-in-database'}) -> - emqx_enhanced_authn_scram_mnesia. - -gen_id(AlreadyExist) -> - ID = list_to_binary(emqx_rule_id:gen()), - case AlreadyExist(ID) of - true -> gen_id(AlreadyExist); - false -> ID - end. - -switch_version(State = #{version := ?VER_1}) -> - State#{version := ?VER_2}; -switch_version(State = #{version := ?VER_2}) -> - State#{version := ?VER_1}; -switch_version(State) -> - State#{version => ?VER_1}. - -split_by_name(Name, Config) -> - {Part1, Part2, true} = lists:foldl( - fun(#{<<"name">> := N} = C, {P1, P2, F0}) -> - F = case N =:= Name of - true -> true; - false -> F0 - end, - case F of - false -> {[C | P1], P2, F}; - true -> {P1, [C | P2], F} - end - end, {[], [], false}, Config), - [Found | NPart2] = lists:reverse(Part2), - {ok, Found, lists:reverse(Part1), NPart2}. - -do_create_authenticator(ChainID, AuthenticatorID, #{name := Name} = Config) -> - Provider = authenticator_provider(Config), - Unique = <>, - case Provider:create(Config#{'_unique' => Unique}) of - {ok, State} -> - Authenticator = #authenticator{id = AuthenticatorID, - name = Name, - provider = Provider, - state = switch_version(State)}, - {ok, Authenticator}; - {error, Reason} -> - {error, Reason} - end. - -do_delete_authenticator(#authenticator{provider = Provider, state = State}) -> - _ = Provider:destroy(State), - ok. - -update_or_create_authenticator(ChainID, AuthenticatorID, #{name := NewName} = Config, CreateWhenNotFound) -> - UpdateFun = - fun(#chain{authenticators = Authenticators} = Chain) -> - case lists:keytake(AuthenticatorID, 1, Authenticators) of - false -> - case CreateWhenNotFound of - true -> - case lists:keymember(NewName, 2, Authenticators) of - true -> - {error, name_has_be_used}; - false -> - case do_create_authenticator(ChainID, AuthenticatorID, Config) of - {ok, Authenticator} -> - NAuthenticators = Authenticators ++ [{AuthenticatorID, NewName, Authenticator}], - true = ets:insert(?CHAIN_TAB, Chain#chain{authenticators = NAuthenticators}), - {ok, serialize_authenticator(Authenticator)}; - {error, Reason} -> - {error, Reason} - end - end; - false -> - {error, {not_found, {authenticator, AuthenticatorID}}} - end; - {value, - {_, _, #authenticator{provider = Provider, - state = #{version := Version} = State} = Authenticator}, - Others} -> - case lists:keymember(NewName, 2, Others) of - true -> - {error, name_has_be_used}; - false -> - case (NewProvider = authenticator_provider(Config)) =:= Provider of - true -> - Unique = <>, - case Provider:update(Config#{'_unique' => Unique}, State) of - {ok, NewState} -> - NewAuthenticator = Authenticator#authenticator{name = NewName, - state = switch_version(NewState)}, - NewAuthenticators = replace_authenticator(AuthenticatorID, NewAuthenticator, Authenticators), - true = ets:insert(?CHAIN_TAB, Chain#chain{authenticators = NewAuthenticators}), - {ok, serialize_authenticator(NewAuthenticator)}; - {error, Reason} -> - {error, Reason} - end; - false -> - Unique = <>, - case NewProvider:create(Config#{'_unique' => Unique}) of - {ok, NewState} -> - NewAuthenticator = Authenticator#authenticator{name = NewName, - provider = NewProvider, - state = switch_version(NewState)}, - NewAuthenticators = replace_authenticator(AuthenticatorID, NewAuthenticator, Authenticators), - true = ets:insert(?CHAIN_TAB, Chain#chain{authenticators = NewAuthenticators}), - _ = Provider:destroy(State), - {ok, serialize_authenticator(NewAuthenticator)}; - {error, Reason} -> - {error, Reason} - end - end - end - end - end, - update_chain(ChainID, UpdateFun). - -replace_authenticator(ID, #authenticator{name = Name} = Authenticator, Authenticators) -> - lists:keyreplace(ID, 1, Authenticators, {ID, Name, Authenticator}). - -do_move_authenticator(AuthenticatorID, Authenticators, Position) when is_binary(AuthenticatorID) -> - case lists:keytake(AuthenticatorID, 1, Authenticators) of - false -> - {error, {not_found, {authenticator, AuthenticatorID}}}; - {value, Authenticator, NAuthenticators} -> - do_move_authenticator(Authenticator, NAuthenticators, Position) - end; - -do_move_authenticator(Authenticator, Authenticators, top) -> - {ok, [Authenticator | Authenticators]}; -do_move_authenticator(Authenticator, Authenticators, bottom) -> - {ok, Authenticators ++ [Authenticator]}; -do_move_authenticator(Authenticator, Authenticators, {before, ID}) -> - insert(Authenticator, Authenticators, ID, []). - -insert(_, [], ID, _) -> - {error, {not_found, {authenticator, ID}}}; -insert(Authenticator, [{ID, _, _} | _] = Authenticators, ID, Acc) -> - {ok, lists:reverse(Acc) ++ [Authenticator | Authenticators]}; -insert(Authenticator, [{_, _, _} = Authenticator0 | More], ID, Acc) -> - insert(Authenticator, More, ID, [Authenticator0 | Acc]). - -update_chain(ChainID, UpdateFun) -> - case ets:lookup(?CHAIN_TAB, ChainID) of - [] -> - {error, {not_found, {chain, ChainID}}}; - [Chain] -> - UpdateFun(Chain) - end. - -call_authenticator(ChainID, AuthenticatorID, Func, Args) -> - UpdateFun = - fun(#chain{authenticators = Authenticators}) -> - case lists:keyfind(AuthenticatorID, 1, Authenticators) of - false -> - {error, {not_found, {authenticator, AuthenticatorID}}}; - {_, _, #authenticator{provider = Provider, state = State}} -> - case erlang:function_exported(Provider, Func, length(Args) + 1) of - true -> - erlang:apply(Provider, Func, Args ++ [State]); - false -> - {error, unsupported_feature} - end - end - end, - update_chain(ChainID, UpdateFun). - -serialize_chain(#chain{id = ID, - authenticators = Authenticators, - created_at = CreatedAt}) -> - #{id => ID, - authenticators => serialize_authenticators(Authenticators), - created_at => CreatedAt}. - -serialize_authenticators(Authenticators) -> - [serialize_authenticator(Authenticator) || {_, _, Authenticator} <- Authenticators]. - -serialize_authenticator(#authenticator{id = ID, - name = Name, - provider = Provider, - state = State}) -> - #{id => ID, name => Name, provider => Provider, state => State}. diff --git a/apps/emqx_authn/src/emqx_authn_api.erl b/apps/emqx_authn/src/emqx_authn_api.erl index 5f2b96b57..5ba1419f0 100644 --- a/apps/emqx_authn/src/emqx_authn_api.erl +++ b/apps/emqx_authn/src/emqx_authn_api.erl @@ -22,37 +22,39 @@ -export([ api_spec/0 , authentication/2 - , authenticators/2 - , authenticators2/2 + , authentication2/2 + , authentication3/2 + , authentication4/2 , move/2 + , move2/2 , import_users/2 + , import_users2/2 , users/2 , users2/2 + , users3/2 + , users4/2 ]). --define(EXAMPLE_1, #{name => <<"example 1">>, - mechanism => <<"password-based">>, - server_type => <<"built-in-database">>, - user_id_type => <<"username">>, +-define(EXAMPLE_1, #{mechanism => <<"password-based">>, + backend => <<"built-in-database">>, + query => <<"SELECT password_hash from built-in-database WHERE username = ${username}">>, password_hash_algorithm => #{ name => <<"sha256">> }}). --define(EXAMPLE_2, #{name => <<"example 2">>, - mechanism => <<"password-based">>, - server_type => <<"http-server">>, +-define(EXAMPLE_2, #{mechanism => <<"password-based">>, + backend => <<"http-server">>, method => <<"post">>, url => <<"http://localhost:80/login">>, headers => #{ <<"content-type">> => <<"application/json">> }, - form_data => #{ + body => #{ <<"username">> => <<"${mqtt-username}">>, <<"password">> => <<"${mqtt-password}">> }}). --define(EXAMPLE_3, #{name => <<"example 3">>, - mechanism => <<"jwt">>, +-define(EXAMPLE_3, #{mechanism => <<"jwt">>, use_jwks => false, algorithm => <<"hmac-based">>, secret => <<"mysecret">>, @@ -61,9 +63,8 @@ <<"username">> => <<"${mqtt-username}">> }}). --define(EXAMPLE_4, #{name => <<"example 4">>, - mechanism => <<"password-based">>, - server_type => <<"mongodb">>, +-define(EXAMPLE_4, #{mechanism => <<"password-based">>, + backend => <<"mongodb">>, server => <<"127.0.0.1:27017">>, database => example, collection => users, @@ -72,13 +73,13 @@ }, password_hash_field => <<"password_hash">>, salt_field => <<"salt">>, + is_superuser_field => <<"is_superuser">>, password_hash_algorithm => <<"sha256">>, salt_position => <<"prefix">> }). --define(EXAMPLE_5, #{name => <<"example 5">>, - mechanism => <<"password-based">>, - server_type => <<"redis">>, +-define(EXAMPLE_5, #{mechanism => <<"password-based">>, + backend => <<"redis">>, server => <<"127.0.0.1:6379">>, database => 0, query => <<"HMGET ${mqtt-username} password_hash salt">>, @@ -86,10 +87,53 @@ salt_position => <<"prefix">> }). +-define(INSTANCE_EXAMPLE_1, maps:merge(?EXAMPLE_1, #{id => <<"password-based:built-in-database">>, + enable => true})). + +-define(INSTANCE_EXAMPLE_2, maps:merge(?EXAMPLE_2, #{id => <<"password-based:http-server">>, + connect_timeout => 5000, + enable_pipelining => true, + headers => #{ + <<"accept">> => <<"application/json">>, + <<"cache-control">> => <<"no-cache">>, + <<"connection">> => <<"keepalive">>, + <<"content-type">> => <<"application/json">>, + <<"keep-alive">> => <<"timeout=5">> + }, + max_retries => 5, + pool_size => 8, + request_timeout => 5000, + retry_interval => 1000, + enable => true})). + +-define(INSTANCE_EXAMPLE_3, maps:merge(?EXAMPLE_3, #{id => <<"jwt">>, + enable => true})). + +-define(INSTANCE_EXAMPLE_4, maps:merge(?EXAMPLE_4, #{id => <<"password-based:mongodb">>, + mongo_type => <<"single">>, + pool_size => 8, + ssl => #{ + enable => false + }, + topology => #{ + max_overflow => 8, + pool_size => 8 + }, + enable => true})). + +-define(INSTANCE_EXAMPLE_5, maps:merge(?EXAMPLE_5, #{id => <<"password-based:redis">>, + auto_reconnect => true, + redis_type => single, + pool_size => 8, + ssl => #{ + enable => false + }, + enable => true})). + -define(ERR_RESPONSE(Desc), #{description => Desc, content => #{ 'application/json' => #{ - schema => minirest:ref(<<"error">>), + schema => minirest:ref(<<"Error">>), examples => #{ example1 => #{ summary => <<"Not Found">>, @@ -107,532 +151,642 @@ api_spec() -> {[ authentication_api() - , authenticators_api() - , authenticators_api2() + , authentication_api2() , move_api() + , authentication_api3() + , authentication_api4() + , move_api2() , import_users_api() + , import_users_api2() , users_api() , users2_api() + , users3_api() + , users4_api() ], definitions()}. authentication_api() -> Metadata = #{ - post => #{ - description => "Enable or disbale authentication", - requestBody => #{ - content => #{ - 'application/json' => #{ - schema => #{ - type => object, - required => [enable], - properties => #{ - enable => #{ - type => boolean, - example => true - } - } - } - } - } - }, - responses => #{ - <<"204">> => #{ - description => <<"No Content">> - }, - <<"400">> => ?ERR_RESPONSE(<<"Bad Request">>) - } - }, - get => #{ - description => "Get status of authentication", - responses => #{ - <<"200">> => #{ - description => <<"OK">>, - content => #{ - 'application/json' => #{ - schema => #{ - type => object, - properties => #{ - enabled => #{ - type => boolean, - example => true - } - } - } - } - } - } - } - } + post => create_authenticator_api_spec(), + get => list_authenticators_api_spec() }, {"/authentication", Metadata, authentication}. -authenticators_api() -> +authentication_api2() -> Metadata = #{ - post => #{ - description => "Create authenticator", - requestBody => #{ - content => #{ - 'application/json' => #{ - schema => minirest:ref(<<"authenticator">>), - examples => #{ - default => #{ - summary => <<"Default">>, - value => emqx_json:encode(?EXAMPLE_1) - }, - http => #{ - summary => <<"Authentication provided by HTTP Server">>, - value => emqx_json:encode(?EXAMPLE_2) - }, - jwt => #{ - summary => <<"JWT Authentication">>, - value => emqx_json:encode(?EXAMPLE_3) - }, - mongodb => #{ - summary => <<"Authentication with MongoDB">>, - value => emqx_json:encode(?EXAMPLE_4) - }, - redis => #{ - summary => <<"Authentication with Redis">>, - value => emqx_json:encode(?EXAMPLE_5) - } - } - } - } - }, - responses => #{ - <<"201">> => #{ - description => <<"Created">>, - content => #{ - 'application/json' => #{ - schema => minirest:ref(<<"returned_authenticator">>), - examples => #{ - %% TODO: return full content - example1 => #{ - summary => <<"Example 1">>, - value => emqx_json:encode(maps:put(id, <<"example 1">>, ?EXAMPLE_1)) - }, - example2 => #{ - summary => <<"Example 2">>, - value => emqx_json:encode(maps:put(id, <<"example 2">>, ?EXAMPLE_2)) - }, - example3 => #{ - summary => <<"Example 3">>, - value => emqx_json:encode(maps:put(id, <<"example 3">>, ?EXAMPLE_3)) - }, - example4 => #{ - summary => <<"Example 4">>, - value => emqx_json:encode(maps:put(id, <<"example 4">>, ?EXAMPLE_4)) - }, - example5 => #{ - summary => <<"Example 4">>, - value => emqx_json:encode(maps:put(id, <<"example 5">>, ?EXAMPLE_5)) - } - } - } - } - }, - <<"400">> => ?ERR_RESPONSE(<<"Bad Request">>), - <<"409">> => ?ERR_RESPONSE(<<"Conflict">>) - } - }, - get => #{ - description => "List authenticators", - responses => #{ - <<"200">> => #{ - description => <<"OK">>, - content => #{ - 'application/json' => #{ - schema => #{ - type => array, - items => minirest:ref(<<"returned_authenticator">>) - }, - examples => #{ - example1 => #{ - summary => <<"Example 1">>, - value => emqx_json:encode([ maps:put(id, <<"example 1">>, ?EXAMPLE_1) - , maps:put(id, <<"example 2">>, ?EXAMPLE_2) - , maps:put(id, <<"example 3">>, ?EXAMPLE_3) - , maps:put(id, <<"example 4">>, ?EXAMPLE_4) - , maps:put(id, <<"example 5">>, ?EXAMPLE_5) - ]) - } - } - } - } - } - } - } + get => find_authenticator_api_spec(), + put => update_authenticator_api_spec(), + delete => delete_authenticator_api_spec() }, - {"/authentication/authenticators", Metadata, authenticators}. + {"/authentication/:id", Metadata, authentication2}. -authenticators_api2() -> +authentication_api3() -> Metadata = #{ - get => #{ - description => "Get authenicator by id", - parameters => [ - #{ - name => id, - in => path, - schema => #{ - type => string - }, - required => true - } - ], - responses => #{ - <<"200">> => #{ - description => <<"OK">>, - content => #{ - 'application/json' => #{ - schema => minirest:ref(<<"returned_authenticator">>), - examples => #{ - example1 => #{ - summary => <<"Example 1">>, - value => emqx_json:encode(maps:put(id, <<"example 1">>, ?EXAMPLE_1)) - }, - example2 => #{ - summary => <<"Example 2">>, - value => emqx_json:encode(maps:put(id, <<"example 2">>, ?EXAMPLE_2)) - }, - example3 => #{ - summary => <<"Example 3">>, - value => emqx_json:encode(maps:put(id, <<"example 3">>, ?EXAMPLE_3)) - }, - example4 => #{ - summary => <<"Example 4">>, - value => emqx_json:encode(maps:put(id, <<"example 4">>, ?EXAMPLE_4)) - }, - example5 => #{ - summary => <<"Example 5">>, - value => emqx_json:encode(maps:put(id, <<"example 5">>, ?EXAMPLE_5)) - } - } - } - } - }, - <<"404">> => ?ERR_RESPONSE(<<"Not Found">>) - } - }, - put => #{ - description => "Update authenticator", - parameters => [ - #{ - name => id, - in => path, - schema => #{ - type => string - }, - required => true - } - ], - requestBody => #{ - content => #{ - 'application/json' => #{ - schema => #{ - oneOf => [ minirest:ref(<<"password_based">>) - , minirest:ref(<<"jwt">>) - , minirest:ref(<<"scram">>) - ] - }, - examples => #{ - example1 => #{ - summary => <<"Example 1">>, - value => emqx_json:encode(?EXAMPLE_1) - }, - example2 => #{ - summary => <<"Example 2">>, - value => emqx_json:encode(?EXAMPLE_2) - } - } - } - } - }, - responses => #{ - <<"200">> => #{ - description => <<"OK">>, - content => #{ - 'application/json' => #{ - schema => minirest:ref(<<"returned_authenticator">>), - examples => #{ - example1 => #{ - summary => <<"Example 1">>, - value => emqx_json:encode(maps:put(id, <<"example 1">>, ?EXAMPLE_1)) - }, - example2 => #{ - summary => <<"Example 2">>, - value => emqx_json:encode(maps:put(id, <<"example 2">>, ?EXAMPLE_2)) - }, - example3 => #{ - summary => <<"Example 3">>, - value => emqx_json:encode(maps:put(id, <<"example 3">>, ?EXAMPLE_3)) - }, - example4 => #{ - summary => <<"Example 4">>, - value => emqx_json:encode(maps:put(id, <<"example 4">>, ?EXAMPLE_4)) - }, - example5 => #{ - summary => <<"Example 5">>, - value => emqx_json:encode(maps:put(id, <<"example 5">>, ?EXAMPLE_5)) - } - } - } - } - }, - <<"400">> => ?ERR_RESPONSE(<<"Bad Request">>), - <<"404">> => ?ERR_RESPONSE(<<"Not Found">>), - <<"409">> => ?ERR_RESPONSE(<<"Conflict">>) - } - }, - delete => #{ - description => "Delete authenticator", - parameters => [ - #{ - name => id, - in => path, - schema => #{ - type => string - }, - required => true - } - ], - responses => #{ - <<"204">> => #{ - description => <<"No Content">> - }, - <<"404">> => ?ERR_RESPONSE(<<"Not Found">>) - } - } + post => create_authenticator_api_spec2(), + get => list_authenticators_api_spec2() }, - {"/authentication/authenticators/:id", Metadata, authenticators2}. + {"/listeners/:listener_id/authentication", Metadata, authentication3}. + +authentication_api4() -> + Metadata = #{ + get => find_authenticator_api_spec2(), + put => update_authenticator_api_spec2(), + delete => delete_authenticator_api_spec2() + }, + {"/listeners/:listener_id/authentication/:id", Metadata, authentication4}. move_api() -> Metadata = #{ - post => #{ - description => "Move authenticator", - parameters => [ - #{ - name => id, - in => path, - schema => #{ - type => string - }, - required => true - } - ], - requestBody => #{ - content => #{ - 'application/json' => #{ - schema => #{ - oneOf => [ - #{ - type => object, - required => [position], - properties => #{ - position => #{ - type => string, - enum => [<<"top">>, <<"bottom">>], - example => <<"top">> - } - } - }, - #{ - type => object, - required => [position], - properties => #{ - position => #{ - type => string, - description => <<"before:">>, - example => <<"before:67e4c9d3">> - } - } - } - ] - } - } - } - }, - responses => #{ - <<"204">> => #{ - description => <<"No Content">> - }, - <<"400">> => ?ERR_RESPONSE(<<"Bad Request">>), - <<"404">> => ?ERR_RESPONSE(<<"Not Found">>) - } - } + post => move_authenticator_api_spec() }, - {"/authentication/authenticators/:id/move", Metadata, move}. + {"/authentication/:id/move", Metadata, move}. + +move_api2() -> + Metadata = #{ + post => move_authenticator_api_spec2() + }, + {"/listeners/:listener_id/authentication/:id/move", Metadata, move2}. import_users_api() -> Metadata = #{ - post => #{ - description => "Import users from json/csv file", - parameters => [ - #{ - name => id, - in => path, - schema => #{ - type => string - }, - required => true + post => import_users_api_spec() + }, + {"/authentication/:id/import_users", Metadata, import_users}. + +import_users_api2() -> + Metadata = #{ + post => import_users_api_spec2() + }, + {"/listeners/:listener_id/authentication/:id/import_users", Metadata, import_users2}. + +users_api() -> + Metadata = #{ + post => create_user_api_spec(), + get => list_users_api_spec() + }, + {"/authentication/:id/users", Metadata, users}. + +users2_api() -> + Metadata = #{ + put => update_user_api_spec(), + get => find_user_api_spec(), + delete => delete_user_api_spec() + }, + {"/authentication/:id/users/:user_id", Metadata, users2}. + +users3_api() -> + Metadata = #{ + post => create_user_api_spec2(), + get => list_users_api_spec2() + }, + {"/listeners/:listener_id/authentication/:id/users", Metadata, users3}. + +users4_api() -> + Metadata = #{ + put => update_user_api_spec2(), + get => find_user_api_spec2(), + delete => delete_user_api_spec2() + }, + {"/listeners/:listener_id/authentication/:id/users/:user_id", Metadata, users4}. + +create_authenticator_api_spec() -> + #{ + description => "Create a authenticator for global authentication", + requestBody => #{ + content => #{ + 'application/json' => #{ + schema => minirest:ref(<<"AuthenticatorConfig">>), + examples => #{ + default => #{ + summary => <<"Default">>, + value => emqx_json:encode(?EXAMPLE_1) + }, + http => #{ + summary => <<"Authentication provided by HTTP Server">>, + value => emqx_json:encode(?EXAMPLE_2) + }, + jwt => #{ + summary => <<"JWT Authentication">>, + value => emqx_json:encode(?EXAMPLE_3) + }, + mongodb => #{ + summary => <<"Authentication with MongoDB">>, + value => emqx_json:encode(?EXAMPLE_4) + }, + redis => #{ + summary => <<"Authentication with Redis">>, + value => emqx_json:encode(?EXAMPLE_5) + } + } } - ], - requestBody => #{ + } + }, + responses => #{ + <<"201">> => #{ + description => <<"Created">>, content => #{ 'application/json' => #{ - schema => #{ - type => object, - required => [filename], - properties => #{ - filename => #{ - type => string - } + schema => minirest:ref(<<"AuthenticatorInstance">>), + examples => #{ + example1 => #{ + summary => <<"Example 1">>, + value => emqx_json:encode(?INSTANCE_EXAMPLE_1) + }, + example2 => #{ + summary => <<"Example 2">>, + value => emqx_json:encode(?INSTANCE_EXAMPLE_2) + }, + example3 => #{ + summary => <<"Example 3">>, + value => emqx_json:encode(?INSTANCE_EXAMPLE_3) + }, + example4 => #{ + summary => <<"Example 4">>, + value => emqx_json:encode(?INSTANCE_EXAMPLE_4) + }, + example5 => #{ + summary => <<"Example 5">>, + value => emqx_json:encode(?INSTANCE_EXAMPLE_5) } } } } }, - responses => #{ - <<"204">> => #{ - description => <<"No Content">> - }, - <<"400">> => ?ERR_RESPONSE(<<"Bad Request">>), - <<"404">> => ?ERR_RESPONSE(<<"Not Found">>) - } - } - }, - {"/authentication/authenticators/:id/import-users", Metadata, import_users}. + <<"400">> => ?ERR_RESPONSE(<<"Bad Request">>), + <<"409">> => ?ERR_RESPONSE(<<"Conflict">>) + } + }. -users_api() -> - Metadata = #{ - post => #{ - description => "Add user", - parameters => [ - #{ - name => id, - in => path, - schema => #{ - type => string - }, - required => true +create_authenticator_api_spec2() -> + Spec = create_authenticator_api_spec(), + Spec#{ + description => "Create a authenticator for listener", + parameters => [ + #{ + name => listener_id, + in => path, + schema => #{ + type => string + }, + required => true + } + ] + }. + +list_authenticators_api_spec() -> + #{ + description => "List authenticators for global authentication", + responses => #{ + <<"200">> => #{ + description => <<"OK">>, + content => #{ + 'application/json' => #{ + schema => #{ + type => array, + items => minirest:ref(<<"AuthenticatorInstance">>) + }, + examples => #{ + example => #{ + summary => <<"Example">>, + value => emqx_json:encode([ ?INSTANCE_EXAMPLE_1 + , ?INSTANCE_EXAMPLE_2 + , ?INSTANCE_EXAMPLE_3 + , ?INSTANCE_EXAMPLE_4 + , ?INSTANCE_EXAMPLE_5 + ])}}}}}}}. + +list_authenticators_api_spec2() -> + Spec = list_authenticators_api_spec(), + Spec#{ + description => "List authenticators for listener", + parameters => [ + #{ + name => listener_id, + in => path, + schema => #{ + type => string + }, + required => true + } + ] + }. + +find_authenticator_api_spec() -> + #{ + description => "Get authenticator by id", + parameters => [ + #{ + name => id, + in => path, + description => "Authenticator id", + schema => #{ + type => string + }, + required => true + } + ], + responses => #{ + <<"200">> => #{ + description => <<"OK">>, + content => #{ + 'application/json' => #{ + schema => minirest:ref(<<"AuthenticatorInstance">>), + examples => #{ + example1 => #{ + summary => <<"Example 1">>, + value => emqx_json:encode(?INSTANCE_EXAMPLE_1) + }, + example2 => #{ + summary => <<"Example 2">>, + value => emqx_json:encode(?INSTANCE_EXAMPLE_2) + }, + example3 => #{ + summary => <<"Example 3">>, + value => emqx_json:encode(?INSTANCE_EXAMPLE_3) + }, + example4 => #{ + summary => <<"Example 4">>, + value => emqx_json:encode(?INSTANCE_EXAMPLE_4) + }, + example5 => #{ + summary => <<"Example 5">>, + value => emqx_json:encode(?INSTANCE_EXAMPLE_5) + } + } + } } - ], - requestBody => #{ + }, + <<"404">> => ?ERR_RESPONSE(<<"Not Found">>) + } + }. + +find_authenticator_api_spec2() -> + Spec = find_authenticator_api_spec(), + Spec#{ + parameters => [ + #{ + name => listener_id, + in => path, + description => "Listener id", + schema => #{ + type => string + }, + required => true + }, + #{ + name => id, + in => path, + description => "Authenticator id", + schema => #{ + type => string + }, + required => true + } + ] + }. + +update_authenticator_api_spec() -> + #{ + description => "Update authenticator", + parameters => [ + #{ + name => id, + in => path, + description => "Authenticator id", + schema => #{ + type => string + }, + required => true + } + ], + requestBody => #{ + content => #{ + 'application/json' => #{ + schema => minirest:ref(<<"AuthenticatorConfig">>), + examples => #{ + example1 => #{ + summary => <<"Example 1">>, + value => emqx_json:encode(?EXAMPLE_1) + }, + example2 => #{ + summary => <<"Example 2">>, + value => emqx_json:encode(?EXAMPLE_2) + }, + example3 => #{ + summary => <<"Example 3">>, + value => emqx_json:encode(?EXAMPLE_3) + }, + example4 => #{ + summary => <<"Example 4">>, + value => emqx_json:encode(?EXAMPLE_4) + }, + example5 => #{ + summary => <<"Example 5">>, + value => emqx_json:encode(?EXAMPLE_5) + } + } + } + } + }, + responses => #{ + <<"200">> => #{ + description => <<"OK">>, + content => #{ + 'application/json' => #{ + schema => minirest:ref(<<"AuthenticatorInstance">>), + examples => #{ + example1 => #{ + summary => <<"Example 1">>, + value => emqx_json:encode(?INSTANCE_EXAMPLE_1) + }, + example2 => #{ + summary => <<"Example 2">>, + value => emqx_json:encode(?INSTANCE_EXAMPLE_2) + }, + example3 => #{ + summary => <<"Example 3">>, + value => emqx_json:encode(?INSTANCE_EXAMPLE_3) + }, + example4 => #{ + summary => <<"Example 4">>, + value => emqx_json:encode(?INSTANCE_EXAMPLE_4) + }, + example5 => #{ + summary => <<"Example 5">>, + value => emqx_json:encode(?INSTANCE_EXAMPLE_5) + } + } + } + } + }, + <<"400">> => ?ERR_RESPONSE(<<"Bad Request">>), + <<"404">> => ?ERR_RESPONSE(<<"Not Found">>), + <<"409">> => ?ERR_RESPONSE(<<"Conflict">>) + } + }. + +update_authenticator_api_spec2() -> + Spec = update_authenticator_api_spec(), + Spec#{ + parameters => [ + #{ + name => listener_id, + in => path, + description => "Listener id", + schema => #{ + type => string + }, + required => true + }, + #{ + name => id, + in => path, + description => "Authenticator id", + schema => #{ + type => string + }, + required => true + } + ] + }. + +delete_authenticator_api_spec() -> + #{ + description => "Delete authenticator", + parameters => [ + #{ + name => id, + in => path, + description => "Authenticator id", + schema => #{ + type => string + }, + required => true + } + ], + responses => #{ + <<"204">> => #{ + description => <<"No Content">> + }, + <<"404">> => ?ERR_RESPONSE(<<"Not Found">>) + } + }. + +delete_authenticator_api_spec2() -> + Spec = delete_authenticator_api_spec(), + Spec#{ + parameters => [ + #{ + name => listener_id, + in => path, + description => "Listener id", + schema => #{ + type => string + }, + required => true + }, + #{ + name => id, + in => path, + description => "Authenticator id", + schema => #{ + type => string + }, + required => true + } + ] + }. + +move_authenticator_api_spec() -> + #{ + description => "Move authenticator", + parameters => [ + #{ + name => id, + in => path, + description => "Authenticator id", + schema => #{ + type => string + }, + required => true + } + ], + requestBody => #{ + content => #{ + 'application/json' => #{ + schema => #{ + oneOf => [ + #{ + type => object, + required => [position], + properties => #{ + position => #{ + type => string, + enum => [<<"top">>, <<"bottom">>], + example => <<"top">> + } + } + }, + #{ + type => object, + required => [position], + properties => #{ + position => #{ + type => string, + description => <<"before:">>, + example => <<"before:password-based:mysql">> + } + } + } + ] + } + } + } + }, + responses => #{ + <<"204">> => #{ + description => <<"No Content">> + }, + <<"400">> => ?ERR_RESPONSE(<<"Bad Request">>), + <<"404">> => ?ERR_RESPONSE(<<"Not Found">>) + } + }. + +move_authenticator_api_spec2() -> + Spec = move_authenticator_api_spec(), + Spec#{ + parameters => [ + #{ + name => listener_id, + in => path, + description => "Listener id", + schema => #{ + type => string + }, + required => true + }, + #{ + name => id, + in => path, + description => "Authenticator id", + schema => #{ + type => string + }, + required => true + } + ] + }. + +import_users_api_spec() -> + #{ + description => "Import users from json/csv file", + parameters => [ + #{ + name => id, + in => path, + description => "Authenticator id", + schema => #{ + type => string + }, + required => true + } + ], + requestBody => #{ + content => #{ + 'application/json' => #{ + schema => #{ + type => object, + required => [filename], + properties => #{ + filename => #{ + type => string + } + } + } + } + } + }, + responses => #{ + <<"204">> => #{ + description => <<"No Content">> + }, + <<"400">> => ?ERR_RESPONSE(<<"Bad Request">>), + <<"404">> => ?ERR_RESPONSE(<<"Not Found">>) + } + }. + +import_users_api_spec2() -> + Spec = import_users_api_spec(), + Spec#{ + parameters => [ + #{ + name => listener_id, + in => path, + description => "Listener id", + schema => #{ + type => string + }, + required => true + }, + #{ + name => id, + in => path, + description => "Authenticator id", + schema => #{ + type => string + }, + required => true + } + ] + }. + +create_user_api_spec() -> + #{ + description => "Add user", + parameters => [ + #{ + name => id, + in => path, + description => "Authenticator id", + schema => #{ + type => string + }, + required => true + } + ], + requestBody => #{ + content => #{ + 'application/json' => #{ + schema => #{ + type => object, + required => [user_id, password], + properties => #{ + user_id => #{ + type => string + }, + password => #{ + type => string + }, + is_superuser => #{ + type => boolean, + default => false + } + } + } + } + } + }, + responses => #{ + <<"201">> => #{ + description => <<"Created">>, content => #{ 'application/json' => #{ schema => #{ type => object, - required => [user_id, password], properties => #{ user_id => #{ type => string }, - password => #{ - type => string - }, - superuser => #{ - type => boolean, - default => false - } - } - } - } - } - }, - responses => #{ - <<"201">> => #{ - description => <<"Created">>, - content => #{ - 'application/json' => #{ - schema => #{ - type => object, - properties => #{ - user_id => #{ - type => string - }, - superuser => #{ - type => boolean - } - } - } - } - } - }, - <<"400">> => ?ERR_RESPONSE(<<"Bad Request">>), - <<"404">> => ?ERR_RESPONSE(<<"Not Found">>) - } - }, - get => #{ - description => "List users", - parameters => [ - #{ - name => id, - in => path, - schema => #{ - type => string - }, - required => true - } - ], - responses => #{ - <<"200">> => #{ - description => <<"OK">>, - content => #{ - 'application/json' => #{ - schema => #{ - type => array, - items => #{ - type => object, - properties => #{ - user_id => #{ - type => string - }, - superuser => #{ - type => boolean - } - } - } - } - } - } - }, - <<"404">> => ?ERR_RESPONSE(<<"Not Found">>) - } - } - }, - {"/authentication/authenticators/:id/users", Metadata, users}. - -users2_api() -> - Metadata = #{ - patch => #{ - description => "Update user", - parameters => [ - #{ - name => id, - in => path, - schema => #{ - type => string - }, - required => true - }, - #{ - name => user_id, - in => path, - schema => #{ - type => string - }, - required => true - } - ], - requestBody => #{ - content => #{ - 'application/json' => #{ - schema => #{ - type => object, - properties => #{ - password => #{ - type => string - }, - superuser => #{ + is_superuser => #{ type => boolean } } @@ -640,116 +794,379 @@ users2_api() -> } } }, - responses => #{ - <<"200">> => #{ - description => <<"OK">>, - content => #{ - 'application/json' => #{ - schema => #{ - type => array, - items => #{ - type => object, - properties => #{ - user_id => #{ - type => string - }, - superuser => #{ - type => boolean - } - } - } - } - } - } - }, - <<"400">> => ?ERR_RESPONSE(<<"Bad Request">>), - <<"404">> => ?ERR_RESPONSE(<<"Not Found">>) - } - }, - get => #{ - description => "Get user info", - parameters => [ - #{ - name => id, - in => path, - schema => #{ - type => string - }, - required => true - }, - #{ - name => user_id, - in => path, - schema => #{ - type => string - }, - required => true - } - ], - responses => #{ - <<"200">> => #{ - description => <<"OK">>, - content => #{ - 'application/json' => #{ - schema => #{ - type => array, - items => #{ - type => object, - properties => #{ - user_id => #{ - type => string - }, - superuser => #{ - type => boolean - } - } - } - } - } - } - }, - <<"404">> => ?ERR_RESPONSE(<<"Not Found">>) - } - }, - delete => #{ - description => "Delete user", - parameters => [ - #{ - name => id, - in => path, - schema => #{ - type => string - }, - required => true - }, - #{ - name => user_id, - in => path, - schema => #{ - type => string - }, - required => true - } - ], - responses => #{ - <<"204">> => #{ - description => <<"No Content">> - }, - <<"404">> => ?ERR_RESPONSE(<<"Not Found">>) - } + <<"400">> => ?ERR_RESPONSE(<<"Bad Request">>), + <<"404">> => ?ERR_RESPONSE(<<"Not Found">>) } - }, - {"/authentication/authenticators/:id/users/:user_id", Metadata, users2}. + }. + +create_user_api_spec2() -> + Spec = create_user_api_spec(), + Spec#{ + parameters => [ + #{ + name => listener_id, + in => path, + description => "Listener id", + schema => #{ + type => string + }, + required => true + }, + #{ + name => id, + in => path, + description => "Authenticator id", + schema => #{ + type => string + }, + required => true + } + ] + }. + +list_users_api_spec() -> + #{ + description => "List users", + parameters => [ + #{ + name => id, + in => path, + description => "Authenticator id", + schema => #{ + type => string + }, + required => true + } + ], + responses => #{ + <<"200">> => #{ + description => <<"OK">>, + content => #{ + 'application/json' => #{ + schema => #{ + type => array, + items => #{ + type => object, + properties => #{ + user_id => #{ + type => string + }, + is_superuser => #{ + type => boolean + } + } + } + } + } + } + }, + <<"404">> => ?ERR_RESPONSE(<<"Not Found">>) + } + }. + +list_users_api_spec2() -> + Spec = list_users_api_spec(), + Spec#{ + parameters => [ + #{ + name => listener_id, + in => path, + description => "Listener id", + schema => #{ + type => string + }, + required => true + }, + #{ + name => id, + in => path, + description => "Authenticator id", + schema => #{ + type => string + }, + required => true + } + ] + }. + +update_user_api_spec() -> + #{ + description => "Update user", + parameters => [ + #{ + name => id, + in => path, + description => "Authenticator id", + schema => #{ + type => string + }, + required => true + }, + #{ + name => user_id, + in => path, + description => "User id", + schema => #{ + type => string + }, + required => true + } + ], + requestBody => #{ + content => #{ + 'application/json' => #{ + schema => #{ + type => object, + properties => #{ + password => #{ + type => string + }, + is_superuser => #{ + type => boolean + } + } + } + } + } + }, + responses => #{ + <<"200">> => #{ + description => <<"OK">>, + content => #{ + 'application/json' => #{ + schema => #{ + type => array, + items => #{ + type => object, + properties => #{ + user_id => #{ + type => string + }, + is_superuser => #{ + type => boolean + } + } + } + } + } + } + }, + <<"400">> => ?ERR_RESPONSE(<<"Bad Request">>), + <<"404">> => ?ERR_RESPONSE(<<"Not Found">>) + } + }. + +update_user_api_spec2() -> + Spec = update_user_api_spec(), + Spec#{ + parameters => [ + #{ + name => listener_id, + in => path, + description => "Listener id", + schema => #{ + type => string + }, + required => true + }, + #{ + name => id, + in => path, + description => "Authenticator id", + schema => #{ + type => string + }, + required => true + }, + #{ + name => user_id, + in => path, + description => "User id", + schema => #{ + type => string + }, + required => true + } + ] + }. + +find_user_api_spec() -> + #{ + description => "Get user info", + parameters => [ + #{ + name => id, + in => path, + description => "Authenticator id", + schema => #{ + type => string + }, + required => true + }, + #{ + name => user_id, + in => path, + description => "User id", + schema => #{ + type => string + }, + required => true + } + ], + responses => #{ + <<"200">> => #{ + description => <<"OK">>, + content => #{ + 'application/json' => #{ + schema => #{ + type => array, + items => #{ + type => object, + properties => #{ + user_id => #{ + type => string + }, + is_superuser => #{ + type => boolean + } + } + } + } + } + } + }, + <<"404">> => ?ERR_RESPONSE(<<"Not Found">>) + } + }. + +find_user_api_spec2() -> + Spec = find_user_api_spec(), + Spec#{ + parameters => [ + #{ + name => listener_id, + in => path, + description => "Listener id", + schema => #{ + type => string + }, + required => true + }, + #{ + name => id, + in => path, + description => "Authenticator id", + schema => #{ + type => string + }, + required => true + }, + #{ + name => user_id, + in => path, + description => "User id", + schema => #{ + type => string + }, + required => true + } + ] + }. + +delete_user_api_spec() -> + #{ + description => "Delete user", + parameters => [ + #{ + name => id, + in => path, + description => "Authenticator id", + schema => #{ + type => string + }, + required => true + }, + #{ + name => user_id, + in => path, + description => "User id", + schema => #{ + type => string + }, + required => true + } + ], + responses => #{ + <<"204">> => #{ + description => <<"No Content">> + }, + <<"404">> => ?ERR_RESPONSE(<<"Not Found">>) + } + }. + +delete_user_api_spec2() -> + Spec = delete_user_api_spec(), + Spec#{ + parameters => [ + #{ + name => listener_id, + in => path, + description => "Listener id", + schema => #{ + type => string + }, + required => true + }, + #{ + name => id, + in => path, + description => "Authenticator id", + schema => #{ + type => string + }, + required => true + }, + #{ + name => user_id, + in => path, + description => "User id", + schema => #{ + type => string + }, + required => true + } + ] + }. + definitions() -> - AuthenticatorDef = #{ - oneOf => [ minirest:ref(<<"password_based">>) - , minirest:ref(<<"jwt">>) - , minirest:ref(<<"scram">>) - ] + AuthenticatorConfigDef = #{ + allOf => [ + #{ + type => object, + properties => #{ + enable => #{ + type => boolean, + default => true, + example => true + } + } + }, + #{ + oneOf => [ minirest:ref(<<"PasswordBasedBuiltInDatabase">>) + , minirest:ref(<<"PasswordBasedMySQL">>) + , minirest:ref(<<"PasswordBasedPostgreSQL">>) + , minirest:ref(<<"PasswordBasedMongoDB">>) + , minirest:ref(<<"PasswordBasedRedis">>) + , minirest:ref(<<"PasswordBasedHTTPServer">>) + , minirest:ref(<<"JWT">>) + , minirest:ref(<<"SCRAMBuiltInDatabase">>) + ] + } + ] }, - ReturnedAuthenticatorDef = #{ + AuthenticatorInstanceDef = #{ allOf => [ #{ type => object, @@ -758,148 +1175,49 @@ definitions() -> type => string } } - }, - #{ - oneOf => [ minirest:ref(<<"password_based">>) - , minirest:ref(<<"jwt">>) - , minirest:ref(<<"scram">>) - ] } - ] - }, - - PasswordBasedDef = #{ - allOf => [ - #{ - type => object, - required => [name, mechanism], - properties => #{ - name => #{ - type => string, - example => "exmaple" - }, - mechanism => #{ - type => string, - enum => [<<"password-based">>], - example => <<"password-based">> - } - } - }, - #{ - oneOf => [ minirest:ref(<<"password_based_built_in_database">>) - , minirest:ref(<<"password_based_mysql">>) - , minirest:ref(<<"password_based_pgsql">>) - , minirest:ref(<<"password_based_mongodb">>) - , minirest:ref(<<"password_based_redis">>) - , minirest:ref(<<"password_based_http_server">>) - ] - } - ] - }, - - JWTDef = #{ - type => object, - required => [name, mechanism], - properties => #{ - name => #{ - type => string, - example => "exmaple" - }, - mechanism => #{ - type => string, - enum => [<<"jwt">>], - example => <<"jwt">> - }, - use_jwks => #{ - type => boolean, - default => false, - example => false - }, - algorithm => #{ - type => string, - enum => [<<"hmac-based">>, <<"public-key">>], - default => <<"hmac-based">>, - example => <<"hmac-based">> - }, - secret => #{ - type => string - }, - secret_base64_encoded => #{ - type => boolean, - default => false - }, - certificate => #{ - type => string - }, - verify_claims => #{ - type => object, - additionalProperties => #{ - type => string - } - }, - ssl => minirest:ref(<<"ssl">>) - } - }, - - SCRAMDef = #{ - type => object, - required => [name, mechanism, server_type], - properties => #{ - name => #{ - type => string, - example => "exmaple" - }, - mechanism => #{ - type => string, - enum => [<<"scram">>], - example => <<"scram">> - }, - server_type => #{ - type => string, - enum => [<<"built-in-database">>], - default => <<"built-in-database">> - }, - algorithm => #{ - type => string, - enum => [<<"sha256">>, <<"sha512">>], - default => <<"sha256">> - }, - iteration_count => #{ - type => integer, - default => 4096 - } - } + ] ++ maps:get(allOf, AuthenticatorConfigDef) }, PasswordBasedBuiltInDatabaseDef = #{ type => object, - required => [server_type], + required => [mechanism, backend], properties => #{ - server_type => #{ + mechanism => #{ + type => string, + enum => [<<"password-based">>], + example => <<"password-based">> + }, + backend => #{ type => string, enum => [<<"built-in-database">>], example => <<"built-in-database">> }, - user_id_type => #{ + query => #{ type => string, - enum => [<<"username">>, <<"clientid">>], - default => <<"username">>, - example => <<"username">> + default => <<"SELECT password_hash from built-in-database WHERE username = ${username}">>, + example => <<"SELECT password_hash from built-in-database WHERE username = ${username}">> }, - password_hash_algorithm => minirest:ref(<<"password_hash_algorithm">>) + password_hash_algorithm => minirest:ref(<<"PasswordHashAlgorithm">>) } }, PasswordBasedMySQLDef = #{ type => object, - required => [ server_type + required => [ mechanism + , backend , server , database , username , password , query], properties => #{ - server_type => #{ + mechanism => #{ + type => string, + enum => [<<"password-based">>], + example => <<"password-based">> + }, + backend => #{ type => string, enum => [<<"mysql">>], example => <<"mysql">> @@ -925,7 +1243,7 @@ definitions() -> type => boolean, default => true }, - ssl => minirest:ref(<<"ssl">>), + ssl => minirest:ref(<<"SSL">>), password_hash_algorithm => #{ type => string, enum => [<<"plain">>, <<"md5">>, <<"sha">>, <<"sha256">>, <<"sha512">>, <<"bcrypt">>], @@ -948,19 +1266,25 @@ definitions() -> } }, - PasswordBasedPgSQLDef = #{ + PasswordBasedPostgreSQLDef = #{ type => object, - required => [ server_type + required => [ mechanism + , backend , server , database , username , password , query], properties => #{ - server_type => #{ + mechanism => #{ type => string, - enum => [<<"pgsql">>], - example => <<"pgsql">> + enum => [<<"password-based">>], + example => <<"password-based">> + }, + backend => #{ + type => string, + enum => [<<"postgresql">>], + example => <<"postgresql">> }, server => #{ type => string, @@ -1002,7 +1326,8 @@ definitions() -> PasswordBasedMongoDBDef = #{ type => object, - required => [ server_type + required => [ mechanism + , backend , server , servers , replica_set_name @@ -1014,10 +1339,15 @@ definitions() -> , password_hash_field ], properties => #{ - server_type => #{ + mechanism => #{ + type => string, + enum => [<<"password-based">>], + example => <<"password-based">> + }, + backend => #{ type => string, enum => [<<"mongodb">>], - example => [<<"mongodb">>] + example => <<"mongodb">> }, server => #{ description => <<"Mutually exclusive with the 'servers' field, only valid in standalone mode">>, @@ -1069,6 +1399,10 @@ definitions() -> type => string, example => <<"salt">> }, + is_superuser_field => #{ + type => string, + example => <<"is_superuser">> + }, password_hash_algorithm => #{ type => string, enum => [<<"plain">>, <<"md5">>, <<"sha">>, <<"sha256">>, <<"sha512">>, <<"bcrypt">>], @@ -1087,7 +1421,8 @@ definitions() -> PasswordBasedRedisDef = #{ type => object, - required => [ server_type + required => [ mechanism + , backend , server , servers , password @@ -1095,10 +1430,15 @@ definitions() -> , query ], properties => #{ - server_type => #{ + mechanism => #{ + type => string, + enum => [<<"password-based">>], + example => <<"password-based">> + }, + backend => #{ type => string, enum => [<<"redis">>], - example => [<<"redis">>] + example => <<"redis">> }, server => #{ description => <<"Mutually exclusive with the 'servers' field, only valid in standalone mode">>, @@ -1153,12 +1493,18 @@ definitions() -> PasswordBasedHTTPServerDef = #{ type => object, - required => [ server_type + required => [ mechanism + , backend , url - , form_data + , body ], properties => #{ - server_type => #{ + mechanism => #{ + type => string, + enum => [<<"password-based">>], + example => <<"password-based">> + }, + backend => #{ type => string, enum => [<<"http-server">>], example => <<"http-server">> @@ -1178,8 +1524,8 @@ definitions() -> type => string } }, - form_data => #{ - type => string + body => #{ + type => object }, connect_timeout => #{ type => integer, @@ -1208,6 +1554,72 @@ definitions() -> } }, + JWTDef = #{ + type => object, + required => [mechanism], + properties => #{ + mechanism => #{ + type => string, + enum => [<<"jwt">>], + example => <<"jwt">> + }, + use_jwks => #{ + type => boolean, + default => false, + example => false + }, + algorithm => #{ + type => string, + enum => [<<"hmac-based">>, <<"public-key">>], + default => <<"hmac-based">>, + example => <<"hmac-based">> + }, + secret => #{ + type => string + }, + secret_base64_encoded => #{ + type => boolean, + default => false + }, + certificate => #{ + type => string + }, + verify_claims => #{ + type => object, + additionalProperties => #{ + type => string + } + }, + ssl => minirest:ref(<<"SSL">>) + } + }, + + SCRAMBuiltInDatabaseDef = #{ + type => object, + required => [mechanism, backend], + properties => #{ + mechanism => #{ + type => string, + enum => [<<"scram">>], + example => <<"scram">> + }, + backend => #{ + type => string, + enum => [<<"built-in-database">>], + example => <<"built-in-database">> + }, + algorithm => #{ + type => string, + enum => [<<"sha256">>, <<"sha512">>], + default => <<"sha256">> + }, + iteration_count => #{ + type => integer, + default => 4096 + } + } + }, + PasswordHashAlgorithmDef = #{ type => object, required => [name], @@ -1273,166 +1685,293 @@ definitions() -> } }, - [ #{<<"authenticator">> => AuthenticatorDef} - , #{<<"returned_authenticator">> => ReturnedAuthenticatorDef} - , #{<<"password_based">> => PasswordBasedDef} - , #{<<"jwt">> => JWTDef} - , #{<<"scram">> => SCRAMDef} - , #{<<"password_based_built_in_database">> => PasswordBasedBuiltInDatabaseDef} - , #{<<"password_based_mysql">> => PasswordBasedMySQLDef} - , #{<<"password_based_pgsql">> => PasswordBasedPgSQLDef} - , #{<<"password_based_mongodb">> => PasswordBasedMongoDBDef} - , #{<<"password_based_redis">> => PasswordBasedRedisDef} - , #{<<"password_based_http_server">> => PasswordBasedHTTPServerDef} - , #{<<"password_hash_algorithm">> => PasswordHashAlgorithmDef} - , #{<<"ssl">> => SSLDef} - , #{<<"error">> => ErrorDef} + [ #{<<"AuthenticatorConfig">> => AuthenticatorConfigDef} + , #{<<"AuthenticatorInstance">> => AuthenticatorInstanceDef} + , #{<<"PasswordBasedBuiltInDatabase">> => PasswordBasedBuiltInDatabaseDef} + , #{<<"PasswordBasedMySQL">> => PasswordBasedMySQLDef} + , #{<<"PasswordBasedPostgreSQL">> => PasswordBasedPostgreSQLDef} + , #{<<"PasswordBasedMongoDB">> => PasswordBasedMongoDBDef} + , #{<<"PasswordBasedRedis">> => PasswordBasedRedisDef} + , #{<<"PasswordBasedHTTPServer">> => PasswordBasedHTTPServerDef} + , #{<<"JWT">> => JWTDef} + , #{<<"SCRAMBuiltInDatabase">> => SCRAMBuiltInDatabaseDef} + , #{<<"PasswordHashAlgorithm">> => PasswordHashAlgorithmDef} + , #{<<"SSL">> => SSLDef} + , #{<<"Error">> => ErrorDef} ]. authentication(post, #{body := Config}) -> - case Config of - #{<<"enable">> := Enable} -> - {ok, _} = emqx_authn:update_config([authentication, enable], {enable, Enable}), - {204}; - _ -> - serialize_error({missing_parameter, enable}) - end; + create_authenticator([authentication], ?GLOBAL, Config); + authentication(get, _Params) -> - Enabled = emqx_authn:is_enabled(), - {200, #{enabled => Enabled}}. + list_authenticators([authentication]). -authenticators(post, #{body := Config}) -> - case emqx_authn:update_config([authentication, authenticators], {create_authenticator, Config}) of - {ok, #{post_config_update := #{emqx_authn := #{id := ID, name := Name}}, - raw_config := RawConfig}} -> - [RawConfig1] = [RC || #{<<"name">> := N} = RC <- RawConfig, N =:= Name], - {200, RawConfig1#{id => ID}}; - {error, {_, _, Reason}} -> - serialize_error(Reason) - end; -authenticators(get, _Params) -> - RawConfig = get_raw_config([authentication, authenticators]), - {ok, Authenticators} = emqx_authn:list_authenticators(?CHAIN), - NAuthenticators = lists:zipwith(fun(#{<<"name">> := Name} = Config, #{id := ID, name := Name}) -> - Config#{id => ID} - end, RawConfig, Authenticators), - {200, NAuthenticators}. +authentication2(get, #{bindings := #{id := AuthenticatorID}}) -> + list_authenticator([authentication], AuthenticatorID); -authenticators2(get, #{bindings := #{id := AuthenticatorID}}) -> - case emqx_authn:lookup_authenticator(?CHAIN, AuthenticatorID) of - {ok, #{id := ID, name := Name}} -> - RawConfig = get_raw_config([authentication, authenticators]), - [RawConfig1] = [RC || #{<<"name">> := N} = RC <- RawConfig, N =:= Name], - {200, RawConfig1#{id => ID}}; +authentication2(put, #{bindings := #{id := AuthenticatorID}, body := Config}) -> + update_authenticator([authentication], ?GLOBAL, AuthenticatorID, Config); + +authentication2(delete, #{bindings := #{id := AuthenticatorID}}) -> + delete_authenticator([authentication], ?GLOBAL, AuthenticatorID). + +authentication3(post, #{bindings := #{listener_id := ListenerID}, body := Config}) -> + case find_listener(ListenerID) of + {ok, {Type, Name}} -> + create_authenticator([listeners, Type, Name, authentication], ListenerID, Config); {error, Reason} -> serialize_error(Reason) end; -authenticators2(put, #{bindings := #{id := AuthenticatorID}, body := Config}) -> - case emqx_authn:update_config([authentication, authenticators], - {update_or_create_authenticator, AuthenticatorID, Config}) of - {ok, #{post_config_update := #{emqx_authn := #{id := ID, name := Name}}, - raw_config := RawConfig}} -> - [RawConfig0] = [RC || #{<<"name">> := N} = RC <- RawConfig, N =:= Name], - {200, RawConfig0#{id => ID}}; - {error, {_, _, Reason}} -> +authentication3(get, #{bindings := #{listener_id := ListenerID}}) -> + case find_listener(ListenerID) of + {ok, {Type, Name}} -> + list_authenticators([listeners, Type, Name, authentication]); + {error, Reason} -> + serialize_error(Reason) + end. + +authentication4(get, #{bindings := #{listener_id := ListenerID, id := AuthenticatorID}}) -> + case find_listener(ListenerID) of + {ok, {Type, Name}} -> + list_authenticator([listeners, Type, Name, authentication], AuthenticatorID); + {error, Reason} -> serialize_error(Reason) end; -authenticators2(delete, #{bindings := #{id := AuthenticatorID}}) -> - case emqx_authn:update_config([authentication, authenticators], {delete_authenticator, AuthenticatorID}) of +authentication4(put, #{bindings := #{listener_id := ListenerID, id := AuthenticatorID}, body := Config}) -> + case find_listener(ListenerID) of + {ok, {Type, Name}} -> + update_authenticator([listeners, Type, Name, authentication], ListenerID, AuthenticatorID, Config); + {error, Reason} -> + serialize_error(Reason) + end; +authentication4(delete, #{bindings := #{listener_id := ListenerID, id := AuthenticatorID}}) -> + case find_listener(ListenerID) of + {ok, {Type, Name}} -> + delete_authenticator([listeners, Type, Name, authentication], ListenerID, AuthenticatorID); + {error, Reason} -> + serialize_error(Reason) + end. + +move(post, #{bindings := #{id := AuthenticatorID}, body := #{<<"position">> := Position}}) -> + move_authenitcator([authentication], ?GLOBAL, AuthenticatorID, Position); +move(post, #{bindings := #{id := _}, body := _}) -> + serialize_error({missing_parameter, position}). + +move2(post, #{bindings := #{listener_id := ListenerID, id := AuthenticatorID}, body := #{<<"position">> := Position}}) -> + case find_listener(ListenerID) of + {ok, {Type, Name}} -> + move_authenitcator([listeners, Type, Name, authentication], ListenerID, AuthenticatorID, Position); + {error, Reason} -> + serialize_error(Reason) + end; +move2(post, #{bindings := #{listener_id := _, id := _}, body := _}) -> + serialize_error({missing_parameter, position}). + +import_users(post, #{bindings := #{id := AuthenticatorID}, body := #{<<"filename">> := Filename}}) -> + case ?AUTHN:import_users(?GLOBAL, AuthenticatorID, Filename) of + ok -> {204}; + {error, Reason} -> serialize_error(Reason) + end; +import_users(post, #{bindings := #{id := _}, body := _}) -> + serialize_error({missing_parameter, filename}). + +import_users2(post, #{bindings := #{listener_id := ListenerID, id := AuthenticatorID}, body := #{<<"filename">> := Filename}}) -> + case ?AUTHN:import_users(ListenerID, AuthenticatorID, Filename) of + ok -> {204}; + {error, Reason} -> serialize_error(Reason) + end; +import_users2(post, #{bindings := #{listener_id := _, id := _}, body := _}) -> + serialize_error({missing_parameter, filename}). + +users(post, #{bindings := #{id := AuthenticatorID}, body := UserInfo}) -> + add_user(?GLOBAL, AuthenticatorID, UserInfo); +users(get, #{bindings := #{id := AuthenticatorID}}) -> + list_users(?GLOBAL, AuthenticatorID). + +users2(put, #{bindings := #{id := AuthenticatorID, + user_id := UserID}, body := UserInfo}) -> + update_user(?GLOBAL, AuthenticatorID, UserID, UserInfo); +users2(get, #{bindings := #{id := AuthenticatorID, user_id := UserID}}) -> + find_user(?GLOBAL, AuthenticatorID, UserID); +users2(delete, #{bindings := #{id := AuthenticatorID, user_id := UserID}}) -> + delete_user(?GLOBAL, AuthenticatorID, UserID). + +users3(post, #{bindings := #{listener_id := ListenerID, + id := AuthenticatorID}, body := UserInfo}) -> + add_user(ListenerID, AuthenticatorID, UserInfo); +users3(get, #{bindings := #{listener_id := ListenerID, + id := AuthenticatorID}}) -> + list_users(ListenerID, AuthenticatorID). + +users4(put, #{bindings := #{listener_id := ListenerID, + id := AuthenticatorID, + user_id := UserID}, body := UserInfo}) -> + update_user(ListenerID, AuthenticatorID, UserID, UserInfo); +users4(get, #{bindings := #{listener_id := ListenerID, + id := AuthenticatorID, + user_id := UserID}}) -> + find_user(ListenerID, AuthenticatorID, UserID); +users4(delete, #{bindings := #{listener_id := ListenerID, + id := AuthenticatorID, + user_id := UserID}}) -> + delete_user(ListenerID, AuthenticatorID, UserID). + +%%------------------------------------------------------------------------------ +%% Internal functions +%%------------------------------------------------------------------------------ + +find_listener(ListenerID) -> + {Type, Name} = emqx_listeners:parse_listener_id(ListenerID), + case emqx_config:find([listeners, Type, Name]) of + {not_found, _, _} -> + {error, {not_found, {listener, ListenerID}}}; + {ok, _} -> + {ok, {Type, Name}} + end. + +create_authenticator(ConfKeyPath, ChainName0, Config) -> + ChainName = to_atom(ChainName0), + case update_config(ConfKeyPath, {create_authenticator, ChainName, Config}) of + {ok, #{post_config_update := #{?AUTHN := #{id := ID}}, + raw_config := AuthenticatorsConfig}} -> + {ok, AuthenticatorConfig} = find_config(ID, AuthenticatorsConfig), + {200, maps:put(id, ID, fill_defaults(AuthenticatorConfig))}; + {error, {_, _, Reason}} -> + serialize_error(Reason) + end. + +list_authenticators(ConfKeyPath) -> + AuthenticatorsConfig = get_raw_config_with_defaults(ConfKeyPath), + NAuthenticators = [maps:put(id, ?AUTHN:generate_id(AuthenticatorConfig), AuthenticatorConfig) + || AuthenticatorConfig <- AuthenticatorsConfig], + {200, NAuthenticators}. + +list_authenticator(ConfKeyPath, AuthenticatorID) -> + AuthenticatorsConfig = get_raw_config_with_defaults(ConfKeyPath), + case find_config(AuthenticatorID, AuthenticatorsConfig) of + {ok, AuthenticatorConfig} -> + {200, AuthenticatorConfig#{id => AuthenticatorID}}; + {error, Reason} -> + serialize_error(Reason) + end. + +update_authenticator(ConfKeyPath, ChainName0, AuthenticatorID, Config) -> + ChainName = to_atom(ChainName0), + case update_config(ConfKeyPath, + {update_authenticator, ChainName, AuthenticatorID, Config}) of + {ok, #{post_config_update := #{?AUTHN := #{id := ID}}, + raw_config := AuthenticatorsConfig}} -> + {ok, AuthenticatorConfig} = find_config(ID, AuthenticatorsConfig), + {200, maps:put(id, ID, fill_defaults(AuthenticatorConfig))}; + {error, {_, _, Reason}} -> + serialize_error(Reason) + end. + +delete_authenticator(ConfKeyPath, ChainName0, AuthenticatorID) -> + ChainName = to_atom(ChainName0), + case update_config(ConfKeyPath, {delete_authenticator, ChainName, AuthenticatorID}) of {ok, _} -> {204}; {error, {_, _, Reason}} -> serialize_error(Reason) end. -move(post, #{bindings := #{id := AuthenticatorID}, body := Body}) -> - case Body of - #{<<"position">> := Position} -> - case emqx_authn:update_config([authentication, authenticators], {move_authenticator, AuthenticatorID, Position}) of - {ok, _} -> {204}; - {error, {_, _, Reason}} -> serialize_error(Reason) - end; - _ -> - serialize_error({missing_parameter, position}) - end. - -import_users(post, #{bindings := #{id := AuthenticatorID}, body := Body}) -> - case Body of - #{<<"filename">> := Filename} -> - case emqx_authn:import_users(?CHAIN, AuthenticatorID, Filename) of - ok -> {204}; - {error, Reason} -> serialize_error(Reason) - end; - _ -> - serialize_error({missing_parameter, filename}) - end. - -users(post, #{bindings := #{id := AuthenticatorID}, body := UserInfo}) -> - case UserInfo of - #{ <<"user_id">> := UserID, <<"password">> := Password} -> - Superuser = maps:get(<<"superuser">>, UserInfo, false), - case emqx_authn:add_user(?CHAIN, AuthenticatorID, #{ user_id => UserID - , password => Password - , superuser => Superuser}) of - {ok, User} -> - {201, User}; - {error, Reason} -> - serialize_error(Reason) - end; - #{<<"user_id">> := _} -> - serialize_error({missing_parameter, password}); - _ -> - serialize_error({missing_parameter, user_id}) - end; -users(get, #{bindings := #{id := AuthenticatorID}}) -> - case emqx_authn:list_users(?CHAIN, AuthenticatorID) of - {ok, Users} -> - {200, Users}; - {error, Reason} -> +move_authenitcator(ConfKeyPath, ChainName0, AuthenticatorID, Position) -> + ChainName = to_atom(ChainName0), + case update_config(ConfKeyPath, {move_authenticator, ChainName, AuthenticatorID, Position}) of + {ok, _} -> + {204}; + {error, {_, _, Reason}} -> serialize_error(Reason) end. -users2(patch, #{bindings := #{id := AuthenticatorID, - user_id := UserID}, - body := UserInfo}) -> - NUserInfo = maps:with([<<"password">>, <<"superuser">>], UserInfo), - case NUserInfo =:= #{} of +add_user(ChainName0, AuthenticatorID, #{<<"user_id">> := UserID, <<"password">> := Password} = UserInfo) -> + ChainName = to_atom(ChainName0), + IsSuperuser = maps:get(<<"is_superuser">>, UserInfo, false), + case ?AUTHN:add_user(ChainName, AuthenticatorID, #{ user_id => UserID + , password => Password + , is_superuser => IsSuperuser}) of + {ok, User} -> + {201, User}; + {error, Reason} -> + serialize_error(Reason) + end; +add_user(_, _, #{<<"user_id">> := _}) -> + serialize_error({missing_parameter, password}); +add_user(_, _, _) -> + serialize_error({missing_parameter, user_id}). + +update_user(ChainName0, AuthenticatorID, UserID, UserInfo) -> + ChainName = to_atom(ChainName0), + case maps:with([<<"password">>, <<"is_superuser">>], UserInfo) =:= #{} of true -> serialize_error({missing_parameter, password}); false -> - case emqx_authn:update_user(?CHAIN, AuthenticatorID, UserID, UserInfo) of + case ?AUTHN:update_user(ChainName, AuthenticatorID, UserID, UserInfo) of {ok, User} -> {200, User}; {error, Reason} -> serialize_error(Reason) end - end; -users2(get, #{bindings := #{id := AuthenticatorID, user_id := UserID}}) -> - case emqx_authn:lookup_user(?CHAIN, AuthenticatorID, UserID) of + end. + +find_user(ChainName0, AuthenticatorID, UserID) -> + ChainName = to_atom(ChainName0), + case ?AUTHN:lookup_user(ChainName, AuthenticatorID, UserID) of {ok, User} -> {200, User}; {error, Reason} -> serialize_error(Reason) - end; -users2(delete, #{bindings := #{id := AuthenticatorID, user_id := UserID}}) -> - case emqx_authn:delete_user(?CHAIN, AuthenticatorID, UserID) of + end. + +delete_user(ChainName0, AuthenticatorID, UserID) -> + ChainName = to_atom(ChainName0), + case ?AUTHN:delete_user(ChainName, AuthenticatorID, UserID) of ok -> {204}; {error, Reason} -> serialize_error(Reason) end. -get_raw_config(ConfKeyPath) -> - %% TODO: call emqx_config:get_raw(ConfKeyPath) directly +list_users(ChainName0, AuthenticatorID) -> + ChainName = to_atom(ChainName0), + case ?AUTHN:list_users(ChainName, AuthenticatorID) of + {ok, Users} -> + {200, Users}; + {error, Reason} -> + serialize_error(Reason) + end. + +update_config(Path, ConfigRequest) -> + emqx:update_config(Path, ConfigRequest, #{rawconf_with_defaults => true}). + +get_raw_config_with_defaults(ConfKeyPath) -> NConfKeyPath = [atom_to_binary(Key, utf8) || Key <- ConfKeyPath], - emqx_map_lib:deep_get(NConfKeyPath, emqx_config:fill_defaults(emqx_config:get_raw([]))). + RawConfig = emqx_map_lib:deep_get(NConfKeyPath, emqx_config:get_raw([]), []), + to_list(fill_defaults(RawConfig)). + +find_config(AuthenticatorID, AuthenticatorsConfig) -> + case [AC || AC <- to_list(AuthenticatorsConfig), AuthenticatorID =:= ?AUTHN:generate_id(AC)] of + [] -> {error, {not_found, {authenticator, AuthenticatorID}}}; + [AuthenticatorConfig] -> {ok, AuthenticatorConfig} + end. + +fill_defaults(Config) -> + #{<<"authentication">> := CheckedConfig} = hocon_schema:check_plain( + ?AUTHN, #{<<"authentication">> => Config}, #{nullable => true, no_conversion => true}), + CheckedConfig. serialize_error({not_found, {authenticator, ID}}) -> {404, #{code => <<"NOT_FOUND">>, message => list_to_binary(io_lib:format("Authenticator '~s' does not exist", [ID]))}}; -serialize_error(name_has_be_used) -> +serialize_error({not_found, {listener, ID}}) -> + {404, #{code => <<"NOT_FOUND">>, + message => list_to_binary(io_lib:format("Listener '~s' does not exist", [ID]))}}; +serialize_error({already_exists, {authenticator, ID}}) -> {409, #{code => <<"ALREADY_EXISTS">>, - message => <<"Name has be used">>}}; + message => list_to_binary( + io_lib:format("Authenticator '~s' already exist", [ID]) + )}}; serialize_error({missing_parameter, Name}) -> {400, #{code => <<"MISSING_PARAMETER">>, message => list_to_binary( @@ -1445,4 +1984,14 @@ serialize_error({invalid_parameter, Name}) -> )}}; serialize_error(Reason) -> {400, #{code => <<"BAD_REQUEST">>, - message => list_to_binary(io_lib:format("Todo: ~p", [Reason]))}}. + message => list_to_binary(io_lib:format("~p", [Reason]))}}. + +to_list(M) when is_map(M) -> + [M]; +to_list(L) when is_list(L) -> + L. + +to_atom(B) when is_binary(B) -> + binary_to_atom(B); +to_atom(A) when is_atom(A) -> + A. \ No newline at end of file diff --git a/apps/emqx_authn/src/emqx_authn_app.erl b/apps/emqx_authn/src/emqx_authn_app.erl index b7f409bc9..016decdd2 100644 --- a/apps/emqx_authn/src/emqx_authn_app.erl +++ b/apps/emqx_authn/src/emqx_authn_app.erl @@ -17,7 +17,6 @@ -module(emqx_authn_app). -include("emqx_authn.hrl"). --include_lib("emqx/include/logger.hrl"). -behaviour(application). @@ -26,33 +25,45 @@ , stop/1 ]). +%%------------------------------------------------------------------------------ +%% APIs +%%------------------------------------------------------------------------------ + start(_StartType, _StartArgs) -> ok = ekka_rlog:wait_for_shards([?AUTH_SHARD], infinity), {ok, Sup} = emqx_authn_sup:start_link(), - emqx_config_handler:add_handler([authentication, authenticators], emqx_authn), - initialize(), + ok = add_providers(), + ok = initialize(), {ok, Sup}. stop(_State) -> + ok = remove_providers(), ok. +%%------------------------------------------------------------------------------ +%% Internal functions +%%------------------------------------------------------------------------------ + +add_providers() -> + _ = [?AUTHN:add_provider(AuthNType, Provider) || {AuthNType, Provider} <- providers()], ok. + +remove_providers() -> + _ = [?AUTHN:remove_provider(AuthNType) || {AuthNType, _} <- providers()], ok. + initialize() -> - AuthNConfig = emqx:get_config([authentication], #{enable => false, - authenticators => []}), - initialize(AuthNConfig). - -initialize(#{enable := Enable, authenticators := AuthenticatorsConfig}) -> - {ok, _} = emqx_authn:create_chain(#{id => ?CHAIN}), - initialize_authenticators(AuthenticatorsConfig), - Enable =:= true andalso emqx_authn:enable(), + ?AUTHN:initialize_authentication(?GLOBAL, emqx:get_raw_config([authentication], [])), + lists:foreach(fun({ListenerID, ListenerConfig}) -> + ?AUTHN:initialize_authentication(ListenerID, maps:get(authentication, ListenerConfig, [])) + end, emqx_listeners:list()), ok. -initialize_authenticators([]) -> - ok; -initialize_authenticators([#{name := Name} = AuthenticatorConfig | More]) -> - case emqx_authn:create_authenticator(?CHAIN, AuthenticatorConfig) of - {ok, _} -> - initialize_authenticators(More); - {error, Reason} -> - ?LOG(error, "Failed to create authenticator '~s': ~p", [Name, Reason]) - end. +providers() -> + [ {{'password-based', 'built-in-database'}, emqx_authn_mnesia} + , {{'password-based', mysql}, emqx_authn_mysql} + , {{'password-based', posgresql}, emqx_authn_pgsql} + , {{'password-based', mongodb}, emqx_authn_mongodb} + , {{'password-based', redis}, emqx_authn_redis} + , {{'password-based', 'http-server'}, emqx_authn_http} + , {jwt, emqx_authn_jwt} + , {{scram, 'built-in-database'}, emqx_enhanced_authn_scram_mnesia} + ]. \ No newline at end of file diff --git a/apps/emqx_authn/src/emqx_authn_schema.erl b/apps/emqx_authn/src/emqx_authn_schema.erl index de0de9fcc..23e412088 100644 --- a/apps/emqx_authn/src/emqx_authn_schema.erl +++ b/apps/emqx_authn/src/emqx_authn_schema.erl @@ -16,53 +16,15 @@ -module(emqx_authn_schema). --include("emqx_authn.hrl"). -include_lib("typerefl/include/types.hrl"). --behaviour(hocon_schema). - --export([ roots/0 - , fields/1 +-export([ common_fields/0 ]). --export([ authenticator_name/1 - ]). - -%% Export it for emqx_gateway_schema module --export([ authenticators/1 - ]). - -roots() -> [ "authentication" ]. - -fields("authentication") -> - [ {enable, fun enable/1} - , {authenticators, fun authenticators/1} +common_fields() -> + [ {enable, fun enable/1} ]. -authenticator_name(type) -> binary(); -authenticator_name(nullable) -> false; -authenticator_name(_) -> undefined. - enable(type) -> boolean(); -enable(default) -> false; +enable(default) -> true; enable(_) -> undefined. - -authenticators(type) -> - hoconsc:array({union, [ hoconsc:ref(emqx_authn_mnesia, config) - , hoconsc:ref(emqx_authn_mysql, config) - , hoconsc:ref(emqx_authn_pgsql, config) - , hoconsc:ref(emqx_authn_mongodb, standalone) - , hoconsc:ref(emqx_authn_mongodb, 'replica-set') - , hoconsc:ref(emqx_authn_mongodb, 'sharded-cluster') - , hoconsc:ref(emqx_authn_redis, standalone) - , hoconsc:ref(emqx_authn_redis, cluster) - , hoconsc:ref(emqx_authn_redis, sentinel) - , hoconsc:ref(emqx_authn_http, get) - , hoconsc:ref(emqx_authn_http, post) - , hoconsc:ref(emqx_authn_jwt, 'hmac-based') - , hoconsc:ref(emqx_authn_jwt, 'public-key') - , hoconsc:ref(emqx_authn_jwt, 'jwks') - , hoconsc:ref(emqx_enhanced_authn_scram_mnesia, config) - ]}); -authenticators(default) -> []; -authenticators(_) -> undefined. diff --git a/apps/emqx_authn/src/emqx_authn_sup.erl b/apps/emqx_authn/src/emqx_authn_sup.erl index 56fcf299a..dd672a7c7 100644 --- a/apps/emqx_authn/src/emqx_authn_sup.erl +++ b/apps/emqx_authn/src/emqx_authn_sup.erl @@ -26,11 +26,5 @@ start_link() -> supervisor:start_link({local, ?MODULE}, ?MODULE, []). init([]) -> - ChildSpecs = [ - #{id => emqx_authn, - start => {emqx_authn, start_link, []}, - restart => permanent, - type => worker, - modules => [emqx_authn]} - ], + ChildSpecs = [], {ok, {{one_for_one, 10, 10}, ChildSpecs}}. diff --git a/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl b/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl index 0ca281aa0..e0f37a50d 100644 --- a/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl +++ b/apps/emqx_authn/src/enhanced_authn/emqx_enhanced_authn_scram_mnesia.erl @@ -20,12 +20,15 @@ -include_lib("typerefl/include/types.hrl"). -behaviour(hocon_schema). +-behaviour(emqx_authentication). --export([ roots/0 +-export([ namespace/0 + , roots/0 , fields/1 ]). --export([ create/1 +-export([ refs/0 + , create/1 , update/2 , authenticate/2 , destroy/1 @@ -50,7 +53,7 @@ , stored_key , server_key , salt - , superuser + , is_superuser }). %%------------------------------------------------------------------------------ @@ -74,19 +77,16 @@ mnesia(copy) -> %% Hocon Schema %%------------------------------------------------------------------------------ +namespace() -> "authn:scram:builtin-db". + roots() -> [config]. fields(config) -> - [ {name, fun emqx_authn_schema:authenticator_name/1} - , {mechanism, {enum, [scram]}} - , {server_type, fun server_type/1} + [ {mechanism, {enum, [scram]}} + , {backend, {enum, ['built-in-database']}} , {algorithm, fun algorithm/1} , {iteration_count, fun iteration_count/1} - ]. - -server_type(type) -> hoconsc:enum(['built-in-database']); -server_type(default) -> 'built-in-database'; -server_type(_) -> undefined. + ] ++ emqx_authn_schema:common_fields(). algorithm(type) -> hoconsc:enum([sha256, sha512]); algorithm(default) -> sha256; @@ -100,6 +100,9 @@ iteration_count(_) -> undefined. %% APIs %%------------------------------------------------------------------------------ +refs() -> + [hoconsc:ref(?MODULE, config)]. + create(#{ algorithm := Algorithm , iteration_count := IterationCount , '_unique' := Unique @@ -144,9 +147,9 @@ add_user(#{user_id := UserID, fun() -> case mnesia:read(?TAB, {UserGroup, UserID}, write) of [] -> - Superuser = maps:get(superuser, UserInfo, false), - add_user(UserID, Password, Superuser, State), - {ok, #{user_id => UserID, superuser => Superuser}}; + IsSuperuser = maps:get(is_superuser, UserInfo, false), + add_user(UserID, Password, IsSuperuser, State), + {ok, #{user_id => UserID, is_superuser => IsSuperuser}}; [_] -> {error, already_exist} end @@ -170,8 +173,8 @@ update_user(UserID, User, case mnesia:read(?TAB, {UserGroup, UserID}, write) of [] -> {error, not_found}; - [#user_info{superuser = Superuser} = UserInfo] -> - UserInfo1 = UserInfo#user_info{superuser = maps:get(superuser, User, Superuser)}, + [#user_info{is_superuser = IsSuperuser} = UserInfo] -> + UserInfo1 = UserInfo#user_info{is_superuser = maps:get(is_superuser, User, IsSuperuser)}, UserInfo2 = case maps:get(password, User, undefined) of undefined -> UserInfo1; @@ -226,36 +229,36 @@ check_client_first_message(Bin, _Cache, #{iteration_count := IterationCount} = S {error, not_authorized} end. -check_client_final_message(Bin, #{superuser := Superuser} = Cache, #{algorithm := Alg}) -> +check_client_final_message(Bin, #{is_superuser := IsSuperuser} = Cache, #{algorithm := Alg}) -> case esasl_scram:check_client_final_message( Bin, Cache#{algorithm => Alg} ) of {ok, ServerFinalMessage} -> - {ok, #{superuser => Superuser}, ServerFinalMessage}; + {ok, #{is_superuser => IsSuperuser}, ServerFinalMessage}; {error, _Reason} -> {error, not_authorized} end. -add_user(UserID, Password, Superuser, State) -> +add_user(UserID, Password, IsSuperuser, State) -> {StoredKey, ServerKey, Salt} = esasl_scram:generate_authentication_info(Password, State), - UserInfo = #user_info{user_id = UserID, - stored_key = StoredKey, - server_key = ServerKey, - salt = Salt, - superuser = Superuser}, + UserInfo = #user_info{user_id = UserID, + stored_key = StoredKey, + server_key = ServerKey, + salt = Salt, + is_superuser = IsSuperuser}, mnesia:write(?TAB, UserInfo, write). retrieve(UserID, #{user_group := UserGroup}) -> case mnesia:dirty_read(?TAB, {UserGroup, UserID}) of - [#user_info{stored_key = StoredKey, - server_key = ServerKey, - salt = Salt, - superuser = Superuser}] -> + [#user_info{stored_key = StoredKey, + server_key = ServerKey, + salt = Salt, + is_superuser = IsSuperuser}] -> {ok, #{stored_key => StoredKey, server_key => ServerKey, salt => Salt, - superuser => Superuser}}; + is_superuser => IsSuperuser}}; [] -> {error, not_found} end. @@ -270,5 +273,5 @@ trans(Fun, Args) -> {aborted, Reason} -> {error, Reason} end. -serialize_user_info(#user_info{user_id = {_, UserID}, superuser = Superuser}) -> - #{user_id => UserID, superuser => Superuser}. +serialize_user_info(#user_info{user_id = {_, UserID}, is_superuser = IsSuperuser}) -> + #{user_id => UserID, is_superuser => IsSuperuser}. diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl index c5cbc0f02..5495b139a 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_http.erl @@ -21,13 +21,16 @@ -include_lib("typerefl/include/types.hrl"). -behaviour(hocon_schema). +-behaviour(emqx_authentication). --export([ roots/0 +-export([ namespace/0 + , roots/0 , fields/1 , validations/0 ]). --export([ create/1 +-export([ refs/0 + , create/1 , update/2 , authenticate/2 , destroy/1 @@ -37,9 +40,11 @@ %% Hocon Schema %%------------------------------------------------------------------------------ +namespace() -> "authn:password-based:http-server". + roots() -> - [ {config, {union, [ hoconsc:t(get) - , hoconsc:t(post) + [ {config, {union, [ hoconsc:ref(?MODULE, get) + , hoconsc:ref(?MODULE, post) ]}} ]. @@ -56,15 +61,15 @@ fields(post) -> ] ++ common_fields(). common_fields() -> - [ {name, fun emqx_authn_schema:authenticator_name/1} - , {mechanism, {enum, ['password-based']}} - , {server_type, {enum, ['http-server']}} + [ {mechanism, {enum, ['password-based']}} + , {backend, {enum, ['http-server']}} , {url, fun url/1} - , {form_data, fun form_data/1} + , {body, fun body/1} , {request_timeout, fun request_timeout/1} - ] ++ maps:to_list(maps:without([ base_url - , pool_type], - maps:from_list(emqx_connector_http:fields(config)))). + ] ++ emqx_authn_schema:common_fields() + ++ maps:to_list(maps:without([ base_url + , pool_type], + maps:from_list(emqx_connector_http:fields(config)))). validations() -> [ {check_ssl_opts, fun check_ssl_opts/1} @@ -92,11 +97,10 @@ headers_no_content_type(converter) -> headers_no_content_type(default) -> default_headers_no_content_type(); headers_no_content_type(_) -> undefined. -%% TODO: Using map() -form_data(type) -> map(); -form_data(nullable) -> false; -form_data(validate) -> [fun check_form_data/1]; -form_data(_) -> undefined. +body(type) -> map(); +body(nullable) -> false; +body(validate) -> [fun check_body/1]; +body(_) -> undefined. request_timeout(type) -> non_neg_integer(); request_timeout(default) -> 5000; @@ -106,10 +110,15 @@ request_timeout(_) -> undefined. %% APIs %%------------------------------------------------------------------------------ +refs() -> + [ hoconsc:ref(?MODULE, get) + , hoconsc:ref(?MODULE, post) + ]. + create(#{ method := Method , url := URL , headers := Headers - , form_data := FormData + , body := Body , request_timeout := RequestTimeout , '_unique' := Unique } = Config) -> @@ -118,8 +127,8 @@ create(#{ method := Method State = #{ method => Method , path => Path , base_query => cow_qs:parse_qs(list_to_binary(Query)) - , headers => normalize_headers(Headers) - , form_data => maps:to_list(FormData) + , headers => maps:to_list(Headers) + , body => maps:to_list(Body) , request_timeout => RequestTimeout , '_unique' => Unique }, @@ -152,16 +161,16 @@ authenticate(Credential, #{'_unique' := Unique, try Request = generate_request(Credential, State), case emqx_resource:query(Unique, {Method, Request, RequestTimeout}) of - {ok, 204, _Headers} -> {ok, #{superuser => false}}; + {ok, 204, _Headers} -> {ok, #{is_superuser => false}}; {ok, 200, Headers, Body} -> ContentType = proplists:get_value(<<"content-type">>, Headers, <<"application/json">>), case safely_parse_body(ContentType, Body) of {ok, NBody} -> %% TODO: Return by user property - {ok, #{superuser => maps:get(<<"superuser">>, NBody, false), + {ok, #{is_superuser => maps:get(<<"is_superuser">>, NBody, false), user_property => NBody}}; {error, _Reason} -> - {ok, #{superuser => false}} + {ok, #{is_superuser => false}} end; {error, _Reason} -> ignore @@ -186,10 +195,10 @@ check_url(URL) -> {error, _} -> false end. -check_form_data(FormData) -> +check_body(Body) -> lists:any(fun({_, V}) -> not is_binary(V) - end, maps:to_list(FormData)). + end, maps:to_list(Body)). default_headers() -> maps:put(<<"content-type">>, @@ -229,24 +238,21 @@ parse_url(URL) -> URIMap end. -normalize_headers(Headers) -> - [{atom_to_binary(K), V} || {K, V} <- maps:to_list(Headers)]. - generate_request(Credential, #{method := Method, path := Path, base_query := BaseQuery, headers := Headers, - form_data := FormData0}) -> - FormData = replace_placeholders(FormData0, Credential), + body := Body0}) -> + Body = replace_placeholders(Body0, Credential), case Method of get -> - NPath = append_query(Path, BaseQuery ++ FormData), + NPath = append_query(Path, BaseQuery ++ Body), {NPath, Headers}; post -> NPath = append_query(Path, BaseQuery), ContentType = proplists:get_value(<<"content-type">>, Headers), - Body = serialize_body(ContentType, FormData), - {NPath, Headers, Body} + NBody = serialize_body(ContentType, Body), + {NPath, Headers, NBody} end. replace_placeholders(KVs, Credential) -> @@ -276,10 +282,10 @@ qs([], Acc) -> qs([{K, V} | More], Acc) -> qs(More, [["&", emqx_http_lib:uri_encode(K), "=", emqx_http_lib:uri_encode(V)] | Acc]). -serialize_body(<<"application/json">>, FormData) -> - emqx_json:encode(FormData); -serialize_body(<<"application/x-www-form-urlencoded">>, FormData) -> - qs(FormData). +serialize_body(<<"application/json">>, Body) -> + emqx_json:encode(Body); +serialize_body(<<"application/x-www-form-urlencoded">>, Body) -> + qs(Body). safely_parse_body(ContentType, Body) -> try parse_body(ContentType, Body) of diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl index bc26bf70e..774d75157 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl @@ -19,12 +19,15 @@ -include_lib("typerefl/include/types.hrl"). -behaviour(hocon_schema). +-behaviour(emqx_authentication). --export([ roots/0 +-export([ namespace/0 + , roots/0 , fields/1 ]). --export([ create/1 +-export([ refs/0 + , create/1 , update/2 , authenticate/2 , destroy/1 @@ -34,10 +37,12 @@ %% Hocon Schema %%------------------------------------------------------------------------------ +namespace() -> "authn:jwt". + roots() -> - [ {config, {union, [ hoconsc:t('hmac-based') - , hoconsc:t('public-key') - , hoconsc:t('jwks') + [ {config, {union, [ hoconsc:mk('hmac-based') + , hoconsc:mk('public-key') + , hoconsc:mk('jwks') ]}} ]. @@ -78,12 +83,11 @@ fields(ssl_disable) -> [ {enable, #{type => false}} ]. common_fields() -> - [ {name, fun emqx_authn_schema:authenticator_name/1} - , {mechanism, {enum, [jwt]}} + [ {mechanism, {enum, [jwt]}} , {verify_claims, fun verify_claims/1} - ]. + ] ++ emqx_authn_schema:common_fields(). -secret(type) -> string(); +secret(type) -> binary(); secret(_) -> undefined. secret_base64_encoded(type) -> boolean(); @@ -130,6 +134,12 @@ verify_claims(_) -> undefined. %% APIs %%------------------------------------------------------------------------------ +refs() -> + [ hoconsc:ref(?MODULE, 'hmac-based') + , hoconsc:ref(?MODULE, 'public-key') + , hoconsc:ref(?MODULE, 'jwks') + ]. + create(#{verify_claims := VerifyClaims} = Config) -> create2(Config#{verify_claims => handle_verify_claims(VerifyClaims)}). @@ -239,7 +249,7 @@ verify(JWS, [JWK | More], VerifyClaims) -> Claims = emqx_json:decode(Payload, [return_maps]), case verify_claims(Claims, VerifyClaims) of ok -> - {ok, #{superuser => maps:get(<<"superuser">>, Claims, false)}}; + {ok, #{is_superuser => maps:get(<<"is_superuser">>, Claims, false)}}; {error, Reason} -> {error, Reason} end; diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl index c525efbf1..b69d613f8 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl @@ -20,10 +20,15 @@ -include_lib("typerefl/include/types.hrl"). -behaviour(hocon_schema). +-behaviour(emqx_authentication). --export([ roots/0, fields/1 ]). +-export([ namespace/0 + , roots/0 + , fields/1 + ]). --export([ create/1 +-export([ refs/0 + , create/1 , update/2 , authenticate/2 , destroy/1 @@ -46,7 +51,7 @@ { user_id :: {user_group(), user_id()} , password_hash :: binary() , salt :: binary() - , superuser :: boolean() + , is_superuser :: boolean() }). -reflect_type([ user_id_type/0 ]). @@ -79,15 +84,16 @@ mnesia(copy) -> %% Hocon Schema %%------------------------------------------------------------------------------ +namespace() -> "authn:password-based:builtin-db". + roots() -> [config]. fields(config) -> - [ {name, fun emqx_authn_schema:authenticator_name/1} - , {mechanism, {enum, ['password-based']}} - , {server_type, {enum, ['built-in-database']}} + [ {mechanism, {enum, ['password-based']}} + , {backend, {enum, ['built-in-database']}} , {user_id_type, fun user_id_type/1} , {password_hash_algorithm, fun password_hash_algorithm/1} - ]; + ] ++ emqx_authn_schema:common_fields(); fields(bcrypt) -> [ {name, {enum, [bcrypt]}} @@ -102,7 +108,8 @@ user_id_type(type) -> user_id_type(); user_id_type(default) -> username; user_id_type(_) -> undefined. -password_hash_algorithm(type) -> {union, [hoconsc:ref(bcrypt), hoconsc:ref(other_algorithms)]}; +password_hash_algorithm(type) -> hoconsc:union([hoconsc:ref(?MODULE, bcrypt), + hoconsc:ref(?MODULE, other_algorithms)]); password_hash_algorithm(default) -> #{<<"name">> => sha256}; password_hash_algorithm(_) -> undefined. @@ -114,6 +121,9 @@ salt_rounds(_) -> undefined. %% APIs %%------------------------------------------------------------------------------ +refs() -> + [hoconsc:ref(?MODULE, config)]. + create(#{ user_id_type := Type , password_hash_algorithm := #{name := bcrypt, salt_rounds := SaltRounds} @@ -148,13 +158,13 @@ authenticate(#{password := Password} = Credential, case mnesia:dirty_read(?TAB, {UserGroup, UserID}) of [] -> ignore; - [#user_info{password_hash = PasswordHash, salt = Salt0, superuser = Superuser}] -> + [#user_info{password_hash = PasswordHash, salt = Salt0, is_superuser = IsSuperuser}] -> Salt = case Algorithm of bcrypt -> PasswordHash; _ -> Salt0 end, case PasswordHash =:= hash(Algorithm, Password, Salt) of - true -> {ok, #{superuser => Superuser}}; + true -> {ok, #{is_superuser => IsSuperuser}}; false -> {error, bad_username_or_password} end end. @@ -187,9 +197,9 @@ add_user(#{user_id := UserID, case mnesia:read(?TAB, {UserGroup, UserID}, write) of [] -> {PasswordHash, Salt} = hash(Password, State), - Superuser = maps:get(superuser, UserInfo, false), - insert_user(UserGroup, UserID, PasswordHash, Salt, Superuser), - {ok, #{user_id => UserID, superuser => Superuser}}; + IsSuperuser = maps:get(is_superuser, UserInfo, false), + insert_user(UserGroup, UserID, PasswordHash, Salt, IsSuperuser), + {ok, #{user_id => UserID, is_superuser => IsSuperuser}}; [_] -> {error, already_exist} end @@ -215,8 +225,8 @@ update_user(UserID, UserInfo, {error, not_found}; [#user_info{ password_hash = PasswordHash , salt = Salt - , superuser = Superuser}] -> - NSuperuser = maps:get(superuser, UserInfo, Superuser), + , is_superuser = IsSuperuser}] -> + NSuperuser = maps:get(is_superuser, UserInfo, IsSuperuser), {NPasswordHash, NSalt} = case maps:get(password, UserInfo, undefined) of undefined -> {PasswordHash, Salt}; @@ -224,7 +234,7 @@ update_user(UserID, UserInfo, hash(Password, State) end, insert_user(UserGroup, UserID, NPasswordHash, NSalt, NSuperuser), - {ok, #{user_id => UserID, superuser => NSuperuser}} + {ok, #{user_id => UserID, is_superuser => NSuperuser}} end end). @@ -280,8 +290,8 @@ import(UserGroup, [#{<<"user_id">> := UserID, <<"password_hash">> := PasswordHash} = UserInfo | More]) when is_binary(UserID) andalso is_binary(PasswordHash) -> Salt = maps:get(<<"salt">>, UserInfo, <<>>), - Superuser = maps:get(<<"superuser">>, UserInfo, false), - insert_user(UserGroup, UserID, PasswordHash, Salt, Superuser), + IsSuperuser = maps:get(<<"is_superuser">>, UserInfo, false), + insert_user(UserGroup, UserID, PasswordHash, Salt, IsSuperuser), import(UserGroup, More); import(_UserGroup, [_ | _More]) -> {error, bad_format}. @@ -295,8 +305,8 @@ import(UserGroup, File, Seq) -> {ok, #{user_id := UserID, password_hash := PasswordHash} = UserInfo} -> Salt = maps:get(salt, UserInfo, <<>>), - Superuser = maps:get(superuser, UserInfo, false), - insert_user(UserGroup, UserID, PasswordHash, Salt, Superuser), + IsSuperuser = maps:get(is_superuser, UserInfo, false), + insert_user(UserGroup, UserID, PasswordHash, Salt, IsSuperuser), import(UserGroup, File, Seq); {error, Reason} -> {error, Reason} @@ -331,10 +341,10 @@ get_user_info_by_seq([PasswordHash | More1], [<<"password_hash">> | More2], Acc) get_user_info_by_seq(More1, More2, Acc#{password_hash => PasswordHash}); get_user_info_by_seq([Salt | More1], [<<"salt">> | More2], Acc) -> get_user_info_by_seq(More1, More2, Acc#{salt => Salt}); -get_user_info_by_seq([<<"true">> | More1], [<<"superuser">> | More2], Acc) -> - get_user_info_by_seq(More1, More2, Acc#{superuser => true}); -get_user_info_by_seq([<<"false">> | More1], [<<"superuser">> | More2], Acc) -> - get_user_info_by_seq(More1, More2, Acc#{superuser => false}); +get_user_info_by_seq([<<"true">> | More1], [<<"is_superuser">> | More2], Acc) -> + get_user_info_by_seq(More1, More2, Acc#{is_superuser => true}); +get_user_info_by_seq([<<"false">> | More1], [<<"is_superuser">> | More2], Acc) -> + get_user_info_by_seq(More1, More2, Acc#{is_superuser => false}); get_user_info_by_seq(_, _, _) -> {error, bad_format}. @@ -358,11 +368,11 @@ hash(Password, #{password_hash_algorithm := Algorithm} = State) -> PasswordHash = hash(Algorithm, Password, Salt), {PasswordHash, Salt}. -insert_user(UserGroup, UserID, PasswordHash, Salt, Superuser) -> +insert_user(UserGroup, UserID, PasswordHash, Salt, IsSuperuser) -> UserInfo = #user_info{user_id = {UserGroup, UserID}, password_hash = PasswordHash, salt = Salt, - superuser = Superuser}, + is_superuser = IsSuperuser}, mnesia:write(?TAB, UserInfo, write). delete_user2(UserInfo) -> @@ -390,5 +400,5 @@ to_binary(B) when is_binary(B) -> to_binary(L) when is_list(L) -> iolist_to_binary(L). -serialize_user_info(#user_info{user_id = {_, UserID}, superuser = Superuser}) -> - #{user_id => UserID, superuser => Superuser}. +serialize_user_info(#user_info{user_id = {_, UserID}, is_superuser = IsSuperuser}) -> + #{user_id => UserID, is_superuser => IsSuperuser}. diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl index 11411b70f..9d77c673c 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl @@ -21,12 +21,15 @@ -include_lib("typerefl/include/types.hrl"). -behaviour(hocon_schema). +-behaviour(emqx_authentication). --export([ roots/0 +-export([ namespace/0 + , roots/0 , fields/1 ]). --export([ create/1 +-export([ refs/0 + , create/1 , update/2 , authenticate/2 , destroy/1 @@ -36,10 +39,12 @@ %% Hocon Schema %%------------------------------------------------------------------------------ +namespace() -> "authn:password-based:mongodb". + roots() -> - [ {config, {union, [ hoconsc:t(standalone) - , hoconsc:t('replica-set') - , hoconsc:t('sharded-cluster') + [ {config, {union, [ hoconsc:mk(standalone) + , hoconsc:mk('replica-set') + , hoconsc:mk('sharded-cluster') ]}} ]. @@ -53,16 +58,16 @@ fields('sharded-cluster') -> common_fields() ++ emqx_connector_mongo:fields(sharded). common_fields() -> - [ {name, fun emqx_authn_schema:authenticator_name/1} - , {mechanism, {enum, ['password-based']}} - , {server_type, {enum, [mongodb]}} + [ {mechanism, {enum, ['password-based']}} + , {backend, {enum, [mongodb]}} , {collection, fun collection/1} , {selector, fun selector/1} , {password_hash_field, fun password_hash_field/1} , {salt_field, fun salt_field/1} + , {is_superuser_field, fun is_superuser_field/1} , {password_hash_algorithm, fun password_hash_algorithm/1} , {salt_position, fun salt_position/1} - ]. + ] ++ emqx_authn_schema:common_fields(). collection(type) -> binary(); collection(nullable) -> false; @@ -80,6 +85,10 @@ salt_field(type) -> binary(); salt_field(nullable) -> true; salt_field(_) -> undefined. +is_superuser_field(type) -> binary(); +is_superuser_field(nullable) -> true; +is_superuser_field(_) -> undefined. + password_hash_algorithm(type) -> {enum, [plain, md5, sha, sha256, sha512, bcrypt]}; password_hash_algorithm(default) -> sha256; password_hash_algorithm(_) -> undefined. @@ -92,6 +101,12 @@ salt_position(_) -> undefined. %% APIs %%------------------------------------------------------------------------------ +refs() -> + [ hoconsc:ref(?MODULE, standalone) + , hoconsc:ref(?MODULE, 'replica-set') + , hoconsc:ref(?MODULE, 'sharded-cluster') + ]. + create(#{ selector := Selector , '_unique' := Unique } = Config) -> @@ -99,6 +114,7 @@ create(#{ selector := Selector State = maps:with([ collection , password_hash_field , salt_field + , is_superuser_field , password_hash_algorithm , salt_position , '_unique'], Config), @@ -139,7 +155,7 @@ authenticate(#{password := Password} = Credential, Doc -> case check_password(Password, Doc, State) of ok -> - {ok, #{superuser => superuser(Doc, State)}}; + {ok, #{is_superuser => is_superuser(Doc, State)}}; {error, {cannot_find_password_hash_field, PasswordHashField}} -> ?LOG(error, "['~s'] Can't find password hash field: ~s", [Unique, PasswordHashField]), {error, bad_username_or_password}; @@ -220,9 +236,9 @@ check_password(Password, end end. -superuser(Doc, #{superuser_field := SuperuserField}) -> - maps:get(SuperuserField, Doc, false); -superuser(_, _) -> +is_superuser(Doc, #{is_superuser_field := IsSuperuserField}) -> + maps:get(IsSuperuserField, Doc, false); +is_superuser(_, _) -> false. hash(Algorithm, Password, Salt, prefix) -> diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl index 3cafdb94e..991bb6aee 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl @@ -21,12 +21,15 @@ -include_lib("typerefl/include/types.hrl"). -behaviour(hocon_schema). +-behaviour(emqx_authentication). --export([ roots/0 +-export([ namespace/0 + , roots/0 , fields/1 ]). --export([ create/1 +-export([ refs/0 + , create/1 , update/2 , authenticate/2 , destroy/1 @@ -36,17 +39,19 @@ %% Hocon Schema %%------------------------------------------------------------------------------ +namespace() -> "authn:password-based:mysql". + roots() -> [config]. fields(config) -> - [ {name, fun emqx_authn_schema:authenticator_name/1} - , {mechanism, {enum, ['password-based']}} - , {server_type, {enum, [mysql]}} + [ {mechanism, {enum, ['password-based']}} + , {backend, {enum, [mysql]}} , {password_hash_algorithm, fun password_hash_algorithm/1} , {salt_position, fun salt_position/1} , {query, fun query/1} , {query_timeout, fun query_timeout/1} - ] ++ emqx_connector_schema_lib:relational_db_fields() + ] ++ emqx_authn_schema:common_fields() + ++ emqx_connector_schema_lib:relational_db_fields() ++ emqx_connector_schema_lib:ssl_fields(). password_hash_algorithm(type) -> {enum, [plain, md5, sha, sha256, sha512, bcrypt]}; @@ -69,6 +74,9 @@ query_timeout(_) -> undefined. %% APIs %%------------------------------------------------------------------------------ +refs() -> + [hoconsc:ref(?MODULE, config)]. + create(#{ password_hash_algorithm := Algorithm , salt_position := SaltPosition , query := Query0 @@ -115,7 +123,7 @@ authenticate(#{password := Password} = Credential, Selected = maps:from_list(lists:zip(Columns, Rows)), case check_password(Password, Selected, State) of ok -> - {ok, #{superuser => maps:get(<<"superuser">>, Selected, false)}}; + {ok, #{is_superuser => maps:get(<<"is_superuser">>, Selected, false)}}; {error, Reason} -> {error, Reason} end; diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl index 5c21d3d6c..c497074de 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl @@ -22,10 +22,15 @@ -include_lib("typerefl/include/types.hrl"). -behaviour(hocon_schema). +-behaviour(emqx_authentication). --export([ roots/0, fields/1 ]). +-export([ namespace/0 + , roots/0 + , fields/1 + ]). --export([ create/1 +-export([ refs/0 + , create/1 , update/2 , authenticate/2 , destroy/1 @@ -35,16 +40,18 @@ %% Hocon Schema %%------------------------------------------------------------------------------ +namespace() -> "authn:password-based:postgresql". + roots() -> [config]. fields(config) -> - [ {name, fun emqx_authn_schema:authenticator_name/1} - , {mechanism, {enum, ['password-based']}} - , {server_type, {enum, [pgsql]}} + [ {mechanism, {enum, ['password-based']}} + , {backend, {enum, [postgresql]}} , {password_hash_algorithm, fun password_hash_algorithm/1} , {salt_position, {enum, [prefix, suffix]}} , {query, fun query/1} - ] ++ emqx_connector_schema_lib:relational_db_fields() + ] ++ emqx_authn_schema:common_fields() + ++ emqx_connector_schema_lib:relational_db_fields() ++ emqx_connector_schema_lib:ssl_fields(). password_hash_algorithm(type) -> {enum, [plain, md5, sha, sha256, sha512, bcrypt]}; @@ -59,6 +66,9 @@ query(_) -> undefined. %% APIs %%------------------------------------------------------------------------------ +refs() -> + [hoconsc:ref(?MODULE, config)]. + create(#{ query := Query0 , password_hash_algorithm := Algorithm , salt_position := SaltPosition @@ -103,7 +113,7 @@ authenticate(#{password := Password} = Credential, Selected = maps:from_list(lists:zip(NColumns, Rows)), case check_password(Password, Selected, State) of ok -> - {ok, #{superuser => maps:get(<<"superuser">>, Selected, false)}}; + {ok, #{is_superuser => maps:get(<<"is_superuser">>, Selected, false)}}; {error, Reason} -> {error, Reason} end; diff --git a/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl b/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl index 1b090b007..949aeeaea 100644 --- a/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl +++ b/apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl @@ -21,12 +21,15 @@ -include_lib("typerefl/include/types.hrl"). -behaviour(hocon_schema). +-behaviour(emqx_authentication). --export([ roots/0 +-export([ namespace/0 + , roots/0 , fields/1 ]). --export([ create/1 +-export([ refs/0 + , create/1 , update/2 , authenticate/2 , destroy/1 @@ -36,10 +39,12 @@ %% Hocon Schema %%------------------------------------------------------------------------------ +namespace() -> "authn:password-based:redis". + roots() -> - [ {config, {union, [ hoconsc:t(standalone) - , hoconsc:t(cluster) - , hoconsc:t(sentinel) + [ {config, {union, [ hoconsc:mk(standalone) + , hoconsc:mk(cluster) + , hoconsc:mk(sentinel) ]}} ]. @@ -53,13 +58,12 @@ fields(sentinel) -> common_fields() ++ emqx_connector_redis:fields(sentinel). common_fields() -> - [ {name, fun emqx_authn_schema:authenticator_name/1} - , {mechanism, {enum, ['password-based']}} - , {server_type, {enum, [redis]}} + [ {mechanism, {enum, ['password-based']}} + , {backend, {enum, [redis]}} , {query, fun query/1} , {password_hash_algorithm, fun password_hash_algorithm/1} , {salt_position, fun salt_position/1} - ]. + ] ++ emqx_authn_schema:common_fields(). query(type) -> string(); query(nullable) -> false; @@ -77,6 +81,12 @@ salt_position(_) -> undefined. %% APIs %%------------------------------------------------------------------------------ +refs() -> + [ hoconsc:ref(?MODULE, standalone) + , hoconsc:ref(?MODULE, cluster) + , hoconsc:ref(?MODULE, sentinel) + ]. + create(#{ query := Query , '_unique' := Unique } = Config) -> @@ -125,7 +135,7 @@ authenticate(#{password := Password} = Credential, Selected = merge(Fields, Values), case check_password(Password, Selected, State) of ok -> - {ok, #{superuser => maps:get("superuser", Selected, false)}}; + {ok, #{is_superuser => maps:get("is_superuser", Selected, false)}}; {error, Reason} -> {error, Reason} end; @@ -170,7 +180,7 @@ check_fields(["password_hash" | More], false) -> check_fields(More, true); check_fields(["salt" | More], HasPassHash) -> check_fields(More, HasPassHash); -check_fields(["superuser" | More], HasPassHash) -> +check_fields(["is_superuser" | More], HasPassHash) -> check_fields(More, HasPassHash); check_fields([Field | _], _) -> error({unsupported_field, Field}). diff --git a/apps/emqx_authn/test/data/user-credentials.csv b/apps/emqx_authn/test/data/user-credentials.csv index 0548308b7..cbadaefbc 100644 --- a/apps/emqx_authn/test/data/user-credentials.csv +++ b/apps/emqx_authn/test/data/user-credentials.csv @@ -1,3 +1,3 @@ -user_id,password_hash,salt,superuser +user_id,password_hash,salt,is_superuser myuser3,b6c743545a7817ae8c8f624371d5f5f0373234bb0ff36b8ffbf19bce0e06ab75,de1024f462fb83910fd13151bd4bd235,true myuser4,ee68c985a69208b6eda8c6c9b4c7c2d2b15ee2352cdd64a903171710a99182e8,ad773b5be9dd0613fe6c2f4d8c403139,false diff --git a/apps/emqx_authn/test/data/user-credentials.json b/apps/emqx_authn/test/data/user-credentials.json index e54501233..94375df22 100644 --- a/apps/emqx_authn/test/data/user-credentials.json +++ b/apps/emqx_authn/test/data/user-credentials.json @@ -3,12 +3,12 @@ "user_id":"myuser1", "password_hash":"c5e46903df45e5dc096dc74657610dbee8deaacae656df88a1788f1847390242", "salt": "e378187547bf2d6f0545a3f441aa4d8a", - "superuser": true + "is_superuser": true }, { "user_id":"myuser2", "password_hash":"f4d17f300b11e522fd33f497c11b126ef1ea5149c74d2220f9a16dc876d4567b", "salt": "6d3f9bd5b54d94b98adbcfe10b6d181f", - "superuser": false + "is_superuser": false } ] diff --git a/apps/emqx_authn/test/emqx_authn_SUITE.erl b/apps/emqx_authn/test/emqx_authn_SUITE.erl index eb7f0291a..74ec397cc 100644 --- a/apps/emqx_authn/test/emqx_authn_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_SUITE.erl @@ -19,97 +19,4 @@ -compile(export_all). -compile(nowarn_export_all). --include_lib("common_test/include/ct.hrl"). --include_lib("eunit/include/eunit.hrl"). - --include("emqx_authn.hrl"). - --define(AUTH, emqx_authn). - -all() -> - emqx_ct:all(?MODULE). - -init_per_suite(Config) -> - application:set_env(ekka, strict_mode, true), - emqx_ct_helpers:start_apps([emqx_authn]), - Config. - -end_per_suite(_) -> - emqx_ct_helpers:stop_apps([emqx_authn]), - ok. - -t_chain(_) -> - ?assertMatch({ok, #{id := ?CHAIN, authenticators := []}}, ?AUTH:lookup_chain(?CHAIN)), - - ChainID = <<"mychain">>, - Chain = #{id => ChainID}, - ?assertMatch({ok, #{id := ChainID, authenticators := []}}, ?AUTH:create_chain(Chain)), - ?assertEqual({error, {already_exists, {chain, ChainID}}}, ?AUTH:create_chain(Chain)), - ?assertMatch({ok, #{id := ChainID, authenticators := []}}, ?AUTH:lookup_chain(ChainID)), - ?assertEqual(ok, ?AUTH:delete_chain(ChainID)), - ?assertMatch({error, {not_found, {chain, ChainID}}}, ?AUTH:lookup_chain(ChainID)), - ok. - -t_authenticator(_) -> - AuthenticatorName1 = <<"myauthenticator1">>, - AuthenticatorConfig1 = #{name => AuthenticatorName1, - mechanism => 'password-based', - server_type => 'built-in-database', - user_id_type => username, - password_hash_algorithm => #{ - name => sha256 - }}, - {ok, #{name := AuthenticatorName1, id := ID1}} = ?AUTH:create_authenticator(?CHAIN, AuthenticatorConfig1), - ?assertMatch({ok, #{name := AuthenticatorName1}}, ?AUTH:lookup_authenticator(?CHAIN, ID1)), - ?assertMatch({ok, [#{name := AuthenticatorName1}]}, ?AUTH:list_authenticators(?CHAIN)), - ?assertEqual({error, name_has_be_used}, ?AUTH:create_authenticator(?CHAIN, AuthenticatorConfig1)), - - AuthenticatorConfig2 = #{name => AuthenticatorName1, - mechanism => jwt, - use_jwks => false, - algorithm => 'hmac-based', - secret => <<"abcdef">>, - secret_base64_encoded => false, - verify_claims => []}, - {ok, #{name := AuthenticatorName1, id := ID1}} = ?AUTH:update_authenticator(?CHAIN, ID1, AuthenticatorConfig2), - - ID2 = <<"random">>, - ?assertEqual({error, {not_found, {authenticator, ID2}}}, ?AUTH:update_authenticator(?CHAIN, ID2, AuthenticatorConfig2)), - ?assertEqual({error, name_has_be_used}, ?AUTH:update_or_create_authenticator(?CHAIN, ID2, AuthenticatorConfig2)), - - AuthenticatorName2 = <<"myauthenticator2">>, - AuthenticatorConfig3 = AuthenticatorConfig2#{name => AuthenticatorName2}, - {ok, #{name := AuthenticatorName2, id := ID2}} = ?AUTH:update_or_create_authenticator(?CHAIN, ID2, AuthenticatorConfig3), - ?assertMatch({ok, #{name := AuthenticatorName2}}, ?AUTH:lookup_authenticator(?CHAIN, ID2)), - {ok, #{name := AuthenticatorName2, id := ID2}} = ?AUTH:update_or_create_authenticator(?CHAIN, ID2, AuthenticatorConfig3#{secret := <<"fedcba">>}), - - ?assertMatch({ok, #{id := ?CHAIN, authenticators := [#{name := AuthenticatorName1}, #{name := AuthenticatorName2}]}}, ?AUTH:lookup_chain(?CHAIN)), - ?assertMatch({ok, [#{name := AuthenticatorName1}, #{name := AuthenticatorName2}]}, ?AUTH:list_authenticators(?CHAIN)), - - ?assertEqual(ok, ?AUTH:move_authenticator(?CHAIN, ID2, top)), - ?assertMatch({ok, [#{name := AuthenticatorName2}, #{name := AuthenticatorName1}]}, ?AUTH:list_authenticators(?CHAIN)), - - ?assertEqual(ok, ?AUTH:move_authenticator(?CHAIN, ID2, bottom)), - ?assertMatch({ok, [#{name := AuthenticatorName1}, #{name := AuthenticatorName2}]}, ?AUTH:list_authenticators(?CHAIN)), - - ?assertEqual(ok, ?AUTH:move_authenticator(?CHAIN, ID2, {before, ID1})), - - ?assertMatch({ok, [#{name := AuthenticatorName2}, #{name := AuthenticatorName1}]}, ?AUTH:list_authenticators(?CHAIN)), - - ?assertEqual({error, {not_found, {authenticator, <<"nonexistent">>}}}, ?AUTH:move_authenticator(?CHAIN, ID2, {before, <<"nonexistent">>})), - - ?assertEqual(ok, ?AUTH:delete_authenticator(?CHAIN, ID1)), - ?assertEqual(ok, ?AUTH:delete_authenticator(?CHAIN, ID2)), - ?assertEqual({ok, []}, ?AUTH:list_authenticators(?CHAIN)), - ok. - -t_authenticate(_) -> - ClientInfo = #{zone => default, - listener => {tcp, default}, - username => <<"myuser">>, - password => <<"mypass">>}, - ?assertEqual({ok, #{superuser => false}}, emqx_access_control:authenticate(ClientInfo)), - ?assertEqual(false, emqx_authn:is_enabled()), - emqx_authn:enable(), - ?assertEqual(true, emqx_authn:is_enabled()), - ?assertEqual({error, not_authorized}, emqx_access_control:authenticate(ClientInfo)). +all() -> emqx_ct:all(?MODULE). \ No newline at end of file diff --git a/apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl b/apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl index ddb2bb209..16c04771d 100644 --- a/apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_jwt_SUITE.erl @@ -19,140 +19,140 @@ -compile(export_all). -compile(nowarn_export_all). --include_lib("common_test/include/ct.hrl"). --include_lib("eunit/include/eunit.hrl"). +% -include_lib("common_test/include/ct.hrl"). +% -include_lib("eunit/include/eunit.hrl"). --include("emqx_authn.hrl"). +% -include("emqx_authn.hrl"). --define(AUTH, emqx_authn). +% -define(AUTH, emqx_authn). all() -> emqx_ct:all(?MODULE). -init_per_suite(Config) -> - emqx_ct_helpers:start_apps([emqx_authn]), - Config. +% init_per_suite(Config) -> +% emqx_ct_helpers:start_apps([emqx_authn]), +% Config. -end_per_suite(_) -> - emqx_ct_helpers:stop_apps([emqx_authn]), - ok. +% end_per_suite(_) -> +% emqx_ct_helpers:stop_apps([emqx_authn]), +% ok. -t_jwt_authenticator(_) -> - AuthenticatorName = <<"myauthenticator">>, - Config = #{name => AuthenticatorName, - mechanism => jwt, - use_jwks => false, - algorithm => 'hmac-based', - secret => <<"abcdef">>, - secret_base64_encoded => false, - verify_claims => []}, - {ok, #{name := AuthenticatorName, id := ID}} = ?AUTH:create_authenticator(?CHAIN, Config), +% t_jwt_authenticator(_) -> +% AuthenticatorName = <<"myauthenticator">>, +% Config = #{name => AuthenticatorName, +% mechanism => jwt, +% use_jwks => false, +% algorithm => 'hmac-based', +% secret => <<"abcdef">>, +% secret_base64_encoded => false, +% verify_claims => []}, +% {ok, #{name := AuthenticatorName, id := ID}} = ?AUTH:create_authenticator(?CHAIN, Config), - Payload = #{<<"username">> => <<"myuser">>}, - JWS = generate_jws('hmac-based', Payload, <<"abcdef">>), - ClientInfo = #{username => <<"myuser">>, - password => JWS}, - ?assertEqual({stop, {ok, #{superuser => false}}}, ?AUTH:authenticate(ClientInfo, ignored)), +% Payload = #{<<"username">> => <<"myuser">>}, +% JWS = generate_jws('hmac-based', Payload, <<"abcdef">>), +% ClientInfo = #{username => <<"myuser">>, +% password => JWS}, +% ?assertEqual({stop, {ok, #{is_superuser => false}}}, ?AUTH:authenticate(ClientInfo, ignored)), - Payload1 = #{<<"username">> => <<"myuser">>, <<"superuser">> => true}, - JWS1 = generate_jws('hmac-based', Payload1, <<"abcdef">>), - ClientInfo1 = #{username => <<"myuser">>, - password => JWS1}, - ?assertEqual({stop, {ok, #{superuser => true}}}, ?AUTH:authenticate(ClientInfo1, ignored)), +% Payload1 = #{<<"username">> => <<"myuser">>, <<"is_superuser">> => true}, +% JWS1 = generate_jws('hmac-based', Payload1, <<"abcdef">>), +% ClientInfo1 = #{username => <<"myuser">>, +% password => JWS1}, +% ?assertEqual({stop, {ok, #{is_superuser => true}}}, ?AUTH:authenticate(ClientInfo1, ignored)), - BadJWS = generate_jws('hmac-based', Payload, <<"bad_secret">>), - ClientInfo2 = ClientInfo#{password => BadJWS}, - ?assertEqual({stop, {error, not_authorized}}, ?AUTH:authenticate(ClientInfo2, ignored)), +% BadJWS = generate_jws('hmac-based', Payload, <<"bad_secret">>), +% ClientInfo2 = ClientInfo#{password => BadJWS}, +% ?assertEqual({stop, {error, not_authorized}}, ?AUTH:authenticate(ClientInfo2, ignored)), - %% secret_base64_encoded - Config2 = Config#{secret => base64:encode(<<"abcdef">>), - secret_base64_encoded => true}, - ?assertMatch({ok, _}, ?AUTH:update_authenticator(?CHAIN, ID, Config2)), - ?assertEqual({stop, {ok, #{superuser => false}}}, ?AUTH:authenticate(ClientInfo, ignored)), +% %% secret_base64_encoded +% Config2 = Config#{secret => base64:encode(<<"abcdef">>), +% secret_base64_encoded => true}, +% ?assertMatch({ok, _}, ?AUTH:update_authenticator(?CHAIN, ID, Config2)), +% ?assertEqual({stop, {ok, #{is_superuser => false}}}, ?AUTH:authenticate(ClientInfo, ignored)), - Config3 = Config#{verify_claims => [{<<"username">>, <<"${mqtt-username}">>}]}, - ?assertMatch({ok, _}, ?AUTH:update_authenticator(?CHAIN, ID, Config3)), - ?assertEqual({stop, {ok, #{superuser => false}}}, ?AUTH:authenticate(ClientInfo, ignored)), - ?assertEqual({stop, {error, bad_username_or_password}}, ?AUTH:authenticate(ClientInfo#{username => <<"otheruser">>}, ok)), +% Config3 = Config#{verify_claims => [{<<"username">>, <<"${mqtt-username}">>}]}, +% ?assertMatch({ok, _}, ?AUTH:update_authenticator(?CHAIN, ID, Config3)), +% ?assertEqual({stop, {ok, #{is_superuser => false}}}, ?AUTH:authenticate(ClientInfo, ignored)), +% ?assertEqual({stop, {error, bad_username_or_password}}, ?AUTH:authenticate(ClientInfo#{username => <<"otheruser">>}, ok)), - %% Expiration - Payload3 = #{ <<"username">> => <<"myuser">> - , <<"exp">> => erlang:system_time(second) - 60}, - JWS3 = generate_jws('hmac-based', Payload3, <<"abcdef">>), - ClientInfo3 = ClientInfo#{password => JWS3}, - ?assertEqual({stop, {error, bad_username_or_password}}, ?AUTH:authenticate(ClientInfo3, ignored)), +% %% Expiration +% Payload3 = #{ <<"username">> => <<"myuser">> +% , <<"exp">> => erlang:system_time(second) - 60}, +% JWS3 = generate_jws('hmac-based', Payload3, <<"abcdef">>), +% ClientInfo3 = ClientInfo#{password => JWS3}, +% ?assertEqual({stop, {error, bad_username_or_password}}, ?AUTH:authenticate(ClientInfo3, ignored)), - Payload4 = #{ <<"username">> => <<"myuser">> - , <<"exp">> => erlang:system_time(second) + 60}, - JWS4 = generate_jws('hmac-based', Payload4, <<"abcdef">>), - ClientInfo4 = ClientInfo#{password => JWS4}, - ?assertEqual({stop, {ok, #{superuser => false}}}, ?AUTH:authenticate(ClientInfo4, ignored)), +% Payload4 = #{ <<"username">> => <<"myuser">> +% , <<"exp">> => erlang:system_time(second) + 60}, +% JWS4 = generate_jws('hmac-based', Payload4, <<"abcdef">>), +% ClientInfo4 = ClientInfo#{password => JWS4}, +% ?assertEqual({stop, {ok, #{is_superuser => false}}}, ?AUTH:authenticate(ClientInfo4, ignored)), - %% Issued At - Payload5 = #{ <<"username">> => <<"myuser">> - , <<"iat">> => erlang:system_time(second) - 60}, - JWS5 = generate_jws('hmac-based', Payload5, <<"abcdef">>), - ClientInfo5 = ClientInfo#{password => JWS5}, - ?assertEqual({stop, {ok, #{superuser => false}}}, ?AUTH:authenticate(ClientInfo5, ignored)), +% %% Issued At +% Payload5 = #{ <<"username">> => <<"myuser">> +% , <<"iat">> => erlang:system_time(second) - 60}, +% JWS5 = generate_jws('hmac-based', Payload5, <<"abcdef">>), +% ClientInfo5 = ClientInfo#{password => JWS5}, +% ?assertEqual({stop, {ok, #{is_superuser => false}}}, ?AUTH:authenticate(ClientInfo5, ignored)), - Payload6 = #{ <<"username">> => <<"myuser">> - , <<"iat">> => erlang:system_time(second) + 60}, - JWS6 = generate_jws('hmac-based', Payload6, <<"abcdef">>), - ClientInfo6 = ClientInfo#{password => JWS6}, - ?assertEqual({stop, {error, bad_username_or_password}}, ?AUTH:authenticate(ClientInfo6, ignored)), +% Payload6 = #{ <<"username">> => <<"myuser">> +% , <<"iat">> => erlang:system_time(second) + 60}, +% JWS6 = generate_jws('hmac-based', Payload6, <<"abcdef">>), +% ClientInfo6 = ClientInfo#{password => JWS6}, +% ?assertEqual({stop, {error, bad_username_or_password}}, ?AUTH:authenticate(ClientInfo6, ignored)), - %% Not Before - Payload7 = #{ <<"username">> => <<"myuser">> - , <<"nbf">> => erlang:system_time(second) - 60}, - JWS7 = generate_jws('hmac-based', Payload7, <<"abcdef">>), - ClientInfo7 = ClientInfo#{password => JWS7}, - ?assertEqual({stop, {ok, #{superuser => false}}}, ?AUTH:authenticate(ClientInfo7, ignored)), +% %% Not Before +% Payload7 = #{ <<"username">> => <<"myuser">> +% , <<"nbf">> => erlang:system_time(second) - 60}, +% JWS7 = generate_jws('hmac-based', Payload7, <<"abcdef">>), +% ClientInfo7 = ClientInfo#{password => JWS7}, +% ?assertEqual({stop, {ok, #{is_superuser => false}}}, ?AUTH:authenticate(ClientInfo7, ignored)), - Payload8 = #{ <<"username">> => <<"myuser">> - , <<"nbf">> => erlang:system_time(second) + 60}, - JWS8 = generate_jws('hmac-based', Payload8, <<"abcdef">>), - ClientInfo8 = ClientInfo#{password => JWS8}, - ?assertEqual({stop, {error, bad_username_or_password}}, ?AUTH:authenticate(ClientInfo8, ignored)), +% Payload8 = #{ <<"username">> => <<"myuser">> +% , <<"nbf">> => erlang:system_time(second) + 60}, +% JWS8 = generate_jws('hmac-based', Payload8, <<"abcdef">>), +% ClientInfo8 = ClientInfo#{password => JWS8}, +% ?assertEqual({stop, {error, bad_username_or_password}}, ?AUTH:authenticate(ClientInfo8, ignored)), - ?assertEqual(ok, ?AUTH:delete_authenticator(?CHAIN, ID)), - ok. +% ?assertEqual(ok, ?AUTH:delete_authenticator(?CHAIN, ID)), +% ok. -t_jwt_authenticator2(_) -> - Dir = code:lib_dir(emqx_authn, test), - PublicKey = list_to_binary(filename:join([Dir, "data/public_key.pem"])), - PrivateKey = list_to_binary(filename:join([Dir, "data/private_key.pem"])), - AuthenticatorName = <<"myauthenticator">>, - Config = #{name => AuthenticatorName, - mechanism => jwt, - use_jwks => false, - algorithm => 'public-key', - certificate => PublicKey, - verify_claims => []}, - {ok, #{name := AuthenticatorName, id := ID}} = ?AUTH:create_authenticator(?CHAIN, Config), +% t_jwt_authenticator2(_) -> +% Dir = code:lib_dir(emqx_authn, test), +% PublicKey = list_to_binary(filename:join([Dir, "data/public_key.pem"])), +% PrivateKey = list_to_binary(filename:join([Dir, "data/private_key.pem"])), +% AuthenticatorName = <<"myauthenticator">>, +% Config = #{name => AuthenticatorName, +% mechanism => jwt, +% use_jwks => false, +% algorithm => 'public-key', +% certificate => PublicKey, +% verify_claims => []}, +% {ok, #{name := AuthenticatorName, id := ID}} = ?AUTH:create_authenticator(?CHAIN, Config), - Payload = #{<<"username">> => <<"myuser">>}, - JWS = generate_jws('public-key', Payload, PrivateKey), - ClientInfo = #{username => <<"myuser">>, - password => JWS}, - ?assertEqual({stop, {ok, #{superuser => false}}}, ?AUTH:authenticate(ClientInfo, ignored)), - ?assertEqual({stop, {error, not_authorized}}, ?AUTH:authenticate(ClientInfo#{password => <<"badpassword">>}, ignored)), +% Payload = #{<<"username">> => <<"myuser">>}, +% JWS = generate_jws('public-key', Payload, PrivateKey), +% ClientInfo = #{username => <<"myuser">>, +% password => JWS}, +% ?assertEqual({stop, {ok, #{is_superuser => false}}}, ?AUTH:authenticate(ClientInfo, ignored)), +% ?assertEqual({stop, {error, not_authorized}}, ?AUTH:authenticate(ClientInfo#{password => <<"badpassword">>}, ignored)), - ?assertEqual(ok, ?AUTH:delete_authenticator(?CHAIN, ID)), - ok. +% ?assertEqual(ok, ?AUTH:delete_authenticator(?CHAIN, ID)), +% ok. -generate_jws('hmac-based', Payload, Secret) -> - JWK = jose_jwk:from_oct(Secret), - Header = #{ <<"alg">> => <<"HS256">> - , <<"typ">> => <<"JWT">> - }, - Signed = jose_jwt:sign(JWK, Header, Payload), - {_, JWS} = jose_jws:compact(Signed), - JWS; -generate_jws('public-key', Payload, PrivateKey) -> - JWK = jose_jwk:from_pem_file(PrivateKey), - Header = #{ <<"alg">> => <<"RS256">> - , <<"typ">> => <<"JWT">> - }, - Signed = jose_jwt:sign(JWK, Header, Payload), - {_, JWS} = jose_jws:compact(Signed), - JWS. +% generate_jws('hmac-based', Payload, Secret) -> +% JWK = jose_jwk:from_oct(Secret), +% Header = #{ <<"alg">> => <<"HS256">> +% , <<"typ">> => <<"JWT">> +% }, +% Signed = jose_jwt:sign(JWK, Header, Payload), +% {_, JWS} = jose_jws:compact(Signed), +% JWS; +% generate_jws('public-key', Payload, PrivateKey) -> +% JWK = jose_jwk:from_pem_file(PrivateKey), +% Header = #{ <<"alg">> => <<"RS256">> +% , <<"typ">> => <<"JWT">> +% }, +% Signed = jose_jwt:sign(JWK, Header, Payload), +% {_, JWS} = jose_jws:compact(Signed), +% JWS. diff --git a/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl b/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl index d6425a89c..959cf0323 100644 --- a/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl +++ b/apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl @@ -19,146 +19,146 @@ -compile(export_all). -compile(nowarn_export_all). --include_lib("common_test/include/ct.hrl"). --include_lib("eunit/include/eunit.hrl"). +% -include_lib("common_test/include/ct.hrl"). +% -include_lib("eunit/include/eunit.hrl"). --include("emqx_authn.hrl"). +% -include("emqx_authn.hrl"). --define(AUTH, emqx_authn). +% -define(AUTH, emqx_authn). all() -> emqx_ct:all(?MODULE). -init_per_suite(Config) -> - emqx_ct_helpers:start_apps([emqx_authn]), - Config. +% init_per_suite(Config) -> +% emqx_ct_helpers:start_apps([emqx_authn]), +% Config. -end_per_suite(_) -> - emqx_ct_helpers:stop_apps([emqx_authn]), - ok. +% end_per_suite(_) -> +% emqx_ct_helpers:stop_apps([emqx_authn]), +% ok. -t_mnesia_authenticator(_) -> - AuthenticatorName = <<"myauthenticator">>, - AuthenticatorConfig = #{name => AuthenticatorName, - mechanism => 'password-based', - server_type => 'built-in-database', - user_id_type => username, - password_hash_algorithm => #{ - name => sha256 - }}, - {ok, #{name := AuthenticatorName, id := ID}} = ?AUTH:create_authenticator(?CHAIN, AuthenticatorConfig), +% t_mnesia_authenticator(_) -> +% AuthenticatorName = <<"myauthenticator">>, +% AuthenticatorConfig = #{name => AuthenticatorName, +% mechanism => 'password-based', +% server_type => 'built-in-database', +% user_id_type => username, +% password_hash_algorithm => #{ +% name => sha256 +% }}, +% {ok, #{name := AuthenticatorName, id := ID}} = ?AUTH:create_authenticator(?CHAIN, AuthenticatorConfig), - UserInfo = #{user_id => <<"myuser">>, - password => <<"mypass">>}, - ?assertMatch({ok, #{user_id := <<"myuser">>}}, ?AUTH:add_user(?CHAIN, ID, UserInfo)), - ?assertMatch({ok, #{user_id := <<"myuser">>}}, ?AUTH:lookup_user(?CHAIN, ID, <<"myuser">>)), +% UserInfo = #{user_id => <<"myuser">>, +% password => <<"mypass">>}, +% ?assertMatch({ok, #{user_id := <<"myuser">>}}, ?AUTH:add_user(?CHAIN, ID, UserInfo)), +% ?assertMatch({ok, #{user_id := <<"myuser">>}}, ?AUTH:lookup_user(?CHAIN, ID, <<"myuser">>)), - ClientInfo = #{zone => external, - username => <<"myuser">>, - password => <<"mypass">>}, - ?assertEqual({stop, {ok, #{superuser => false}}}, ?AUTH:authenticate(ClientInfo, ignored)), - ?AUTH:enable(), - ?assertEqual({ok, #{superuser => false}}, emqx_access_control:authenticate(ClientInfo)), +% ClientInfo = #{zone => external, +% username => <<"myuser">>, +% password => <<"mypass">>}, +% ?assertEqual({stop, {ok, #{is_superuser => false}}}, ?AUTH:authenticate(ClientInfo, ignored)), +% ?AUTH:enable(), +% ?assertEqual({ok, #{is_superuser => false}}, emqx_access_control:authenticate(ClientInfo)), - ClientInfo2 = ClientInfo#{username => <<"baduser">>}, - ?assertEqual({stop, {error, not_authorized}}, ?AUTH:authenticate(ClientInfo2, ignored)), - ?assertEqual({error, not_authorized}, emqx_access_control:authenticate(ClientInfo2)), +% ClientInfo2 = ClientInfo#{username => <<"baduser">>}, +% ?assertEqual({stop, {error, not_authorized}}, ?AUTH:authenticate(ClientInfo2, ignored)), +% ?assertEqual({error, not_authorized}, emqx_access_control:authenticate(ClientInfo2)), - ClientInfo3 = ClientInfo#{password => <<"badpass">>}, - ?assertEqual({stop, {error, bad_username_or_password}}, ?AUTH:authenticate(ClientInfo3, ignored)), - ?assertEqual({error, bad_username_or_password}, emqx_access_control:authenticate(ClientInfo3)), +% ClientInfo3 = ClientInfo#{password => <<"badpass">>}, +% ?assertEqual({stop, {error, bad_username_or_password}}, ?AUTH:authenticate(ClientInfo3, ignored)), +% ?assertEqual({error, bad_username_or_password}, emqx_access_control:authenticate(ClientInfo3)), - UserInfo2 = UserInfo#{password => <<"mypass2">>}, - ?assertMatch({ok, #{user_id := <<"myuser">>}}, ?AUTH:update_user(?CHAIN, ID, <<"myuser">>, UserInfo2)), - ClientInfo4 = ClientInfo#{password => <<"mypass2">>}, - ?assertEqual({stop, {ok, #{superuser => false}}}, ?AUTH:authenticate(ClientInfo4, ignored)), +% UserInfo2 = UserInfo#{password => <<"mypass2">>}, +% ?assertMatch({ok, #{user_id := <<"myuser">>}}, ?AUTH:update_user(?CHAIN, ID, <<"myuser">>, UserInfo2)), +% ClientInfo4 = ClientInfo#{password => <<"mypass2">>}, +% ?assertEqual({stop, {ok, #{is_superuser => false}}}, ?AUTH:authenticate(ClientInfo4, ignored)), - ?assertMatch({ok, #{user_id := <<"myuser">>}}, ?AUTH:update_user(?CHAIN, ID, <<"myuser">>, #{superuser => true})), - ?assertEqual({stop, {ok, #{superuser => true}}}, ?AUTH:authenticate(ClientInfo4, ignored)), +% ?assertMatch({ok, #{user_id := <<"myuser">>}}, ?AUTH:update_user(?CHAIN, ID, <<"myuser">>, #{is_superuser => true})), +% ?assertEqual({stop, {ok, #{is_superuser => true}}}, ?AUTH:authenticate(ClientInfo4, ignored)), - ?assertEqual(ok, ?AUTH:delete_user(?CHAIN, ID, <<"myuser">>)), - ?assertEqual({error, not_found}, ?AUTH:lookup_user(?CHAIN, ID, <<"myuser">>)), +% ?assertEqual(ok, ?AUTH:delete_user(?CHAIN, ID, <<"myuser">>)), +% ?assertEqual({error, not_found}, ?AUTH:lookup_user(?CHAIN, ID, <<"myuser">>)), - ?assertMatch({ok, #{user_id := <<"myuser">>}}, ?AUTH:add_user(?CHAIN, ID, UserInfo)), - ?assertMatch({ok, #{user_id := <<"myuser">>}}, ?AUTH:lookup_user(?CHAIN, ID, <<"myuser">>)), - ?assertEqual(ok, ?AUTH:delete_authenticator(?CHAIN, ID)), +% ?assertMatch({ok, #{user_id := <<"myuser">>}}, ?AUTH:add_user(?CHAIN, ID, UserInfo)), +% ?assertMatch({ok, #{user_id := <<"myuser">>}}, ?AUTH:lookup_user(?CHAIN, ID, <<"myuser">>)), +% ?assertEqual(ok, ?AUTH:delete_authenticator(?CHAIN, ID)), - {ok, #{name := AuthenticatorName, id := ID1}} = ?AUTH:create_authenticator(?CHAIN, AuthenticatorConfig), - ?assertMatch({error, not_found}, ?AUTH:lookup_user(?CHAIN, ID1, <<"myuser">>)), - ?assertEqual(ok, ?AUTH:delete_authenticator(?CHAIN, ID1)), - ok. +% {ok, #{name := AuthenticatorName, id := ID1}} = ?AUTH:create_authenticator(?CHAIN, AuthenticatorConfig), +% ?assertMatch({error, not_found}, ?AUTH:lookup_user(?CHAIN, ID1, <<"myuser">>)), +% ?assertEqual(ok, ?AUTH:delete_authenticator(?CHAIN, ID1)), +% ok. -t_import(_) -> - AuthenticatorName = <<"myauthenticator">>, - AuthenticatorConfig = #{name => AuthenticatorName, - mechanism => 'password-based', - server_type => 'built-in-database', - user_id_type => username, - password_hash_algorithm => #{ - name => sha256 - }}, - {ok, #{name := AuthenticatorName, id := ID}} = ?AUTH:create_authenticator(?CHAIN, AuthenticatorConfig), +% t_import(_) -> +% AuthenticatorName = <<"myauthenticator">>, +% AuthenticatorConfig = #{name => AuthenticatorName, +% mechanism => 'password-based', +% server_type => 'built-in-database', +% user_id_type => username, +% password_hash_algorithm => #{ +% name => sha256 +% }}, +% {ok, #{name := AuthenticatorName, id := ID}} = ?AUTH:create_authenticator(?CHAIN, AuthenticatorConfig), - Dir = code:lib_dir(emqx_authn, test), - ?assertEqual(ok, ?AUTH:import_users(?CHAIN, ID, filename:join([Dir, "data/user-credentials.json"]))), - ?assertEqual(ok, ?AUTH:import_users(?CHAIN, ID, filename:join([Dir, "data/user-credentials.csv"]))), - ?assertMatch({ok, #{user_id := <<"myuser1">>}}, ?AUTH:lookup_user(?CHAIN, ID, <<"myuser1">>)), - ?assertMatch({ok, #{user_id := <<"myuser3">>}}, ?AUTH:lookup_user(?CHAIN, ID, <<"myuser3">>)), +% Dir = code:lib_dir(emqx_authn, test), +% ?assertEqual(ok, ?AUTH:import_users(?CHAIN, ID, filename:join([Dir, "data/user-credentials.json"]))), +% ?assertEqual(ok, ?AUTH:import_users(?CHAIN, ID, filename:join([Dir, "data/user-credentials.csv"]))), +% ?assertMatch({ok, #{user_id := <<"myuser1">>}}, ?AUTH:lookup_user(?CHAIN, ID, <<"myuser1">>)), +% ?assertMatch({ok, #{user_id := <<"myuser3">>}}, ?AUTH:lookup_user(?CHAIN, ID, <<"myuser3">>)), - ClientInfo1 = #{username => <<"myuser1">>, - password => <<"mypassword1">>}, - ?assertEqual({stop, {ok, #{superuser => true}}}, ?AUTH:authenticate(ClientInfo1, ignored)), +% ClientInfo1 = #{username => <<"myuser1">>, +% password => <<"mypassword1">>}, +% ?assertEqual({stop, {ok, #{is_superuser => true}}}, ?AUTH:authenticate(ClientInfo1, ignored)), - ClientInfo2 = ClientInfo1#{username => <<"myuser2">>, - password => <<"mypassword2">>}, - ?assertEqual({stop, {ok, #{superuser => false}}}, ?AUTH:authenticate(ClientInfo2, ignored)), +% ClientInfo2 = ClientInfo1#{username => <<"myuser2">>, +% password => <<"mypassword2">>}, +% ?assertEqual({stop, {ok, #{is_superuser => false}}}, ?AUTH:authenticate(ClientInfo2, ignored)), - ClientInfo3 = ClientInfo1#{username => <<"myuser3">>, - password => <<"mypassword3">>}, - ?assertEqual({stop, {ok, #{superuser => true}}}, ?AUTH:authenticate(ClientInfo3, ignored)), +% ClientInfo3 = ClientInfo1#{username => <<"myuser3">>, +% password => <<"mypassword3">>}, +% ?assertEqual({stop, {ok, #{is_superuser => true}}}, ?AUTH:authenticate(ClientInfo3, ignored)), - ?assertEqual(ok, ?AUTH:delete_authenticator(?CHAIN, ID)), - ok. +% ?assertEqual(ok, ?AUTH:delete_authenticator(?CHAIN, ID)), +% ok. -t_multi_mnesia_authenticator(_) -> - AuthenticatorName1 = <<"myauthenticator1">>, - AuthenticatorConfig1 = #{name => AuthenticatorName1, - mechanism => 'password-based', - server_type => 'built-in-database', - user_id_type => username, - password_hash_algorithm => #{ - name => sha256 - }}, - AuthenticatorName2 = <<"myauthenticator2">>, - AuthenticatorConfig2 = #{name => AuthenticatorName2, - mechanism => 'password-based', - server_type => 'built-in-database', - user_id_type => clientid, - password_hash_algorithm => #{ - name => sha256 - }}, - {ok, #{name := AuthenticatorName1, id := ID1}} = ?AUTH:create_authenticator(?CHAIN, AuthenticatorConfig1), - {ok, #{name := AuthenticatorName2, id := ID2}} = ?AUTH:create_authenticator(?CHAIN, AuthenticatorConfig2), +% t_multi_mnesia_authenticator(_) -> +% AuthenticatorName1 = <<"myauthenticator1">>, +% AuthenticatorConfig1 = #{name => AuthenticatorName1, +% mechanism => 'password-based', +% server_type => 'built-in-database', +% user_id_type => username, +% password_hash_algorithm => #{ +% name => sha256 +% }}, +% AuthenticatorName2 = <<"myauthenticator2">>, +% AuthenticatorConfig2 = #{name => AuthenticatorName2, +% mechanism => 'password-based', +% server_type => 'built-in-database', +% user_id_type => clientid, +% password_hash_algorithm => #{ +% name => sha256 +% }}, +% {ok, #{name := AuthenticatorName1, id := ID1}} = ?AUTH:create_authenticator(?CHAIN, AuthenticatorConfig1), +% {ok, #{name := AuthenticatorName2, id := ID2}} = ?AUTH:create_authenticator(?CHAIN, AuthenticatorConfig2), - ?assertMatch({ok, #{user_id := <<"myuser">>}}, - ?AUTH:add_user(?CHAIN, ID1, - #{user_id => <<"myuser">>, - password => <<"mypass1">>})), - ?assertMatch({ok, #{user_id := <<"myclient">>}}, - ?AUTH:add_user(?CHAIN, ID2, - #{user_id => <<"myclient">>, - password => <<"mypass2">>})), +% ?assertMatch({ok, #{user_id := <<"myuser">>}}, +% ?AUTH:add_user(?CHAIN, ID1, +% #{user_id => <<"myuser">>, +% password => <<"mypass1">>})), +% ?assertMatch({ok, #{user_id := <<"myclient">>}}, +% ?AUTH:add_user(?CHAIN, ID2, +% #{user_id => <<"myclient">>, +% password => <<"mypass2">>})), - ClientInfo1 = #{username => <<"myuser">>, - clientid => <<"myclient">>, - password => <<"mypass1">>}, - ?assertEqual({stop, {ok, #{superuser => false}}}, ?AUTH:authenticate(ClientInfo1, ignored)), - ?assertEqual(ok, ?AUTH:move_authenticator(?CHAIN, ID2, top)), +% ClientInfo1 = #{username => <<"myuser">>, +% clientid => <<"myclient">>, +% password => <<"mypass1">>}, +% ?assertEqual({stop, {ok, #{is_superuser => false}}}, ?AUTH:authenticate(ClientInfo1, ignored)), +% ?assertEqual(ok, ?AUTH:move_authenticator(?CHAIN, ID2, top)), - ?assertEqual({stop, {error, bad_username_or_password}}, ?AUTH:authenticate(ClientInfo1, ignored)), - ClientInfo2 = ClientInfo1#{password => <<"mypass2">>}, - ?assertEqual({stop, {ok, #{superuser => false}}}, ?AUTH:authenticate(ClientInfo2, ignored)), +% ?assertEqual({stop, {error, bad_username_or_password}}, ?AUTH:authenticate(ClientInfo1, ignored)), +% ClientInfo2 = ClientInfo1#{password => <<"mypass2">>}, +% ?assertEqual({stop, {ok, #{is_superuser => false}}}, ?AUTH:authenticate(ClientInfo2, ignored)), - ?assertEqual(ok, ?AUTH:delete_authenticator(?CHAIN, ID1)), - ?assertEqual(ok, ?AUTH:delete_authenticator(?CHAIN, ID2)), - ok. +% ?assertEqual(ok, ?AUTH:delete_authenticator(?CHAIN, ID1)), +% ?assertEqual(ok, ?AUTH:delete_authenticator(?CHAIN, ID2)), +% ok. diff --git a/apps/emqx_authz/etc/authorization_rules.conf b/apps/emqx_authz/etc/acl.conf similarity index 100% rename from apps/emqx_authz/etc/authorization_rules.conf rename to apps/emqx_authz/etc/acl.conf diff --git a/apps/emqx_authz/etc/emqx_authz.conf b/apps/emqx_authz/etc/emqx_authz.conf index 8eadab38b..ed4ad573c 100644 --- a/apps/emqx_authz/etc/emqx_authz.conf +++ b/apps/emqx_authz/etc/emqx_authz.conf @@ -2,72 +2,62 @@ authorization { sources = [ # { # type: http - # config: { - # url: "https://emqx.com" - # headers: { - # Accept: "application/json" - # Content-Type: "application/json" - # } + # url: "https://emqx.com" + # headers: { + # Accept: "application/json" + # Content-Type: "application/json" # } # }, # { # type: mysql - # config: { - # server: "127.0.0.1:3306" - # database: mqtt - # pool_size: 1 - # username: root - # password: public - # auto_reconnect: true - # ssl: { - # enable: true - # cacertfile: "{{ platform_etc_dir }}/certs/cacert.pem" - # certfile: "{{ platform_etc_dir }}/certs/client-cert.pem" - # keyfile: "{{ platform_etc_dir }}/certs/client-key.pem" - # } + # server: "127.0.0.1:3306" + # database: mqtt + # pool_size: 1 + # username: root + # password: public + # auto_reconnect: true + # ssl: { + # enable: true + # cacertfile: "{{ platform_etc_dir }}/certs/cacert.pem" + # certfile: "{{ platform_etc_dir }}/certs/client-cert.pem" + # keyfile: "{{ platform_etc_dir }}/certs/client-key.pem" # } # sql: "select ipaddress, username, clientid, action, permission, topic from mqtt_authz where ipaddr = '%a' or username = '%u' or clientid = '%c'" # }, # { # type: pgsql - # config: { - # server: "127.0.0.1:5432" - # database: mqtt - # pool_size: 1 - # username: root - # password: public - # auto_reconnect: true - # ssl: {enable: false} - # } + # server: "127.0.0.1:5432" + # database: mqtt + # pool_size: 1 + # username: root + # password: public + # auto_reconnect: true + # ssl: {enable: false} # sql: "select ipaddress, username, clientid, action, permission, topic from mqtt_authz where ipaddr = '%a' or username = '%u' or username = '$all' or clientid = '%c'" # }, # { # type: redis - # config: { - # server: "127.0.0.1:6379" - # database: 0 - # pool_size: 1 - # password: public - # auto_reconnect: true - # ssl: {enable: false} - # } + # server: "127.0.0.1:6379" + # database: 0 + # pool_size: 1 + # password: public + # auto_reconnect: true + # ssl: {enable: false} # cmd: "HGETALL mqtt_authz:%u" # }, # { # type: mongo - # config: { - # mongo_type: single - # server: "127.0.0.1:27017" - # pool_size: 1 - # database: mqtt - # ssl: {enable: false} - # } + # mongo_type: single + # server: "127.0.0.1:27017" + # pool_size: 1 + # database: mqtt + # ssl: {enable: false} # collection: mqtt_authz # find: { "$or": [ { "username": "%u" }, { "clientid": "%c" } ] } # }, { type: file - path: "{{ platform_etc_dir }}/authorization_rules.conf" + path: "{{ platform_etc_dir }}/acl.conf" } ] } diff --git a/apps/emqx_authz/src/emqx_authz.erl b/apps/emqx_authz/src/emqx_authz.erl index 3a39a2984..e0e584806 100644 --- a/apps/emqx_authz/src/emqx_authz.erl +++ b/apps/emqx_authz/src/emqx_authz.erl @@ -30,13 +30,16 @@ , lookup/0 , lookup/1 , move/2 + , move/3 , update/2 + , update/3 , authorize/5 ]). -export([post_config_update/4, pre_config_update/2]). -define(CONF_KEY_PATH, [authorization, sources]). +-define(SOURCE_TYPES, [file, http, mongo, mysql, pgsql, redis]). -spec(register_metrics() -> ok). register_metrics() -> @@ -45,88 +48,123 @@ register_metrics() -> init() -> ok = register_metrics(), emqx_config_handler:add_handler(?CONF_KEY_PATH, ?MODULE), - NSources = [init_source(Source) || Source <- emqx:get_config(?CONF_KEY_PATH, [])], + Sources = emqx:get_config(?CONF_KEY_PATH, []), + ok = check_dup_types(Sources), + NSources = [init_source(Source) || Source <- Sources], ok = emqx_hooks:add('client.authorize', {?MODULE, authorize, [NSources]}, -1). lookup() -> {_M, _F, [A]}= find_action_in_hooks(), A. -lookup(Id) -> - try find_source_by_id(Id, lookup()) of +lookup(Type) -> + try find_source_by_type(atom(Type), lookup()) of {_, Source} -> Source catch error:Reason -> {error, Reason} end. -move(Id, Position) -> - emqx:update_config(?CONF_KEY_PATH, {move, Id, Position}). +move(Type, Cmd) -> + move(Type, Cmd, #{}). + +move(Type, #{<<"before">> := Before}, Opts) -> + emqx:update_config(?CONF_KEY_PATH, {move, atom(Type), #{<<"before">> => atom(Before)}}, Opts); +move(Type, #{<<"after">> := After}, Opts) -> + emqx:update_config(?CONF_KEY_PATH, {move, atom(Type), #{<<"after">> => atom(After)}}, Opts); +move(Type, Position, Opts) -> + emqx:update_config(?CONF_KEY_PATH, {move, atom(Type), Position}, Opts). update(Cmd, Sources) -> - emqx:update_config(?CONF_KEY_PATH, {Cmd, Sources}). + update(Cmd, Sources, #{}). -pre_config_update({move, Id, <<"top">>}, Conf) when is_list(Conf) -> - {Index, _} = find_source_by_id(Id), +update({replace_once, Type}, Sources, Opts) -> + emqx:update_config(?CONF_KEY_PATH, {{replace_once, atom(Type)}, Sources}, Opts); +update({delete_once, Type}, Sources, Opts) -> + emqx:update_config(?CONF_KEY_PATH, {{delete_once, atom(Type)}, Sources}, Opts); +update(Cmd, Sources, Opts) -> + emqx:update_config(?CONF_KEY_PATH, {Cmd, Sources}, Opts). + +pre_config_update({move, Type, <<"top">>}, Conf) when is_list(Conf) -> + {Index, _} = find_source_by_type(Type), {List1, List2} = lists:split(Index, Conf), - {ok, [lists:nth(Index, Conf)] ++ lists:droplast(List1) ++ List2}; + NConf = [lists:nth(Index, Conf)] ++ lists:droplast(List1) ++ List2, + ok = check_dup_types(NConf), + {ok, NConf}; -pre_config_update({move, Id, <<"bottom">>}, Conf) when is_list(Conf) -> - {Index, _} = find_source_by_id(Id), +pre_config_update({move, Type, <<"bottom">>}, Conf) when is_list(Conf) -> + {Index, _} = find_source_by_type(Type), {List1, List2} = lists:split(Index, Conf), - {ok, lists:droplast(List1) ++ List2 ++ [lists:nth(Index, Conf)]}; + NConf = lists:droplast(List1) ++ List2 ++ [lists:nth(Index, Conf)], + ok = check_dup_types(NConf), + {ok, NConf}; -pre_config_update({move, Id, #{<<"before">> := BeforeId}}, Conf) when is_list(Conf) -> - {Index1, _} = find_source_by_id(Id), +pre_config_update({move, Type, #{<<"before">> := Before}}, Conf) when is_list(Conf) -> + {Index1, _} = find_source_by_type(Type), Conf1 = lists:nth(Index1, Conf), - {Index2, _} = find_source_by_id(BeforeId), + {Index2, _} = find_source_by_type(Before), Conf2 = lists:nth(Index2, Conf), {List1, List2} = lists:split(Index2, Conf), - {ok, lists:delete(Conf1, lists:droplast(List1)) - ++ [Conf1] ++ [Conf2] - ++ lists:delete(Conf1, List2)}; + NConf = lists:delete(Conf1, lists:droplast(List1)) + ++ [Conf1] ++ [Conf2] + ++ lists:delete(Conf1, List2), + ok = check_dup_types(NConf), + {ok, NConf}; -pre_config_update({move, Id, #{<<"after">> := AfterId}}, Conf) when is_list(Conf) -> - {Index1, _} = find_source_by_id(Id), +pre_config_update({move, Type, #{<<"after">> := After}}, Conf) when is_list(Conf) -> + {Index1, _} = find_source_by_type(Type), Conf1 = lists:nth(Index1, Conf), - {Index2, _} = find_source_by_id(AfterId), + {Index2, _} = find_source_by_type(After), {List1, List2} = lists:split(Index2, Conf), - {ok, lists:delete(Conf1, List1) - ++ [Conf1] - ++ lists:delete(Conf1, List2)}; + NConf = lists:delete(Conf1, List1) + ++ [Conf1] + ++ lists:delete(Conf1, List2), + ok = check_dup_types(NConf), + {ok, NConf}; pre_config_update({head, Sources}, Conf) when is_list(Sources), is_list(Conf) -> + NConf = Sources ++ Conf, + ok = check_dup_types(NConf), {ok, Sources ++ Conf}; pre_config_update({tail, Sources}, Conf) when is_list(Sources), is_list(Conf) -> + NConf = Conf ++ Sources, + ok = check_dup_types(NConf), {ok, Conf ++ Sources}; -pre_config_update({{replace_once, Id}, Source}, Conf) when is_map(Source), is_list(Conf) -> - {Index, _} = find_source_by_id(Id), +pre_config_update({{replace_once, Type}, Source}, Conf) when is_map(Source), is_list(Conf) -> + {Index, _} = find_source_by_type(Type), {List1, List2} = lists:split(Index, Conf), - {ok, lists:droplast(List1) ++ [Source] ++ List2}; + NConf = lists:droplast(List1) ++ [Source] ++ List2, + ok = check_dup_types(NConf), + {ok, NConf}; +pre_config_update({{delete_once, Type}, _Source}, Conf) when is_list(Conf) -> + {_, Source} = find_source_by_type(Type), + NConf = lists:delete(Source, Conf), + ok = check_dup_types(NConf), + {ok, NConf}; pre_config_update({_, Sources}, _Conf) when is_list(Sources)-> %% overwrite the entire config! {ok, Sources}. post_config_update(_, undefined, _Conf, _AppEnvs) -> ok; -post_config_update({move, Id, <<"top">>}, _NewSources, _OldSources, _AppEnvs) -> +post_config_update({move, Type, <<"top">>}, _NewSources, _OldSources, _AppEnvs) -> InitedSources = lookup(), - {Index, Source} = find_source_by_id(Id, InitedSources), + {Index, Source} = find_source_by_type(Type, InitedSources), {Sources1, Sources2 } = lists:split(Index, InitedSources), Sources3 = [Source] ++ lists:droplast(Sources1) ++ Sources2, ok = emqx_hooks:put('client.authorize', {?MODULE, authorize, [Sources3]}, -1), ok = emqx_authz_cache:drain_cache(); -post_config_update({move, Id, <<"bottom">>}, _NewSources, _OldSources, _AppEnvs) -> +post_config_update({move, Type, <<"bottom">>}, _NewSources, _OldSources, _AppEnvs) -> InitedSources = lookup(), - {Index, Source} = find_source_by_id(Id, InitedSources), + {Index, Source} = find_source_by_type(Type, InitedSources), {Sources1, Sources2 } = lists:split(Index, InitedSources), Sources3 = lists:droplast(Sources1) ++ Sources2 ++ [Source], ok = emqx_hooks:put('client.authorize', {?MODULE, authorize, [Sources3]}, -1), ok = emqx_authz_cache:drain_cache(); -post_config_update({move, Id, #{<<"before">> := BeforeId}}, _NewSources, _OldSources, _AppEnvs) -> +post_config_update({move, Type, #{<<"before">> := Before}}, _NewSources, _OldSources, _AppEnvs) -> InitedSources = lookup(), - {_, Source0} = find_source_by_id(Id, InitedSources), - {Index, Source1} = find_source_by_id(BeforeId, InitedSources), + {_, Source0} = find_source_by_type(Type, InitedSources), + {Index, Source1} = find_source_by_type(Before, InitedSources), {Sources1, Sources2} = lists:split(Index, InitedSources), Sources3 = lists:delete(Source0, lists:droplast(Sources1)) ++ [Source0] ++ [Source1] @@ -134,10 +172,10 @@ post_config_update({move, Id, #{<<"before">> := BeforeId}}, _NewSources, _OldSou ok = emqx_hooks:put('client.authorize', {?MODULE, authorize, [Sources3]}, -1), ok = emqx_authz_cache:drain_cache(); -post_config_update({move, Id, #{<<"after">> := AfterId}}, _NewSources, _OldSources, _AppEnvs) -> +post_config_update({move, Type, #{<<"after">> := After}}, _NewSources, _OldSources, _AppEnvs) -> InitedSources = lookup(), - {_, Source} = find_source_by_id(Id, InitedSources), - {Index, _} = find_source_by_id(AfterId, InitedSources), + {_, Source} = find_source_by_type(Type, InitedSources), + {Index, _} = find_source_by_type(After, InitedSources), {Sources1, Sources2} = lists:split(Index, InitedSources), Sources3 = lists:delete(Source, Sources1) ++ [Source] @@ -155,20 +193,30 @@ post_config_update({tail, Sources}, _NewSources, _OldConf, _AppEnvs) -> emqx_hooks:put('client.authorize', {?MODULE, authorize, [lookup() ++ InitedSources]}, -1), ok = emqx_authz_cache:drain_cache(); -post_config_update({{replace_once, Id}, Source}, _NewSources, _OldConf, _AppEnvs) when is_map(Source) -> +post_config_update({{replace_once, Type}, #{type := Type} = Source}, _NewSources, _OldConf, _AppEnvs) when is_map(Source) -> OldInitedSources = lookup(), - {Index, OldSource} = find_source_by_id(Id, OldInitedSources), + {Index, OldSource} = find_source_by_type(Type, OldInitedSources), case maps:get(type, OldSource, undefined) of undefined -> ok; + file -> ok; _ -> #{annotations := #{id := Id}} = OldSource, ok = emqx_resource:remove(Id) end, {OldSources1, OldSources2 } = lists:split(Index, OldInitedSources), - InitedSources = [init_source(R#{annotations => #{id => Id}}) || R <- check_sources([Source])], + InitedSources = [init_source(R) || R <- check_sources([Source])], ok = emqx_hooks:put('client.authorize', {?MODULE, authorize, [lists:droplast(OldSources1) ++ InitedSources ++ OldSources2]}, -1), ok = emqx_authz_cache:drain_cache(); - +post_config_update({{delete_once, Type}, _Source}, _NewSources, _OldConf, _AppEnvs) -> + OldInitedSources = lookup(), + {_, OldSource} = find_source_by_type(Type, OldInitedSources), + case OldSource of + #{annotations := #{id := Id}} -> + ok = emqx_resource:remove(Id); + _ -> ok + end, + ok = emqx_hooks:put('client.authorize', {?MODULE, authorize, [lists:delete(OldSource, OldInitedSources)]}, -1), + ok = emqx_authz_cache:drain_cache(); post_config_update(_, NewSources, _OldConf, _AppEnvs) -> %% overwrite the entire config! OldInitedSources = lookup(), @@ -181,52 +229,34 @@ post_config_update(_, NewSources, _OldConf, _AppEnvs) -> ok = emqx_authz_cache:drain_cache(). %%-------------------------------------------------------------------- -%% Internal functions +%% Initialize source %%-------------------------------------------------------------------- -check_sources(RawSources) -> - {ok, Conf} = hocon:binary(jsx:encode(#{<<"authorization">> => #{<<"sources">> => RawSources}}), #{format => richmap}), - CheckConf = hocon_schema:check(emqx_authz_schema, Conf, #{atom_key => true}), - #{authorization:= #{sources := Sources}} = hocon_schema:richmap_to_map(CheckConf), - Sources. - -find_source_by_id(Id) -> find_source_by_id(Id, lookup()). -find_source_by_id(Id, Sources) -> find_source_by_id(Id, Sources, 1). -find_source_by_id(_SourceId, [], _N) -> error(not_found_rule); -find_source_by_id(SourceId, [ Source = #{annotations := #{id := Id}} | Tail], N) -> - case SourceId =:= Id of - true -> {N, Source}; - false -> find_source_by_id(SourceId, Tail, N + 1) - end. - -find_action_in_hooks() -> - Callbacks = emqx_hooks:lookup('client.authorize'), - [Action] = [Action || {callback,{?MODULE, authorize, _} = Action, _, _} <- Callbacks ], - Action. - -gen_id(Type) -> - iolist_to_binary([io_lib:format("~s_~s",[?APP, Type]), "_", integer_to_list(erlang:system_time())]). - -create_resource(#{type := DB, - config := Config, - annotations := #{id := ResourceID}}) -> - case emqx_resource:update(ResourceID, connector_module(DB), Config, []) of - {ok, _} -> ResourceID; - {error, Reason} -> {error, Reason} - end; -create_resource(#{type := DB, - config := Config}) -> - ResourceID = gen_id(DB), - case emqx_resource:create(ResourceID, connector_module(DB), Config) of - {ok, already_created} -> ResourceID; - {ok, _} -> ResourceID; - {error, Reason} -> {error, Reason} +check_dup_types(Sources) -> + check_dup_types(Sources, ?SOURCE_TYPES). +check_dup_types(_Sources, []) -> ok; +check_dup_types(Sources, [T0 | Tail]) -> + case lists:foldl(fun (#{type := T1}, AccIn) -> + case T0 =:= T1 of + true -> AccIn + 1; + false -> AccIn + end; + (#{<<"type">> := T1}, AccIn) -> + case T0 =:= atom(T1) of + true -> AccIn + 1; + false -> AccIn + end + end, 0, Sources) > 1 of + true -> + ?LOG(error, "The type is duplicated in the Authorization source"), + {error, authz_source_dup}; + false -> check_dup_types(Sources, Tail) end. init_source(#{enable := true, - type := file, - path := Path - } = Source) -> + type := file, + path := Path + } = Source) -> Rules = case file:consult(Path) of {ok, Terms} -> [emqx_authz_rule:compile(Term) || Term <- Terms]; @@ -240,35 +270,28 @@ init_source(#{enable := true, ?LOG(alert, "Failed to read ~s: ~p", [Path, Reason]), error(Reason) end, - Source#{annotations => - #{id => gen_id(file), - rules => Rules - }}; + Source#{annotations => #{rules => Rules}}; init_source(#{enable := true, - type := http, - config := #{url := Url} = Config - } = Source) -> - NConfig = maps:merge(Config, #{base_url => maps:remove(query, Url)}), - case create_resource(Source#{config := NConfig}) of + type := http, + url := Url + } = Source) -> + NSource= maps:put(base_url, maps:remove(query, Url), Source), + case create_resource(NSource) of {error, Reason} -> error({load_config_error, Reason}); - Id -> Source#{annotations => - #{id => Id} - } + Id -> Source#{annotations => #{id => Id}} end; init_source(#{enable := true, - type := DB - } = Source) when DB =:= redis; + type := DB + } = Source) when DB =:= redis; DB =:= mongo -> case create_resource(Source) of {error, Reason} -> error({load_config_error, Reason}); - Id -> Source#{annotations => - #{id => Id} - } + Id -> Source#{annotations => #{id => Id}} end; init_source(#{enable := true, - type := DB, - sql := SQL - } = Source) when DB =:= mysql; + type := DB, + sql := SQL + } = Source) when DB =:= mysql; DB =:= pgsql -> Mod = authz_module(DB), case create_resource(Source) of @@ -323,8 +346,56 @@ do_authorize(Client, PubSub, Topic, Matched -> Matched end. +%%-------------------------------------------------------------------- +%% Internal function +%%-------------------------------------------------------------------- + +check_sources(RawSources) -> + Schema = #{roots => emqx_authz_schema:fields("authorization"), fields => #{}}, + Conf = #{<<"sources">> => RawSources}, + #{sources := Sources} = hocon_schema:check_plain(Schema, Conf, #{atom_key => true}), + Sources. + +find_source_by_type(Type) -> find_source_by_type(Type, lookup()). +find_source_by_type(Type, Sources) -> find_source_by_type(Type, Sources, 1). +find_source_by_type(_, [], _N) -> error(not_found_source); +find_source_by_type(Type, [ Source = #{type := T} | Tail], N) -> + case Type =:= T of + true -> {N, Source}; + false -> find_source_by_type(Type, Tail, N + 1) + end. + +find_action_in_hooks() -> + Callbacks = emqx_hooks:lookup('client.authorize'), + [Action] = [Action || {callback,{?MODULE, authorize, _} = Action, _, _} <- Callbacks ], + Action. + +gen_id(Type) -> + iolist_to_binary([io_lib:format("~s_~s",[?APP, Type])]). + +create_resource(#{type := DB, + annotations := #{id := ResourceID}} = Source) -> + case emqx_resource:update(ResourceID, connector_module(DB), Source, []) of + {ok, _} -> ResourceID; + {error, Reason} -> {error, Reason} + end; +create_resource(#{type := DB} = Source) -> + ResourceID = gen_id(DB), + case emqx_resource:create(ResourceID, connector_module(DB), Source) of + {ok, already_created} -> ResourceID; + {ok, _} -> ResourceID; + {error, Reason} -> {error, Reason} + end. + authz_module(Type) -> list_to_existing_atom("emqx_authz_" ++ atom_to_list(Type)). connector_module(Type) -> list_to_existing_atom("emqx_connector_" ++ atom_to_list(Type)). + +atom(B) when is_binary(B) -> + try binary_to_existing_atom(B, utf8) + catch + _ -> binary_to_atom(B) + end; +atom(A) when is_atom(A) -> A. diff --git a/apps/emqx_authz/src/emqx_authz_api.erl b/apps/emqx_authz/src/emqx_authz_api.erl deleted file mode 100644 index 1646a9af2..000000000 --- a/apps/emqx_authz/src/emqx_authz_api.erl +++ /dev/null @@ -1,528 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_authz_api). - --behavior(minirest_api). - --include("emqx_authz.hrl"). - --define(EXAMPLE_RETURNED_RULE1, - #{principal => <<"all">>, - permission => <<"allow">>, - action => <<"all">>, - topics => [<<"#">>], - annotations => #{id => 1} - }). - - --define(EXAMPLE_RETURNED_RULES, - #{rules => [?EXAMPLE_RETURNED_RULE1 - ] - }). - --define(EXAMPLE_RULE1, #{principal => <<"all">>, - permission => <<"allow">>, - action => <<"all">>, - topics => [<<"#">>]}). - --export([ api_spec/0 - , rules/2 - , rule/2 - , move_rule/2 - ]). - -api_spec() -> - {[ rules_api() - , rule_api() - , move_rule_api() - ], definitions()}. - -definitions() -> emqx_authz_api_schema:definitions(). - -rules_api() -> - Metadata = #{ - get => #{ - description => "List authorization rules", - parameters => [ - #{ - name => page, - in => query, - schema => #{ - type => integer - }, - required => false - }, - #{ - name => limit, - in => query, - schema => #{ - type => integer - }, - required => false - } - ], - responses => #{ - <<"200">> => #{ - description => <<"OK">>, - content => #{ - 'application/json' => #{ - schema => #{ - type => object, - required => [rules], - properties => #{rules => #{ - type => array, - items => minirest:ref(<<"returned_rules">>) - } - } - }, - examples => #{ - rules => #{ - summary => <<"Rules">>, - value => jsx:encode(?EXAMPLE_RETURNED_RULES) - } - } - } - } - } - } - }, - post => #{ - description => "Add new rule", - requestBody => #{ - content => #{ - 'application/json' => #{ - schema => minirest:ref(<<"rules">>), - examples => #{ - simple_rule => #{ - summary => <<"Rules">>, - value => jsx:encode(?EXAMPLE_RULE1) - } - } - } - } - }, - responses => #{ - <<"204">> => #{description => <<"Created">>}, - <<"400">> => #{ - description => <<"Bad Request">>, - content => #{ - 'application/json' => #{ - schema => minirest:ref(<<"error">>), - examples => #{ - example1 => #{ - summary => <<"Bad Request">>, - value => #{ - code => <<"BAD_REQUEST">>, - message => <<"Bad Request">> - } - } - } - } - } - } - } - }, - put => #{ - - description => "Update all rules", - requestBody => #{ - content => #{ - 'application/json' => #{ - schema => #{ - type => array, - items => minirest:ref(<<"returned_rules">>) - }, - examples => #{ - rules => #{ - summary => <<"Rules">>, - value => jsx:encode([?EXAMPLE_RULE1]) - } - } - } - } - }, - responses => #{ - <<"204">> => #{description => <<"Created">>}, - <<"400">> => #{ - description => <<"Bad Request">>, - content => #{ - 'application/json' => #{ - schema => minirest:ref(<<"error">>), - examples => #{ - example1 => #{ - summary => <<"Bad Request">>, - value => #{ - code => <<"BAD_REQUEST">>, - message => <<"Bad Request">> - } - } - } - } - } - } - } - } - }, - {"/authorization", Metadata, rules}. - -rule_api() -> - Metadata = #{ - get => #{ - description => "List authorization rules", - parameters => [ - #{ - name => id, - in => path, - schema => #{ - type => string - }, - required => true - } - ], - responses => #{ - <<"200">> => #{ - description => <<"OK">>, - content => #{ - 'application/json' => #{ - schema => minirest:ref(<<"returned_rules">>), - examples => #{ - rules => #{ - summary => <<"Rules">>, - value => jsx:encode(?EXAMPLE_RETURNED_RULE1) - } - } - } - } - }, - <<"404">> => #{ - description => <<"Bad Request">>, - content => #{ - 'application/json' => #{ - schema => minirest:ref(<<"error">>), - examples => #{ - example1 => #{ - summary => <<"Not Found">>, - value => #{ - code => <<"NOT_FOUND">>, - message => <<"rule xxx not found">> - } - } - } - } - } - } - } - }, - put => #{ - description => "Update rule", - parameters => [ - #{ - name => id, - in => path, - schema => #{ - type => string - }, - required => true - } - ], - requestBody => #{ - content => #{ - 'application/json' => #{ - schema => minirest:ref(<<"rules">>), - examples => #{ - simple_rule => #{ - summary => <<"Rules">>, - value => jsx:encode(?EXAMPLE_RULE1) - } - } - } - } - }, - responses => #{ - <<"204">> => #{description => <<"No Content">>}, - <<"404">> => #{ - description => <<"Bad Request">>, - content => #{ - 'application/json' => #{ - schema => minirest:ref(<<"error">>), - examples => #{ - example1 => #{ - summary => <<"Not Found">>, - value => #{ - code => <<"NOT_FOUND">>, - message => <<"rule xxx not found">> - } - } - } - } - } - }, - <<"400">> => #{ - description => <<"Bad Request">>, - content => #{ - 'application/json' => #{ - schema => minirest:ref(<<"error">>), - examples => #{ - example1 => #{ - summary => <<"Bad Request">>, - value => #{ - code => <<"BAD_REQUEST">>, - message => <<"Bad Request">> - } - } - } - } - } - } - } - }, - delete => #{ - description => "Delete rule", - parameters => [ - #{ - name => id, - in => path, - schema => #{ - type => string - }, - required => true - } - ], - responses => #{ - <<"204">> => #{description => <<"No Content">>}, - <<"400">> => #{ - description => <<"Bad Request">>, - content => #{ - 'application/json' => #{ - schema => minirest:ref(<<"error">>), - examples => #{ - example1 => #{ - summary => <<"Bad Request">>, - value => #{ - code => <<"BAD_REQUEST">>, - message => <<"Bad Request">> - } - } - } - } - } - } - } - } - }, - {"/authorization/:id", Metadata, rule}. - -move_rule_api() -> - Metadata = #{ - post => #{ - description => "Change the order of rules", - parameters => [ - #{ - name => id, - in => path, - schema => #{ - type => string - }, - required => true - } - ], - requestBody => #{ - content => #{ - 'application/json' => #{ - schema => #{ - type => object, - required => [position], - properties => #{ - position => #{ - oneOf => [ - #{type => string, - enum => [<<"top">>, <<"bottom">>] - }, - #{type => object, - required => ['after'], - properties => #{ - 'after' => #{ - type => string - } - } - }, - #{type => object, - required => ['before'], - properties => #{ - 'before' => #{ - type => string - } - } - } - ] - } - } - } - } - } - }, - responses => #{ - <<"204">> => #{ - description => <<"No Content">> - }, - <<"404">> => #{ - description => <<"Bad Request">>, - content => #{ - 'application/json' => #{ - schema => minirest:ref(<<"error">>), - examples => #{ - example1 => #{ - summary => <<"Not Found">>, - value => #{ - code => <<"NOT_FOUND">>, - message => <<"rule xxx not found">> - } - } - } - } - } - }, - <<"400">> => #{ - description => <<"Bad Request">>, - content => #{ - 'application/json' => #{ - schema => minirest:ref(<<"error">>), - examples => #{ - example1 => #{ - summary => <<"Bad Request">>, - value => #{ - code => <<"BAD_REQUEST">>, - message => <<"Bad Request">> - } - } - } - } - } - } - } - } - }, - {"/authorization/:id/move", Metadata, move_rule}. - -rules(get, #{query_string := Query}) -> - Rules = lists:foldl(fun (#{type := _Type, enable := true, config := #{server := Server} = Config, annotations := #{id := Id}} = Rule, AccIn) -> - NRule = case emqx_resource:health_check(Id) of - ok -> - Rule#{config => Config#{server => emqx_connector_schema_lib:ip_port_to_string(Server)}, - annotations => #{id => Id, - status => healthy}}; - _ -> - Rule#{config => Config#{server => emqx_connector_schema_lib:ip_port_to_string(Server)}, - annotations => #{id => Id, - status => unhealthy}} - end, - lists:append(AccIn, [NRule]); - (#{type := _Type, enable := true, annotations := #{id := Id}} = Rule, AccIn) -> - NRule = case emqx_resource:health_check(Id) of - ok -> - Rule#{annotations => #{id => Id, - status => healthy}}; - _ -> - Rule#{annotations => #{id => Id, - status => unhealthy}} - end, - lists:append(AccIn, [NRule]); - (Rule, AccIn) -> - lists:append(AccIn, [Rule]) - end, [], emqx_authz:lookup()), - case maps:is_key(<<"page">>, Query) andalso maps:is_key(<<"limit">>, Query) of - true -> - Page = maps:get(<<"page">>, Query), - Limit = maps:get(<<"limit">>, Query), - Index = (binary_to_integer(Page) - 1) * binary_to_integer(Limit), - {_, Rules1} = lists:split(Index, Rules), - case binary_to_integer(Limit) < length(Rules1) of - true -> - {Rules2, _} = lists:split(binary_to_integer(Limit), Rules1), - {200, #{rules => Rules2}}; - false -> {200, #{rules => Rules1}} - end; - false -> {200, #{rules => Rules}} - end; -rules(post, #{body := RawConfig}) -> - case emqx_authz:update(head, [RawConfig]) of - {ok, _} -> {204}; - {error, Reason} -> - {400, #{code => <<"BAD_REQUEST">>, - messgae => atom_to_binary(Reason)}} - end; -rules(put, #{body := RawConfig}) -> - case emqx_authz:update(replace, RawConfig) of - {ok, _} -> {204}; - {error, Reason} -> - {400, #{code => <<"BAD_REQUEST">>, - messgae => atom_to_binary(Reason)}} - end. - -rule(get, #{bindings := #{id := Id}}) -> - case emqx_authz:lookup(Id) of - {error, Reason} -> {404, #{messgae => atom_to_binary(Reason)}}; - #{type := file} = Rule -> {200, Rule}; - #{config := #{server := Server} = Config} = Rule -> - case emqx_resource:health_check(Id) of - ok -> - {200, Rule#{config => Config#{server => emqx_connector_schema_lib:ip_port_to_string(Server)}, - annotations => #{id => Id, - status => healthy}}}; - _ -> - {200, Rule#{config => Config#{server => emqx_connector_schema_lib:ip_port_to_string(Server)}, - annotations => #{id => Id, - status => unhealthy}}} - end; - Rule -> - case emqx_resource:health_check(Id) of - ok -> - {200, Rule#{annotations => #{id => Id, - status => healthy}}}; - _ -> - {200, Rule#{annotations => #{id => Id, - status => unhealthy}}} - end - end; -rule(put, #{bindings := #{id := RuleId}, body := RawConfig}) -> - case emqx_authz:update({replace_once, RuleId}, RawConfig) of - {ok, _} -> {204}; - {error, not_found_rule} -> - {404, #{code => <<"NOT_FOUND">>, - messgae => <<"rule ", RuleId/binary, " not found">>}}; - {error, Reason} -> - {400, #{code => <<"BAD_REQUEST">>, - messgae => atom_to_binary(Reason)}} - end; -rule(delete, #{bindings := #{id := RuleId}}) -> - case emqx_authz:update({replace_once, RuleId}, #{}) of - {ok, _} -> {204}; - {error, Reason} -> - {400, #{code => <<"BAD_REQUEST">>, - messgae => atom_to_binary(Reason)}} - end. -move_rule(post, #{bindings := #{id := RuleId}, body := Body}) -> - #{<<"position">> := Position} = Body, - case emqx_authz:move(RuleId, Position) of - {ok, _} -> {204}; - {error, not_found_rule} -> - {404, #{code => <<"NOT_FOUND">>, - messgae => <<"rule ", RuleId/binary, " not found">>}}; - {error, Reason} -> - {400, #{code => <<"BAD_REQUEST">>, - messgae => atom_to_binary(Reason)}} - end. diff --git a/apps/emqx_authz/src/emqx_authz_api_schema.erl b/apps/emqx_authz/src/emqx_authz_api_schema.erl index 64ecc58eb..09f145075 100644 --- a/apps/emqx_authz/src/emqx_authz_api_schema.erl +++ b/apps/emqx_authz/src/emqx_authz_api_schema.erl @@ -19,137 +19,491 @@ -export([definitions/0]). definitions() -> - RetruenedRules = #{ + RetruenedSources = #{ allOf => [ #{type => object, properties => #{ annotations => #{ type => object, - required => [id], + required => [status], properties => #{ id => #{ type => string }, - principal => minirest:ref(<<"principal">>) + status => #{ + type => string, + example => <<"healthy">> + } } } } } - , minirest:ref(<<"rules">>) + , minirest:ref(<<"sources">>) ] }, - Rules = #{ - oneOf => [ minirest:ref(<<"simple_rule">>) - % , minirest:ref(<<"connector_redis">>) + Sources = #{ + oneOf => [ minirest:ref(<<"http">>) + , minirest:ref(<<"mongo_single">>) + , minirest:ref(<<"mongo_rs">>) + , minirest:ref(<<"mongo_sharded">>) + , minirest:ref(<<"mysql">>) + , minirest:ref(<<"pgsql">>) + , minirest:ref(<<"redis_single">>) + , minirest:ref(<<"redis_sentinel">>) + , minirest:ref(<<"redis_cluster">>) + , minirest:ref(<<"file">>) ] }, - % ConnectorRedis = #{ - % type => object, - % required => [principal, type, enable, config, cmd] - % properties => #{ - % principal => minirest:ref(<<"principal">>), - % type => #{ - % type => string, - % enum => [<<"redis">>], - % example => <<"redis">> - % }, - % enable => #{ - % type => boolean, - % example => true - % } - % config => #{ - % type => - % } - % } - % } - SimpleRule = #{ + SSL = #{ + type => object, + required => [enable], + properties => #{ + enable => #{type => boolean, example => true}, + cacertfile => #{type => string}, + keyfile => #{type => string}, + certfile => #{type => string}, + verify => #{type => boolean, example => false} + } + }, + HTTP = #{ type => object, - required => [principal, permission, action, topics], + required => [ type + , enable + , method + , headers + , request_timeout + , connect_timeout + , max_retries + , retry_interval + , pool_type + , pool_size + , enable_pipelining + , ssl + ], properties => #{ - action => #{ + type => #{ type => string, - enum => [<<"publish">>, <<"subscribe">>, <<"all">>], - example => <<"publish">> + enum => [<<"http">>], + example => <<"http">> }, - permission => #{ + enable => #{ + type => boolean, + example => true + }, + url => #{ type => string, - enum => [<<"allow">>, <<"deny">>], - example => <<"allow">> + example => <<"https://emqx.com">> }, - topics => #{ - type => array, - items => #{ - oneOf => [ #{type => string, example => <<"#">>} - , #{type => object, - required => [eq], - properties => #{ - eq => #{type => string} - }, - example => #{eq => <<"#">>} - } - ] - } + method => #{ + type => string, + enum => [<<"get">>, <<"post">>, <<"put">>], + example => <<"get">> }, - principal => minirest:ref(<<"principal">>) + headers => #{type => object}, + body => #{type => object}, + connect_timeout => #{type => integer}, + max_retries => #{type => integer}, + retry_interval => #{type => integer}, + pool_type => #{ + type => string, + enum => [<<"random">>, <<"hash">>], + example => <<"random">> + }, + pool_size => #{type => integer}, + enable_pipelining => #{type => boolean}, + ssl => minirest:ref(<<"ssl">>) } }, - Principal = #{ - oneOf => [ minirest:ref(<<"principal_username">>) - , minirest:ref(<<"principal_clientid">>) - , minirest:ref(<<"principal_ipaddress">>) - , #{type => string, enum=>[<<"all">>], example => <<"all">>} - , #{type => object, - required => ['and'], - properties => #{'and' => #{type => array, - items => #{oneOf => [ minirest:ref(<<"principal_username">>) - , minirest:ref(<<"principal_clientid">>) - , minirest:ref(<<"principal_ipaddress">>) - ]}}}, - example => #{'and' => [#{username => <<"emqx">>}, #{clientid => <<"emqx">>}]} - } - , #{type => object, - required => ['or'], - properties => #{'and' => #{type => array, - items => #{oneOf => [ minirest:ref(<<"principal_username">>) - , minirest:ref(<<"principal_clientid">>) - , minirest:ref(<<"principal_ipaddress">>) - ]}}}, - example => #{'or' => [#{username => <<"emqx">>}, #{clientid => <<"emqx">>}]} - } - ] - }, - PrincipalUsername = #{type => object, - required => [username], - properties => #{username => #{type => string}}, - example => #{username => <<"emqx">>} - }, - PrincipalClientid = #{type => object, - required => [clientid], - properties => #{clientid => #{type => string}}, - example => #{clientid => <<"emqx">>} - }, - PrincipalIpaddress = #{type => object, - required => [ipaddress], - properties => #{ipaddress => #{type => string}}, - example => #{ipaddress => <<"127.0.0.1">>} - }, - ErrorDef = #{ + MongoSingle= #{ type => object, + required => [ type + , enable + , collection + , find + , mongo_type + , server + , pool_size + , username + , password + , auth_source + , database + , topology + , ssl + ], properties => #{ - code => #{ + type => #{ type => string, - example => <<"BAD_REQUEST">> + enum => [<<"mongo">>], + example => <<"mongo">> }, - message => #{ - type => string + enable => #{ + type => boolean, + example => true + }, + collection => #{type => string}, + find => #{type => object}, + mongo_type => #{type => string, + enum => [<<"single">>], + example => <<"single">>}, + server => #{type => string, example => <<"127.0.0.1:27017">>}, + pool_size => #{type => integer}, + username => #{type => string}, + password => #{type => string}, + auth_source => #{type => string}, + database => #{type => string}, + topology => #{type => object, + properties => #{ + pool_size => #{type => integer}, + max_overflow => #{type => integer}, + overflow_ttl => #{type => integer}, + overflow_check_period => #{type => integer}, + local_threshold_ms => #{type => integer}, + connect_timeout_ms => #{type => integer}, + socket_timeout_ms => #{type => integer}, + server_selection_timeout_ms => #{type => integer}, + wait_queue_timeout_ms => #{type => integer}, + heartbeat_frequency_ms => #{type => integer}, + min_heartbeat_frequency_ms => #{type => integer} + } + }, + ssl => minirest:ref(<<"ssl">>) + } + }, + MongoRs= #{ + type => object, + required => [ type + , enable + , collection + , find + , mongo_type + , servers + , replica_set_name + , pool_size + , username + , password + , auth_source + , database + , topology + , ssl + ], + properties => #{ + type => #{ + type => string, + enum => [<<"mongo">>], + example => <<"mongo">> + }, + enable => #{ + type => boolean, + example => true + }, + collection => #{type => string}, + find => #{type => object}, + mongo_type => #{type => string, + enum => [<<"rs">>], + example => <<"rs">>}, + servers => #{type => array, + items => #{type => string,example => <<"127.0.0.1:27017">>}}, + replica_set_name => #{type => string}, + pool_size => #{type => integer}, + username => #{type => string}, + password => #{type => string}, + auth_source => #{type => string}, + database => #{type => string}, + topology => #{type => object, + properties => #{ + pool_size => #{type => integer}, + max_overflow => #{type => integer}, + overflow_ttl => #{type => integer}, + overflow_check_period => #{type => integer}, + local_threshold_ms => #{type => integer}, + connect_timeout_ms => #{type => integer}, + socket_timeout_ms => #{type => integer}, + server_selection_timeout_ms => #{type => integer}, + wait_queue_timeout_ms => #{type => integer}, + heartbeat_frequency_ms => #{type => integer}, + min_heartbeat_frequency_ms => #{type => integer} + } + }, + ssl => minirest:ref(<<"ssl">>) + } + }, + MongoSharded = #{ + type => object, + required => [ type + , enable + , collection + , find + , mongo_type + , servers + , pool_size + , username + , password + , auth_source + , database + , topology + , ssl + ], + properties => #{ + type => #{ + type => string, + enum => [<<"mongo">>], + example => <<"mongo">> + }, + enable => #{ + type => boolean, + example => true + }, + collection => #{type => string}, + find => #{type => object}, + mongo_type => #{type => string, + enum => [<<"sharded">>], + example => <<"sharded">>}, + servers => #{type => array, + items => #{type => string,example => <<"127.0.0.1:27017">>}}, + pool_size => #{type => integer}, + username => #{type => string}, + password => #{type => string}, + auth_source => #{type => string}, + database => #{type => string}, + topology => #{type => object, + properties => #{ + pool_size => #{type => integer}, + max_overflow => #{type => integer}, + overflow_ttl => #{type => integer}, + overflow_check_period => #{type => integer}, + local_threshold_ms => #{type => integer}, + connect_timeout_ms => #{type => integer}, + socket_timeout_ms => #{type => integer}, + server_selection_timeout_ms => #{type => integer}, + wait_queue_timeout_ms => #{type => integer}, + heartbeat_frequency_ms => #{type => integer}, + min_heartbeat_frequency_ms => #{type => integer} + } + }, + ssl => minirest:ref(<<"ssl">>) + } + }, + Mysql = #{ + type => object, + required => [ type + , enable + , sql + , server + , database + , pool_size + , username + , password + , auto_reconnect + , ssl + ], + properties => #{ + type => #{ + type => string, + enum => [<<"mysql">>], + example => <<"mysql">> + }, + enable => #{ + type => boolean, + example => true + }, + sql => #{type => string}, + server => #{type => string, + example => <<"127.0.0.1:3306">> + }, + database => #{type => string}, + pool_size => #{type => integer}, + username => #{type => string}, + password => #{type => string}, + auto_reconnect => #{type => boolean, + example => true + }, + ssl => minirest:ref(<<"ssl">>) + } + }, + Pgsql = #{ + type => object, + required => [ type + , enable + , sql + , server + , database + , pool_size + , username + , password + , auto_reconnect + , ssl + ], + properties => #{ + type => #{ + type => string, + enum => [<<"pgsql">>], + example => <<"pgsql">> + }, + enable => #{ + type => boolean, + example => true + }, + sql => #{type => string}, + server => #{type => string, + example => <<"127.0.0.1:5432">> + }, + database => #{type => string}, + pool_size => #{type => integer}, + username => #{type => string}, + password => #{type => string}, + auto_reconnect => #{type => boolean, + example => true + }, + ssl => minirest:ref(<<"ssl">>) + } + }, + RedisSingle = #{ + type => object, + required => [ type + , enable + , cmd + , server + , redis_type + , pool_size + , auto_reconnect + , ssl + ], + properties => #{ + type => #{ + type => string, + enum => [<<"redis">>], + example => <<"redis">> + }, + enable => #{ + type => boolean, + example => true + }, + cmd => #{ + type => string, + example => <<"HGETALL mqtt_authz">> + }, + server => #{type => string, example => <<"127.0.0.1:3306">>}, + redis_type => #{type => string, + enum => [<<"single">>], + example => <<"single">>}, + pool_size => #{type => integer}, + auto_reconnect => #{type => boolean, example => true}, + password => #{type => string}, + database => #{type => integer}, + ssl => minirest:ref(<<"ssl">>) + } + }, + RedisSentinel= #{ + type => object, + required => [ type + , enable + , cmd + , servers + , redis_type + , sentinel + , pool_size + , auto_reconnect + , ssl + ], + properties => #{ + type => #{ + type => string, + enum => [<<"redis">>], + example => <<"redis">> + }, + enable => #{ + type => boolean, + example => true + }, + cmd => #{ + type => string, + example => <<"HGETALL mqtt_authz">> + }, + servers => #{type => array, + items => #{type => string,example => <<"127.0.0.1:3306">>}}, + redis_type => #{type => string, + enum => [<<"sentinel">>], + example => <<"sentinel">>}, + sentinel => #{type => string}, + pool_size => #{type => integer}, + auto_reconnect => #{type => boolean, example => true}, + password => #{type => string}, + database => #{type => integer}, + ssl => minirest:ref(<<"ssl">>) + } + }, + RedisCluster= #{ + type => object, + required => [ type + , enable + , cmd + , servers + , redis_type + , pool_size + , auto_reconnect + , ssl], + properties => #{ + type => #{ + type => string, + enum => [<<"redis">>], + example => <<"redis">> + }, + enable => #{ + type => boolean, + example => true + }, + cmd => #{ + type => string, + example => <<"HGETALL mqtt_authz">> + }, + servers => #{type => array, + items => #{type => string, example => <<"127.0.0.1:3306">>}}, + redis_type => #{type => string, + enum => [<<"cluster">>], + example => <<"cluster">>}, + pool_size => #{type => integer}, + auto_reconnect => #{type => boolean, example => true}, + password => #{type => string}, + database => #{type => integer}, + ssl => minirest:ref(<<"ssl">>) + } + }, + File = #{ + type => object, + required => [type, enable, rules], + properties => #{ + type => #{ + type => string, + enum => [<<"redis">>], + example => <<"redis">> + }, + enable => #{ + type => boolean, + example => true + }, + rules => #{ + type => array, + items => #{ + type => string, + example => <<"{allow,{username,\"^dashboard?\"},subscribe,[\"$SYS/#\"]}.">> + } + }, + path => #{ + type => string, + example => <<"/path/to/authorizaiton_rules.conf">> } } }, - [ #{<<"returned_rules">> => RetruenedRules} - , #{<<"rules">> => Rules} - , #{<<"simple_rule">> => SimpleRule} - , #{<<"principal">> => Principal} - , #{<<"principal_username">> => PrincipalUsername} - , #{<<"principal_clientid">> => PrincipalClientid} - , #{<<"principal_ipaddress">> => PrincipalIpaddress} - , #{<<"error">> => ErrorDef} + [ #{<<"returned_sources">> => RetruenedSources} + , #{<<"sources">> => Sources} + , #{<<"ssl">> => SSL} + , #{<<"http">> => HTTP} + , #{<<"mongo_single">> => MongoSingle} + , #{<<"mongo_rs">> => MongoRs} + , #{<<"mongo_sharded">> => MongoSharded} + , #{<<"mysql">> => Mysql} + , #{<<"pgsql">> => Pgsql} + , #{<<"redis_single">> => RedisSingle} + , #{<<"redis_sentinel">> => RedisSentinel} + , #{<<"redis_cluster">> => RedisCluster} + , #{<<"file">> => File} ]. diff --git a/apps/emqx_authz/src/emqx_authz_api_settings.erl b/apps/emqx_authz/src/emqx_authz_api_settings.erl new file mode 100644 index 000000000..ac48fafd9 --- /dev/null +++ b/apps/emqx_authz/src/emqx_authz_api_settings.erl @@ -0,0 +1,61 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_authz_api_settings). + +-behavior(minirest_api). + +-export([ api_spec/0 + , settings/2 + ]). + +api_spec() -> + {[settings_api()], []}. + +authorization_settings() -> + maps:remove(<<"sources">>, emqx:get_raw_config([authorization], #{})). + +conf_schema() -> + emqx_mgmt_api_configs:gen_schema(authorization_settings()). + +settings_api() -> + Metadata = #{ + get => #{ + description => "Get authorization settings", + responses => #{<<"200">> => emqx_mgmt_util:schema(conf_schema())} + }, + put => #{ + description => "Update authorization settings", + requestBody => emqx_mgmt_util:schema(conf_schema()), + responses => #{ + <<"200">> => emqx_mgmt_util:schema(conf_schema()), + <<"400">> => emqx_mgmt_util:bad_request() + } + } + }, + {"/authorization/settings", Metadata, settings}. + +settings(get, _Params) -> + {200, authorization_settings()}; + +settings(put, #{body := #{<<"no_match">> := NoMatch, + <<"deny_action">> := DenyAction, + <<"cache">> := Cache}}) -> + {ok, _} = emqx:update_config([authorization, no_match], NoMatch), + {ok, _} = emqx:update_config([authorization, deny_action], DenyAction), + {ok, _} = emqx:update_config([authorization, cache], Cache), + ok = emqx_authz_cache:drain_cache(), + {200, authorization_settings()}. diff --git a/apps/emqx_authz/src/emqx_authz_api_sources.erl b/apps/emqx_authz/src/emqx_authz_api_sources.erl new file mode 100644 index 000000000..209bbc01f --- /dev/null +++ b/apps/emqx_authz/src/emqx_authz_api_sources.erl @@ -0,0 +1,493 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_authz_api_sources). + +-behavior(minirest_api). + +-include("emqx_authz.hrl"). +-include_lib("emqx/include/logger.hrl"). + +-define(EXAMPLE_REDIS, + #{type=> redis, + enable => true, + server => <<"127.0.0.1:3306">>, + redis_type => single, + pool_size => 1, + auto_reconnect => true, + cmd => <<"HGETALL mqtt_authz">>}). +-define(EXAMPLE_FILE, + #{type=> file, + enable => true, + rules => [<<"{allow,{username,\"^dashboard?\"},subscribe,[\"$SYS/#\"]}.">>, + <<"{allow,{ipaddr,\"127.0.0.1\"},all,[\"$SYS/#\",\"#\"]}.">> + ]}). + +-define(EXAMPLE_RETURNED_REDIS, + maps:put(annotations, #{status => healthy}, ?EXAMPLE_REDIS) + ). +-define(EXAMPLE_RETURNED_FILE, + maps:put(annotations, #{status => healthy}, ?EXAMPLE_FILE) + ). + +-define(EXAMPLE_RETURNED, + #{sources => [ ?EXAMPLE_RETURNED_REDIS + , ?EXAMPLE_RETURNED_FILE + ] + }). + +-export([ api_spec/0 + , sources/2 + , source/2 + , move_source/2 + ]). + +api_spec() -> + {[ sources_api() + , source_api() + , move_source_api() + ], definitions()}. + +definitions() -> emqx_authz_api_schema:definitions(). + +sources_api() -> + Metadata = #{ + get => #{ + description => "List authorization sources", + responses => #{ + <<"200">> => #{ + description => <<"OK">>, + content => #{ + 'application/json' => #{ + schema => #{ + type => object, + required => [sources], + properties => #{sources => #{ + type => array, + items => minirest:ref(<<"returned_sources">>) + } + } + }, + examples => #{ + sources => #{ + summary => <<"Sources">>, + value => jsx:encode(?EXAMPLE_RETURNED) + } + } + } + } + } + } + }, + post => #{ + description => "Add new source", + requestBody => #{ + content => #{ + 'application/json' => #{ + schema => minirest:ref(<<"sources">>), + examples => #{ + redis => #{ + summary => <<"Redis">>, + value => jsx:encode(?EXAMPLE_REDIS) + }, + file => #{ + summary => <<"File">>, + value => jsx:encode(?EXAMPLE_FILE) + } + } + } + } + }, + responses => #{ + <<"204">> => #{description => <<"Created">>}, + <<"400">> => emqx_mgmt_util:bad_request() + } + }, + put => #{ + description => "Update all sources", + requestBody => #{ + content => #{ + 'application/json' => #{ + schema => #{ + type => array, + items => minirest:ref(<<"returned_sources">>) + }, + examples => #{ + redis => #{ + summary => <<"Redis">>, + value => jsx:encode(?EXAMPLE_REDIS) + }, + file => #{ + summary => <<"File">>, + value => jsx:encode(?EXAMPLE_FILE) + } + } + } + } + }, + responses => #{ + <<"204">> => #{description => <<"Created">>}, + <<"400">> => emqx_mgmt_util:bad_request() + } + } + }, + {"/authorization/sources", Metadata, sources}. + +source_api() -> + Metadata = #{ + get => #{ + description => "List authorization sources", + parameters => [ + #{ + name => type, + in => path, + schema => #{ + type => string + }, + required => true + } + ], + responses => #{ + <<"200">> => #{ + description => <<"OK">>, + content => #{ + 'application/json' => #{ + schema => minirest:ref(<<"returned_sources">>), + examples => #{ + redis => #{ + summary => <<"Redis">>, + value => jsx:encode(?EXAMPLE_RETURNED_REDIS) + }, + file => #{ + summary => <<"File">>, + value => jsx:encode(?EXAMPLE_RETURNED_FILE) + } + } + } + } + }, + <<"404">> => emqx_mgmt_util:bad_request(<<"Not Found">>) + } + }, + put => #{ + description => "Update source", + parameters => [ + #{ + name => type, + in => path, + schema => #{ + type => string + }, + required => true + } + ], + requestBody => #{ + content => #{ + 'application/json' => #{ + schema => minirest:ref(<<"sources">>), + examples => #{ + redis => #{ + summary => <<"Redis">>, + value => jsx:encode(?EXAMPLE_REDIS) + }, + file => #{ + summary => <<"File">>, + value => jsx:encode(?EXAMPLE_FILE) + } + } + } + } + }, + responses => #{ + <<"204">> => #{description => <<"No Content">>}, + <<"404">> => emqx_mgmt_util:bad_request(<<"Not Found">>), + <<"400">> => emqx_mgmt_util:bad_request() + } + }, + delete => #{ + description => "Delete source", + parameters => [ + #{ + name => type, + in => path, + schema => #{ + type => string + }, + required => true + } + ], + responses => #{ + <<"204">> => #{description => <<"No Content">>}, + <<"400">> => emqx_mgmt_util:bad_request() + } + } + }, + {"/authorization/sources/:type", Metadata, source}. + +move_source_api() -> + Metadata = #{ + post => #{ + description => "Change the order of sources", + parameters => [ + #{ + name => type, + in => path, + schema => #{ + type => string + }, + required => true + } + ], + requestBody => #{ + content => #{ + 'application/json' => #{ + schema => #{ + type => object, + required => [position], + properties => #{ + position => #{ + oneOf => [ + #{type => string, + enum => [<<"top">>, <<"bottom">>] + }, + #{type => object, + required => ['after'], + properties => #{ + 'after' => #{ + type => string + } + } + }, + #{type => object, + required => ['before'], + properties => #{ + 'before' => #{ + type => string + } + } + } + ] + } + } + } + } + } + }, + responses => #{ + <<"204">> => #{ + description => <<"No Content">> + }, + <<"404">> => emqx_mgmt_util:bad_request(<<"Not Found">>), + <<"400">> => emqx_mgmt_util:bad_request() + } + } + }, + {"/authorization/sources/:type/move", Metadata, move_source}. + +sources(get, _) -> + Sources = lists:foldl(fun (#{type := file, enable := Enable, path := Path}, AccIn) -> + {ok, Rules} = file:consult(Path), + lists:append(AccIn, [#{type => file, + enable => Enable, + rules => [ iolist_to_binary(io_lib:format("~p.", [R])) || R <- Rules], + annotations => #{status => healthy} + }]); + (#{enable := false} = Source, AccIn) -> + lists:append(AccIn, [Source#{annotations => #{status => unhealthy}}]); + (#{type := _Type, annotations := #{id := Id}} = Source, AccIn) -> + NSource0 = case maps:get(server, Source, undefined) of + undefined -> Source; + Server -> + Source#{server => emqx_connector_schema_lib:ip_port_to_string(Server)} + end, + NSource1 = case maps:get(servers, Source, undefined) of + undefined -> NSource0; + Servers -> + NSource0#{servers => [emqx_connector_schema_lib:ip_port_to_string(Server) || Server <- Servers]} + end, + NSource2 = case emqx_resource:health_check(Id) of + ok -> + NSource1#{annotations => #{status => healthy}}; + _ -> + NSource1#{annotations => #{status => unhealthy}} + end, + lists:append(AccIn, [read_cert(NSource2)]); + (Source, AccIn) -> + lists:append(AccIn, [Source#{annotations => #{status => healthy}}]) + end, [], emqx_authz:lookup()), + {200, #{sources => Sources}}; +sources(post, #{body := #{<<"type">> := <<"file">>, <<"rules">> := Rules, <<"enable">> := Enable}}) when is_list(Rules) -> + {ok, Filename} = write_file(filename:join([emqx:get_config([node, data_dir]), "acl.conf"]), + erlang:list_to_bitstring([<> || Rule <- Rules]) + ), + case emqx_authz:update(head, [#{type => file, enable => Enable, path => Filename}]) of + {ok, _} -> {204}; + {error, Reason} -> + {400, #{code => <<"BAD_REQUEST">>, + messgae => atom_to_binary(Reason)}} + end; +sources(post, #{body := Body}) when is_map(Body) -> + case emqx_authz:update(head, [write_cert(Body)]) of + {ok, _} -> {204}; + {error, Reason} -> + {400, #{code => <<"BAD_REQUEST">>, + messgae => atom_to_binary(Reason)}} + end; +sources(put, #{body := Body}) when is_list(Body) -> + NBody = [ begin + case Source of + #{<<"type">> := <<"file">>, <<"rules">> := Rules, <<"enable">> := Enable} -> + {ok, Filename} = write_file(filename:join([emqx:get_config([node, data_dir]), "acl.conf"]), + erlang:list_to_bitstring([<> || Rule <- Rules]) + ), + #{type => file, enable => Enable, path => Filename}; + _ -> write_cert(Source) + end + end || Source <- Body], + case emqx_authz:update(replace, NBody) of + {ok, _} -> {204}; + {error, Reason} -> + {400, #{code => <<"BAD_REQUEST">>, + messgae => atom_to_binary(Reason)}} + end. + +source(get, #{bindings := #{type := Type}}) -> + case emqx_authz:lookup(Type) of + {error, Reason} -> {404, #{messgae => atom_to_binary(Reason)}}; + #{type := file, enable := Enable, path := Path}-> + {ok, Rules} = file:consult(Path), + {200, #{type => file, + enable => Enable, + rules => [ iolist_to_binary(io_lib:format("~p.", [R])) || R <- Rules], + annotations => #{status => healthy} + } + }; + #{enable := false} = Source -> {200, Source#{annotations => #{status => unhealthy}}}; + #{annotations := #{id := Id}} = Source -> + NSource0 = case maps:get(server, Source, undefined) of + undefined -> Source; + Server -> + Source#{server => emqx_connector_schema_lib:ip_port_to_string(Server)} + end, + NSource1 = case maps:get(servers, Source, undefined) of + undefined -> NSource0; + Servers -> + NSource0#{servers => [emqx_connector_schema_lib:ip_port_to_string(Server) || Server <- Servers]} + end, + NSource2 = case emqx_resource:health_check(Id) of + ok -> + NSource1#{annotations => #{status => healthy}}; + _ -> + NSource1#{annotations => #{status => unhealthy}} + end, + {200, read_cert(NSource2)} + end; +source(put, #{bindings := #{type := <<"file">>}, body := #{<<"type">> := <<"file">>, <<"rules">> := Rules, <<"enable">> := Enable}}) -> + {ok, Filename} = write_file(maps:get(path, emqx_authz:lookup(file), ""), + erlang:list_to_bitstring([<> || Rule <- Rules]) + ), + case emqx_authz:update({replace_once, file}, #{type => file, enable => Enable, path => Filename}) of + {ok, _} -> {204}; + {error, Reason} -> + {400, #{code => <<"BAD_REQUEST">>, + messgae => atom_to_binary(Reason)}} + end; +source(put, #{bindings := #{type := Type}, body := Body}) when is_map(Body) -> + case emqx_authz:update({replace_once, Type}, write_cert(Body)) of + {ok, _} -> {204}; + {error, not_found_source} -> + {404, #{code => <<"NOT_FOUND">>, + messgae => <<"source ", Type/binary, " not found">>}}; + {error, Reason} -> + {400, #{code => <<"BAD_REQUEST">>, + messgae => atom_to_binary(Reason)}} + end; +source(delete, #{bindings := #{type := Type}}) -> + case emqx_authz:update({delete_once, Type}, #{}) of + {ok, _} -> {204}; + {error, Reason} -> + {400, #{code => <<"BAD_REQUEST">>, + messgae => atom_to_binary(Reason)}} + end. +move_source(post, #{bindings := #{type := Type}, body := #{<<"position">> := Position}}) -> + case emqx_authz:move(Type, Position) of + {ok, _} -> {204}; + {error, not_found_source} -> + {404, #{code => <<"NOT_FOUND">>, + messgae => <<"source ", Type/binary, " not found">>}}; + {error, Reason} -> + {400, #{code => <<"BAD_REQUEST">>, + messgae => atom_to_binary(Reason)}} + end. + +read_cert(#{ssl := #{enable := true} = SSL} = Source) -> + CaCert = case file:read_file(maps:get(cacertfile, SSL, "")) of + {ok, CaCert0} -> CaCert0; + _ -> "" + end, + Cert = case file:read_file(maps:get(certfile, SSL, "")) of + {ok, Cert0} -> Cert0; + _ -> "" + end, + Key = case file:read_file(maps:get(keyfile, SSL, "")) of + {ok, Key0} -> Key0; + _ -> "" + end, + Source#{ssl => SSL#{cacertfile => CaCert, + certfile => Cert, + keyfile => Key + } + }; +read_cert(Source) -> Source. + +write_cert(#{<<"ssl">> := #{<<"enable">> := true} = SSL} = Source) -> + CertPath = filename:join([emqx:get_config([node, data_dir]), "certs"]), + CaCert = case maps:is_key(<<"cacertfile">>, SSL) of + true -> + {ok, CaCertFile} = write_file(filename:join([CertPath, "cacert-" ++ emqx_rule_id:gen() ++".pem"]), + maps:get(<<"cacertfile">>, SSL)), + CaCertFile; + false -> "" + end, + Cert = case maps:is_key(<<"certfile">>, SSL) of + true -> + {ok, CertFile} = write_file(filename:join([CertPath, "cert-" ++ emqx_rule_id:gen() ++".pem"]), + maps:get(<<"certfile">>, SSL)), + CertFile; + false -> "" + end, + Key = case maps:is_key(<<"keyfile">>, SSL) of + true -> + {ok, KeyFile} = write_file(filename:join([CertPath, "key-" ++ emqx_rule_id:gen() ++".pem"]), + maps:get(<<"keyfile">>, SSL)), + KeyFile; + false -> "" + end, + Source#{<<"ssl">> => SSL#{<<"cacertfile">> => CaCert, + <<"certfile">> => Cert, + <<"keyfile">> => Key + } + }; +write_cert(Source) -> Source. + +write_file(Filename, Bytes) -> + ok = filelib:ensure_dir(Filename), + case file:write_file(Filename, Bytes) of + ok -> {ok, iolist_to_binary(Filename)}; + {error, Reason} -> + ?LOG(error, "Write File ~p Error: ~p", [Filename, Reason]), + error(Reason) + end. diff --git a/apps/emqx_authz/src/emqx_authz_http.erl b/apps/emqx_authz/src/emqx_authz_http.erl index c95d200e1..93aa634f3 100644 --- a/apps/emqx_authz/src/emqx_authz_http.erl +++ b/apps/emqx_authz/src/emqx_authz_http.erl @@ -35,12 +35,12 @@ description() -> authorize(Client, PubSub, Topic, #{type := http, - config := #{url := #{path := Path} = Url, - headers := Headers, - method := Method, - request_timeout := RequestTimeout} = Config, + url := #{path := Path} = Url, + headers := Headers, + method := Method, + request_timeout := RequestTimeout, annotations := #{id := ResourceID} - }) -> + } = Source) -> Request = case Method of get -> Query = maps:get(query, Url, ""), @@ -49,7 +49,7 @@ authorize(Client, PubSub, Topic, _ -> Body0 = serialize_body( maps:get('Accept', Headers, <<"application/json">>), - maps:get(body, Config, #{}) + maps:get(body, Source, #{}) ), Body1 = replvar(Body0, PubSub, Topic, Client), Path1 = replvar(Path, PubSub, Topic, Client), diff --git a/apps/emqx_authz/src/emqx_authz_schema.erl b/apps/emqx_authz/src/emqx_authz_schema.erl index 7fb60bae2..b90d522e8 100644 --- a/apps/emqx_authz/src/emqx_authz_schema.erl +++ b/apps/emqx_authz/src/emqx_authz_schema.erl @@ -13,14 +13,32 @@ -type permission() :: allow | deny. -type url() :: emqx_http_lib:uri_map(). --export([ roots/0 +-export([ namespace/0 + , roots/0 , fields/1 ]). -roots() -> ["authorization"]. +namespace() -> authz. + +%% @doc authorization schema is not exported +%% but directly used by emqx_schema +roots() -> []. fields("authorization") -> - [ {sources, sources()} + [ {sources, #{type => union_array( + [ hoconsc:ref(?MODULE, file) + , hoconsc:ref(?MODULE, http_get) + , hoconsc:ref(?MODULE, http_post) + , hoconsc:ref(?MODULE, mongo_single) + , hoconsc:ref(?MODULE, mongo_rs) + , hoconsc:ref(?MODULE, mongo_sharded) + , hoconsc:ref(?MODULE, mysql) + , hoconsc:ref(?MODULE, pgsql) + , hoconsc:ref(?MODULE, redis_single) + , hoconsc:ref(?MODULE, redis_sentinel) + , hoconsc:ref(?MODULE, redis_cluster) + ])} + } ]; fields(file) -> [ {type, #{type => file}} @@ -34,17 +52,12 @@ fields(file) -> end }} ]; -fields(http) -> +fields(http_get) -> [ {type, #{type => http}} , {enable, #{type => boolean(), default => true}} - , {config, #{type => hoconsc:union([ hoconsc:ref(?MODULE, http_get) - , hoconsc:ref(?MODULE, http_post) - ])} - } - ]; -fields(http_get) -> - [ {url, #{type => url()}} + , {url, #{type => url()}} + , {method, #{type => get, default => get }} , {headers, #{type => map(), default => #{ <<"accept">> => <<"application/json">> , <<"cache-control">> => <<"no-cache">> @@ -64,11 +77,15 @@ fields(http_get) -> end } } - , {method, #{type => get, default => get }} , {request_timeout, #{type => timeout(), default => 30000 }} ] ++ proplists:delete(base_url, emqx_connector_http:fields(config)); fields(http_post) -> - [ {url, #{type => url()}} + [ {type, #{type => http}} + , {enable, #{type => boolean(), + default => true}} + , {url, #{type => url()}} + , {method, #{type => hoconsc:enum([post, put]), + default => get}} , {headers, #{type => map(), default => #{ <<"accept">> => <<"application/json">> , <<"cache-control">> => <<"no-cache">> @@ -90,54 +107,42 @@ fields(http_post) -> end } } - , {method, #{type => hoconsc:enum([post, put]), - default => get}} + , {request_timeout, #{type => timeout(), default => 30000 }} , {body, #{type => map(), nullable => true } } ] ++ proplists:delete(base_url, emqx_connector_http:fields(config)); -fields(mongo) -> - connector_fields(mongo) ++ +fields(mongo_single) -> + connector_fields(mongo, single) ++ + [ {collection, #{type => atom()}} + , {find, #{type => map()}} + ]; +fields(mongo_rs) -> + connector_fields(mongo, rs) ++ + [ {collection, #{type => atom()}} + , {find, #{type => map()}} + ]; +fields(mongo_sharded) -> + connector_fields(mongo, sharded) ++ [ {collection, #{type => atom()}} , {find, #{type => map()}} ]; -fields(redis) -> - connector_fields(redis) ++ - [ {cmd, query()} ]; fields(mysql) -> connector_fields(mysql) ++ [ {sql, query()} ]; fields(pgsql) -> connector_fields(pgsql) ++ [ {sql, query()} ]; -fields(username) -> - [{username, #{type => binary()}}]; -fields(clientid) -> - [{clientid, #{type => binary()}}]; -fields(ipaddress) -> - [{ipaddress, #{type => string()}}]; -fields(andlist) -> - [{'and', #{type => union_array( - [ hoconsc:ref(?MODULE, username) - , hoconsc:ref(?MODULE, clientid) - , hoconsc:ref(?MODULE, ipaddress) - ]) - } - } - ]; -fields(orlist) -> - [{'or', #{type => union_array( - [ hoconsc:ref(?MODULE, username) - , hoconsc:ref(?MODULE, clientid) - , hoconsc:ref(?MODULE, ipaddress) - ]) - } - } - ]; -fields(eq_topic) -> - [{eq, #{type => binary()}}]. - +fields(redis_single) -> + connector_fields(redis, single) ++ + [ {cmd, query()} ]; +fields(redis_sentinel) -> + connector_fields(redis, sentinel) ++ + [ {cmd, query()} ]; +fields(redis_cluster) -> + connector_fields(redis, cluster) ++ + [ {cmd, query()} ]. %%-------------------------------------------------------------------- %% Internal functions @@ -146,17 +151,6 @@ fields(eq_topic) -> union_array(Item) when is_list(Item) -> hoconsc:array(hoconsc:union(Item)). -sources() -> - #{type => union_array( - [ hoconsc:ref(?MODULE, file) - , hoconsc:ref(?MODULE, http) - , hoconsc:ref(?MODULE, mysql) - , hoconsc:ref(?MODULE, pgsql) - , hoconsc:ref(?MODULE, redis) - , hoconsc:ref(?MODULE, mongo) - ]) - }. - query() -> #{type => binary(), validator => fun(S) -> @@ -168,6 +162,8 @@ query() -> }. connector_fields(DB) -> + connector_fields(DB, config). +connector_fields(DB, Fields) -> Mod0 = io_lib:format("~s_~s",[emqx_connector, DB]), Mod = try list_to_existing_atom(Mod0) @@ -180,4 +176,4 @@ connector_fields(DB) -> [ {type, #{type => DB}} , {enable, #{type => boolean(), default => true}} - ] ++ Mod:roots(). + ] ++ Mod:fields(Fields). diff --git a/apps/emqx_authz/test/emqx_authz_SUITE.erl b/apps/emqx_authz/test/emqx_authz_SUITE.erl index ef7644a65..f2cb01d05 100644 --- a/apps/emqx_authz/test/emqx_authz_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_SUITE.erl @@ -61,120 +61,141 @@ init_per_testcase(_, Config) -> Config. -define(SOURCE1, #{<<"type">> => <<"http">>, - <<"config">> => #{ - <<"url">> => <<"https://fake.com:443/">>, - <<"headers">> => #{}, - <<"method">> => <<"get">>, - <<"request_timeout">> => 5000} - }). + <<"enable">> => true, + <<"url">> => <<"https://fake.com:443/">>, + <<"headers">> => #{}, + <<"method">> => <<"get">>, + <<"request_timeout">> => 5000 + }). -define(SOURCE2, #{<<"type">> => <<"mongo">>, - <<"config">> => #{ - <<"mongo_type">> => <<"single">>, - <<"server">> => <<"127.0.0.1:27017">>, - <<"pool_size">> => 1, - <<"database">> => <<"mqtt">>, - <<"ssl">> => #{<<"enable">> => false}}, - <<"collection">> => <<"fake">>, - <<"find">> => #{<<"a">> => <<"b">>} - }). + <<"enable">> => true, + <<"mongo_type">> => <<"single">>, + <<"server">> => <<"127.0.0.1:27017">>, + <<"pool_size">> => 1, + <<"database">> => <<"mqtt">>, + <<"ssl">> => #{<<"enable">> => false}, + <<"collection">> => <<"fake">>, + <<"find">> => #{<<"a">> => <<"b">>} + }). -define(SOURCE3, #{<<"type">> => <<"mysql">>, - <<"config">> => #{ - <<"server">> => <<"127.0.0.1:27017">>, - <<"pool_size">> => 1, - <<"database">> => <<"mqtt">>, - <<"username">> => <<"xx">>, - <<"password">> => <<"ee">>, - <<"auto_reconnect">> => true, - <<"ssl">> => #{<<"enable">> => false}}, - <<"sql">> => <<"abcb">> - }). + <<"enable">> => true, + <<"server">> => <<"127.0.0.1:27017">>, + <<"pool_size">> => 1, + <<"database">> => <<"mqtt">>, + <<"username">> => <<"xx">>, + <<"password">> => <<"ee">>, + <<"auto_reconnect">> => true, + <<"ssl">> => #{<<"enable">> => false}, + <<"sql">> => <<"abcb">> + }). -define(SOURCE4, #{<<"type">> => <<"pgsql">>, - <<"config">> => #{ - <<"server">> => <<"127.0.0.1:27017">>, - <<"pool_size">> => 1, - <<"database">> => <<"mqtt">>, - <<"username">> => <<"xx">>, - <<"password">> => <<"ee">>, - <<"auto_reconnect">> => true, - <<"ssl">> => #{<<"enable">> => false}}, - <<"sql">> => <<"abcb">> - }). + <<"enable">> => true, + <<"server">> => <<"127.0.0.1:27017">>, + <<"pool_size">> => 1, + <<"database">> => <<"mqtt">>, + <<"username">> => <<"xx">>, + <<"password">> => <<"ee">>, + <<"auto_reconnect">> => true, + <<"ssl">> => #{<<"enable">> => false}, + <<"sql">> => <<"abcb">> + }). -define(SOURCE5, #{<<"type">> => <<"redis">>, - <<"config">> => #{ - <<"server">> => <<"127.0.0.1:27017">>, - <<"pool_size">> => 1, - <<"database">> => 0, - <<"password">> => <<"ee">>, - <<"auto_reconnect">> => true, - <<"ssl">> => #{<<"enable">> => false}}, - <<"cmd">> => <<"HGETALL mqtt_authz:%u">> - }). + <<"enable">> => true, + <<"server">> => <<"127.0.0.1:27017">>, + <<"pool_size">> => 1, + <<"database">> => 0, + <<"password">> => <<"ee">>, + <<"auto_reconnect">> => true, + <<"ssl">> => #{<<"enable">> => false}, + <<"cmd">> => <<"HGETALL mqtt_authz:%u">> + }). +-define(SOURCE6, #{<<"type">> => <<"file">>, + <<"enable">> => true, + <<"path">> => emqx_ct_helpers:deps_path(emqx_authz, "etc/acl.conf") + }). + %%------------------------------------------------------------------------------ %% Testcases %%------------------------------------------------------------------------------ t_update_source(_) -> - {ok, _} = emqx_authz:update(replace, [?SOURCE2]), + {ok, _} = emqx_authz:update(replace, [?SOURCE3]), + {ok, _} = emqx_authz:update(head, [?SOURCE2]), {ok, _} = emqx_authz:update(head, [?SOURCE1]), - {ok, _} = emqx_authz:update(tail, [?SOURCE3]), + {ok, _} = emqx_authz:update(tail, [?SOURCE4]), + {ok, _} = emqx_authz:update(tail, [?SOURCE5]), + {ok, _} = emqx_authz:update(tail, [?SOURCE6]), - ?assertMatch([#{type := http}, #{type := mongo}, #{type := mysql}], emqx:get_config([authorization, sources], [])), + ?assertMatch([ #{type := http, enable := true} + , #{type := mongo, enable := true} + , #{type := mysql, enable := true} + , #{type := pgsql, enable := true} + , #{type := redis, enable := true} + , #{type := file, enable := true} + ], emqx:get_config([authorization, sources], [])), - [#{annotations := #{id := Id1}, type := http}, - #{annotations := #{id := Id2}, type := mongo}, - #{annotations := #{id := Id3}, type := mysql} - ] = emqx_authz:lookup(), + {ok, _} = emqx_authz:update({replace_once, http}, ?SOURCE1#{<<"enable">> := false}), + {ok, _} = emqx_authz:update({replace_once, mongo}, ?SOURCE2#{<<"enable">> := false}), + {ok, _} = emqx_authz:update({replace_once, mysql}, ?SOURCE3#{<<"enable">> := false}), + {ok, _} = emqx_authz:update({replace_once, pgsql}, ?SOURCE4#{<<"enable">> := false}), + {ok, _} = emqx_authz:update({replace_once, redis}, ?SOURCE5#{<<"enable">> := false}), + {ok, _} = emqx_authz:update({replace_once, file}, ?SOURCE6#{<<"enable">> := false}), - {ok, _} = emqx_authz:update({replace_once, Id1}, ?SOURCE5), - {ok, _} = emqx_authz:update({replace_once, Id3}, ?SOURCE4), - ?assertMatch([#{type := redis}, #{type := mongo}, #{type := pgsql}], emqx:get_config([authorization, sources], [])), - - [#{annotations := #{id := Id1}, type := redis}, - #{annotations := #{id := Id2}, type := mongo}, - #{annotations := #{id := Id3}, type := pgsql} - ] = emqx_authz:lookup(), + ?assertMatch([ #{type := http, enable := false} + , #{type := mongo, enable := false} + , #{type := mysql, enable := false} + , #{type := pgsql, enable := false} + , #{type := redis, enable := false} + , #{type := file, enable := false} + ], emqx:get_config([authorization, sources], [])), {ok, _} = emqx_authz:update(replace, []). t_move_source(_) -> - {ok, _} = emqx_authz:update(replace, [?SOURCE1, ?SOURCE2, ?SOURCE3, ?SOURCE4, ?SOURCE5]), - [#{annotations := #{id := Id1}}, - #{annotations := #{id := Id2}}, - #{annotations := #{id := Id3}}, - #{annotations := #{id := Id4}}, - #{annotations := #{id := Id5}} - ] = emqx_authz:lookup(), - - {ok, _} = emqx_authz:move(Id4, <<"top">>), - ?assertMatch([#{annotations := #{id := Id4}}, - #{annotations := #{id := Id1}}, - #{annotations := #{id := Id2}}, - #{annotations := #{id := Id3}}, - #{annotations := #{id := Id5}} + {ok, _} = emqx_authz:update(replace, [?SOURCE1, ?SOURCE2, ?SOURCE3, ?SOURCE4, ?SOURCE5, ?SOURCE6]), + ?assertMatch([ #{type := http} + , #{type := mongo} + , #{type := mysql} + , #{type := pgsql} + , #{type := redis} + , #{type := file} ], emqx_authz:lookup()), - {ok, _} = emqx_authz:move(Id1, <<"bottom">>), - ?assertMatch([#{annotations := #{id := Id4}}, - #{annotations := #{id := Id2}}, - #{annotations := #{id := Id3}}, - #{annotations := #{id := Id5}}, - #{annotations := #{id := Id1}} + {ok, _} = emqx_authz:move(pgsql, <<"top">>), + ?assertMatch([ #{type := pgsql} + , #{type := http} + , #{type := mongo} + , #{type := mysql} + , #{type := redis} + , #{type := file} ], emqx_authz:lookup()), - {ok, _} = emqx_authz:move(Id3, #{<<"before">> => Id4}), - ?assertMatch([#{annotations := #{id := Id3}}, - #{annotations := #{id := Id4}}, - #{annotations := #{id := Id2}}, - #{annotations := #{id := Id5}}, - #{annotations := #{id := Id1}} + {ok, _} = emqx_authz:move(http, <<"bottom">>), + ?assertMatch([ #{type := pgsql} + , #{type := mongo} + , #{type := mysql} + , #{type := redis} + , #{type := file} + , #{type := http} ], emqx_authz:lookup()), - {ok, _} = emqx_authz:move(Id2, #{<<"after">> => Id1}), - ?assertMatch([#{annotations := #{id := Id3}}, - #{annotations := #{id := Id4}}, - #{annotations := #{id := Id5}}, - #{annotations := #{id := Id1}}, - #{annotations := #{id := Id2}} + {ok, _} = emqx_authz:move(mysql, #{<<"before">> => pgsql}), + ?assertMatch([ #{type := mysql} + , #{type := pgsql} + , #{type := mongo} + , #{type := redis} + , #{type := file} + , #{type := http} ], emqx_authz:lookup()), + + {ok, _} = emqx_authz:move(mongo, #{<<"after">> => http}), + ?assertMatch([ #{type := mysql} + , #{type := pgsql} + , #{type := redis} + , #{type := file} + , #{type := http} + , #{type := mongo} + ], emqx_authz:lookup()), + ok. diff --git a/apps/emqx_authz/test/emqx_authz_api_SUITE.erl b/apps/emqx_authz/test/emqx_authz_api_SUITE.erl deleted file mode 100644 index 8d92413b3..000000000 --- a/apps/emqx_authz/test/emqx_authz_api_SUITE.erl +++ /dev/null @@ -1,266 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_authz_api_SUITE). - --compile(nowarn_export_all). --compile(export_all). - --include("emqx_authz.hrl"). --include_lib("eunit/include/eunit.hrl"). --include_lib("common_test/include/ct.hrl"). - --define(CONF_DEFAULT, <<"authorization: {sources: []}">>). - --import(emqx_ct_http, [ request_api/3 - , request_api/5 - , get_http_data/1 - , create_default_app/0 - , delete_default_app/0 - , default_auth_header/0 - , auth_header/2 - ]). - --define(HOST, "http://127.0.0.1:18083/"). --define(API_VERSION, "v5"). --define(BASE_PATH, "api"). - --define(SOURCE1, #{<<"type">> => <<"http">>, - <<"config">> => #{ - <<"url">> => <<"https://fake.com:443/">>, - <<"headers">> => #{}, - <<"method">> => <<"get">>, - <<"request_timeout">> => 5000} - }). --define(SOURCE2, #{<<"type">> => <<"mongo">>, - <<"config">> => #{ - <<"mongo_type">> => <<"single">>, - <<"server">> => <<"127.0.0.1:27017">>, - <<"pool_size">> => 1, - <<"database">> => <<"mqtt">>, - <<"ssl">> => #{<<"enable">> => false}}, - <<"collection">> => <<"fake">>, - <<"find">> => #{<<"a">> => <<"b">>} - }). --define(SOURCE3, #{<<"type">> => <<"mysql">>, - <<"config">> => #{ - <<"server">> => <<"127.0.0.1:27017">>, - <<"pool_size">> => 1, - <<"database">> => <<"mqtt">>, - <<"username">> => <<"xx">>, - <<"password">> => <<"ee">>, - <<"auto_reconnect">> => true, - <<"ssl">> => #{<<"enable">> => false}}, - <<"sql">> => <<"abcb">> - }). --define(SOURCE4, #{<<"type">> => <<"pgsql">>, - <<"config">> => #{ - <<"server">> => <<"127.0.0.1:27017">>, - <<"pool_size">> => 1, - <<"database">> => <<"mqtt">>, - <<"username">> => <<"xx">>, - <<"password">> => <<"ee">>, - <<"auto_reconnect">> => true, - <<"ssl">> => #{<<"enable">> => false}}, - <<"sql">> => <<"abcb">> - }). --define(SOURCE5, #{<<"type">> => <<"redis">>, - <<"config">> => #{ - <<"server">> => <<"127.0.0.1:27017">>, - <<"pool_size">> => 1, - <<"database">> => 0, - <<"password">> => <<"ee">>, - <<"auto_reconnect">> => true, - <<"ssl">> => #{<<"enable">> => false}}, - <<"cmd">> => <<"HGETALL mqtt_authz:%u">> - }). - -all() -> - emqx_ct:all(?MODULE). - -groups() -> - []. - -init_per_suite(Config) -> - meck:new(emqx_schema, [non_strict, passthrough, no_history, no_link]), - meck:expect(emqx_schema, fields, fun("authorization") -> - meck:passthrough(["authorization"]) ++ - emqx_authz_schema:fields("authorization"); - (F) -> meck:passthrough([F]) - end), - - meck:new(emqx_resource, [non_strict, passthrough, no_history, no_link]), - meck:expect(emqx_resource, create, fun(_, _, _) -> {ok, meck_data} end), - meck:expect(emqx_resource, update, fun(_, _, _, _) -> {ok, meck_data} end), - meck:expect(emqx_resource, health_check, fun(_) -> ok end), - meck:expect(emqx_resource, remove, fun(_) -> ok end ), - - ok = emqx_config:init_load(emqx_authz_schema, ?CONF_DEFAULT), - - ok = emqx_ct_helpers:start_apps([emqx_authz, emqx_dashboard], fun set_special_configs/1), - {ok, _} = emqx:update_config([authorization, cache, enable], false), - {ok, _} = emqx:update_config([authorization, no_match], deny), - - Config. - -end_per_suite(_Config) -> - {ok, _} = emqx_authz:update(replace, []), - emqx_ct_helpers:stop_apps([emqx_resource, emqx_authz, emqx_dashboard]), - meck:unload(emqx_resource), - meck:unload(emqx_schema), - ok. - -set_special_configs(emqx_dashboard) -> - Config = #{ - default_username => <<"admin">>, - default_password => <<"public">>, - listeners => [#{ - protocol => http, - port => 18083 - }] - }, - emqx_config:put([emqx_dashboard], Config), - ok; -set_special_configs(emqx_authz) -> - emqx_config:put([authorization], #{rules => []}), - ok; -set_special_configs(_App) -> - ok. - -%%------------------------------------------------------------------------------ -%% Testcases -%%------------------------------------------------------------------------------ - -t_api(_) -> - {ok, 200, Result1} = request(get, uri(["authorization"]), []), - ?assertEqual([], get_rules(Result1)), - - lists:foreach(fun(_) -> - {ok, 204, _} = request(post, uri(["authorization"]), ?SOURCE1) - end, lists:seq(1, 20)), - {ok, 200, Result2} = request(get, uri(["authorization"]), []), - ?assertEqual(20, length(get_rules(Result2))), - - lists:foreach(fun(Page) -> - Query = "?page=" ++ integer_to_list(Page) ++ "&&limit=10", - Url = uri(["authorization" ++ Query]), - {ok, 200, Result} = request(get, Url, []), - ?assertEqual(10, length(get_rules(Result))) - end, lists:seq(1, 2)), - - {ok, 204, _} = request(put, uri(["authorization"]), [?SOURCE1, ?SOURCE2, ?SOURCE3, ?SOURCE4]), - - {ok, 200, Result3} = request(get, uri(["authorization"]), []), - Rules = get_rules(Result3), - ?assertEqual(4, length(Rules)), - ?assertMatch([ #{<<"type">> := <<"http">>} - , #{<<"type">> := <<"mongo">>} - , #{<<"type">> := <<"mysql">>} - , #{<<"type">> := <<"pgsql">>} - ], Rules), - - #{<<"annotations">> := #{<<"id">> := Id}} = lists:nth(2, Rules), - - {ok, 204, _} = request(put, uri(["authorization", binary_to_list(Id)]), ?SOURCE5), - - {ok, 200, Result4} = request(get, uri(["authorization", binary_to_list(Id)]), []), - ?assertMatch(#{<<"type">> := <<"redis">>}, jsx:decode(Result4)), - - lists:foreach(fun(#{<<"annotations">> := #{<<"id">> := Id0}}) -> - {ok, 204, _} = request(delete, uri(["authorization", binary_to_list(Id0)]), []) - end, Rules), - {ok, 200, Result5} = request(get, uri(["authorization"]), []), - ?assertEqual([], get_rules(Result5)), - ok. - -t_move_rule(_) -> - {ok, _} = emqx_authz:update(replace, [?SOURCE1, ?SOURCE2, ?SOURCE3, ?SOURCE4, ?SOURCE5]), - [#{annotations := #{id := Id1}}, - #{annotations := #{id := Id2}}, - #{annotations := #{id := Id3}}, - #{annotations := #{id := Id4}}, - #{annotations := #{id := Id5}} - ] = emqx_authz:lookup(), - - {ok, 204, _} = request(post, uri(["authorization", Id4, "move"]), - #{<<"position">> => <<"top">>}), - ?assertMatch([#{annotations := #{id := Id4}}, - #{annotations := #{id := Id1}}, - #{annotations := #{id := Id2}}, - #{annotations := #{id := Id3}}, - #{annotations := #{id := Id5}} - ], emqx_authz:lookup()), - - {ok, 204, _} = request(post, uri(["authorization", Id1, "move"]), - #{<<"position">> => <<"bottom">>}), - ?assertMatch([#{annotations := #{id := Id4}}, - #{annotations := #{id := Id2}}, - #{annotations := #{id := Id3}}, - #{annotations := #{id := Id5}}, - #{annotations := #{id := Id1}} - ], emqx_authz:lookup()), - - {ok, 204, _} = request(post, uri(["authorization", Id3, "move"]), - #{<<"position">> => #{<<"before">> => Id4}}), - ?assertMatch([#{annotations := #{id := Id3}}, - #{annotations := #{id := Id4}}, - #{annotations := #{id := Id2}}, - #{annotations := #{id := Id5}}, - #{annotations := #{id := Id1}} - ], emqx_authz:lookup()), - - {ok, 204, _} = request(post, uri(["authorization", Id2, "move"]), - #{<<"position">> => #{<<"after">> => Id1}}), - ?assertMatch([#{annotations := #{id := Id3}}, - #{annotations := #{id := Id4}}, - #{annotations := #{id := Id5}}, - #{annotations := #{id := Id1}}, - #{annotations := #{id := Id2}} - ], emqx_authz:lookup()), - - ok. - -%%-------------------------------------------------------------------- -%% HTTP Request -%%-------------------------------------------------------------------- - -request(Method, Url, Body) -> - Request = case Body of - [] -> {Url, [auth_header_()]}; - _ -> {Url, [auth_header_()], "application/json", jsx:encode(Body)} - end, - ct:pal("Method: ~p, Request: ~p", [Method, Request]), - case httpc:request(Method, Request, [], [{body_format, binary}]) of - {error, socket_closed_remotely} -> - {error, socket_closed_remotely}; - {ok, {{"HTTP/1.1", Code, _}, _Headers, Return} } -> - {ok, Code, Return}; - {ok, {Reason, _, _}} -> - {error, Reason} - end. - -uri() -> uri([]). -uri(Parts) when is_list(Parts) -> - NParts = [E || E <- Parts], - ?HOST ++ filename:join([?BASE_PATH, ?API_VERSION | NParts]). - -get_rules(Result) -> - maps:get(<<"rules">>, jsx:decode(Result), []). - -auth_header_() -> - Username = <<"admin">>, - Password = <<"public">>, - {ok, Token} = emqx_dashboard_admin:sign_token(Username, Password), - {"Authorization", "Bearer " ++ binary_to_list(Token)}. diff --git a/apps/emqx_authz/test/emqx_authz_api_settings_SUITE.erl b/apps/emqx_authz/test/emqx_authz_api_settings_SUITE.erl new file mode 100644 index 000000000..1db9fff2b --- /dev/null +++ b/apps/emqx_authz/test/emqx_authz_api_settings_SUITE.erl @@ -0,0 +1,135 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_authz_api_settings_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include("emqx_authz.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +-define(CONF_DEFAULT, <<"authorization: {sources: []}">>). + +-import(emqx_ct_http, [ request_api/3 + , request_api/5 + , get_http_data/1 + , create_default_app/0 + , delete_default_app/0 + , default_auth_header/0 + , auth_header/2 + ]). + +-define(HOST, "http://127.0.0.1:18083/"). +-define(API_VERSION, "v5"). +-define(BASE_PATH, "api"). + +all() -> + emqx_ct:all(?MODULE). + +groups() -> + []. + +init_per_suite(Config) -> + ok = emqx_ct_helpers:start_apps([emqx_authz, emqx_dashboard], fun set_special_configs/1), + {ok, _} = emqx:update_config([authorization, cache, enable], false), + {ok, _} = emqx:update_config([authorization, no_match], deny), + + Config. + +end_per_suite(_Config) -> + emqx_ct_helpers:stop_apps([emqx_resource, emqx_authz, emqx_dashboard]), + ok. + +set_special_configs(emqx_dashboard) -> + Config = #{ + default_username => <<"admin">>, + default_password => <<"public">>, + listeners => [#{ + protocol => http, + port => 18083 + }] + }, + emqx_config:put([emqx_dashboard], Config), + ok; +set_special_configs(_App) -> + ok. + +%%------------------------------------------------------------------------------ +%% Testcases +%%------------------------------------------------------------------------------ + +t_api(_) -> + Settings1 = #{<<"no_match">> => <<"deny">>, + <<"deny_action">> => <<"disconnect">>, + <<"cache">> => #{ + <<"enable">> => false, + <<"max_size">> => 32, + <<"ttl">> => 60000 + } + }, + + {ok, 200, Result1} = request(put, uri(["authorization", "settings"]), Settings1), + {ok, 200, Result1} = request(get, uri(["authorization", "settings"]), []), + ?assertEqual(Settings1, jsx:decode(Result1)), + + Settings2 = #{<<"no_match">> => <<"allow">>, + <<"deny_action">> => <<"ignore">>, + <<"cache">> => #{ + <<"enable">> => true, + <<"max_size">> => 32, + <<"ttl">> => 60000 + } + }, + + {ok, 200, Result2} = request(put, uri(["authorization", "settings"]), Settings2), + {ok, 200, Result2} = request(get, uri(["authorization", "settings"]), []), + ?assertEqual(Settings2, jsx:decode(Result2)), + + ok. + +%%-------------------------------------------------------------------- +%% HTTP Request +%%-------------------------------------------------------------------- + +request(Method, Url, Body) -> + Request = case Body of + [] -> {Url, [auth_header_()]}; + _ -> {Url, [auth_header_()], "application/json", jsx:encode(Body)} + end, + ct:pal("Method: ~p, Request: ~p", [Method, Request]), + case httpc:request(Method, Request, [], [{body_format, binary}]) of + {error, socket_closed_remotely} -> + {error, socket_closed_remotely}; + {ok, {{"HTTP/1.1", Code, _}, _Headers, Return} } -> + {ok, Code, Return}; + {ok, {Reason, _, _}} -> + {error, Reason} + end. + +uri() -> uri([]). +uri(Parts) when is_list(Parts) -> + NParts = [E || E <- Parts], + ?HOST ++ filename:join([?BASE_PATH, ?API_VERSION | NParts]). + +get_sources(Result) -> + maps:get(<<"sources">>, jsx:decode(Result), []). + +auth_header_() -> + Username = <<"admin">>, + Password = <<"public">>, + {ok, Token} = emqx_dashboard_admin:sign_token(Username, Password), + {"Authorization", "Bearer " ++ binary_to_list(Token)}. diff --git a/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl b/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl new file mode 100644 index 000000000..8c37189c9 --- /dev/null +++ b/apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl @@ -0,0 +1,305 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_authz_api_sources_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include("emqx_authz.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +-define(CONF_DEFAULT, <<"authorization: {sources: []}">>). + +-import(emqx_ct_http, [ request_api/3 + , request_api/5 + , get_http_data/1 + , create_default_app/0 + , delete_default_app/0 + , default_auth_header/0 + , auth_header/2 + ]). + +-define(HOST, "http://127.0.0.1:18083/"). +-define(API_VERSION, "v5"). +-define(BASE_PATH, "api"). + +-define(SOURCE1, #{<<"type">> => <<"http">>, + <<"enable">> => true, + <<"url">> => <<"https://fake.com:443/">>, + <<"headers">> => #{}, + <<"method">> => <<"get">>, + <<"request_timeout">> => 5000 + }). +-define(SOURCE2, #{<<"type">> => <<"mongo">>, + <<"enable">> => true, + <<"mongo_type">> => <<"sharded">>, + <<"servers">> => [<<"127.0.0.1:27017">>, + <<"192.168.0.1:27017">> + ], + <<"pool_size">> => 1, + <<"database">> => <<"mqtt">>, + <<"ssl">> => #{<<"enable">> => false}, + <<"collection">> => <<"fake">>, + <<"find">> => #{<<"a">> => <<"b">>} + }). +-define(SOURCE3, #{<<"type">> => <<"mysql">>, + <<"enable">> => true, + <<"server">> => <<"127.0.0.1:3306">>, + <<"pool_size">> => 1, + <<"database">> => <<"mqtt">>, + <<"username">> => <<"xx">>, + <<"password">> => <<"ee">>, + <<"auto_reconnect">> => true, + <<"ssl">> => #{<<"enable">> => false}, + <<"sql">> => <<"abcb">> + }). +-define(SOURCE4, #{<<"type">> => <<"pgsql">>, + <<"enable">> => true, + <<"server">> => <<"127.0.0.1:5432">>, + <<"pool_size">> => 1, + <<"database">> => <<"mqtt">>, + <<"username">> => <<"xx">>, + <<"password">> => <<"ee">>, + <<"auto_reconnect">> => true, + <<"ssl">> => #{<<"enable">> => false}, + <<"sql">> => <<"abcb">> + }). +-define(SOURCE5, #{<<"type">> => <<"redis">>, + <<"enable">> => true, + <<"servers">> => [<<"127.0.0.1:6379">>, + <<"127.0.0.1:6380">> + ], + <<"pool_size">> => 1, + <<"database">> => 0, + <<"password">> => <<"ee">>, + <<"auto_reconnect">> => true, + <<"ssl">> => #{<<"enable">> => false}, + <<"cmd">> => <<"HGETALL mqtt_authz:%u">> + }). +-define(SOURCE6, #{<<"type">> => <<"file">>, + <<"enable">> => true, + <<"rules">> => + [<<"{allow,{username,\"^dashboard?\"},subscribe,[\"$SYS/#\"]}.">>, + <<"{allow,{ipaddr,\"127.0.0.1\"},all,[\"$SYS/#\",\"#\"]}.">> + ] + }). + +all() -> + emqx_ct:all(?MODULE). + +groups() -> + []. + +init_per_suite(Config) -> + meck:new(emqx_schema, [non_strict, passthrough, no_history, no_link]), + meck:expect(emqx_schema, fields, fun("authorization") -> + meck:passthrough(["authorization"]) ++ + emqx_authz_schema:fields("authorization"); + (F) -> meck:passthrough([F]) + end), + + meck:new(emqx_resource, [non_strict, passthrough, no_history, no_link]), + meck:expect(emqx_resource, create, fun(_, _, _) -> {ok, meck_data} end), + meck:expect(emqx_resource, update, fun(_, _, _, _) -> {ok, meck_data} end), + meck:expect(emqx_resource, health_check, fun(_) -> ok end), + meck:expect(emqx_resource, remove, fun(_) -> ok end ), + + ok = emqx_config:init_load(emqx_authz_schema, ?CONF_DEFAULT), + + ok = emqx_ct_helpers:start_apps([emqx_authz, emqx_dashboard], fun set_special_configs/1), + {ok, _} = emqx:update_config([authorization, cache, enable], false), + {ok, _} = emqx:update_config([authorization, no_match], deny), + + Config. + +end_per_suite(_Config) -> + {ok, _} = emqx_authz:update(replace, []), + emqx_ct_helpers:stop_apps([emqx_resource, emqx_authz, emqx_dashboard]), + meck:unload(emqx_resource), + meck:unload(emqx_schema), + ok. + +set_special_configs(emqx_dashboard) -> + Config = #{ + default_username => <<"admin">>, + default_password => <<"public">>, + listeners => [#{ + protocol => http, + port => 18083 + }] + }, + emqx_config:put([emqx_dashboard], Config), + ok; +set_special_configs(emqx_authz) -> + emqx_config:put([authorization], #{sources => []}), + ok; +set_special_configs(_App) -> + ok. + +init_per_testcase(t_api, Config) -> + meck:new(emqx_rule_id, [non_strict, passthrough, no_history, no_link]), + meck:expect(emqx_rule_id, gen, fun() -> "fake" end), + + meck:new(emqx, [non_strict, passthrough, no_history, no_link]), + meck:expect(emqx, get_config, fun([node, data_dir]) -> + % emqx_ct_helpers:deps_path(emqx_authz, "test"); + {data_dir, Data} = lists:keyfind(data_dir, 1, Config), + Data; + (C) -> meck:passthrough([C]) + end), + Config; +init_per_testcase(_, Config) -> Config. + +end_per_testcase(t_api, _Config) -> + meck:unload(emqx_rule_id), + meck:unload(emqx), + ok; +end_per_testcase(_, _Config) -> ok. + +%%------------------------------------------------------------------------------ +%% Testcases +%%------------------------------------------------------------------------------ + +t_api(_) -> + {ok, 200, Result1} = request(get, uri(["authorization", "sources"]), []), + ?assertEqual([], get_sources(Result1)), + + {ok, 204, _} = request(put, uri(["authorization", "sources"]), [?SOURCE2, ?SOURCE3, ?SOURCE4, ?SOURCE5, ?SOURCE6]), + {ok, 204, _} = request(post, uri(["authorization", "sources"]), ?SOURCE1), + + {ok, 200, Result2} = request(get, uri(["authorization", "sources"]), []), + Sources = get_sources(Result2), + ?assertMatch([ #{<<"type">> := <<"http">>} + , #{<<"type">> := <<"mongo">>} + , #{<<"type">> := <<"mysql">>} + , #{<<"type">> := <<"pgsql">>} + , #{<<"type">> := <<"redis">>} + , #{<<"type">> := <<"file">>} + ], Sources), + ?assert(filelib:is_file(filename:join([emqx:get_config([node, data_dir]), "acl.conf"]))), + + {ok, 204, _} = request(put, uri(["authorization", "sources", "http"]), ?SOURCE1#{<<"enable">> := false}), + {ok, 200, Result3} = request(get, uri(["authorization", "sources", "http"]), []), + ?assertMatch(#{<<"type">> := <<"http">>, <<"enable">> := false}, jsx:decode(Result3)), + + {ok, 204, _} = request(put, uri(["authorization", "sources", "mongo"]), + ?SOURCE2#{<<"ssl">> := #{ + <<"enable">> => true, + <<"cacertfile">> => <<"fake cacert file">>, + <<"certfile">> => <<"fake cert file">>, + <<"keyfile">> => <<"fake key file">>, + <<"verify">> => false + }}), + {ok, 200, Result4} = request(get, uri(["authorization", "sources", "mongo"]), []), + ?assertMatch(#{<<"type">> := <<"mongo">>, + <<"ssl">> := #{<<"enable">> := true, + <<"cacertfile">> := <<"fake cacert file">>, + <<"certfile">> := <<"fake cert file">>, + <<"keyfile">> := <<"fake key file">>, + <<"verify">> := false + } + }, jsx:decode(Result4)), + ?assert(filelib:is_file(filename:join([emqx:get_config([node, data_dir]), "certs", "cacert-fake.pem"]))), + ?assert(filelib:is_file(filename:join([emqx:get_config([node, data_dir]), "certs", "cert-fake.pem"]))), + ?assert(filelib:is_file(filename:join([emqx:get_config([node, data_dir]), "certs", "key-fake.pem"]))), + + lists:foreach(fun(#{<<"type">> := Type}) -> + {ok, 204, _} = request(delete, uri(["authorization", "sources", binary_to_list(Type)]), []) + end, Sources), + {ok, 200, Result5} = request(get, uri(["authorization", "sources"]), []), + ?assertEqual([], get_sources(Result5)), + ok. + +t_move_source(_) -> + {ok, _} = emqx_authz:update(replace, [?SOURCE1, ?SOURCE2, ?SOURCE3, ?SOURCE4, ?SOURCE5]), + ?assertMatch([ #{type := http} + , #{type := mongo} + , #{type := mysql} + , #{type := pgsql} + , #{type := redis} + ], emqx_authz:lookup()), + + {ok, 204, _} = request(post, uri(["authorization", "sources", "pgsql", "move"]), + #{<<"position">> => <<"top">>}), + ?assertMatch([ #{type := pgsql} + , #{type := http} + , #{type := mongo} + , #{type := mysql} + , #{type := redis} + ], emqx_authz:lookup()), + + {ok, 204, _} = request(post, uri(["authorization", "sources", "http", "move"]), + #{<<"position">> => <<"bottom">>}), + ?assertMatch([ #{type := pgsql} + , #{type := mongo} + , #{type := mysql} + , #{type := redis} + , #{type := http} + ], emqx_authz:lookup()), + + {ok, 204, _} = request(post, uri(["authorization", "sources", "mysql", "move"]), + #{<<"position">> => #{<<"before">> => <<"pgsql">>}}), + ?assertMatch([ #{type := mysql} + , #{type := pgsql} + , #{type := mongo} + , #{type := redis} + , #{type := http} + ], emqx_authz:lookup()), + + {ok, 204, _} = request(post, uri(["authorization", "sources", "mongo", "move"]), + #{<<"position">> => #{<<"after">> => <<"http">>}}), + ?assertMatch([ #{type := mysql} + , #{type := pgsql} + , #{type := redis} + , #{type := http} + , #{type := mongo} + ], emqx_authz:lookup()), + + ok. + +%%-------------------------------------------------------------------- +%% HTTP Request +%%-------------------------------------------------------------------- + +request(Method, Url, Body) -> + Request = case Body of + [] -> {Url, [auth_header_()]}; + _ -> {Url, [auth_header_()], "application/json", jsx:encode(Body)} + end, + ct:pal("Method: ~p, Request: ~p", [Method, Request]), + case httpc:request(Method, Request, [], [{body_format, binary}]) of + {error, socket_closed_remotely} -> + {error, socket_closed_remotely}; + {ok, {{"HTTP/1.1", Code, _}, _Headers, Return} } -> + {ok, Code, Return}; + {ok, {Reason, _, _}} -> + {error, Reason} + end. + +uri() -> uri([]). +uri(Parts) when is_list(Parts) -> + NParts = [E || E <- Parts], + ?HOST ++ filename:join([?BASE_PATH, ?API_VERSION | NParts]). + +get_sources(Result) -> + maps:get(<<"sources">>, jsx:decode(Result), []). + +auth_header_() -> + Username = <<"admin">>, + Password = <<"public">>, + {ok, Token} = emqx_dashboard_admin:sign_token(Username, Password), + {"Authorization", "Bearer " ++ binary_to_list(Token)}. diff --git a/apps/emqx_authz/test/emqx_authz_http_SUITE.erl b/apps/emqx_authz/test/emqx_authz_http_SUITE.erl index fad5e9580..17763d993 100644 --- a/apps/emqx_authz/test/emqx_authz_http_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_http_SUITE.erl @@ -47,12 +47,11 @@ init_per_suite(Config) -> {ok, _} = emqx:update_config([authorization, cache, enable], false), {ok, _} = emqx:update_config([authorization, no_match], deny), Rules = [#{<<"type">> => <<"http">>, - <<"config">> => #{ - <<"url">> => <<"https://fake.com:443/">>, - <<"headers">> => #{}, - <<"method">> => <<"get">>, - <<"request_timeout">> => 5000 - }} + <<"url">> => <<"https://fake.com:443/">>, + <<"headers">> => #{}, + <<"method">> => <<"get">>, + <<"request_timeout">> => 5000 + } ], {ok, _} = emqx_authz:update(replace, Rules), Config. diff --git a/apps/emqx_authz/test/emqx_authz_mongo_SUITE.erl b/apps/emqx_authz/test/emqx_authz_mongo_SUITE.erl index db111ce83..8f4a6f29f 100644 --- a/apps/emqx_authz/test/emqx_authz_mongo_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_mongo_SUITE.erl @@ -47,12 +47,11 @@ init_per_suite(Config) -> {ok, _} = emqx:update_config([authorization, cache, enable], false), {ok, _} = emqx:update_config([authorization, no_match], deny), Rules = [#{<<"type">> => <<"mongo">>, - <<"config">> => #{ - <<"mongo_type">> => <<"single">>, - <<"server">> => <<"127.0.0.1:27017">>, - <<"pool_size">> => 1, - <<"database">> => <<"mqtt">>, - <<"ssl">> => #{<<"enable">> => false}}, + <<"mongo_type">> => <<"single">>, + <<"server">> => <<"127.0.0.1:27017">>, + <<"pool_size">> => 1, + <<"database">> => <<"mqtt">>, + <<"ssl">> => #{<<"enable">> => false}, <<"collection">> => <<"fake">>, <<"find">> => #{<<"a">> => <<"b">>} }], diff --git a/apps/emqx_authz/test/emqx_authz_mysql_SUITE.erl b/apps/emqx_authz/test/emqx_authz_mysql_SUITE.erl index 0675e1caf..1173b0e3e 100644 --- a/apps/emqx_authz/test/emqx_authz_mysql_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_mysql_SUITE.erl @@ -48,14 +48,13 @@ init_per_suite(Config) -> {ok, _} = emqx:update_config([authorization, cache, enable], false), {ok, _} = emqx:update_config([authorization, no_match], deny), Rules = [#{<<"type">> => <<"mysql">>, - <<"config">> => #{ - <<"server">> => <<"127.0.0.1:27017">>, - <<"pool_size">> => 1, - <<"database">> => <<"mqtt">>, - <<"username">> => <<"xx">>, - <<"password">> => <<"ee">>, - <<"auto_reconnect">> => true, - <<"ssl">> => #{<<"enable">> => false}}, + <<"server">> => <<"127.0.0.1:27017">>, + <<"pool_size">> => 1, + <<"database">> => <<"mqtt">>, + <<"username">> => <<"xx">>, + <<"password">> => <<"ee">>, + <<"auto_reconnect">> => true, + <<"ssl">> => #{<<"enable">> => false}, <<"sql">> => <<"abcb">> }], {ok, _} = emqx_authz:update(replace, Rules), diff --git a/apps/emqx_authz/test/emqx_authz_pgsql_SUITE.erl b/apps/emqx_authz/test/emqx_authz_pgsql_SUITE.erl index 6880ab405..24c2e7b35 100644 --- a/apps/emqx_authz/test/emqx_authz_pgsql_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_pgsql_SUITE.erl @@ -48,14 +48,13 @@ init_per_suite(Config) -> {ok, _} = emqx:update_config([authorization, cache, enable], false), {ok, _} = emqx:update_config([authorization, no_match], deny), Rules = [#{<<"type">> => <<"pgsql">>, - <<"config">> => #{ - <<"server">> => <<"127.0.0.1:27017">>, - <<"pool_size">> => 1, - <<"database">> => <<"mqtt">>, - <<"username">> => <<"xx">>, - <<"password">> => <<"ee">>, - <<"auto_reconnect">> => true, - <<"ssl">> => #{<<"enable">> => false}}, + <<"server">> => <<"127.0.0.1:27017">>, + <<"pool_size">> => 1, + <<"database">> => <<"mqtt">>, + <<"username">> => <<"xx">>, + <<"password">> => <<"ee">>, + <<"auto_reconnect">> => true, + <<"ssl">> => #{<<"enable">> => false}, <<"sql">> => <<"abcb">> }], {ok, _} = emqx_authz:update(replace, Rules), diff --git a/apps/emqx_authz/test/emqx_authz_redis_SUITE.erl b/apps/emqx_authz/test/emqx_authz_redis_SUITE.erl index 09682761d..9949e8b51 100644 --- a/apps/emqx_authz/test/emqx_authz_redis_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_redis_SUITE.erl @@ -47,13 +47,12 @@ init_per_suite(Config) -> {ok, _} = emqx:update_config([authorization, cache, enable], false), {ok, _} = emqx:update_config([authorization, no_match], deny), Rules = [#{<<"type">> => <<"redis">>, - <<"config">> => #{ - <<"server">> => <<"127.0.0.1:27017">>, - <<"pool_size">> => 1, - <<"database">> => 0, - <<"password">> => <<"ee">>, - <<"auto_reconnect">> => true, - <<"ssl">> => #{<<"enable">> => false}}, + <<"server">> => <<"127.0.0.1:27017">>, + <<"pool_size">> => 1, + <<"database">> => 0, + <<"password">> => <<"ee">>, + <<"auto_reconnect">> => true, + <<"ssl">> => #{<<"enable">> => false}, <<"cmd">> => <<"HGETALL mqtt_authz:%u">> }], {ok, _} = emqx_authz:update(replace, Rules), diff --git a/apps/emqx_auto_subscribe/etc/emqx_auto_subscribe.conf b/apps/emqx_auto_subscribe/etc/emqx_auto_subscribe.conf index c91b77aa1..f6d041dab 100644 --- a/apps/emqx_auto_subscribe/etc/emqx_auto_subscribe.conf +++ b/apps/emqx_auto_subscribe/etc/emqx_auto_subscribe.conf @@ -4,10 +4,12 @@ auto_subscribe { # { # topic = "/c/${clientid}", # qos = 0 - # }, + # rh = 0 + # rap = 0 + # nl = 0 + # } # { # topic = "/u/${username}", - # qos = 1 # }, # { # topic = "/h/${host}", @@ -15,15 +17,12 @@ auto_subscribe { # }, # { # topic = "/p/${port}", - # qos = 0 # }, # { # topic = "/topic/abc", - # qos = 0 # }, # { # topic = "/client/${clientid}/username/${username}/host/${host}/port/${port}", - # qos = 0 # } ] } diff --git a/apps/emqx_auto_subscribe/src/emqx_auto_subscribe_api.erl b/apps/emqx_auto_subscribe/src/emqx_auto_subscribe_api.erl index d55444dba..97c9674b9 100644 --- a/apps/emqx_auto_subscribe/src/emqx_auto_subscribe_api.erl +++ b/apps/emqx_auto_subscribe/src/emqx_auto_subscribe_api.erl @@ -23,6 +23,7 @@ -export([auto_subscribe/2]). -define(EXCEED_LIMIT, 'EXCEED_LIMIT'). +-define(BAD_REQUEST, 'BAD_REQUEST'). api_spec() -> {[auto_subscribe_api()], []}. @@ -30,7 +31,7 @@ api_spec() -> schema() -> emqx_mgmt_util:schema( emqx_mgmt_api_configs:gen_schema( - emqx:get_raw_config([auto_subscribe, topics]))). + emqx:get_raw_config([auto_subscribe, topics])), <<"">>). auto_subscribe_api() -> Metadata = #{ @@ -43,6 +44,8 @@ auto_subscribe_api() -> 'requestBody' => schema(), responses => #{ <<"200">> => schema(), + <<"400">> => emqx_mgmt_util:error_schema( + <<"Request body required">>, [?BAD_REQUEST]), <<"409">> => emqx_mgmt_util:error_schema( <<"Auto Subscribe topics max limit">>, [?EXCEED_LIMIT])}} }, @@ -53,6 +56,8 @@ auto_subscribe_api() -> auto_subscribe(get, _) -> {200, emqx_auto_subscribe:list()}; +auto_subscribe(put, #{body := #{}}) -> + {400, #{code => ?BAD_REQUEST, message => <<"Request body required">>}}; auto_subscribe(put, #{body := Params}) -> case emqx_auto_subscribe:update(Params) of {error, quota_exceeded} -> diff --git a/apps/emqx_auto_subscribe/src/emqx_auto_subscribe_placeholder.erl b/apps/emqx_auto_subscribe/src/emqx_auto_subscribe_placeholder.erl index cbe881bde..70779770d 100644 --- a/apps/emqx_auto_subscribe/src/emqx_auto_subscribe_placeholder.erl +++ b/apps/emqx_auto_subscribe/src/emqx_auto_subscribe_placeholder.erl @@ -23,16 +23,17 @@ generate(Topics) when is_list(Topics) -> [generate(Topic) || Topic <- Topics]; -generate(#{qos := Qos, topic := Topic}) when is_binary(Topic) -> - #{qos => Qos, placeholder => generate(Topic, [])}. +generate(T0 = #{topic := Topic}) -> + T = maps:without([topic], T0), + T#{placeholder => generate(Topic, [])}. -spec(to_topic_table(list(), map(), map()) -> list()). -to_topic_table(PlaceHolders, ClientInfo, ConnInfo) -> +to_topic_table(PHs, ClientInfo, ConnInfo) -> [begin Topic0 = to_topic(PlaceHolder, ClientInfo, ConnInfo, []), {Topic, Opts} = emqx_topic:parse(Topic0), - {Topic, Opts#{qos => Qos}} - end || #{qos := Qos, placeholder := PlaceHolder} <- PlaceHolders]. + {Topic, Opts#{qos => Qos, rh => RH, rap => RAP, nl => NL}} + end || #{qos := Qos, rh := RH, rap := RAP, nl := NL, placeholder := PlaceHolder} <- PHs]. %%-------------------------------------------------------------------- %% internal diff --git a/apps/emqx_auto_subscribe/src/emqx_auto_subscribe_schema.erl b/apps/emqx_auto_subscribe/src/emqx_auto_subscribe_schema.erl index 73ae262a1..5b781455d 100644 --- a/apps/emqx_auto_subscribe/src/emqx_auto_subscribe_schema.erl +++ b/apps/emqx_auto_subscribe/src/emqx_auto_subscribe_schema.erl @@ -19,16 +19,30 @@ -include_lib("typerefl/include/types.hrl"). --export([ roots/0 +-export([ namespace/0 + , roots/0 , fields/1]). +namespace() -> "auto_subscribe". + roots() -> ["auto_subscribe"]. fields("auto_subscribe") -> - [ {topics, hoconsc:array(hoconsc:ref(?MODULE, "topic"))}]; + [ {topics, hoconsc:array(hoconsc:ref(?MODULE, "topic"))} + ]; fields("topic") -> - [ {topic, emqx_schema:t(binary())} - , {qos, emqx_schema:t(integer(), undefined, 0)} + [ {topic, sc(binary(), #{})} + , {qos, sc(typerefl:union([0, 1, 2]), #{default => 0})} + , {rh, sc(typerefl:union([0, 1, 2]), #{default => 0})} + , {rap, sc(typerefl:union([0, 1]), #{default => 0})} + , {nl, sc(typerefl:union([0, 1]), #{default => 0})} ]. + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- + +sc(Type, Meta) -> + hoconsc:mk(Type, Meta). diff --git a/apps/emqx_data_bridge/.gitignore b/apps/emqx_bridge/.gitignore similarity index 100% rename from apps/emqx_data_bridge/.gitignore rename to apps/emqx_bridge/.gitignore diff --git a/apps/emqx_data_bridge/README.md b/apps/emqx_bridge/README.md similarity index 95% rename from apps/emqx_data_bridge/README.md rename to apps/emqx_bridge/README.md index 8f76f17a5..0f274eea1 100644 --- a/apps/emqx_data_bridge/README.md +++ b/apps/emqx_bridge/README.md @@ -1,4 +1,4 @@ -# emqx_data_bridge +# emqx_bridge EMQ X Data Bridge is an application that managing the resources (see emqx_resource) used by emqx rule engine. diff --git a/apps/emqx_bridge/etc/emqx_bridge.conf b/apps/emqx_bridge/etc/emqx_bridge.conf new file mode 100644 index 000000000..08873228d --- /dev/null +++ b/apps/emqx_bridge/etc/emqx_bridge.conf @@ -0,0 +1,49 @@ +##-------------------------------------------------------------------- +## EMQ X Bridge +##-------------------------------------------------------------------- + +#bridges.mqtt.my_mqtt_bridge { +# server = "127.0.0.1:1883" +# proto_ver = "v4" +# ## the clientid will be the concatenation of `clientid_prefix` and ids in `in` and `out`. +# clientid_prefix = "bridge_client:" +# username = "username1" +# password = "" +# clean_start = true +# keepalive = 300 +# retry_interval = "30s" +# max_inflight = 32 +# reconnect_interval = "30s" +# bridge_mode = true +# replayq { +# dir = "{{ platform_data_dir }}/replayq/bridge_mqtt/" +# seg_bytes = "100MB" +# offload = false +# max_total_bytes = "1GB" +# } +# ssl { +# enable = false +# keyfile = "{{ platform_etc_dir }}/certs/client-key.pem" +# certfile = "{{ platform_etc_dir }}/certs/client-cert.pem" +# cacertfile = "{{ platform_etc_dir }}/certs/cacert.pem" +# } +# ## we will create one MQTT connection for each element of the `in` +# in: [{ +# id = "pull_msgs_from_aws" +# subscribe_remote_topic = "aws/#" +# subscribe_qos = 1 +# local_topic = "from_aws/${topic}" +# payload = "${payload}" +# qos = "${qos}" +# retain = "${retain}" +# }] +# ## we will create one MQTT connection for each element of the `out` +# out: [{ +# id = "push_msgs_to_aws" +# subscribe_local_topic = "emqx/#" +# remote_topic = "from_emqx/${topic}" +# payload = "${payload}" +# qos = 1 +# retain = false +# }] +#} diff --git a/apps/emqx_data_bridge/rebar.config b/apps/emqx_bridge/rebar.config similarity index 73% rename from apps/emqx_data_bridge/rebar.config rename to apps/emqx_bridge/rebar.config index cf4cfcf1b..3fd6b41e0 100644 --- a/apps/emqx_data_bridge/rebar.config +++ b/apps/emqx_bridge/rebar.config @@ -3,5 +3,5 @@ {shell, [ % {config, "config/sys.config"}, - {apps, [emqx_data_bridge]} + {apps, [emqx_bridge]} ]}. diff --git a/apps/emqx_data_bridge/src/emqx_data_bridge.app.src b/apps/emqx_bridge/src/emqx_bridge.app.src similarity index 69% rename from apps/emqx_data_bridge/src/emqx_data_bridge.app.src rename to apps/emqx_bridge/src/emqx_bridge.app.src index 84486da19..9c0f6b779 100644 --- a/apps/emqx_data_bridge/src/emqx_data_bridge.app.src +++ b/apps/emqx_bridge/src/emqx_bridge.app.src @@ -1,12 +1,13 @@ -{application, emqx_data_bridge, +{application, emqx_bridge, [{description, "An OTP application"}, {vsn, "0.1.0"}, {registered, []}, - {mod, {emqx_data_bridge_app, []}}, + {mod, {emqx_bridge_app, []}}, {applications, [kernel, stdlib, - emqx + emqx, + emqx_connector ]}, {env,[]}, {modules, []}, diff --git a/apps/emqx_data_bridge/src/emqx_data_bridge.erl b/apps/emqx_bridge/src/emqx_bridge.erl similarity index 78% rename from apps/emqx_data_bridge/src/emqx_data_bridge.erl rename to apps/emqx_bridge/src/emqx_bridge.erl index 52cea80fb..75ebfac0c 100644 --- a/apps/emqx_data_bridge/src/emqx_data_bridge.erl +++ b/apps/emqx_bridge/src/emqx_bridge.erl @@ -13,7 +13,7 @@ %% See the License for the specific language governing permissions and %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_data_bridge). +-module(emqx_bridge). -export([ load_bridges/0 , resource_type/1 @@ -27,15 +27,17 @@ ]). load_bridges() -> - Bridges = emqx:get_config([emqx_data_bridge, bridges], []), - emqx_data_bridge_monitor:ensure_all_started(Bridges). + Bridges = emqx:get_config([bridges], #{}), + emqx_bridge_monitor:ensure_all_started(Bridges). +resource_type(mqtt) -> emqx_connector_mqtt; resource_type(mysql) -> emqx_connector_mysql; resource_type(pgsql) -> emqx_connector_pgsql; resource_type(mongo) -> emqx_connector_mongo; resource_type(redis) -> emqx_connector_redis; resource_type(ldap) -> emqx_connector_ldap. +bridge_type(emqx_connector_mqtt) -> mqtt; bridge_type(emqx_connector_mysql) -> mysql; bridge_type(emqx_connector_pgsql) -> pgsql; bridge_type(emqx_connector_mongo) -> mongo; @@ -43,13 +45,14 @@ bridge_type(emqx_connector_redis) -> redis; bridge_type(emqx_connector_ldap) -> ldap. name_to_resource_id(BridgeName) -> - <<"bridge:", BridgeName/binary>>. + Name = bin(BridgeName), + <<"bridge:", Name/binary>>. resource_id_to_name(<<"bridge:", BridgeName/binary>> = _ResourceId) -> BridgeName. list_bridges() -> - emqx_resource_api:list_instances(fun emqx_data_bridge:is_bridge/1). + emqx_resource_api:list_instances(fun emqx_bridge:is_bridge/1). is_bridge(#{id := <<"bridge:", _/binary>>}) -> true; @@ -57,7 +60,11 @@ is_bridge(_Data) -> false. config_key_path() -> - [emqx_data_bridge, bridges]. + [emqx_bridge, bridges]. update_config(ConfigReq) -> emqx:update_config(config_key_path(), ConfigReq). + +bin(Bin) when is_binary(Bin) -> Bin; +bin(Str) when is_list(Str) -> list_to_binary(Str); +bin(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8). diff --git a/apps/emqx_data_bridge/src/emqx_data_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl similarity index 83% rename from apps/emqx_data_bridge/src/emqx_data_bridge_api.erl rename to apps/emqx_bridge/src/emqx_bridge_api.erl index 6fe75e4ce..c10875e55 100644 --- a/apps/emqx_data_bridge/src/emqx_data_bridge_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_api.erl @@ -13,7 +13,7 @@ %% See the License for the specific language governing permissions and %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_data_bridge_api). +-module(emqx_bridge_api). -rest_api(#{ name => list_data_bridges , method => 'GET' @@ -61,10 +61,10 @@ list_bridges(_Binding, _Params) -> {200, #{code => 0, data => [format_api_reply(Data) || - Data <- emqx_data_bridge:list_bridges()]}}. + Data <- emqx_bridge:list_bridges()]}}. get_bridge(#{name := Name}, _Params) -> - case emqx_resource:get_instance(emqx_data_bridge:name_to_resource_id(Name)) of + case emqx_resource:get_instance(emqx_bridge:name_to_resource_id(Name)) of {ok, Data} -> {200, #{code => 0, data => format_api_reply(emqx_resource_api:format_data(Data))}}; {error, not_found} -> @@ -75,8 +75,8 @@ create_bridge(#{name := Name}, Params) -> Config = proplists:get_value(<<"config">>, Params), BridgeType = proplists:get_value(<<"type">>, Params), case emqx_resource:check_and_create( - emqx_data_bridge:name_to_resource_id(Name), - emqx_data_bridge:resource_type(atom(BridgeType)), maps:from_list(Config)) of + emqx_bridge:name_to_resource_id(Name), + emqx_bridge:resource_type(atom(BridgeType)), maps:from_list(Config)) of {ok, already_created} -> {400, #{code => 102, message => <<"bridge already created: ", Name/binary>>}}; {ok, Data} -> @@ -91,8 +91,8 @@ update_bridge(#{name := Name}, Params) -> Config = proplists:get_value(<<"config">>, Params), BridgeType = proplists:get_value(<<"type">>, Params), case emqx_resource:check_and_update( - emqx_data_bridge:name_to_resource_id(Name), - emqx_data_bridge:resource_type(atom(BridgeType)), maps:from_list(Config), []) of + emqx_bridge:name_to_resource_id(Name), + emqx_bridge:resource_type(atom(BridgeType)), maps:from_list(Config), []) of {ok, Data} -> update_config_and_reply(Name, BridgeType, Config, Data); {error, not_found} -> @@ -104,26 +104,26 @@ update_bridge(#{name := Name}, Params) -> end. delete_bridge(#{name := Name}, _Params) -> - case emqx_resource:remove(emqx_data_bridge:name_to_resource_id(Name)) of + case emqx_resource:remove(emqx_bridge:name_to_resource_id(Name)) of ok -> delete_config_and_reply(Name); {error, Reason} -> {500, #{code => 102, message => emqx_resource_api:stringnify(Reason)}} end. format_api_reply(#{resource_type := Type, id := Id, config := Conf, status := Status}) -> - #{type => emqx_data_bridge:bridge_type(Type), - name => emqx_data_bridge:resource_id_to_name(Id), + #{type => emqx_bridge:bridge_type(Type), + name => emqx_bridge:resource_id_to_name(Id), config => Conf, status => Status}. % format_conf(#{resource_type := Type, id := Id, config := Conf}) -> -% #{type => Type, name => emqx_data_bridge:resource_id_to_name(Id), +% #{type => Type, name => emqx_bridge:resource_id_to_name(Id), % config => Conf}. % get_all_configs() -> -% [format_conf(Data) || Data <- emqx_data_bridge:list_bridges()]. +% [format_conf(Data) || Data <- emqx_bridge:list_bridges()]. update_config_and_reply(Name, BridgeType, Config, Data) -> - case emqx_data_bridge:update_config({update, ?BRIDGE(Name, BridgeType, Config)}) of + case emqx_bridge:update_config({update, ?BRIDGE(Name, BridgeType, Config)}) of {ok, _} -> {200, #{code => 0, data => format_api_reply( emqx_resource_api:format_data(Data))}}; @@ -132,7 +132,7 @@ update_config_and_reply(Name, BridgeType, Config, Data) -> end. delete_config_and_reply(Name) -> - case emqx_data_bridge:update_config({delete, Name}) of + case emqx_bridge:update_config({delete, Name}) of {ok, _} -> {200, #{code => 0, data => #{}}}; {error, Reason} -> {500, #{code => 102, message => emqx_resource_api:stringnify(Reason)}} diff --git a/apps/emqx_data_bridge/src/emqx_data_bridge_app.erl b/apps/emqx_bridge/src/emqx_bridge_app.erl similarity index 87% rename from apps/emqx_data_bridge/src/emqx_data_bridge_app.erl rename to apps/emqx_bridge/src/emqx_bridge_app.erl index 859952480..cfefe118f 100644 --- a/apps/emqx_data_bridge/src/emqx_data_bridge_app.erl +++ b/apps/emqx_bridge/src/emqx_bridge_app.erl @@ -13,7 +13,7 @@ %% See the License for the specific language governing permissions and %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_data_bridge_app). +-module(emqx_bridge_app). -behaviour(application). @@ -22,9 +22,9 @@ -export([start/2, stop/1, pre_config_update/2]). start(_StartType, _StartArgs) -> - {ok, Sup} = emqx_data_bridge_sup:start_link(), - ok = emqx_data_bridge:load_bridges(), - emqx_config_handler:add_handler(emqx_data_bridge:config_key_path(), ?MODULE), + {ok, Sup} = emqx_bridge_sup:start_link(), + ok = emqx_bridge:load_bridges(), + emqx_config_handler:add_handler(emqx_bridge:config_key_path(), ?MODULE), {ok, Sup}. stop(_State) -> diff --git a/apps/emqx_data_bridge/src/emqx_data_bridge_monitor.erl b/apps/emqx_bridge/src/emqx_bridge_monitor.erl similarity index 84% rename from apps/emqx_data_bridge/src/emqx_data_bridge_monitor.erl rename to apps/emqx_bridge/src/emqx_bridge_monitor.erl index 4917833ec..d76af5fb9 100644 --- a/apps/emqx_data_bridge/src/emqx_data_bridge_monitor.erl +++ b/apps/emqx_bridge/src/emqx_bridge_monitor.erl @@ -15,7 +15,7 @@ %%-------------------------------------------------------------------- %% This process monitors all the data bridges, and try to restart a bridge %% when one of it stopped. --module(emqx_data_bridge_monitor). +-module(emqx_bridge_monitor). -behaviour(gen_server). @@ -65,14 +65,18 @@ code_change(_OldVsn, State, _Extra) -> %%============================================================================ load_bridges(Configs) -> - lists:foreach(fun load_bridge/1, Configs). + lists:foreach(fun({Type, NamedConf}) -> + lists:foreach(fun({Name, Conf}) -> + load_bridge(Name, Type, Conf) + end, maps:to_list(NamedConf)) + end, maps:to_list(Configs)). %% TODO: move this monitor into emqx_resource %% emqx_resource:check_and_create_local(ResourceId, ResourceType, Config, #{keep_retry => true}). -load_bridge(#{name := Name, type := Type, config := Config}) -> +load_bridge(Name, Type, Config) -> case emqx_resource:create_local( - emqx_data_bridge:name_to_resource_id(Name), - emqx_data_bridge:resource_type(Type), Config) of + emqx_bridge:name_to_resource_id(Name), + emqx_bridge:resource_type(Type), Config) of {ok, already_created} -> ok; {ok, _} -> ok; {error, Reason} -> diff --git a/apps/emqx_bridge/src/emqx_bridge_schema.erl b/apps/emqx_bridge/src/emqx_bridge_schema.erl new file mode 100644 index 000000000..beb0f282c --- /dev/null +++ b/apps/emqx_bridge/src/emqx_bridge_schema.erl @@ -0,0 +1,17 @@ +-module(emqx_bridge_schema). + +-export([roots/0, fields/1]). + +%%====================================================================================== +%% Hocon Schema Definitions + +roots() -> ["bridges"]. + +fields("bridges") -> + [{mqtt, hoconsc:ref(?MODULE, "mqtt")}]; + +fields("mqtt") -> + [{"$name", hoconsc:ref(?MODULE, "mqtt_bridge")}]; + +fields("mqtt_bridge") -> + emqx_connector_mqtt:fields("config"). diff --git a/apps/emqx_data_bridge/src/emqx_data_bridge_sup.erl b/apps/emqx_bridge/src/emqx_bridge_sup.erl similarity index 86% rename from apps/emqx_data_bridge/src/emqx_data_bridge_sup.erl rename to apps/emqx_bridge/src/emqx_bridge_sup.erl index a699a72a0..fd12b1a99 100644 --- a/apps/emqx_data_bridge/src/emqx_data_bridge_sup.erl +++ b/apps/emqx_bridge/src/emqx_bridge_sup.erl @@ -13,7 +13,7 @@ %% See the License for the specific language governing permissions and %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_data_bridge_sup). +-module(emqx_bridge_sup). -behaviour(supervisor). @@ -31,11 +31,11 @@ init([]) -> intensity => 10, period => 10}, ChildSpecs = [ - #{id => emqx_data_bridge_monitor, - start => {emqx_data_bridge_monitor, start_link, []}, + #{id => emqx_bridge_monitor, + start => {emqx_bridge_monitor, start_link, []}, restart => permanent, type => worker, - modules => [emqx_data_bridge_monitor]} + modules => [emqx_bridge_monitor]} ], {ok, {SupFlags, ChildSpecs}}. diff --git a/apps/emqx_bridge_mqtt/.gitignore b/apps/emqx_bridge_mqtt/.gitignore deleted file mode 100644 index bf9523be5..000000000 --- a/apps/emqx_bridge_mqtt/.gitignore +++ /dev/null @@ -1,21 +0,0 @@ -.eunit -deps -*.o -*.beam -*.plt -erl_crash.dump -ebin/*.beam -rel -_build -.concrete/DEV_MODE -.rebar -.erlang.mk -data -ebin -emqx_bridge_mqtt.d -*.rendered -.rebar3/ -*.coverdata -rebar.lock -.DS_Store -Mnesia.*/ \ No newline at end of file diff --git a/apps/emqx_bridge_mqtt/README.md b/apps/emqx_bridge_mqtt/README.md deleted file mode 100644 index 812645627..000000000 --- a/apps/emqx_bridge_mqtt/README.md +++ /dev/null @@ -1,265 +0,0 @@ -# EMQ Bridge MQTT - -The concept of **Bridge** means that EMQ X supports forwarding messages -of one of its own topics to another MQTT Broker in some way. - -**Bridge** differs from **Cluster** in that the bridge does not -replicate the topic trie and routing tables and only forwards MQTT -messages based on bridging rules. - -At present, the bridging methods supported by EMQ X are as follows: - -- RPC bridge: RPC Bridge only supports message forwarding and does not - support subscribing to the topic of remote nodes to synchronize - data; -- MQTT Bridge: MQTT Bridge supports both forwarding and data - synchronization through subscription topic. - -These concepts are shown below: - -![bridge](docs/images/bridge.png) - -In addition, the EMQ X message broker supports multi-node bridge mode interconnection - -``` - --------- --------- --------- -Publisher --> | Node1 | --Bridge Forward--> | Node2 | --Bridge Forward--> | Node3 | --> Subscriber - --------- --------- --------- -``` - -In EMQ X, bridge is configured by modifying `etc/emqx.conf`. EMQ X distinguishes between different bridges based on different names. E.g - -``` -## Bridge address: node name for local bridge, host:port for remote. -bridge.mqtt.aws.address = 127.0.0.1:1883 -``` - -This configuration declares a bridge named `aws` and specifies that it is bridged to the MQTT broker of 127.0.0.1:1883 by MQTT mode. - -In case of creating multiple bridges, it is convenient to replicate all configuration items of the first bridge, and modify the bridge name and other configuration items if necessary (such as bridge.$name.address, where $name refers to the name of bridge) - -The next two sections describe how to create a bridge in RPC and MQTT mode respectively and create a forwarding rule that forwards the messages from sensors. Assuming that two EMQ X nodes are running on two hosts: - - -| Name | Node | MQTT Port | -|------|-------------------|-----------| -| emqx1| emqx1@192.168.1.1.| 1883 | -| emqx2| emqx2@192.168.1.2 | 1883 | - - -## EMQ X RPC Bridge Configuration - -The following is the basic configuration of RPC bridging. A simplest RPC bridging only requires the following three items - -``` -## Bridge Address: Use node name (nodename@host) for rpc bridging, and host:port for mqtt connection -bridge.mqtt.emqx2.address = "emqx2@192.168.1.2" - -## Forwarding topics of the message -bridge.mqtt.emqx2.forwards = "sensor1/#,sensor2/#" - -## bridged mountpoint -bridge.mqtt.emqx2.mountpoint = "bridge/emqx2/${node}/" -``` - -If the messages received by the local node emqx1 matches the topic `sersor1/#` or `sensor2/#`, these messages will be forwarded to the `sensor1/#` or `sensor2/#` topic of the remote node emqx2. - -`forwards` is used to specify topics. Messages of the in `forwards` specified topics on local node are forwarded to the remote node. - -`mountpoint` is used to add a topic prefix when forwarding a message. To use `mountpoint`, the `forwards` directive must be set. In the above example, a message with the topic `sensor1/hello` received by the local node will be forwarded to the remote node with the topic `bridge/emqx2/emqx1@192.168.1.1/sensor1/hello`. - -Limitations of RPC bridging: - -1. The RPC bridge of emqx can only forward local messages to the remote node, and cannot synchronize the messages of the remote node to the local node; - -2. RPC bridge can only bridge two EMQ X broker together and cannot bridge EMQ X broker to other MQTT brokers. - -## EMQ X MQTT Bridge Configuration - -EMQ X 3.0 officially introduced MQTT bridge, so that EMQ X can bridge any MQTT broker. Because of the characteristics of the MQTT protocol, EMQ X can subscribe to the remote mqtt broker's topic through MQTT bridge, and then synchronize the remote MQTT broker's message to the local. - -EMQ X MQTT bridging principle: Create an MQTT client on the EMQ X broker, and connect this MQTT client to the remote MQTT broker. Therefore, in the MQTT bridge configuration, following fields may be set for the EMQ X to connect to the remote broker as an mqtt client - -``` -## Bridge Address: Use node name for rpc bridging, use host:port for mqtt connection -bridge.mqtt.emqx2.address = "192.168.1.2:1883" - -## Bridged Protocol Version -## Enumeration value: mqttv3 | mqttv4 | mqttv5 -bridge.mqtt.emqx2.proto_ver = "mqttv4" - -## mqtt client's clientid -bridge.mqtt.emqx2.clientid = "bridge_emq" - -## mqtt client's clean_start field -## Note: Some MQTT Brokers need to set the clean_start value as `true` -bridge.mqtt.emqx2.clean_start = true - -## mqtt client's username field -bridge.mqtt.emqx2.username = "user" - -## mqtt client's password field -bridge.mqtt.emqx2.password = "passwd" - -## Whether the mqtt client uses ssl to connect to a remote serve or not -bridge.mqtt.emqx2.ssl = off - -## CA Certificate of Client SSL Connection (PEM format) -bridge.mqtt.emqx2.cacertfile = "etc/certs/cacert.pem" - -## SSL certificate of Client SSL connection -bridge.mqtt.emqx2.certfile = "etc/certs/client-cert.pem" - -## Key file of Client SSL connection -bridge.mqtt.emqx2.keyfile = "etc/certs/client-key.pem" - -## SSL encryption -bridge.mqtt.emqx2.ciphers = "ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384" - -## TTLS PSK password -## Note 'listener.ssl.external.ciphers' and 'listener.ssl.external.psk_ciphers' cannot be configured at the same time -## -## See 'https://tools.ietf.org/html/rfc4279#section-2'. -## bridge.mqtt.emqx2.psk_ciphers = "PSK-AES128-CBC-SHA,PSK-AES256-CBC-SHA,PSK-3DES-EDE-CBC-SHA,PSK-RC4-SHA" - -## Client's heartbeat interval -bridge.mqtt.emqx2.keepalive = 60s - -## Supported TLS version -bridge.mqtt.emqx2.tls_versions = "tlsv1.3,tlsv1.2,tlsv1.1,tlsv1" - -## Forwarding topics of the message -bridge.mqtt.emqx2.forwards = "sensor1/#,sensor2/#" - -## Bridged mountpoint -bridge.mqtt.emqx2.mountpoint = "bridge/emqx2/${node}/" - -## Subscription topic for bridging -bridge.mqtt.emqx2.subscription.1.topic = "cmd/topic1" - -## Subscription qos for bridging -bridge.mqtt.emqx2.subscription.1.qos = 1 - -## Subscription topic for bridging -bridge.mqtt.emqx2.subscription.2.topic = "cmd/topic2" - -## Subscription qos for bridging -bridge.mqtt.emqx2.subscription.2.qos = 1 - -## Bridging reconnection interval -## Default: 30s -bridge.mqtt.emqx2.reconnect_interval = 30s - -## QoS1 message retransmission interval -bridge.mqtt.emqx2.retry_interval = 20s - -## Inflight Size. -bridge.mqtt.emqx2.max_inflight_batches = 32 -``` - -## Bridge Cache Configuration - -The bridge of EMQ X has a message caching mechanism. The caching mechanism is applicable to both RPC bridging and MQTT bridging. When the bridge is disconnected (such as when the network connection is unstable), the messages with a topic specified in `forwards` can be cached to the local message queue. Until the bridge is restored, these messages are re-forwarded to the remote node. The configuration of the cache queue is as follows - -``` -## emqx_bridge internal number of messages used for batch -bridge.mqtt.emqx2.queue.batch_count_limit = 32 - -## emqx_bridge internal number of message bytes used for batch -bridge.mqtt.emqx2.queue.batch_bytes_limit = 1000MB - -## The path for placing replayq queue. If it is not specified, then replayq will run in `mem-only` mode and messages will not be cached on disk. -bridge.mqtt.emqx2.queue.replayq_dir = data/emqx_emqx2_bridge/ - -## Replayq data segment size -bridge.mqtt.emqx2.queue.replayq_seg_bytes = 10MB -``` - -`bridge.mqtt.emqx2.queue.replayq_dir` is a configuration parameter for specifying the path of the bridge storage queue. - -`bridge.mqtt.emqx2.queue.replayq_seg_bytes` is used to specify the size of the largest single file of the message queue that is cached on disk. If the message queue size exceeds the specified value, a new file is created to store the message queue. - -## CLI for EMQ X Bridge MQTT - -CLI for EMQ X Bridge MQTT: - -``` bash -$ cd emqx1/ && ./bin/emqx_ctl bridges -bridges list # List bridges -bridges start # Start a bridge -bridges stop # Stop a bridge -bridges forwards # Show a bridge forward topic -bridges add-forward # Add bridge forward topic -bridges del-forward # Delete bridge forward topic -bridges subscriptions # Show a bridge subscriptions topic -bridges add-subscription # Add bridge subscriptions topic -``` - -List all bridge states - -``` bash -$ ./bin/emqx_ctl bridges list -name: emqx status: Stopped $ ./bin/emqx_ctl bridges list -name: emqx status: Stopped -``` - -Start the specified bridge - -``` bash -$ ./bin/emqx_ctl bridges start emqx -Start bridge successfully. -``` - -Stop the specified bridge - -``` bash -$ ./bin/emqx_ctl bridges stop emqx -Stop bridge successfully. -``` -List the forwarding topics for the specified bridge - -``` bash -$ ./bin/emqx_ctl bridges forwards emqx -topic: topic1/# -topic: topic2/# -``` - -Add a forwarding topic for the specified bridge - -``` bash -$ ./bin/emqx_ctl bridges add-forwards emqx topic3/# -Add-forward topic successfully. -``` - -Delete the forwarding topic for the specified bridge - - -``` bash -$ ./bin/emqx_ctl bridges del-forwards emqx topic3/# -Del-forward topic successfully. -``` - -List subscriptions for the specified bridge - -``` bash -$ ./bin/emqx_ctl bridges subscriptions emqx -topic: cmd/topic1, qos: 1 -topic: cmd/topic2, qos: 1 -``` - -Add a subscription topic for the specified bridge - -``` bash -$ ./bin/emqx_ctl bridges add-subscription emqx cmd/topic3 1 -Add-subscription topic successfully. -``` - -Delete the subscription topic for the specified bridge - -``` bash -$ ./bin/emqx_ctl bridges del-subscription emqx cmd/topic3 -Del-subscription topic successfully. -``` - -Note: In case of creating multiple bridges, it is convenient to replicate all configuration items of the first bridge, and modify the bridge name and other configuration items if necessary. - diff --git a/apps/emqx_bridge_mqtt/docs/guide.rst b/apps/emqx_bridge_mqtt/docs/guide.rst deleted file mode 100644 index a1c2a9126..000000000 --- a/apps/emqx_bridge_mqtt/docs/guide.rst +++ /dev/null @@ -1,286 +0,0 @@ - -EMQ Bridge MQTT -=============== - -The concept of **Bridge** means that EMQ X supports forwarding messages -of one of its own topics to another MQTT Broker in some way. - -**Bridge** differs from **Cluster** in that the bridge does not -replicate the topic trie and routing tables and only forwards MQTT -messages based on bridging rules. - -At present, the bridging methods supported by EMQ X are as follows: - - -* RPC bridge: RPC Bridge only supports message forwarding and does not - support subscribing to the topic of remote nodes to synchronize - data; -* MQTT Bridge: MQTT Bridge supports both forwarding and data - synchronization through subscription topic. - -These concepts are shown below: - - -.. image:: images/bridge.png - :target: images/bridge.png - :alt: bridge - - -In addition, the EMQ X message broker supports multi-node bridge mode interconnection - -.. code-block:: - - --------- --------- --------- - Publisher --> | Node1 | --Bridge Forward--> | Node2 | --Bridge Forward--> | Node3 | --> Subscriber - --------- --------- --------- - -In EMQ X, bridge is configured by modifying ``etc/emqx.conf``. EMQ X distinguishes between different bridges based on different names. E.g - -.. code-block:: - - ## Bridge address: node name for local bridge, host:port for remote. - bridge.mqtt.aws.address = "127.0.0.1:1883" - -This configuration declares a bridge named ``aws`` and specifies that it is bridged to the MQTT broker of 127.0.0.1:1883 by MQTT mode. - -In case of creating multiple bridges, it is convenient to replicate all configuration items of the first bridge, and modify the bridge name and other configuration items if necessary (such as bridge.$name.address, where $name refers to the name of bridge) - -The next two sections describe how to create a bridge in RPC and MQTT mode respectively and create a forwarding rule that forwards the messages from sensors. Assuming that two EMQ X nodes are running on two hosts: - -.. list-table:: - :header-rows: 1 - - * - Name - - Node - - MQTT Port - * - emqx1 - - emqx1@192.168.1.1. - - 1883 - * - emqx2 - - emqx2@192.168.1.2 - - 1883 - - -EMQ X RPC Bridge Configuration ------------------------------- - -The following is the basic configuration of RPC bridging. A simplest RPC bridging only requires the following three items - -.. code-block:: - - ## Bridge Address: Use node name (nodename@host) for rpc bridging, and host:port for mqtt connection - bridge.mqtt.emqx2.address = "emqx2@192.168.1.2" - - ## Forwarding topics of the message - bridge.mqtt.emqx2.forwards = "sensor1/#,sensor2/#" - - ## bridged mountpoint - bridge.mqtt.emqx2.mountpoint = "bridge/emqx2/${node}/" - -If the messages received by the local node emqx1 matches the topic ``sersor1/#`` or ``sensor2/#``\ , these messages will be forwarded to the ``sensor1/#`` or ``sensor2/#`` topic of the remote node emqx2. - -``forwards`` is used to specify topics. Messages of the in ``forwards`` specified topics on local node are forwarded to the remote node. - -``mountpoint`` is used to add a topic prefix when forwarding a message. To use ``mountpoint``\ , the ``forwards`` directive must be set. In the above example, a message with the topic ``sensor1/hello`` received by the local node will be forwarded to the remote node with the topic ``bridge/emqx2/emqx1@192.168.1.1/sensor1/hello``. - -Limitations of RPC bridging: - - -#. - The RPC bridge of emqx can only forward local messages to the remote node, and cannot synchronize the messages of the remote node to the local node; - -#. - RPC bridge can only bridge two EMQ X broker together and cannot bridge EMQ X broker to other MQTT brokers. - -EMQ X MQTT Bridge Configuration -------------------------------- - -EMQ X 3.0 officially introduced MQTT bridge, so that EMQ X can bridge any MQTT broker. Because of the characteristics of the MQTT protocol, EMQ X can subscribe to the remote mqtt broker's topic through MQTT bridge, and then synchronize the remote MQTT broker's message to the local. - -EMQ X MQTT bridging principle: Create an MQTT client on the EMQ X broker, and connect this MQTT client to the remote MQTT broker. Therefore, in the MQTT bridge configuration, following fields may be set for the EMQ X to connect to the remote broker as an mqtt client - -.. code-block:: - - ## Bridge Address: Use node name for rpc bridging, use host:port for mqtt connection - bridge.mqtt.emqx2.address = "192.168.1.2:1883" - - ## Bridged Protocol Version - ## Enumeration value: mqttv3 | mqttv4 | mqttv5 - bridge.mqtt.emqx2.proto_ver = "mqttv4" - - ## mqtt client's clientid - bridge.mqtt.emqx2.clientid = "bridge_emq" - - ## mqtt client's clean_start field - ## Note: Some MQTT Brokers need to set the clean_start value as `true` - bridge.mqtt.emqx2.clean_start = true - - ## mqtt client's username field - bridge.mqtt.emqx2.username = "user" - - ## mqtt client's password field - bridge.mqtt.emqx2.password = "passwd" - - ## Whether the mqtt client uses ssl to connect to a remote serve or not - bridge.mqtt.emqx2.ssl = off - - ## CA Certificate of Client SSL Connection (PEM format) - bridge.mqtt.emqx2.cacertfile = "etc/certs/cacert.pem" - - ## SSL certificate of Client SSL connection - bridge.mqtt.emqx2.certfile = "etc/certs/client-cert.pem" - - ## Key file of Client SSL connection - bridge.mqtt.emqx2.keyfile = "etc/certs/client-key.pem" - - ## TTLS PSK password - ## Note 'listener.ssl.external.ciphers' and 'listener.ssl.external.psk_ciphers' cannot be configured at the same time - ## - ## See 'https://tools.ietf.org/html/rfc4279#section-2'. - ## bridge.mqtt.emqx2.psk_ciphers = "PSK-AES128-CBC-SHA,PSK-AES256-CBC-SHA,PSK-3DES-EDE-CBC-SHA,PSK-RC4-SHA" - - ## Client's heartbeat interval - bridge.mqtt.emqx2.keepalive = 60s - - ## Supported TLS version - bridge.mqtt.emqx2.tls_versions = "tlsv1.2" - - ## SSL encryption - bridge.mqtt.emqx2.ciphers = "ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384" - - ## Forwarding topics of the message - bridge.mqtt.emqx2.forwards = "sensor1/#,sensor2/#" - - ## Bridged mountpoint - bridge.mqtt.emqx2.mountpoint = "bridge/emqx2/${node}/" - - ## Subscription topic for bridging - bridge.mqtt.emqx2.subscription.1.topic = "cmd/topic1" - - ## Subscription qos for bridging - bridge.mqtt.emqx2.subscription.1.qos = 1 - - ## Subscription topic for bridging - bridge.mqtt.emqx2.subscription.2.topic = "cmd/topic2" - - ## Subscription qos for bridging - bridge.mqtt.emqx2.subscription.2.qos = 1 - - ## Bridging reconnection interval - ## Default: 30s - bridge.mqtt.emqx2.reconnect_interval = 30s - - ## QoS1 message retransmission interval - bridge.mqtt.emqx2.retry_interval = 20s - - ## Inflight Size. - bridge.mqtt.emqx2.max_inflight_batches = 32 - -Bridge Cache Configuration --------------------------- - -The bridge of EMQ X has a message caching mechanism. The caching mechanism is applicable to both RPC bridging and MQTT bridging. When the bridge is disconnected (such as when the network connection is unstable), the messages with a topic specified in ``forwards`` can be cached to the local message queue. Until the bridge is restored, these messages are re-forwarded to the remote node. The configuration of the cache queue is as follows - -.. code-block:: - - ## emqx_bridge internal number of messages used for batch - bridge.mqtt.emqx2.queue.batch_count_limit = 32 - - ## emqx_bridge internal number of message bytes used for batch - bridge.mqtt.emqx2.queue.batch_bytes_limit = 1000MB - - ## The path for placing replayq queue. If it is not specified, then replayq will run in `mem-only` mode and messages will not be cached on disk. - bridge.mqtt.emqx2.queue.replayq_dir = "data/emqx_emqx2_bridge/" - - ## Replayq data segment size - bridge.mqtt.emqx2.queue.replayq_seg_bytes = 10MB - -``bridge.mqtt.emqx2.queue.replayq_dir`` is a configuration parameter for specifying the path of the bridge storage queue. - -``bridge.mqtt.emqx2.queue.replayq_seg_bytes`` is used to specify the size of the largest single file of the message queue that is cached on disk. If the message queue size exceeds the specified value, a new file is created to store the message queue. - -CLI for EMQ X Bridge MQTT -------------------------- - -CLI for EMQ X Bridge MQTT: - -.. code-block:: bash - - $ cd emqx1/ && ./bin/emqx_ctl bridges - bridges list # List bridges - bridges start # Start a bridge - bridges stop # Stop a bridge - bridges forwards # Show a bridge forward topic - bridges add-forward # Add bridge forward topic - bridges del-forward # Delete bridge forward topic - bridges subscriptions # Show a bridge subscriptions topic - bridges add-subscription # Add bridge subscriptions topic - -List all bridge states - -.. code-block:: bash - - $ ./bin/emqx_ctl bridges list - name: emqx status: Stopped $ ./bin/emqx_ctl bridges list - name: emqx status: Stopped - -Start the specified bridge - -.. code-block:: bash - - $ ./bin/emqx_ctl bridges start emqx - Start bridge successfully. - -Stop the specified bridge - -.. code-block:: bash - - $ ./bin/emqx_ctl bridges stop emqx - Stop bridge successfully. - -List the forwarding topics for the specified bridge - -.. code-block:: bash - - $ ./bin/emqx_ctl bridges forwards emqx - topic: topic1/# - topic: topic2/# - -Add a forwarding topic for the specified bridge - -.. code-block:: bash - - $ ./bin/emqx_ctl bridges add-forwards emqx topic3/# - Add-forward topic successfully. - -Delete the forwarding topic for the specified bridge - -.. code-block:: bash - - $ ./bin/emqx_ctl bridges del-forwards emqx topic3/# - Del-forward topic successfully. - -List subscriptions for the specified bridge - -.. code-block:: bash - - $ ./bin/emqx_ctl bridges subscriptions emqx - topic: cmd/topic1, qos: 1 - topic: cmd/topic2, qos: 1 - -Add a subscription topic for the specified bridge - -.. code-block:: bash - - $ ./bin/emqx_ctl bridges add-subscription emqx cmd/topic3 1 - Add-subscription topic successfully. - -Delete the subscription topic for the specified bridge - -.. code-block:: bash - - $ ./bin/emqx_ctl bridges del-subscription emqx cmd/topic3 - Del-subscription topic successfully. - -Note: In case of creating multiple bridges, it is convenient to replicate all configuration items of the first bridge, and modify the bridge name and other configuration items if necessary. - diff --git a/apps/emqx_bridge_mqtt/docs/images/bridge.png b/apps/emqx_bridge_mqtt/docs/images/bridge.png deleted file mode 100644 index 9bb9c024c..000000000 Binary files a/apps/emqx_bridge_mqtt/docs/images/bridge.png and /dev/null differ diff --git a/apps/emqx_bridge_mqtt/etc/emqx_bridge_mqtt.conf b/apps/emqx_bridge_mqtt/etc/emqx_bridge_mqtt.conf deleted file mode 100644 index 247c09d8d..000000000 --- a/apps/emqx_bridge_mqtt/etc/emqx_bridge_mqtt.conf +++ /dev/null @@ -1,56 +0,0 @@ -##==================================================================== -## Configuration for EMQ X MQTT Broker Bridge -##==================================================================== - -bridge_mqtt: [ - # { - # name: "mqtt1" - # start_type: auto - # forwards: ["test/#"], - # forward_mountpoint: "" - # reconnect_interval: "30s" - # batch_size: 100 - # queue { - # replayq_dir: "{{ platform_data_dir }}/replayq/bridge_mqtt/" - # replayq_seg_bytes: "100MB" - # replayq_offload_mode: false - # replayq_max_total_bytes: "1GB" - # }, - # config { - # conn_type: mqtt - # address: "127.0.0.1:1883" - # proto_ver: v4 - # bridge_mode: true - # clientid: "client1" - # clean_start: true - # username: "username1" - # password: "" - # keepalive: 300 - # subscriptions: [{ - # topic: "t/#" - # qos: 1 - # }] - # receive_mountpoint: "" - # retry_interval: "30s" - # max_inflight: 32 - # } - # }, - # { - # name: "rpc1" - # start_type: auto - # forwards: ["test/#"], - # forward_mountpoint: "" - # reconnect_interval: "30s" - # batch_size: 100 - # queue { - # replayq_dir: "{{ platform_data_dir }}/replayq/bridge_mqtt/" - # replayq_seg_bytes: "100MB" - # replayq_offload_mode: false - # replayq_max_total_bytes: "1GB" - # }, - # config { - # conn_type: rpc - # node: "emqx@127.0.0.1" - # } - # } -] diff --git a/apps/emqx_bridge_mqtt/include/emqx_bridge_mqtt.hrl b/apps/emqx_bridge_mqtt/include/emqx_bridge_mqtt.hrl deleted file mode 100644 index 531518668..000000000 --- a/apps/emqx_bridge_mqtt/include/emqx_bridge_mqtt.hrl +++ /dev/null @@ -1,18 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --define(APP, emqx_bridge_mqtt). - diff --git a/apps/emqx_bridge_mqtt/rebar.config b/apps/emqx_bridge_mqtt/rebar.config deleted file mode 100644 index 37ac5b034..000000000 --- a/apps/emqx_bridge_mqtt/rebar.config +++ /dev/null @@ -1,19 +0,0 @@ -{deps, []}. -{edoc_opts, [{preprocess, true}]}. -{erl_opts, [warn_unused_vars, - warn_shadow_vars, - warn_unused_import, - warn_obsolete_guard, - debug_info]}. - -{xref_checks, [undefined_function_calls, undefined_functions, - locals_not_used, deprecated_function_calls, - warnings_as_errors, deprecated_functions]}. -{cover_enabled, true}. -{cover_opts, [verbose]}. -{cover_export_enabled, true}. - -{shell, [ - % {config, "config/sys.config"}, - {apps, [emqx, emqx_bridge_mqtt]} -]}. diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src deleted file mode 100644 index afac20404..000000000 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src +++ /dev/null @@ -1,14 +0,0 @@ -{application, emqx_bridge_mqtt, - [{description, "EMQ X Bridge to MQTT Broker"}, - {vsn, "5.0.0"}, % strict semver, bump manually! - {modules, []}, - {registered, []}, - {applications, [kernel,stdlib,replayq,emqtt,emqx]}, - {mod, {emqx_bridge_mqtt_app, []}}, - {env, []}, - {licenses, ["Apache-2.0"]}, - {maintainers, ["EMQ X Team "]}, - {links, [{"Homepage", "https://emqx.io/"}, - {"Github", "https://github.com/emqx/emqx-bridge-mqtt"} - ]} - ]}. diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_app.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_app.erl deleted file mode 100644 index a145009c9..000000000 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_app.erl +++ /dev/null @@ -1,31 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_bridge_mqtt_app). - --behaviour(application). - --export([start/2, stop/1]). - -start(_StartType, _StartArgs) -> - emqx_ctl:register_command(bridges, {emqx_bridge_mqtt_cli, cli}, []), - emqx_bridge_worker:register_metrics(), - emqx_bridge_mqtt_sup:start_link(). - -stop(_State) -> - emqx_ctl:unregister_command(bridges), - ok. - diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_cli.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_cli.erl deleted file mode 100644 index a76ea3a8c..000000000 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_cli.erl +++ /dev/null @@ -1,92 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_bridge_mqtt_cli). - --include("emqx_bridge_mqtt.hrl"). - --import(lists, [foreach/2]). - --export([cli/1]). - -cli(["list"]) -> - foreach(fun({Name, State0}) -> - State = case State0 of - connected -> <<"Running">>; - _ -> <<"Stopped">> - end, - emqx_ctl:print("name: ~s status: ~s~n", [Name, State]) - end, emqx_bridge_mqtt_sup:bridges()); - -cli(["start", Name]) -> - emqx_ctl:print("~s.~n", [try emqx_bridge_worker:ensure_started(Name) of - ok -> <<"Start bridge successfully">>; - connected -> <<"Bridge already started">>; - _ -> <<"Start bridge failed">> - catch - _Error:_Reason -> - <<"Start bridge failed">> - end]); - -cli(["stop", Name]) -> - emqx_ctl:print("~s.~n", [try emqx_bridge_worker:ensure_stopped(Name) of - ok -> <<"Stop bridge successfully">>; - _ -> <<"Stop bridge failed">> - catch - _Error:_Reason -> - <<"Stop bridge failed">> - end]); - -cli(["forwards", Name]) -> - foreach(fun(Topic) -> - emqx_ctl:print("topic: ~s~n", [Topic]) - end, emqx_bridge_worker:get_forwards(Name)); - -cli(["add-forward", Name, Topic]) -> - ok = emqx_bridge_worker:ensure_forward_present(Name, iolist_to_binary(Topic)), - emqx_ctl:print("Add-forward topic successfully.~n"); - -cli(["del-forward", Name, Topic]) -> - ok = emqx_bridge_worker:ensure_forward_absent(Name, iolist_to_binary(Topic)), - emqx_ctl:print("Del-forward topic successfully.~n"); - -cli(["subscriptions", Name]) -> - foreach(fun({Topic, Qos}) -> - emqx_ctl:print("topic: ~s, qos: ~p~n", [Topic, Qos]) - end, emqx_bridge_worker:get_subscriptions(Name)); - -cli(["add-subscription", Name, Topic, Qos]) -> - case emqx_bridge_worker:ensure_subscription_present(Name, Topic, list_to_integer(Qos)) of - ok -> emqx_ctl:print("Add-subscription topic successfully.~n"); - {error, Reason} -> emqx_ctl:print("Add-subscription failed reason: ~p.~n", [Reason]) - end; - -cli(["del-subscription", Name, Topic]) -> - ok = emqx_bridge_worker:ensure_subscription_absent(Name, Topic), - emqx_ctl:print("Del-subscription topic successfully.~n"); - -cli(_) -> - emqx_ctl:usage([{"bridges list", "List bridges"}, - {"bridges start ", "Start a bridge"}, - {"bridges stop ", "Stop a bridge"}, - {"bridges forwards ", "Show a bridge forward topic"}, - {"bridges add-forward ", "Add bridge forward topic"}, - {"bridges del-forward ", "Delete bridge forward topic"}, - {"bridges subscriptions ", "Show a bridge subscriptions topic"}, - {"bridges add-subscription ", "Add bridge subscriptions topic"}, - {"bridges del-subscription ", "Delete bridge subscriptions topic"}]). - - diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_schema.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_schema.erl deleted file mode 100644 index 925bfa403..000000000 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_schema.erl +++ /dev/null @@ -1,87 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_bridge_mqtt_schema). - --include_lib("typerefl/include/types.hrl"). - --behaviour(hocon_schema). - --export([ roots/0 - , fields/1]). - -roots() -> [array("bridge_mqtt")]. - -array(Name) -> {Name, hoconsc:array(hoconsc:ref(Name))}. - -fields("bridge_mqtt") -> - [ {name, emqx_schema:t(string(), undefined, true)} - , {start_type, fun start_type/1} - , {forwards, fun forwards/1} - , {forward_mountpoint, emqx_schema:t(string())} - , {reconnect_interval, emqx_schema:t(emqx_schema:duration_ms(), undefined, "30s")} - , {batch_size, emqx_schema:t(integer(), undefined, 100)} - , {queue, emqx_schema:t(hoconsc:ref(?MODULE, "queue"))} - , {config, hoconsc:union([hoconsc:ref(?MODULE, "mqtt"), hoconsc:ref(?MODULE, "rpc")])} - ]; - -fields("mqtt") -> - [ {conn_type, fun conn_type/1} - , {address, emqx_schema:t(string(), undefined, "127.0.0.1:1883")} - , {proto_ver, fun proto_ver/1} - , {bridge_mode, emqx_schema:t(boolean(), undefined, true)} - , {clientid, emqx_schema:t(string())} - , {username, emqx_schema:t(string())} - , {password, emqx_schema:t(string())} - , {clean_start, emqx_schema:t(boolean(), undefined, true)} - , {keepalive, emqx_schema:t(integer(), undefined, 300)} - , {subscriptions, hoconsc:array("subscriptions")} - , {receive_mountpoint, emqx_schema:t(string())} - , {retry_interval, emqx_schema:t(emqx_schema:duration_ms(), undefined, "30s")} - , {max_inflight, emqx_schema:t(integer(), undefined, 32)} - ]; - -fields("rpc") -> - [ {conn_type, fun conn_type/1} - , {node, emqx_schema:t(atom(), undefined, 'emqx@127.0.0.1')} - ]; - -fields("subscriptions") -> - [ {topic, #{type => binary(), nullable => false}} - , {qos, emqx_schema:t(integer(), undefined, 1)} - ]; - -fields("queue") -> - [ {replayq_dir, hoconsc:union([boolean(), string()])} - , {replayq_seg_bytes, emqx_schema:t(emqx_schema:bytesize(), undefined, "100MB")} - , {replayq_offload_mode, emqx_schema:t(boolean(), undefined, false)} - , {replayq_max_total_bytes, emqx_schema:t(emqx_schema:bytesize(), undefined, "1024MB")} - ]. - -conn_type(type) -> hoconsc:enum([mqtt, rpc]); -conn_type(_) -> undefined. - -proto_ver(type) -> hoconsc:enum([v3, v4, v5]); -proto_ver(default) -> v4; -proto_ver(_) -> undefined. - -start_type(type) -> hoconsc:enum([auto, manual]); -start_type(default) -> auto; -start_type(_) -> undefined. - -forwards(type) -> hoconsc:array(binary()); -forwards(default) -> []; -forwards(_) -> undefined. diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_sup.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_sup.erl deleted file mode 100644 index 4207067fe..000000000 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_sup.erl +++ /dev/null @@ -1,72 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_bridge_mqtt_sup). --behaviour(supervisor). - --include("emqx_bridge_mqtt.hrl"). --include_lib("emqx/include/logger.hrl"). - - -%% APIs --export([ start_link/0 - ]). - --export([ create_bridge/1 - , drop_bridge/1 - , bridges/0 - ]). - -%% supervisor callbacks --export([init/1]). - --define(WORKER_SUP, emqx_bridge_worker_sup). - -start_link() -> - supervisor:start_link({local, ?MODULE}, ?MODULE, []). - -init([]) -> - BridgesConf = emqx:get_config([?APP, bridges], []), - BridgeSpec = lists:map(fun bridge_spec/1, BridgesConf), - SupFlag = #{strategy => one_for_one, - intensity => 100, - period => 10}, - {ok, {SupFlag, BridgeSpec}}. - -bridge_spec(Config) -> - Name = list_to_atom(maps:get(name, Config)), - #{id => Name, - start => {emqx_bridge_worker, start_link, [Config]}, - restart => permanent, - shutdown => 5000, - type => worker, - modules => [emqx_bridge_worker]}. - --spec(bridges() -> [{node(), map()}]). -bridges() -> - [{Name, emqx_bridge_worker:status(Name)} || {Name, _Pid, _, _} <- supervisor:which_children(?MODULE)]. - -create_bridge(Config) -> - supervisor:start_child(?MODULE, bridge_spec(Config)). - -drop_bridge(Name) -> - case supervisor:terminate_child(?MODULE, Name) of - ok -> - supervisor:delete_child(?MODULE, Name); - {error, Error} -> - ?LOG(error, "Delete bridge failed, error : ~p", [Error]), - {error, Error} - end. diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_msg.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_msg.erl deleted file mode 100644 index 91fd18bf4..000000000 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_msg.erl +++ /dev/null @@ -1,99 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_bridge_msg). - --export([ to_binary/1 - , from_binary/1 - , to_export/3 - , to_broker_msgs/1 - , to_broker_msg/1 - , to_broker_msg/2 - , estimate_size/1 - ]). - --export_type([msg/0]). - --include_lib("emqx/include/emqx.hrl"). - --include_lib("emqx_bridge_mqtt/include/emqx_bridge_mqtt.hrl"). --include_lib("emqtt/include/emqtt.hrl"). - - --type msg() :: emqx_types:message(). --type exp_msg() :: emqx_types:message() | #mqtt_msg{}. - -%% @doc Make export format: -%% 1. Mount topic to a prefix -%% 2. Fix QoS to 1 -%% @end -%% Shame that we have to know the callback module here -%% would be great if we can get rid of #mqtt_msg{} record -%% and use #message{} in all places. --spec to_export(emqx_bridge_rpc | emqx_bridge_worker, - undefined | binary(), msg()) -> exp_msg(). -to_export(emqx_bridge_mqtt, Mountpoint, - #message{topic = Topic, - payload = Payload, - flags = Flags, - qos = QoS - }) -> - Retain = maps:get(retain, Flags, false), - #mqtt_msg{qos = QoS, - retain = Retain, - topic = topic(Mountpoint, Topic), - props = #{}, - payload = Payload}; -to_export(_Module, Mountpoint, - #message{topic = Topic} = Msg) -> - Msg#message{topic = topic(Mountpoint, Topic)}. - -%% @doc Make `binary()' in order to make iodata to be persisted on disk. --spec to_binary(msg()) -> binary(). -to_binary(Msg) -> term_to_binary(Msg). - -%% @doc Unmarshal binary into `msg()'. --spec from_binary(binary()) -> msg(). -from_binary(Bin) -> binary_to_term(Bin). - -%% @doc Estimate the size of a message. -%% Count only the topic length + payload size --spec estimate_size(msg()) -> integer(). -estimate_size(#message{topic = Topic, payload = Payload}) -> - size(Topic) + size(Payload). - -%% @doc By message/batch receiver, transform received batch into -%% messages to deliver to local brokers. -to_broker_msgs(Batch) -> lists:map(fun to_broker_msg/1, Batch). - -to_broker_msg(#message{} = Msg) -> - %% internal format from another EMQX node via rpc - Msg; -to_broker_msg(Msg) -> - to_broker_msg(Msg, undefined). -to_broker_msg(#{qos := QoS, dup := Dup, retain := Retain, topic := Topic, - properties := Props, payload := Payload}, Mountpoint) -> - %% published from remote node over a MQTT connection - set_headers(Props, - emqx_message:set_flags(#{dup => Dup, retain => Retain}, - emqx_message:make(bridge, QoS, topic(Mountpoint, Topic), Payload))). - -set_headers(undefined, Msg) -> - Msg; -set_headers(Val, Msg) -> - emqx_message:set_headers(Val, Msg). -topic(undefined, Topic) -> Topic; -topic(Prefix, Topic) -> emqx_topic:prepend(Prefix, Topic). diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_rpc.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_rpc.erl deleted file mode 100644 index 33511cc03..000000000 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_rpc.erl +++ /dev/null @@ -1,95 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - -%% @doc This module implements EMQX Bridge transport layer based on gen_rpc. - --module(emqx_bridge_rpc). - --export([ start/1 - , send/2 - , stop/1 - ]). - -%% Internal exports --export([ handle_send/1 - , heartbeat/2 - ]). - --type ack_ref() :: emqx_bridge_worker:ack_ref(). --type batch() :: emqx_bridge_worker:batch(). --define(HEARTBEAT_INTERVAL, timer:seconds(1)). - --define(RPC, emqx_rpc). - -start(#{node := RemoteNode}) -> - case poke(RemoteNode) of - ok -> - Pid = proc_lib:spawn_link(?MODULE, heartbeat, [self(), RemoteNode]), - {ok, #{client_pid => Pid, remote_node => RemoteNode}}; - Error -> - Error - end. - -stop(#{client_pid := Pid}) when is_pid(Pid) -> - Ref = erlang:monitor(process, Pid), - unlink(Pid), - Pid ! stop, - receive - {'DOWN', Ref, process, Pid, _Reason} -> - ok - after - 1000 -> - exit(Pid, kill) - end, - ok. - -%% @doc Callback for `emqx_bridge_connect' behaviour --spec send(#{remote_node := atom(), _ => _}, batch()) -> {ok, ack_ref()} | {error, any()}. -send(#{remote_node := RemoteNode}, Batch) -> - case ?RPC:call(RemoteNode, ?MODULE, handle_send, [Batch]) of - ok -> - Ref = make_ref(), - self() ! {batch_ack, Ref}, - {ok, Ref}; - {badrpc, Reason} -> {error, Reason} - end. - -%% @doc Handle send on receiver side. --spec handle_send(batch()) -> ok. -handle_send(Batch) -> - lists:foreach(fun(Msg) -> emqx_broker:publish(Msg) end, Batch). - -%% @hidden Heartbeat loop -heartbeat(Parent, RemoteNode) -> - Interval = ?HEARTBEAT_INTERVAL, - receive - stop -> exit(normal) - after - Interval -> - case poke(RemoteNode) of - ok -> - ?MODULE:heartbeat(Parent, RemoteNode); - {error, Reason} -> - Parent ! {disconnected, self(), Reason}, - exit(normal) - end - end. - -poke(RemoteNode) -> - case ?RPC:call(RemoteNode, erlang, node, []) of - RemoteNode -> ok; - {badrpc, Reason} -> {error, Reason} - end. diff --git a/apps/emqx_bridge_mqtt/test/emqx_bridge_rpc_tests.erl b/apps/emqx_bridge_mqtt/test/emqx_bridge_rpc_tests.erl deleted file mode 100644 index cbd80ba3d..000000000 --- a/apps/emqx_bridge_mqtt/test/emqx_bridge_rpc_tests.erl +++ /dev/null @@ -1,42 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_bridge_rpc_tests). --include_lib("eunit/include/eunit.hrl"). - -send_and_ack_test() -> - %% delegate from emqx_rpc to rpc for unit test - meck:new(emqx_rpc, [passthrough, no_history]), - meck:expect(emqx_rpc, call, 4, - fun(Node, Module, Fun, Args) -> - rpc:call(Node, Module, Fun, Args) - end), - meck:expect(emqx_rpc, cast, 4, - fun(Node, Module, Fun, Args) -> - rpc:cast(Node, Module, Fun, Args) - end), - meck:new(emqx_bridge_worker, [passthrough, no_history]), - try - {ok, #{client_pid := Pid, remote_node := Node}} = emqx_bridge_rpc:start(#{node => node()}), - {ok, Ref} = emqx_bridge_rpc:send(#{remote_node => Node}, []), - receive - {batch_ack, Ref} -> - ok - end, - ok = emqx_bridge_rpc:stop( #{client_pid => Pid}) - after - meck:unload(emqx_rpc) - end. diff --git a/apps/emqx_bridge_mqtt/test/emqx_bridge_stub_conn.erl b/apps/emqx_bridge_mqtt/test/emqx_bridge_stub_conn.erl deleted file mode 100644 index 4c2fde6dd..000000000 --- a/apps/emqx_bridge_mqtt/test/emqx_bridge_stub_conn.erl +++ /dev/null @@ -1,38 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_bridge_stub_conn). - --export([ start/1 - , send/2 - , stop/1 - ]). - --type ack_ref() :: emqx_bridge_worker:ack_ref(). --type batch() :: emqx_bridge_worker:batch(). - -start(#{client_pid := Pid} = Cfg) -> - Pid ! {self(), ?MODULE, ready}, - {ok, Cfg}. - -stop(_) -> ok. - -%% @doc Callback for `emqx_bridge_connect' behaviour --spec send(_, batch()) -> {ok, ack_ref()} | {error, any()}. -send(#{client_pid := Pid}, Batch) -> - Ref = make_ref(), - Pid ! {stub_message, self(), Ref, Batch}, - {ok, Ref}. diff --git a/apps/emqx_bridge_mqtt/test/emqx_bridge_worker_SUITE.erl b/apps/emqx_bridge_mqtt/test/emqx_bridge_worker_SUITE.erl deleted file mode 100644 index f3f5d5ceb..000000000 --- a/apps/emqx_bridge_mqtt/test/emqx_bridge_worker_SUITE.erl +++ /dev/null @@ -1,372 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_bridge_worker_SUITE). - --compile(export_all). --compile(nowarn_export_all). - --include_lib("eunit/include/eunit.hrl"). --include_lib("common_test/include/ct.hrl"). --include_lib("emqx/include/emqx_mqtt.hrl"). --include_lib("emqx/include/emqx.hrl"). --include_lib("snabbkaffe/include/snabbkaffe.hrl"). - --define(wait(For, Timeout), emqx_ct_helpers:wait_for(?FUNCTION_NAME, ?LINE, fun() -> For end, Timeout)). - --define(SNK_WAIT(WHAT), ?assertMatch({ok, _}, ?block_until(#{?snk_kind := WHAT}, 2000, 1000))). - -receive_messages(Count) -> - receive_messages(Count, []). - -receive_messages(0, Msgs) -> - Msgs; -receive_messages(Count, Msgs) -> - receive - {publish, Msg} -> - receive_messages(Count-1, [Msg|Msgs]); - _Other -> - receive_messages(Count, Msgs) - after 1000 -> - Msgs - end. - -all() -> - lists:filtermap( - fun({FunName, _Arity}) -> - case atom_to_list(FunName) of - "t_" ++ _ -> {true, FunName}; - _ -> false - end - end, - ?MODULE:module_info(exports)). - -init_per_suite(Config) -> - case node() of - nonode@nohost -> net_kernel:start(['emqx@127.0.0.1', longnames]); - _ -> ok - end, - emqx_ct_helpers:start_apps([emqx_bridge_mqtt]), - emqx_logger:set_log_level(error), - [{log_level, error} | Config]. - -end_per_suite(_Config) -> - emqx_ct_helpers:stop_apps([emqx_bridge_mqtt]). - -init_per_testcase(_TestCase, Config) -> - ok = snabbkaffe:start_trace(), - Config. - -end_per_testcase(_TestCase, _Config) -> - ok = snabbkaffe:stop(). - -t_rpc_mngr(_Config) -> - Name = "rpc_name", - Cfg = #{ - name => Name, - forwards => [<<"mngr">>], - forward_mountpoint => <<"forwarded">>, - start_type => auto, - config => #{ - conn_type => rpc, - node => node() - } - }, - {ok, Pid} = emqx_bridge_mqtt_sup:create_bridge(Cfg), - ?assertEqual([<<"mngr">>], emqx_bridge_worker:get_forwards(Name)), - ?assertEqual(ok, emqx_bridge_worker:ensure_forward_present(Name, "mngr")), - ?assertEqual(ok, emqx_bridge_worker:ensure_forward_present(Name, "mngr2")), - ?assertEqual([<<"mngr2">>, <<"mngr">>], emqx_bridge_worker:get_forwards(Name)), - ?assertEqual(ok, emqx_bridge_worker:ensure_forward_absent(Name, "mngr2")), - ?assertEqual(ok, emqx_bridge_worker:ensure_forward_absent(Name, "mngr3")), - ?assertEqual([<<"mngr">>], emqx_bridge_worker:get_forwards(Name)), - ?assertEqual({error, no_remote_subscription_support}, - emqx_bridge_worker:ensure_subscription_present(Name, <<"t">>, 0)), - ?assertEqual({error, no_remote_subscription_support}, - emqx_bridge_worker:ensure_subscription_absent(Name, <<"t">>)), - ok = emqx_bridge_worker:stop(Pid). - -t_mqtt_mngr(_Config) -> - Name = "mqtt_name", - Cfg = #{ - name => Name, - forwards => [<<"mngr">>], - forward_mountpoint => <<"forwarded">>, - start_type => auto, - config => #{ - address => "127.0.0.1:1883", - conn_type => mqtt, - clientid => <<"client1">>, - keepalive => 300, - subscriptions => [#{topic => <<"t/#">>, qos => 1}] - } - }, - {ok, Pid} = emqx_bridge_mqtt_sup:create_bridge(Cfg), - ?assertEqual([<<"mngr">>], emqx_bridge_worker:get_forwards(Name)), - ?assertEqual(ok, emqx_bridge_worker:ensure_forward_present(Name, "mngr")), - ?assertEqual(ok, emqx_bridge_worker:ensure_forward_present(Name, "mngr2")), - ?assertEqual([<<"mngr2">>, <<"mngr">>], emqx_bridge_worker:get_forwards(Name)), - ?assertEqual(ok, emqx_bridge_worker:ensure_forward_absent(Name, "mngr2")), - ?assertEqual(ok, emqx_bridge_worker:ensure_forward_absent(Name, "mngr3")), - ?assertEqual([<<"mngr">>], emqx_bridge_worker:get_forwards(Name)), - ?assertEqual(ok, emqx_bridge_worker:ensure_subscription_present(Name, <<"t">>, 0)), - ?assertEqual(ok, emqx_bridge_worker:ensure_subscription_absent(Name, <<"t">>)), - ?assertEqual([{<<"t/#">>,1}], emqx_bridge_worker:get_subscriptions(Name)), - ok = emqx_bridge_worker:stop(Pid). - -%% A loopback RPC to local node -t_rpc(_Config) -> - Name = "rpc", - Cfg = #{ - name => Name, - forwards => [<<"t_rpc/#">>], - forward_mountpoint => <<"forwarded">>, - start_type => auto, - config => #{ - conn_type => rpc, - node => node() - } - }, - {ok, Pid} = emqx_bridge_mqtt_sup:create_bridge(Cfg), - {ok, ConnPid} = emqtt:start_link([{clientid, <<"ClientId">>}]), - {ok, _Props} = emqtt:connect(ConnPid), - {ok, _Props, [1]} = emqtt:subscribe(ConnPid, {<<"forwarded/t_rpc/one">>, ?QOS_1}), - timer:sleep(100), - {ok, _PacketId} = emqtt:publish(ConnPid, <<"t_rpc/one">>, <<"hello">>, ?QOS_1), - timer:sleep(100), - ?assertEqual(1, length(receive_messages(1))), - emqtt:disconnect(ConnPid), - emqx_bridge_worker:stop(Pid). - -%% Full data loopback flow explained: -%% mqtt-client ----> local-broker ---(local-subscription)---> -%% bridge(export) --- (mqtt-connection)--> local-broker ---(remote-subscription) --> -%% bridge(import) --> mqtt-client -t_mqtt(_Config) -> - SendToTopic = <<"t_mqtt/one">>, - SendToTopic2 = <<"t_mqtt/two">>, - SendToTopic3 = <<"t_mqtt/three">>, - Mountpoint = <<"forwarded/${node}/">>, - Name = "mqtt", - Cfg = #{ - name => Name, - forwards => [SendToTopic], - forward_mountpoint => Mountpoint, - start_type => auto, - config => #{ - address => "127.0.0.1:1883", - conn_type => mqtt, - clientid => <<"client1">>, - keepalive => 300, - subscriptions => [#{topic => SendToTopic2, qos => 1}], - receive_mountpoint => <<"receive/aws/">> - }, - queue => #{ - replayq_dir => "data/t_mqtt/", - replayq_seg_bytes => 10000, - batch_bytes_limit => 1000, - batch_count_limit => 10 - } - }, - {ok, Pid} = emqx_bridge_mqtt_sup:create_bridge(Cfg), - ?assertEqual([{SendToTopic2, 1}], emqx_bridge_worker:get_subscriptions(Name)), - ok = emqx_bridge_worker:ensure_subscription_present(Name, SendToTopic3, _QoS = 1), - ?assertEqual([{SendToTopic3, 1},{SendToTopic2, 1}], - emqx_bridge_worker:get_subscriptions(Name)), - {ok, ConnPid} = emqtt:start_link([{clientid, <<"client-1">>}]), - {ok, _Props} = emqtt:connect(ConnPid), - emqtt:subscribe(ConnPid, <<"forwarded/+/t_mqtt/one">>, 1), - %% message from a different client, to avoid getting terminated by no-local - Max = 10, - Msgs = lists:seq(1, Max), - lists:foreach(fun(I) -> - {ok, _PacketId} = emqtt:publish(ConnPid, SendToTopic, integer_to_binary(I), ?QOS_1) - end, Msgs), - ?assertEqual(10, length(receive_messages(200))), - - emqtt:subscribe(ConnPid, <<"receive/aws/t_mqtt/two">>, 1), - %% message from a different client, to avoid getting terminated by no-local - Max = 10, - Msgs = lists:seq(1, Max), - lists:foreach(fun(I) -> - {ok, _PacketId} = emqtt:publish(ConnPid, SendToTopic2, integer_to_binary(I), ?QOS_1) - end, Msgs), - ?assertEqual(10, length(receive_messages(200))), - - emqtt:disconnect(ConnPid), - ok = emqx_bridge_worker:stop(Pid). - -t_stub_normal(Config) when is_list(Config) -> - Name = "stub_normal", - Cfg = #{ - name => Name, - forwards => [<<"t_stub_normal/#">>], - forward_mountpoint => <<"forwarded">>, - start_type => auto, - config => #{ - conn_type => emqx_bridge_stub_conn, - client_pid => self() - } - }, - {ok, Pid} = emqx_bridge_mqtt_sup:create_bridge(Cfg), - receive - {Pid, emqx_bridge_stub_conn, ready} -> ok - after - 5000 -> - error(timeout) - end, - {ok, ConnPid} = emqtt:start_link([{clientid, <<"ClientId">>}]), - {ok, _} = emqtt:connect(ConnPid), - {ok, _PacketId} = emqtt:publish(ConnPid, <<"t_stub_normal/one">>, <<"hello">>, ?QOS_1), - receive - {stub_message, WorkerPid, BatchRef, _Batch} -> - WorkerPid ! {batch_ack, BatchRef}, - ok - after - 5000 -> - error(timeout) - end, - ?SNK_WAIT(inflight_drained), - ?SNK_WAIT(replayq_drained), - emqtt:disconnect(ConnPid), - ok = emqx_bridge_worker:stop(Pid). - -t_stub_overflow(_Config) -> - Topic = <<"t_stub_overflow/one">>, - MaxInflight = 20, - Name = "stub_overflow", - Cfg = #{ - name => Name, - forwards => [<<"t_stub_overflow/one">>], - forward_mountpoint => <<"forwarded">>, - start_type => auto, - max_inflight => MaxInflight, - config => #{ - conn_type => emqx_bridge_stub_conn, - client_pid => self() - } - }, - {ok, Worker} = emqx_bridge_mqtt_sup:create_bridge(Cfg), - {ok, ConnPid} = emqtt:start_link([{clientid, <<"ClientId">>}]), - {ok, _} = emqtt:connect(ConnPid), - lists:foreach( - fun(I) -> - Data = integer_to_binary(I), - _ = emqtt:publish(ConnPid, Topic, Data, ?QOS_1) - end, lists:seq(1, MaxInflight * 2)), - ?SNK_WAIT(inflight_full), - Acks = stub_receive(MaxInflight), - lists:foreach(fun({Pid, Ref}) -> Pid ! {batch_ack, Ref} end, Acks), - Acks2 = stub_receive(MaxInflight), - lists:foreach(fun({Pid, Ref}) -> Pid ! {batch_ack, Ref} end, Acks2), - ?SNK_WAIT(inflight_drained), - ?SNK_WAIT(replayq_drained), - emqtt:disconnect(ConnPid), - ok = emqx_bridge_worker:stop(Worker). - -t_stub_random_order(_Config) -> - Topic = <<"t_stub_random_order/a">>, - MaxInflight = 10, - Name = "stub_random_order", - Cfg = #{ - name => Name, - forwards => [Topic], - forward_mountpoint => <<"forwarded">>, - start_type => auto, - max_inflight => MaxInflight, - config => #{ - conn_type => emqx_bridge_stub_conn, - client_pid => self() - } - }, - {ok, Worker} = emqx_bridge_mqtt_sup:create_bridge(Cfg), - ClientId = <<"ClientId">>, - {ok, ConnPid} = emqtt:start_link([{clientid, ClientId}]), - {ok, _} = emqtt:connect(ConnPid), - lists:foreach( - fun(I) -> - Data = integer_to_binary(I), - _ = emqtt:publish(ConnPid, Topic, Data, ?QOS_1) - end, lists:seq(1, MaxInflight)), - Acks = stub_receive(MaxInflight), - lists:foreach(fun({Pid, Ref}) -> Pid ! {batch_ack, Ref} end, - lists:reverse(Acks)), - ?SNK_WAIT(inflight_drained), - ?SNK_WAIT(replayq_drained), - emqtt:disconnect(ConnPid), - ok = emqx_bridge_worker:stop(Worker). - -t_stub_retry_inflight(_Config) -> - Topic = <<"to_stub_retry_inflight/a">>, - MaxInflight = 10, - Name = "stub_retry_inflight", - Cfg = #{ - name => Name, - forwards => [Topic], - forward_mountpoint => <<"forwarded">>, - reconnect_interval => 10, - start_type => auto, - max_inflight => MaxInflight, - config => #{ - conn_type => emqx_bridge_stub_conn, - client_pid => self() - } - }, - {ok, Worker} = emqx_bridge_mqtt_sup:create_bridge(Cfg), - ClientId = <<"ClientId2">>, - case ?block_until(#{?snk_kind := connected, inflight := 0}, 2000, 1000) of - {ok, #{inflight := 0}} -> ok; - Other -> ct:fail("~p", [Other]) - end, - {ok, ConnPid} = emqtt:start_link([{clientid, ClientId}]), - {ok, _} = emqtt:connect(ConnPid), - lists:foreach( - fun(I) -> - Data = integer_to_binary(I), - _ = emqtt:publish(ConnPid, Topic, Data, ?QOS_1) - end, lists:seq(1, MaxInflight)), - %% receive acks but do not ack - Acks1 = stub_receive(MaxInflight), - ?assertEqual(MaxInflight, length(Acks1)), - %% simulate a disconnect - Worker ! {disconnected, self(), test}, - ?SNK_WAIT(disconnected), - case ?block_until(#{?snk_kind := connected, inflight := MaxInflight}, 2000, 20) of - {ok, _} -> ok; - Error -> ct:fail("~p", [Error]) - end, - %% expect worker to retry inflight, so to receive acks again - Acks2 = stub_receive(MaxInflight), - ?assertEqual(MaxInflight, length(Acks2)), - lists:foreach(fun({Pid, Ref}) -> Pid ! {batch_ack, Ref} end, - lists:reverse(Acks2)), - ?SNK_WAIT(inflight_drained), - ?SNK_WAIT(replayq_drained), - emqtt:disconnect(ConnPid), - ok = emqx_bridge_worker:stop(Worker). - -stub_receive(N) -> - stub_receive(N, []). - -stub_receive(0, Acc) -> lists:reverse(Acc); -stub_receive(N, Acc) -> - receive - {stub_message, WorkerPid, BatchRef, _Batch} -> - stub_receive(N - 1, [{WorkerPid, BatchRef} | Acc]) - after - 5000 -> - lists:reverse(Acc) - end. diff --git a/apps/emqx_connector/rebar.config b/apps/emqx_connector/rebar.config index cbeff37eb..fd2329cbd 100644 --- a/apps/emqx_connector/rebar.config +++ b/apps/emqx_connector/rebar.config @@ -17,7 +17,8 @@ %% By accident, We have always been using the upstream fork due to %% eredis_cluster's dependency getting resolved earlier. %% Here we pin 1.5.2 to avoid surprises in the future. - {poolboy, {git, "https://github.com/emqx/poolboy.git", {tag, "1.5.2"}}} + {poolboy, {git, "https://github.com/emqx/poolboy.git", {tag, "1.5.2"}}}, + {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.4.3"}}} ]}. {shell, [ diff --git a/apps/emqx_connector/src/emqx_connector.app.src b/apps/emqx_connector/src/emqx_connector.app.src index 5e1ca2ca8..f4481dc2c 100644 --- a/apps/emqx_connector/src/emqx_connector.app.src +++ b/apps/emqx_connector/src/emqx_connector.app.src @@ -13,7 +13,8 @@ epgsql, mysql, mongodb, - emqx + emqx, + emqtt ]}, {env,[]}, {modules, []}, diff --git a/apps/emqx_connector/src/emqx_connector_app.erl b/apps/emqx_connector/src/emqx_connector_app.erl index 64e6b8109..4de078076 100644 --- a/apps/emqx_connector/src/emqx_connector_app.erl +++ b/apps/emqx_connector/src/emqx_connector_app.erl @@ -21,6 +21,7 @@ -export([start/2, stop/1]). start(_StartType, _StartArgs) -> + emqx_connector_mqtt_worker:register_metrics(), emqx_connector_sup:start_link(). stop(_State) -> diff --git a/apps/emqx_connector/src/emqx_connector_http.erl b/apps/emqx_connector/src/emqx_connector_http.erl index 572b2a4e8..159562f33 100644 --- a/apps/emqx_connector/src/emqx_connector_http.erl +++ b/apps/emqx_connector/src/emqx_connector_http.erl @@ -58,9 +58,7 @@ fields(config) -> , {pool_type, fun pool_type/1} , {pool_size, fun pool_size/1} , {enable_pipelining, fun enable_pipelining/1} - , {ssl_opts, #{type => hoconsc:ref(?MODULE, ssl_opts), - default => #{}}} - ]; + ] ++ emqx_connector_schema_lib:ssl_fields(); fields(ssl_opts) -> [ {cacertfile, fun cacertfile/1} @@ -200,12 +198,11 @@ check_ssl_opts(Conf) -> check_ssl_opts(URLFrom, Conf) -> #{schema := Scheme} = hocon_schema:get_value(URLFrom, Conf), - SSLOpts = hocon_schema:get_value("ssl_opts", Conf), - case {Scheme, maps:size(SSLOpts)} of - {http, 0} -> true; - {http, _} -> false; - {https, 0} -> false; - {https, _} -> true + SSL= hocon_schema:get_value("ssl", Conf), + case {Scheme, maps:get(enable, SSL, false)} of + {http, false} -> true; + {https, true} -> true; + {_, _} -> false end. update_path(BasePath, {Path, Headers}) -> diff --git a/apps/emqx_connector/src/emqx_connector_mongo.erl b/apps/emqx_connector/src/emqx_connector_mongo.erl index 88dfb2b72..906b57fb3 100644 --- a/apps/emqx_connector/src/emqx_connector_mongo.erl +++ b/apps/emqx_connector/src/emqx_connector_mongo.erl @@ -83,9 +83,7 @@ mongo_fields() -> nullable => true}} , {database, fun emqx_connector_schema_lib:database/1} , {topology, #{type => hoconsc:ref(?MODULE, topology), - default => #{}}} - %% TODO: Does the ref type support nullable=ture ? - % nullable => true}} + nullable => true}} ] ++ emqx_connector_schema_lib:ssl_fields(). @@ -178,7 +176,7 @@ do_start(InstId, Opts0, Config = #{mongo_type := Type, ]; false -> [{ssl, false}] end, - Topology= maps:get(topology, Config, #{}), + Topology= maps:get(topology, Config, #{}), Opts = Opts0 ++ [{pool_size, PoolSize}, {options, init_topology_options(maps:to_list(Topology), [])}, @@ -244,15 +242,8 @@ init_worker_options([_ | R], Acc) -> init_worker_options(R, Acc); init_worker_options([], Acc) -> Acc. -host_port(HostPort) -> - case string:split(HostPort, ":") of - [Host, Port] -> - {ok, Host1} = inet:parse_address(Host), - [{host, Host1}, {port, list_to_integer(Port)}]; - [Host] -> - {ok, Host1} = inet:parse_address(Host), - [{host, Host1}] - end. +host_port({Host, Port}) -> + [{host, Host}, {port, Port}]. server(type) -> server(); server(validator) -> [?NOT_EMPTY("the value of the field 'server' cannot be empty")]; diff --git a/apps/emqx_connector/src/emqx_connector_mqtt.erl b/apps/emqx_connector/src/emqx_connector_mqtt.erl new file mode 100644 index 000000000..6631fd23a --- /dev/null +++ b/apps/emqx_connector/src/emqx_connector_mqtt.erl @@ -0,0 +1,214 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_connector_mqtt). + +-include_lib("typerefl/include/types.hrl"). +-include_lib("emqx_resource/include/emqx_resource_behaviour.hrl"). + +-behaviour(supervisor). + +%% API and callbacks for supervisor +-export([ start_link/0 + , init/1 + , create_bridge/1 + , drop_bridge/1 + , bridges/0 + ]). + +%% callbacks of behaviour emqx_resource +-export([ on_start/2 + , on_stop/2 + , on_query/4 + , on_health_check/2 + ]). + +-behaviour(hocon_schema). + +-export([ roots/0 + , fields/1]). + +%%===================================================================== +%% Hocon schema +roots() -> + [{config, #{type => hoconsc:ref(?MODULE, "config")}}]. + +fields("config") -> + emqx_connector_mqtt_schema:fields("config"). + +%% =================================================================== +%% supervisor APIs +start_link() -> + supervisor:start_link({local, ?MODULE}, ?MODULE, []). + +init([]) -> + SupFlag = #{strategy => one_for_one, + intensity => 100, + period => 10}, + {ok, {SupFlag, []}}. + +bridge_spec(Config) -> + #{id => maps:get(name, Config), + start => {emqx_connector_mqtt_worker, start_link, [Config]}, + restart => permanent, + shutdown => 5000, + type => worker, + modules => [emqx_connector_mqtt_worker]}. + +-spec(bridges() -> [{node(), map()}]). +bridges() -> + [{Name, emqx_connector_mqtt_worker:status(Name)} + || {Name, _Pid, _, _} <- supervisor:which_children(?MODULE)]. + +create_bridge(Config) -> + supervisor:start_child(?MODULE, bridge_spec(Config)). + +drop_bridge(Name) -> + case supervisor:terminate_child(?MODULE, Name) of + ok -> + supervisor:delete_child(?MODULE, Name); + {error, Error} -> + {error, Error} + end. + +%% =================================================================== +on_start(InstId, Conf) -> + logger:info("starting mqtt connector: ~p, ~p", [InstId, Conf]), + NamePrefix = binary_to_list(InstId), + BasicConf = basic_config(Conf), + InitRes = {ok, #{name_prefix => NamePrefix, baisc_conf => BasicConf, sub_bridges => []}}, + InOutConfigs = check_channel_id_dup(maps:get(in, Conf, []) ++ maps:get(out, Conf, [])), + lists:foldl(fun + (_InOutConf, {error, Reason}) -> + {error, Reason}; + (InOutConf, {ok, #{sub_bridges := SubBridges} = Res}) -> + case create_channel(InOutConf, NamePrefix, BasicConf) of + {error, Reason} -> {error, Reason}; + {ok, Name} -> {ok, Res#{sub_bridges => [Name | SubBridges]}} + end + end, InitRes, InOutConfigs). + +on_stop(InstId, #{}) -> + logger:info("stopping mqtt connector: ~p", [InstId]), + case ?MODULE:drop_bridge(InstId) of + ok -> ok; + {error, not_found} -> ok; + {error, Reason} -> + logger:error("stop bridge failed, error: ~p", [Reason]) + end. + +%% TODO: let the emqx_resource trigger on_query/4 automatically according to the +%% `in` and `out` config +on_query(InstId, {create_channel, Conf}, _AfterQuery, #{name_prefix := Prefix, + baisc_conf := BasicConf}) -> + logger:debug("create channel to connector: ~p, conf: ~p", [InstId, Conf]), + create_channel(Conf, Prefix, BasicConf); +on_query(InstId, {publish_to_local, Msg}, _AfterQuery, _State) -> + logger:debug("publish to local node, connector: ~p, msg: ~p", [InstId, Msg]); +on_query(InstId, {publish_to_remote, Msg}, _AfterQuery, _State) -> + logger:debug("publish to remote node, connector: ~p, msg: ~p", [InstId, Msg]). + +on_health_check(_InstId, #{sub_bridges := NameList} = State) -> + Results = [{Name, emqx_connector_mqtt_worker:ping(Name)} || Name <- NameList], + case lists:all(fun({_, pong}) -> true; ({_, _}) -> false end, Results) of + true -> {ok, State}; + false -> {error, {some_sub_bridge_down, Results}, State} + end. + +check_channel_id_dup(Confs) -> + lists:foreach(fun(#{id := Id}) -> + case length([Id || #{id := Id0} <- Confs, Id0 == Id]) of + 1 -> ok; + L when L > 1 -> error({mqtt_bridge_conf, {duplicate_id_found, Id}}) + end + end, Confs), + Confs. + +%% this is an `in` bridge +create_channel(#{subscribe_remote_topic := _, id := BridgeId} = InConf, NamePrefix, + #{clientid_prefix := ClientPrefix} = BasicConf) -> + logger:info("creating 'in' channel for: ~p", [BridgeId]), + create_sub_bridge(BasicConf#{name => bridge_name(NamePrefix, BridgeId), + clientid => clientid(ClientPrefix, BridgeId), + subscriptions => InConf, forwards => undefined}); +%% this is an `out` bridge +create_channel(#{subscribe_local_topic := _, id := BridgeId} = OutConf, NamePrefix, + #{clientid_prefix := ClientPrefix} = BasicConf) -> + logger:info("creating 'out' channel for: ~p", [BridgeId]), + create_sub_bridge(BasicConf#{name => bridge_name(NamePrefix, BridgeId), + clientid => clientid(ClientPrefix, BridgeId), + subscriptions => undefined, forwards => OutConf}). + +create_sub_bridge(#{name := Name} = Conf) -> + case ?MODULE:create_bridge(Conf) of + {ok, _Pid} -> + start_sub_bridge(Name); + {error, {already_started, _Pid}} -> + ok; + {error, Reason} -> + {error, Reason} + end. + +start_sub_bridge(Name) -> + case emqx_connector_mqtt_worker:ensure_started(Name) of + ok -> {ok, Name}; + {error, Reason} -> {error, Reason} + end. + +basic_config(#{ + server := Server, + reconnect_interval := ReconnIntv, + proto_ver := ProtoVer, + bridge_mode := BridgeMod, + clientid_prefix := ClientIdPrefix, + username := User, + password := Password, + clean_start := CleanStart, + keepalive := KeepAlive, + retry_interval := RetryIntv, + max_inflight := MaxInflight, + replayq := ReplayQ, + ssl := #{enable := EnableSsl} = Ssl}) -> + #{ + replayq => ReplayQ, + %% connection opts + server => Server, + reconnect_interval => ReconnIntv, + proto_ver => ProtoVer, + bridge_mode => BridgeMod, + clientid_prefix => ClientIdPrefix, + username => User, + password => Password, + clean_start => CleanStart, + keepalive => KeepAlive, + retry_interval => RetryIntv, + max_inflight => MaxInflight, + ssl => EnableSsl, + ssl_opts => maps:to_list(maps:remove(enable, Ssl)), + if_record_metrics => true + }. + +bridge_name(Prefix, Id) -> + list_to_atom(str(Prefix) ++ ":" ++ str(Id)). + +clientid(Prefix, Id) -> + list_to_binary(str(Prefix) ++ str(Id)). + +str(A) when is_atom(A) -> + atom_to_list(A); +str(B) when is_binary(B) -> + binary_to_list(B); +str(S) when is_list(S) -> + S. diff --git a/apps/emqx_connector/src/emqx_connector_redis.erl b/apps/emqx_connector/src/emqx_connector_redis.erl index 4fe26381e..44b036f39 100644 --- a/apps/emqx_connector/src/emqx_connector_redis.erl +++ b/apps/emqx_connector/src/emqx_connector_redis.erl @@ -19,9 +19,13 @@ -include_lib("typerefl/include/types.hrl"). -include_lib("emqx_resource/include/emqx_resource_behaviour.hrl"). --type server() :: emqx_schema:ip_port(). +-type server() :: tuple(). + -reflect_type([server/0]). --typerefl_from_string({server/0, emqx_connector_schema_lib, to_ip_port}). + +-typerefl_from_string({server/0, ?MODULE, to_server}). + +-export([to_server/1]). -export([roots/0, fields/1]). @@ -168,3 +172,9 @@ redis_fields() -> default => 0}} , {auto_reconnect, fun emqx_connector_schema_lib:auto_reconnect/1} ]. + +to_server(Server) -> + case string:tokens(Server, ":") of + [Host, Port] -> {ok, {Host, list_to_integer(Port)}}; + _ -> {error, Server} + end. \ No newline at end of file diff --git a/apps/emqx_connector/src/emqx_connector_schema_lib.erl b/apps/emqx_connector/src/emqx_connector_schema_lib.erl index 5f9472cca..6dcc564af 100644 --- a/apps/emqx_connector/src/emqx_connector_schema_lib.erl +++ b/apps/emqx_connector/src/emqx_connector_schema_lib.erl @@ -53,24 +53,18 @@ -export([roots/0, fields/1]). -roots() -> [ssl_on, ssl_off]. +roots() -> ["ssl"]. -fields(ssl_on) -> - [ {enable, #{type => true}} +fields("ssl") -> + [ {enable, #{type => boolean(), default => false}} , {cacertfile, fun cacertfile/1} , {keyfile, fun keyfile/1} , {certfile, fun certfile/1} , {verify, fun verify/1} - ]; - -fields(ssl_off) -> - [ {enable, #{type => false}} ]. + ]. ssl_fields() -> - [ {ssl, #{type => hoconsc:union( - [ hoconsc:ref(?MODULE, ssl_on) - , hoconsc:ref(?MODULE, ssl_off) - ]), + [ {ssl, #{type => hoconsc:ref(?MODULE, "ssl"), default => #{<<"enable">> => false} } } @@ -142,7 +136,9 @@ to_ip_port(Str) -> _ -> {error, Str} end. -ip_port_to_string({Ip, Port}) -> +ip_port_to_string({Ip, Port}) when is_list(Ip) -> + iolist_to_binary([Ip, ":", integer_to_list(Port)]); +ip_port_to_string({Ip, Port}) when is_tuple(Ip) -> iolist_to_binary([inet:ntoa(Ip), ":", integer_to_list(Port)]). to_servers(Str) -> diff --git a/apps/emqx_connector/src/emqx_connector_sup.erl b/apps/emqx_connector/src/emqx_connector_sup.erl index 603b9a8ad..a24a97b8f 100644 --- a/apps/emqx_connector/src/emqx_connector_sup.erl +++ b/apps/emqx_connector/src/emqx_connector_sup.erl @@ -28,9 +28,19 @@ start_link() -> init([]) -> SupFlags = #{strategy => one_for_all, - intensity => 0, - period => 1}, - ChildSpecs = [], + intensity => 5, + period => 20}, + ChildSpecs = [ + child_spec(emqx_connector_mqtt) + ], {ok, {SupFlags, ChildSpecs}}. +child_spec(Mod) -> + #{id => Mod, + start => {Mod, start_link, []}, + restart => permanent, + shutdown => 3000, + type => supervisor, + modules => [Mod]}. + %% internal functions diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_mod.erl similarity index 78% rename from apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.erl rename to apps/emqx_connector/src/mqtt/emqx_connector_mqtt_mod.erl index 8d442463b..3de7feac4 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.erl +++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_mod.erl @@ -16,11 +16,12 @@ %% @doc This module implements EMQX Bridge transport layer on top of MQTT protocol --module(emqx_bridge_mqtt). +-module(emqx_connector_mqtt_mod). -export([ start/1 , send/2 , stop/1 + , ping/1 ]). -export([ ensure_subscribed/3 @@ -33,9 +34,6 @@ , handle_disconnected/2 ]). --export([ check_subscriptions/1 - ]). - -include_lib("emqx/include/logger.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). @@ -50,15 +48,11 @@ start(Config) -> Parent = self(), - Address = maps:get(address, Config), + {Host, Port} = maps:get(server, Config), Mountpoint = maps:get(receive_mountpoint, Config, undefined), - Subscriptions = maps:get(subscriptions, Config, []), - Subscriptions1 = check_subscriptions(Subscriptions), - Handlers = make_hdlr(Parent, Mountpoint), - {Host, Port} = case string:tokens(Address, ":") of - [H] -> {H, 1883}; - [H, P] -> {H, list_to_integer(P)} - end, + Subscriptions = maps:get(subscriptions, Config, undefined), + Vars = emqx_connector_mqtt_msg:make_pub_vars(Mountpoint, Subscriptions), + Handlers = make_hdlr(Parent, Vars), Config1 = Config#{ msg_handler => Handlers, host => Host, @@ -66,13 +60,13 @@ start(Config) -> force_ping => true, proto_ver => maps:get(proto_ver, Config, v4) }, - case emqtt:start_link(without_config(Config1)) of + case emqtt:start_link(process_config(Config1)) of {ok, Pid} -> case emqtt:connect(Pid) of {ok, _} -> try - Subscriptions2 = subscribe_remote_topics(Pid, Subscriptions1), - {ok, #{client_pid => Pid, subscriptions => Subscriptions2}} + ok = subscribe_remote_topics(Pid, Subscriptions), + {ok, #{client_pid => Pid, subscriptions => Subscriptions}} catch throw : Reason -> ok = stop(#{client_pid => Pid}), @@ -90,6 +84,9 @@ stop(#{client_pid := Pid}) -> safe_stop(Pid, fun() -> emqtt:stop(Pid) end, 1000), ok. +ping(#{client_pid := Pid}) -> + emqtt:ping(Pid). + ensure_subscribed(#{client_pid := Pid, subscriptions := Subs} = Conn, Topic, QoS) when is_pid(Pid) -> case emqtt:subscribe(Pid, Topic, QoS) of {ok, _, _} -> Conn#{subscriptions => [{Topic, QoS}|Subs]}; @@ -158,33 +155,29 @@ handle_puback(#{packet_id := PktId, reason_code := RC}, Parent) RC =:= ?RC_NO_MATCHING_SUBSCRIBERS -> Parent ! {batch_ack, PktId}, ok; handle_puback(#{packet_id := PktId, reason_code := RC}, _Parent) -> - ?LOG(warning, "Publish ~p to remote node falied, reason_code: ~p", [PktId, RC]). + ?LOG(warning, "publish ~p to remote node falied, reason_code: ~p", [PktId, RC]). -handle_publish(Msg, Mountpoint) -> - emqx_broker:publish(emqx_bridge_msg:to_broker_msg(Msg, Mountpoint)). +handle_publish(Msg, undefined) -> + ?LOG(error, "cannot publish to local broker as 'bridge.mqtt..in' not configured, msg: ~p", [Msg]); +handle_publish(Msg, Vars) -> + ?LOG(debug, "publish to local broker, msg: ~p, vars: ~p", [Msg, Vars]), + emqx_broker:publish(emqx_connector_mqtt_msg:to_broker_msg(Msg, Vars)). handle_disconnected(Reason, Parent) -> Parent ! {disconnected, self(), Reason}. -make_hdlr(Parent, Mountpoint) -> +make_hdlr(Parent, Vars) -> #{puback => {fun ?MODULE:handle_puback/2, [Parent]}, - publish => {fun ?MODULE:handle_publish/2, [Mountpoint]}, + publish => {fun ?MODULE:handle_publish/2, [Vars]}, disconnected => {fun ?MODULE:handle_disconnected/2, [Parent]} }. -subscribe_remote_topics(ClientPid, Subscriptions) -> - lists:map(fun({Topic, Qos}) -> - case emqtt:subscribe(ClientPid, Topic, Qos) of - {ok, _, _} -> {Topic, Qos}; - Error -> throw(Error) - end - end, Subscriptions). +subscribe_remote_topics(_ClientPid, undefined) -> ok; +subscribe_remote_topics(ClientPid, #{subscribe_remote_topic := FromTopic, subscribe_qos := QoS}) -> + case emqtt:subscribe(ClientPid, FromTopic, QoS) of + {ok, _, _} -> ok; + Error -> throw(Error) + end. -without_config(Config) -> - maps:without([conn_type, address, receive_mountpoint, subscriptions], Config). - -check_subscriptions(Subscriptions) -> - lists:map(fun(#{qos := QoS, topic := Topic}) -> - true = emqx_topic:validate({filter, Topic}), - {Topic, QoS} - end, Subscriptions). +process_config(Config) -> + maps:without([conn_type, address, receive_mountpoint, subscriptions, name], Config). diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl new file mode 100644 index 000000000..5f076ed9e --- /dev/null +++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl @@ -0,0 +1,125 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_connector_mqtt_msg). + +-export([ to_binary/1 + , from_binary/1 + , make_pub_vars/2 + , to_remote_msg/2 + , to_broker_msg/2 + , estimate_size/1 + ]). + +-export_type([msg/0]). + +-include_lib("emqx/include/emqx.hrl"). + +-include_lib("emqtt/include/emqtt.hrl"). + + +-type msg() :: emqx_types:message(). +-type exp_msg() :: emqx_types:message() | #mqtt_msg{}. + +-type variables() :: #{ + mountpoint := undefined | binary(), + topic := binary(), + qos := original | integer(), + retain := original | boolean(), + payload := binary() +}. + +make_pub_vars(_, undefined) -> undefined; +make_pub_vars(Mountpoint, #{payload := _, qos := _, retain := _, remote_topic := Topic} = Conf) -> + Conf#{topic => Topic, mountpoint => Mountpoint}; +make_pub_vars(Mountpoint, #{payload := _, qos := _, retain := _, local_topic := Topic} = Conf) -> + Conf#{topic => Topic, mountpoint => Mountpoint}. + +%% @doc Make export format: +%% 1. Mount topic to a prefix +%% 2. Fix QoS to 1 +%% @end +%% Shame that we have to know the callback module here +%% would be great if we can get rid of #mqtt_msg{} record +%% and use #message{} in all places. +-spec to_remote_msg(msg() | map(), variables()) + -> exp_msg(). +to_remote_msg(#message{flags = Flags0} = Msg, Vars) -> + Retain0 = maps:get(retain, Flags0, false), + MapMsg = maps:put(retain, Retain0, emqx_message:to_map(Msg)), + to_remote_msg(MapMsg, Vars); +to_remote_msg(MapMsg, #{topic := TopicToken, payload := PayloadToken, + qos := QoSToken, retain := RetainToken, mountpoint := Mountpoint}) when is_map(MapMsg) -> + Topic = replace_vars_in_str(TopicToken, MapMsg), + Payload = replace_vars_in_str(PayloadToken, MapMsg), + QoS = replace_simple_var(QoSToken, MapMsg), + Retain = replace_simple_var(RetainToken, MapMsg), + #mqtt_msg{qos = QoS, + retain = Retain, + topic = topic(Mountpoint, Topic), + props = #{}, + payload = Payload}; +to_remote_msg(#message{topic = Topic} = Msg, #{mountpoint := Mountpoint}) -> + Msg#message{topic = topic(Mountpoint, Topic)}. + +%% published from remote node over a MQTT connection +to_broker_msg(#{dup := Dup, properties := Props} = MapMsg, + #{topic := TopicToken, payload := PayloadToken, + qos := QoSToken, retain := RetainToken, mountpoint := Mountpoint}) -> + Topic = replace_vars_in_str(TopicToken, MapMsg), + Payload = replace_vars_in_str(PayloadToken, MapMsg), + QoS = replace_simple_var(QoSToken, MapMsg), + Retain = replace_simple_var(RetainToken, MapMsg), + set_headers(Props, + emqx_message:set_flags(#{dup => Dup, retain => Retain}, + emqx_message:make(bridge, QoS, topic(Mountpoint, Topic), Payload))). + +%% Replace a string contains vars to another string in which the placeholders are replace by the +%% corresponding values. For example, given "a: ${var}", if the var=1, the result string will be: +%% "a: 1". +replace_vars_in_str(Tokens, Data) when is_list(Tokens) -> + emqx_plugin_libs_rule:proc_tmpl(Tokens, Data, #{return => full_binary}); +replace_vars_in_str(Val, _Data) -> + Val. + +%% Replace a simple var to its value. For example, given "${var}", if the var=1, then the result +%% value will be an integer 1. +replace_simple_var(Tokens, Data) when is_list(Tokens) -> + [Var] = emqx_plugin_libs_rule:proc_tmpl(Tokens, Data, #{return => rawlist}), + Var; +replace_simple_var(Val, _Data) -> + Val. + +%% @doc Make `binary()' in order to make iodata to be persisted on disk. +-spec to_binary(msg()) -> binary(). +to_binary(Msg) -> term_to_binary(Msg). + +%% @doc Unmarshal binary into `msg()'. +-spec from_binary(binary()) -> msg(). +from_binary(Bin) -> binary_to_term(Bin). + +%% @doc Estimate the size of a message. +%% Count only the topic length + payload size +-spec estimate_size(msg()) -> integer(). +estimate_size(#message{topic = Topic, payload = Payload}) -> + size(Topic) + size(Payload). + +set_headers(undefined, Msg) -> + Msg; +set_headers(Val, Msg) -> + emqx_message:set_headers(Val, Msg). +topic(undefined, Topic) -> Topic; +topic(Prefix, Topic) -> emqx_topic:prepend(Prefix, Topic). diff --git a/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl new file mode 100644 index 000000000..ed7fd4408 --- /dev/null +++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl @@ -0,0 +1,78 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_connector_mqtt_schema). + +-include_lib("typerefl/include/types.hrl"). + +-behaviour(hocon_schema). + +-export([ roots/0 + , fields/1]). + +roots() -> + [{config, #{type => hoconsc:ref(?MODULE, "config")}}]. + +fields("config") -> + [ {server, hoconsc:mk(emqx_schema:ip_port(), #{default => "127.0.0.1:1883"})} + , {reconnect_interval, hoconsc:mk(emqx_schema:duration_ms(), #{default => "30s"})} + , {proto_ver, fun proto_ver/1} + , {bridge_mode, hoconsc:mk(boolean(), #{default => true})} + , {clientid_prefix, hoconsc:mk(string(), #{default => ""})} + , {username, hoconsc:mk(string())} + , {password, hoconsc:mk(string())} + , {clean_start, hoconsc:mk(boolean(), #{default => true})} + , {keepalive, hoconsc:mk(integer(), #{default => 300})} + , {retry_interval, hoconsc:mk(emqx_schema:duration_ms(), #{default => "30s"})} + , {max_inflight, hoconsc:mk(integer(), #{default => 32})} + , {replayq, hoconsc:mk(hoconsc:ref(?MODULE, "replayq"))} + , {in, hoconsc:mk(hoconsc:array(hoconsc:ref(?MODULE, "in")), #{default => []})} + , {out, hoconsc:mk(hoconsc:array(hoconsc:ref(?MODULE, "out")), #{default => []})} + ] ++ emqx_connector_schema_lib:ssl_fields(); + +fields("in") -> + [ {subscribe_remote_topic, #{type => binary(), nullable => false}} + , {local_topic, hoconsc:mk(binary(), #{default => <<"${topic}">>})} + , {subscribe_qos, hoconsc:mk(qos(), #{default => 1})} + ] ++ common_inout_confs(); + +fields("out") -> + [ {subscribe_local_topic, #{type => binary(), nullable => false}} + , {remote_topic, hoconsc:mk(binary(), #{default => <<"${topic}">>})} + ] ++ common_inout_confs(); + +fields("replayq") -> + [ {dir, hoconsc:union([boolean(), string()])} + , {seg_bytes, hoconsc:mk(emqx_schema:bytesize(), #{default => "100MB"})} + , {offload, hoconsc:mk(boolean(), #{default => false})} + , {max_total_bytes, hoconsc:mk(emqx_schema:bytesize(), #{default => "1024MB"})} + ]. + +common_inout_confs() -> + [{id, #{type => binary(), nullable => false}}] ++ publish_confs(). + +publish_confs() -> + [ {qos, hoconsc:mk(qos(), #{default => <<"${qos}">>})} + , {retain, hoconsc:mk(hoconsc:union([boolean(), binary()]), #{default => <<"${retain}">>})} + , {payload, hoconsc:mk(binary(), #{default => <<"${payload}">>})} + ]. + +qos() -> + hoconsc:union([typerefl:integer(0), typerefl:integer(1), typerefl:integer(2), binary()]). + +proto_ver(type) -> hoconsc:enum([v3, v4, v5]); +proto_ver(default) -> v4; +proto_ver(_) -> undefined. diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_worker.erl b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl similarity index 68% rename from apps/emqx_bridge_mqtt/src/emqx_bridge_worker.erl rename to apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl index 630fb4443..6ced719df 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_worker.erl +++ b/apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl @@ -19,7 +19,7 @@ %% to remote MQTT node/cluster via `connection' transport layer. %% In case `REMOTE' is also an EMQX node, `connection' is recommended to be %% the `gen_rpc' based implementation `emqx_bridge_rpc'. Otherwise `connection' -%% has to be `emqx_bridge_mqtt'. +%% has to be `emqx_connector_mqtt_mod'. %% %% ``` %% +------+ +--------+ @@ -59,7 +59,7 @@ %% NOTES: %% * Local messages are all normalised to QoS-1 when exporting to remote --module(emqx_bridge_worker). +-module(emqx_connector_mqtt_worker). -behaviour(gen_statem). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). @@ -86,16 +86,13 @@ -export([ ensure_started/1 , ensure_stopped/1 , status/1 + , ping/1 ]). -export([ get_forwards/1 - , ensure_forward_present/2 - , ensure_forward_absent/2 ]). -export([ get_subscriptions/1 - , ensure_subscription_present/3 - , ensure_subscription_absent/2 ]). %% Internal @@ -109,7 +106,7 @@ -type id() :: atom() | string() | pid(). -type qos() :: emqx_mqtt_types:qos(). -type config() :: map(). --type batch() :: [emqx_bridge_msg:exp_msg()]. +-type batch() :: [emqx_connector_mqtt_msg:exp_msg()]. -type ack_ref() :: term(). -type topic() :: emqx_topic:topic(). @@ -135,12 +132,12 @@ %% mountpoint: The topic mount point for messages sent to remote node/cluster %% `undefined', `<<>>' or `""' to disable %% forwards: Local topics to subscribe. -%% queue.batch_bytes_limit: Max number of bytes to collect in a batch for each +%% replayq.batch_bytes_limit: Max number of bytes to collect in a batch for each %% send call towards emqx_bridge_connect -%% queue.batch_count_limit: Max number of messages to collect in a batch for +%% replayq.batch_count_limit: Max number of messages to collect in a batch for %% each send call towards emqx_bridge_connect -%% queue.replayq_dir: Directory where replayq should persist messages -%% queue.replayq_seg_bytes: Size in bytes for each replayq segment file +%% replayq.dir: Directory where replayq should persist messages +%% replayq.seg_bytes: Size in bytes for each replayq segment file %% %% Find more connection specific configs in the callback modules %% of emqx_bridge_connect behaviour. @@ -169,6 +166,11 @@ status(Pid) when is_pid(Pid) -> status(Name) -> gen_statem:call(name(Name), status). +ping(Pid) when is_pid(Pid) -> + gen_statem:call(Pid, ping); +ping(Name) -> + gen_statem:call(name(Name), ping). + %% @doc Return all forwards (local subscriptions). -spec get_forwards(id()) -> [topic()]. get_forwards(Name) -> gen_statem:call(name(Name), get_forwards, timer:seconds(1000)). @@ -177,47 +179,21 @@ get_forwards(Name) -> gen_statem:call(name(Name), get_forwards, timer:seconds(10 -spec get_subscriptions(id()) -> [{emqx_topic:topic(), qos()}]. get_subscriptions(Name) -> gen_statem:call(name(Name), get_subscriptions). -%% @doc Add a new forward (local topic subscription). --spec ensure_forward_present(id(), topic()) -> ok. -ensure_forward_present(Name, Topic) -> - gen_statem:call(name(Name), {ensure_forward_present, topic(Topic)}). - -%% @doc Ensure a forward topic is deleted. --spec ensure_forward_absent(id(), topic()) -> ok. -ensure_forward_absent(Name, Topic) -> - gen_statem:call(name(Name), {ensure_forward_absent, topic(Topic)}). - -%% @doc Ensure subscribed to remote topic. -%% NOTE: only applicable when connection module is emqx_bridge_mqtt -%% return `{error, no_remote_subscription_support}' otherwise. --spec ensure_subscription_present(id(), topic(), qos()) -> ok | {error, any()}. -ensure_subscription_present(Name, Topic, QoS) -> - gen_statem:call(name(Name), {ensure_subscription_present, topic(Topic), QoS}). - -%% @doc Ensure unsubscribed from remote topic. -%% NOTE: only applicable when connection module is emqx_bridge_mqtt --spec ensure_subscription_absent(id(), topic()) -> ok. -ensure_subscription_absent(Name, Topic) -> - gen_statem:call(name(Name), {ensure_subscription_absent, topic(Topic)}). - callback_mode() -> [state_functions]. %% @doc Config should be a map(). -init(Opts) -> +init(#{name := Name} = ConnectOpts) -> + ?LOG(info, "starting bridge worker for ~p", [Name]), erlang:process_flag(trap_exit, true), - ConnectOpts = maps:get(config, Opts), - ConnectModule = conn_type(maps:get(conn_type, ConnectOpts)), - Forwards = maps:get(forwards, Opts, []), - Queue = open_replayq(maps:get(queue, Opts, #{})), - State = init_opts(Opts), + Queue = open_replayq(Name, maps:get(replayq, ConnectOpts, #{})), + State = init_state(ConnectOpts), self() ! idle, - {ok, idle, State#{connect_module => ConnectModule, - connect_opts => ConnectOpts, - forwards => Forwards, - replayq => Queue - }}. + {ok, idle, State#{ + connect_opts => pre_process_opts(ConnectOpts), + replayq => Queue + }}. -init_opts(Opts) -> +init_state(Opts) -> IfRecordMetrics = maps:get(if_record_metrics, Opts, true), ReconnDelayMs = maps:get(reconnect_interval, Opts, ?DEFAULT_RECONNECT_DELAY_MS), StartType = maps:get(start_type, Opts, manual), @@ -235,17 +211,39 @@ init_opts(Opts) -> if_record_metrics => IfRecordMetrics, name => Name}. -open_replayq(QCfg) -> - Dir = maps:get(replayq_dir, QCfg, undefined), - SegBytes = maps:get(replayq_seg_bytes, QCfg, ?DEFAULT_SEG_BYTES), +open_replayq(Name, QCfg) -> + Dir = maps:get(dir, QCfg, undefined), + SegBytes = maps:get(seg_bytes, QCfg, ?DEFAULT_SEG_BYTES), MaxTotalSize = maps:get(max_total_size, QCfg, ?DEFAULT_MAX_TOTAL_SIZE), QueueConfig = case Dir =:= undefined orelse Dir =:= "" of true -> #{mem_only => true}; - false -> #{dir => Dir, seg_bytes => SegBytes, max_total_size => MaxTotalSize} + false -> #{dir => filename:join([Dir, node(), Name]), + seg_bytes => SegBytes, max_total_size => MaxTotalSize} end, - replayq:open(QueueConfig#{sizer => fun emqx_bridge_msg:estimate_size/1, + replayq:open(QueueConfig#{sizer => fun emqx_connector_mqtt_msg:estimate_size/1, marshaller => fun ?MODULE:msg_marshaller/1}). +pre_process_opts(#{subscriptions := InConf, forwards := OutConf} = ConnectOpts) -> + ConnectOpts#{subscriptions => pre_process_in_out(InConf), + forwards => pre_process_in_out(OutConf)}. + +pre_process_in_out(undefined) -> undefined; +pre_process_in_out(Conf) when is_map(Conf) -> + Conf1 = pre_process_conf(local_topic, Conf), + Conf2 = pre_process_conf(remote_topic, Conf1), + Conf3 = pre_process_conf(payload, Conf2), + Conf4 = pre_process_conf(qos, Conf3), + pre_process_conf(retain, Conf4). + +pre_process_conf(Key, Conf) -> + case maps:find(Key, Conf) of + error -> Conf; + {ok, Val} when is_binary(Val) -> + Conf#{Key => emqx_plugin_libs_rule:preproc_tmpl(Val)}; + {ok, Val} -> + Conf#{Key => Val} + end. + code_change(_Vsn, State, Data, _Extra) -> {ok, State, Data}. @@ -311,28 +309,18 @@ connected(Type, Content, State) -> %% Common handlers common(StateName, {call, From}, status, _State) -> {keep_state_and_data, [{reply, From, StateName}]}; +common(_StateName, {call, From}, ping, #{connection := Conn} =_State) -> + Reply = emqx_connector_mqtt_mod:ping(Conn), + {keep_state_and_data, [{reply, From, Reply}]}; common(_StateName, {call, From}, ensure_stopped, #{connection := undefined} = _State) -> {keep_state_and_data, [{reply, From, ok}]}; -common(_StateName, {call, From}, ensure_stopped, #{connection := Conn, - connect_module := ConnectModule} = State) -> - Reply = ConnectModule:stop(Conn), +common(_StateName, {call, From}, ensure_stopped, #{connection := Conn} = State) -> + Reply = emqx_connector_mqtt_mod:stop(Conn), {next_state, idle, State#{connection => undefined}, [{reply, From, Reply}]}; -common(_StateName, {call, From}, get_forwards, #{forwards := Forwards}) -> +common(_StateName, {call, From}, get_forwards, #{connect_opts := #{forwards := Forwards}}) -> {keep_state_and_data, [{reply, From, Forwards}]}; common(_StateName, {call, From}, get_subscriptions, #{connection := Connection}) -> - {keep_state_and_data, [{reply, From, maps:get(subscriptions, Connection, [])}]}; -common(_StateName, {call, From}, {ensure_forward_present, Topic}, State) -> - {Result, NewState} = do_ensure_forward_present(Topic, State), - {keep_state, NewState, [{reply, From, Result}]}; -common(_StateName, {call, From}, {ensure_subscription_present, Topic, QoS}, State) -> - {Result, NewState} = do_ensure_subscription_present(Topic, QoS, State), - {keep_state, NewState, [{reply, From, Result}]}; -common(_StateName, {call, From}, {ensure_forward_absent, Topic}, State) -> - {Result, NewState} = do_ensure_forward_absent(Topic, State), - {keep_state, NewState, [{reply, From, Result}]}; -common(_StateName, {call, From}, {ensure_subscription_absent, Topic}, State) -> - {Result, NewState} = do_ensure_subscription_absent(Topic, State), - {keep_state, NewState, [{reply, From, Result}]}; + {keep_state_and_data, [{reply, From, maps:get(subscriptions, Connection, #{})}]}; common(_StateName, info, {deliver, _, Msg}, State = #{replayq := Q, if_record_metrics := IfRecordMetric}) -> Msgs = collect([Msg]), @@ -349,76 +337,21 @@ common(StateName, Type, Content, #{name := Name} = State) -> [Name, Type, StateName, Content]), {keep_state, State}. -do_ensure_forward_present(Topic, #{forwards := Forwards, name := Name} = State) -> - case is_topic_present(Topic, Forwards) of - true -> - {ok, State}; - false -> - R = subscribe_local_topic(Topic, Name), - {R, State#{forwards => [Topic | Forwards]}} - end. - -do_ensure_subscription_present(_Topic, _QoS, #{connection := undefined} = State) -> - {{error, no_connection}, State}; -do_ensure_subscription_present(_Topic, _QoS, #{connect_module := emqx_bridge_rpc} = State) -> - {{error, no_remote_subscription_support}, State}; -do_ensure_subscription_present(Topic, QoS, #{connect_module := ConnectModule, - connection := Conn} = State) -> - case is_topic_present(Topic, maps:get(subscriptions, Conn, [])) of - true -> - {ok, State}; - false -> - case ConnectModule:ensure_subscribed(Conn, Topic, QoS) of - {error, Error} -> - {{error, Error}, State}; - Conn1 -> - {ok, State#{connection => Conn1}} - end - end. - -do_ensure_forward_absent(Topic, #{forwards := Forwards} = State) -> - case is_topic_present(Topic, Forwards) of - true -> - R = do_unsubscribe(Topic), - {R, State#{forwards => lists:delete(Topic, Forwards)}}; - false -> - {ok, State} - end. -do_ensure_subscription_absent(_Topic, #{connection := undefined} = State) -> - {{error, no_connection}, State}; -do_ensure_subscription_absent(_Topic, #{connect_module := emqx_bridge_rpc} = State) -> - {{error, no_remote_subscription_support}, State}; -do_ensure_subscription_absent(Topic, #{connect_module := ConnectModule, - connection := Conn} = State) -> - case is_topic_present(Topic, maps:get(subscriptions, Conn, [])) of - true -> - case ConnectModule:ensure_unsubscribed(Conn, Topic) of - {error, Error} -> - {{error, Error}, State}; - Conn1 -> - {ok, State#{connection => Conn1}} - end; - false -> - {ok, State} - end. - -is_topic_present(Topic, Topics) -> - lists:member(Topic, Topics) orelse false =/= lists:keyfind(Topic, 1, Topics). - -do_connect(#{forwards := Forwards, - connect_module := ConnectModule, - connect_opts := ConnectOpts, +do_connect(#{connect_opts := ConnectOpts = #{forwards := Forwards}, inflight := Inflight, name := Name} = State) -> - ok = subscribe_local_topics(Forwards, Name), - case ConnectModule:start(ConnectOpts) of + case Forwards of + undefined -> ok; + #{subscribe_local_topic := Topic} -> subscribe_local_topic(Topic, Name) + end, + case emqx_connector_mqtt_mod:start(ConnectOpts) of {ok, Conn} -> ?tp(info, connected, #{name => Name, inflight => length(Inflight)}), {ok, State#{connection => Conn}}; {error, Reason} -> ConnectOpts1 = obfuscate(ConnectOpts), - ?LOG(error, "Failed to connect with module=~p\n" - "config=~p\nreason:~p", [ConnectModule, ConnectOpts1, Reason]), + ?LOG(error, "Failed to connect \n" + "config=~p\nreason:~p", [ConnectOpts1, Reason]), {error, Reason, State} end. @@ -441,22 +374,19 @@ retry_inflight(State, [#{q_ack_ref := QAckRef, batch := Batch} | Rest] = OldInf) {error, State1#{inflight := NewInf ++ OldInf}} end. -pop_and_send(#{inflight := Inflight, max_inflight := Max } = State) -> +pop_and_send(#{inflight := Inflight, max_inflight := Max} = State) -> pop_and_send_loop(State, Max - length(Inflight)). pop_and_send_loop(State, 0) -> ?tp(debug, inflight_full, #{}), {ok, State}; -pop_and_send_loop(#{replayq := Q, connect_module := Module} = State, N) -> +pop_and_send_loop(#{replayq := Q} = State, N) -> case replayq:is_empty(Q) of true -> ?tp(debug, replayq_drained, #{}), {ok, State}; false -> - BatchSize = case Module of - emqx_bridge_rpc -> maps:get(batch_size, State); - _ -> 1 - end, + BatchSize = 1, Opts = #{count_limit => BatchSize, bytes_limit => 999999999}, {Q1, QAckRef, Batch} = replayq:pop(Q, Opts), case do_send(State#{replayq := Q1}, QAckRef, Batch) of @@ -466,16 +396,20 @@ pop_and_send_loop(#{replayq := Q, connect_module := Module} = State, N) -> end. %% Assert non-empty batch because we have a is_empty check earlier. +do_send(#{connect_opts := #{forwards := undefined}}, _QAckRef, Batch) -> + ?LOG(error, "cannot forward messages to remote broker as 'bridge.mqtt..in' not configured, msg: ~p", [Batch]); do_send(#{inflight := Inflight, - connect_module := Module, connection := Connection, mountpoint := Mountpoint, + connect_opts := #{forwards := Forwards}, if_record_metrics := IfRecordMetrics} = State, QAckRef, [_ | _] = Batch) -> + Vars = emqx_connector_mqtt_msg:make_pub_vars(Mountpoint, Forwards), ExportMsg = fun(Message) -> bridges_metrics_inc(IfRecordMetrics, 'bridge.mqtt.message_sent'), - emqx_bridge_msg:to_export(Module, Mountpoint, Message) + emqx_connector_mqtt_msg:to_remote_msg(Message, Vars) end, - case Module:send(Connection, [ExportMsg(M) || M <- Batch]) of + ?LOG(debug, "publish to remote broker, msg: ~p, vars: ~p", [Batch, Vars]), + case emqx_connector_mqtt_mod:send(Connection, [ExportMsg(M) || M <- Batch]) of {ok, Refs} -> {ok, State#{inflight := Inflight ++ [#{q_ack_ref => QAckRef, send_ack_ref => map_set(Refs), @@ -530,9 +464,6 @@ drop_acked_batches(Q, [#{send_ack_ref := Refs, All end. -subscribe_local_topics(Topics, Name) -> - lists:foreach(fun(Topic) -> subscribe_local_topic(Topic, Name) end, Topics). - subscribe_local_topic(Topic, Name) -> do_subscribe(Topic, Name). @@ -549,32 +480,25 @@ validate(RawTopic) -> do_subscribe(RawTopic, Name) -> TopicFilter = validate(RawTopic), - {Topic, SubOpts} = emqx_topic:parse(TopicFilter, #{qos => ?QOS_1}), + {Topic, SubOpts} = emqx_topic:parse(TopicFilter, #{qos => ?QOS_2}), emqx_broker:subscribe(Topic, Name, SubOpts). -do_unsubscribe(RawTopic) -> - TopicFilter = validate(RawTopic), - {Topic, _SubOpts} = emqx_topic:parse(TopicFilter), - emqx_broker:unsubscribe(Topic). - -disconnect(#{connection := Conn, - connect_module := Module - } = State) when Conn =/= undefined -> - Module:stop(Conn), +disconnect(#{connection := Conn} = State) when Conn =/= undefined -> + emqx_connector_mqtt_mod:stop(Conn), State#{connection => undefined}; disconnect(State) -> State. %% Called only when replayq needs to dump it to disk. -msg_marshaller(Bin) when is_binary(Bin) -> emqx_bridge_msg:from_binary(Bin); -msg_marshaller(Msg) -> emqx_bridge_msg:to_binary(Msg). +msg_marshaller(Bin) when is_binary(Bin) -> emqx_connector_mqtt_msg:from_binary(Bin); +msg_marshaller(Msg) -> emqx_connector_mqtt_msg:to_binary(Msg). format_mountpoint(undefined) -> undefined; format_mountpoint(Prefix) -> binary:replace(iolist_to_binary(Prefix), <<"${node}">>, atom_to_binary(node(), utf8)). -name(Id) -> list_to_atom(lists:concat([?MODULE, "_", Id])). +name(Id) -> list_to_atom(str(Id)). register_metrics() -> lists:foreach(fun emqx_metrics:ensure/1, @@ -603,9 +527,9 @@ obfuscate(Map) -> is_sensitive(password) -> true; is_sensitive(_) -> false. -conn_type(rpc) -> - emqx_bridge_rpc; -conn_type(mqtt) -> - emqx_bridge_mqtt; -conn_type(Mod) when is_atom(Mod) -> - Mod. +str(A) when is_atom(A) -> + atom_to_list(A); +str(B) when is_binary(B) -> + binary_to_list(B); +str(S) when is_list(S) -> + S. \ No newline at end of file diff --git a/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_tests.erl b/apps/emqx_connector/test/emqx_connector_mqtt_tests.erl similarity index 87% rename from apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_tests.erl rename to apps/emqx_connector/test/emqx_connector_mqtt_tests.erl index 5babe0ed9..0f4d651c9 100644 --- a/apps/emqx_bridge_mqtt/test/emqx_bridge_mqtt_tests.erl +++ b/apps/emqx_connector/test/emqx_connector_mqtt_tests.erl @@ -14,7 +14,7 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_bridge_mqtt_tests). +-module(emqx_connector_mqtt_tests). -include_lib("eunit/include/eunit.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). @@ -37,11 +37,11 @@ send_and_ack_test() -> try Max = 1, Batch = lists:seq(1, Max), - {ok, Conn} = emqx_bridge_mqtt:start(#{address => "127.0.0.1:1883"}), + {ok, Conn} = emqx_connector_mqtt_mod:start(#{server => {{127,0,0,1}, 1883}}), % %% return last packet id as batch reference - {ok, _AckRef} = emqx_bridge_mqtt:send(Conn, Batch), + {ok, _AckRef} = emqx_connector_mqtt_mod:send(Conn, Batch), - ok = emqx_bridge_mqtt:stop(Conn) + ok = emqx_connector_mqtt_mod:stop(Conn) after meck:unload(emqtt) end. diff --git a/apps/emqx_bridge_mqtt/test/emqx_bridge_worker_tests.erl b/apps/emqx_connector/test/emqx_connector_mqtt_worker_tests.erl similarity index 55% rename from apps/emqx_bridge_mqtt/test/emqx_bridge_worker_tests.erl rename to apps/emqx_connector/test/emqx_connector_mqtt_worker_tests.erl index ffa2e9ee5..090106cef 100644 --- a/apps/emqx_bridge_mqtt/test/emqx_bridge_worker_tests.erl +++ b/apps/emqx_connector/test/emqx_connector_mqtt_worker_tests.erl @@ -14,14 +14,14 @@ %% limitations under the License. %%-------------------------------------------------------------------- --module(emqx_bridge_worker_tests). +-module(emqx_connector_mqtt_worker_tests). -include_lib("eunit/include/eunit.hrl"). -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). -define(BRIDGE_NAME, test). --define(BRIDGE_REG_NAME, emqx_bridge_worker_test). +-define(BRIDGE_REG_NAME, emqx_connector_mqtt_worker_test). -define(WAIT(PATTERN, TIMEOUT), receive PATTERN -> @@ -31,7 +31,6 @@ error(timeout) end). -%% stub callbacks -export([start/1, send/2, stop/1]). start(#{connect_result := Result, test_pid := Pid, test_ref := Ref}) -> @@ -49,33 +48,41 @@ stop(_Pid) -> ok. %% bridge worker should retry connecting remote node indefinitely % reconnect_test() -> % emqx_metrics:start_link(), -% emqx_bridge_worker:register_metrics(), +% emqx_connector_mqtt_worker:register_metrics(), % Ref = make_ref(), % Config = make_config(Ref, self(), {error, test}), -% {ok, Pid} = emqx_bridge_worker:start_link(?BRIDGE_NAME, Config), +% {ok, Pid} = emqx_connector_mqtt_worker:start_link(?BRIDGE_NAME, Config), % %% assert name registered % ?assertEqual(Pid, whereis(?BRIDGE_REG_NAME)), % ?WAIT({connection_start_attempt, Ref}, 1000), % %% expect same message again % ?WAIT({connection_start_attempt, Ref}, 1000), -% ok = emqx_bridge_worker:stop(?BRIDGE_REG_NAME), +% ok = emqx_connector_mqtt_worker:stop(?BRIDGE_REG_NAME), % emqx_metrics:stop(), % ok. %% connect first, disconnect, then connect again disturbance_test() -> - emqx_metrics:start_link(), - emqx_bridge_worker:register_metrics(), - Ref = make_ref(), - TestPid = self(), - Config = make_config(Ref, TestPid, {ok, #{client_pid => TestPid}}), - {ok, Pid} = emqx_bridge_worker:start_link(Config#{name => disturbance}), - ?assertEqual(Pid, whereis(emqx_bridge_worker_disturbance)), - ?WAIT({connection_start_attempt, Ref}, 1000), - Pid ! {disconnected, TestPid, test}, - ?WAIT({connection_start_attempt, Ref}, 1000), - emqx_metrics:stop(), - ok = emqx_bridge_worker:stop(Pid). + meck:new(emqx_connector_mqtt_mod, [passthrough, no_history]), + meck:expect(emqx_connector_mqtt_mod, start, 1, fun(Conf) -> start(Conf) end), + meck:expect(emqx_connector_mqtt_mod, send, 2, fun(SendFun, Batch) -> send(SendFun, Batch) end), + meck:expect(emqx_connector_mqtt_mod, stop, 1, fun(Pid) -> stop(Pid) end), + try + emqx_metrics:start_link(), + emqx_connector_mqtt_worker:register_metrics(), + Ref = make_ref(), + TestPid = self(), + Config = make_config(Ref, TestPid, {ok, #{client_pid => TestPid}}), + {ok, Pid} = emqx_connector_mqtt_worker:start_link(Config#{name => bridge_disturbance}), + ?assertEqual(Pid, whereis(bridge_disturbance)), + ?WAIT({connection_start_attempt, Ref}, 1000), + Pid ! {disconnected, TestPid, test}, + ?WAIT({connection_start_attempt, Ref}, 1000), + emqx_metrics:stop(), + ok = emqx_connector_mqtt_worker:stop(Pid) + after + meck:unload(emqx_connector_mqtt_mod) + end. % % %% buffer should continue taking in messages when disconnected % buffer_when_disconnected_test_() -> @@ -96,40 +103,47 @@ disturbance_test() -> % Config0 = make_config(Ref, false, {ok, #{client_pid => undefined}}), % Config = Config0#{reconnect_delay_ms => 100}, % emqx_metrics:start_link(), -% emqx_bridge_worker:register_metrics(), -% {ok, Pid} = emqx_bridge_worker:start_link(?BRIDGE_NAME, Config), +% emqx_connector_mqtt_worker:register_metrics(), +% {ok, Pid} = emqx_connector_mqtt_worker:start_link(?BRIDGE_NAME, Config), % Sender ! {bridge, Pid}, % Receiver ! {bridge, Pid}, % ?assertEqual(Pid, whereis(?BRIDGE_REG_NAME)), % Pid ! {disconnected, Ref, test}, % ?WAIT({'DOWN', SenderMref, process, Sender, normal}, 5000), % ?WAIT({'DOWN', ReceiverMref, process, Receiver, normal}, 1000), -% ok = emqx_bridge_worker:stop(?BRIDGE_REG_NAME), +% ok = emqx_connector_mqtt_worker:stop(?BRIDGE_REG_NAME), % emqx_metrics:stop(). manual_start_stop_test() -> - emqx_metrics:start_link(), - emqx_bridge_worker:register_metrics(), - Ref = make_ref(), - TestPid = self(), - BridgeName = manual_start_stop, - Config0 = make_config(Ref, TestPid, {ok, #{client_pid => TestPid}}), - Config = Config0#{start_type := manual}, - {ok, Pid} = emqx_bridge_worker:start_link(Config#{name => BridgeName}), - %% call ensure_started again should yeld the same result - ok = emqx_bridge_worker:ensure_started(BridgeName), - emqx_bridge_worker:ensure_stopped(BridgeName), - emqx_metrics:stop(), - ok = emqx_bridge_worker:stop(Pid). + meck:new(emqx_connector_mqtt_mod, [passthrough, no_history]), + meck:expect(emqx_connector_mqtt_mod, start, 1, fun(Conf) -> start(Conf) end), + meck:expect(emqx_connector_mqtt_mod, send, 2, fun(SendFun, Batch) -> send(SendFun, Batch) end), + meck:expect(emqx_connector_mqtt_mod, stop, 1, fun(Pid) -> stop(Pid) end), + try + emqx_metrics:start_link(), + emqx_connector_mqtt_worker:register_metrics(), + Ref = make_ref(), + TestPid = self(), + BridgeName = manual_start_stop, + Config0 = make_config(Ref, TestPid, {ok, #{client_pid => TestPid}}), + Config = Config0#{start_type := manual}, + {ok, Pid} = emqx_connector_mqtt_worker:start_link(Config#{name => BridgeName}), + %% call ensure_started again should yeld the same result + ok = emqx_connector_mqtt_worker:ensure_started(BridgeName), + emqx_connector_mqtt_worker:ensure_stopped(BridgeName), + emqx_metrics:stop(), + ok = emqx_connector_mqtt_worker:stop(Pid) + after + meck:unload(emqx_connector_mqtt_mod) + end. make_config(Ref, TestPid, Result) -> #{ start_type => auto, + subscriptions => undefined, + forwards => undefined, reconnect_interval => 50, - config => #{ - test_pid => TestPid, - test_ref => Ref, - conn_type => ?MODULE, - connect_result => Result - } + test_pid => TestPid, + test_ref => Ref, + connect_result => Result }. diff --git a/apps/emqx_dashboard/etc/emqx_dashboard.conf b/apps/emqx_dashboard/etc/emqx_dashboard.conf index 31c95a9ee..70b1d1d71 100644 --- a/apps/emqx_dashboard/etc/emqx_dashboard.conf +++ b/apps/emqx_dashboard/etc/emqx_dashboard.conf @@ -11,35 +11,30 @@ emqx_dashboard { token_expired_time = 60m listeners = [ { + protocol = http num_acceptors = 4 max_connections = 512 - protocol = http port = 18083 backlog = 512 - send_timeout = 15s - send_timeout_close = true + send_timeout = 5s inet6 = false ipv6_v6only = false } -## , -## { -## protocol: https -## port: 18084 -## acceptors: 2 -## backlog: 512 -## send_timeout: 15s -## send_timeout_close: true -## inet6: false -## ipv6_v6only: false -## certfile = "etc/certs/cert.pem" -## keyfile = "etc/certs/key.pem" -## cacertfile = "etc/certs/cacert.pem" -## verify = verify_peer -## tls_versions = "tlsv1.3,tlsv1.2,tlsv1.1,tlsv1" -## ciphers = "TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256,TLS_CHACHA20_POLY1305_SHA256,TLS_AES_128_CCM_SHA256,TLS_AES_128_CCM_8_SHA256,ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-AES256-SHA384,ECDHE-RSA-AES256-SHA384,ECDHE-ECDSA-DES-CBC3-SHA,ECDH-ECDSA-AES256-GCM-SHA384,ECDH-RSA-AES256-GCM-SHA384,ECDH-ECDSA-AES256-SHA384,ECDH-RSA-AES256-SHA384,DHE-DSS-AES256-GCM-SHA384,DHE-DSS-AES256-SHA256,AES256-GCM-SHA384,AES256-SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES128-SHA256,ECDHE-RSA-AES128-SHA256,ECDH-ECDSA-AES128-GCM-SHA256,ECDH-RSA-AES128-GCM-SHA256,ECDH-ECDSA-AES128-SHA256,ECDH-RSA-AES128-SHA256,DHE-DSS-AES128-GCM-SHA256,DHE-DSS-AES128-SHA256,AES128-GCM-SHA256,AES128-SHA256,ECDHE-ECDSA-AES256-SHA,ECDHE-RSA-AES256-SHA,DHE-DSS-AES256-SHA,ECDH-ECDSA-AES256-SHA,ECDH-RSA-AES256-SHA,AES256-SHA,ECDHE-ECDSA-AES128-SHA,ECDHE-RSA-AES128-SHA,DHE-DSS-AES128-SHA,ECDH-ECDSA-AES128-SHA,ECDH-RSA-AES128-SHA,AES128-SHA" -## fail_if_no_peer_cert = true -## inet6 = false -## ipv6_v6only = false -## } + # , + # { + # protocol = https + # port = 18084 + # num_acceptors = 2 + # backlog = 512 + # send_timeout = 5s + # inet6 = false + # ipv6_v6only = false + # certfile = "etc/certs/cert.pem" + # keyfile = "etc/certs/key.pem" + # cacertfile = "etc/certs/cacert.pem" + # verify = verify_peer + # versions = ["tlsv1.3","tlsv1.2","tlsv1.1","tlsv1"] + # ciphers = ["TLS_AES_256_GCM_SHA384","TLS_AES_128_GCM_SHA256","TLS_CHACHA20_POLY1305_SHA256","TLS_AES_128_CCM_SHA256","TLS_AES_128_CCM_8_SHA256","ECDHE-ECDSA-AES256-GCM-SHA384","ECDHE-RSA-AES256-GCM-SHA384","ECDHE-ECDSA-AES256-SHA384","ECDHE-RSA-AES256-SHA384","ECDHE-ECDSA-DES-CBC3-SHA","ECDH-ECDSA-AES256-GCM-SHA384","ECDH-RSA-AES256-GCM-SHA384","ECDH-ECDSA-AES256-SHA384","ECDH-RSA-AES256-SHA384","DHE-DSS-AES256-GCM-SHA384","DHE-DSS-AES256-SHA256","AES256-GCM-SHA384","AES256-SHA256","ECDHE-ECDSA-AES128-GCM-SHA256","ECDHE-RSA-AES128-GCM-SHA256","ECDHE-ECDSA-AES128-SHA256","ECDHE-RSA-AES128-SHA256","ECDH-ECDSA-AES128-GCM-SHA256","ECDH-RSA-AES128-GCM-SHA256","ECDH-ECDSA-AES128-SHA256","ECDH-RSA-AES128-SHA256","DHE-DSS-AES128-GCM-SHA256","DHE-DSS-AES128-SHA256","AES128-GCM-SHA256","AES128-SHA256","ECDHE-ECDSA-AES256-SHA","ECDHE-RSA-AES256-SHA","DHE-DSS-AES256-SHA","ECDH-ECDSA-AES256-SHA","ECDH-RSA-AES256-SHA","AES256-SHA","ECDHE-ECDSA-AES128-SHA","ECDHE-RSA-AES128-SHA","DHE-DSS-AES128-SHA","ECDH-ECDSA-AES128-SHA","ECDH-RSA-AES128-SHA","AES128-SHA"] + # } ] } diff --git a/apps/emqx_dashboard/src/emqx_dashboard.erl b/apps/emqx_dashboard/src/emqx_dashboard.erl index fb0e25564..d109dd445 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard.erl @@ -20,9 +20,7 @@ -export([ start_listeners/0 - , stop_listeners/0 - , start_listener/1 - , stop_listener/1]). + , stop_listeners/0]). %% Authorization -export([authorize_appid/1]). @@ -36,15 +34,8 @@ %%-------------------------------------------------------------------- start_listeners() -> - lists:foreach(fun start_listener/1, listeners()). - -stop_listeners() -> - lists:foreach(fun stop_listener/1, listeners()). - -start_listener({Proto, Port, Options}) -> {ok, _} = application:ensure_all_started(minirest), Authorization = {?MODULE, authorize_appid}, - RanchOptions = ranch_opts(Port, Options), GlobalSpec = #{ openapi => "3.0.0", info => #{title => "EMQ X Dashboard API", version => "5.0.0"}, @@ -56,20 +47,33 @@ start_listener({Proto, Port, Options}) -> type => apiKey, name => "authorization", in => header}}}}, - Dispatch = [{"/", cowboy_static, {priv_file, emqx_dashboard, "www/index.html"}}, - {"/static/[...]", cowboy_static, {priv_dir, emqx_dashboard, "www/static"}}, - {'_', cowboy_static, {priv_file, emqx_dashboard, "www/index.html"}}], - Minirest = #{ - protocol => Proto, + Dispatch = [ + {"/", cowboy_static, {priv_file, emqx_dashboard, "www/index.html"}}, + {"/static/[...]", cowboy_static, {priv_dir, emqx_dashboard, "www/static"}}, + {'_', cowboy_static, {priv_file, emqx_dashboard, "www/index.html"}} + ], + BaseMinirest = #{ base_path => ?BASE_PATH, modules => minirest_api:find_api_modules(apps()), authorization => Authorization, security => [#{application => []}], swagger_global_spec => GlobalSpec, - dispatch => Dispatch}, - MinirestOptions = maps:merge(Minirest, RanchOptions), - {ok, _} = minirest:start(listener_name(Proto), MinirestOptions), - ?ULOG("Start ~p listener on ~p successfully.~n", [listener_name(Proto), Port]). + dispatch => Dispatch + }, + [begin + Minirest = maps:put(protocol, Protocol, BaseMinirest), + {ok, _} = minirest:start(Name, RanchOptions, Minirest), + ?ULOG("Start listener ~s on ~p successfully.~n", [Name, Port]) + end || {Name, Protocol, Port, RanchOptions} <- listeners()]. + +stop_listeners() -> + [begin + ok = minirest:stop(Name), + ?ULOG("Stop listener ~s on ~p successfully.~n", [Name, Port]) + end || {Name, _, Port, _} <- listeners()]. + +%%-------------------------------------------------------------------- +%% internal apps() -> [App || {App, _, _} <- application:loaded_applications(), @@ -78,33 +82,61 @@ apps() -> _ -> false end]. -ranch_opts(Port, Options0) -> - Options = lists:foldl( - fun - ({K, _V}, Acc) when K =:= max_connections orelse K =:= num_acceptors -> Acc; - ({inet6, true}, Acc) -> [inet6 | Acc]; - ({inet6, false}, Acc) -> Acc; - ({ipv6_v6only, true}, Acc) -> [{ipv6_v6only, true} | Acc]; - ({ipv6_v6only, false}, Acc) -> Acc; - ({K, V}, Acc)-> - [{K, V} | Acc] - end, [], Options0), - maps:from_list([{port, Port} | Options]). - -stop_listener({Proto, Port, _}) -> - ?ULOG("Stop dashboard listener on ~s successfully.~n", [format(Port)]), - minirest:stop(listener_name(Proto)). - listeners() -> - [{Protocol, Port, maps:to_list(maps:without([protocol, port], Map))} - || Map = #{protocol := Protocol,port := Port} - <- emqx:get_config([emqx_dashboard, listeners], [])]. + [begin + Protocol = maps:get(protocol, ListenerOptions, http), + Port = maps:get(port, ListenerOptions, 18083), + Name = listener_name(Protocol, Port), + RanchOptions = ranch_opts(maps:without([protocol], ListenerOptions)), + {Name, Protocol, Port, RanchOptions} + end || ListenerOptions <- emqx_config:get([emqx_dashboard, listeners], [])]. -listener_name(Proto) -> - list_to_atom(atom_to_list(Proto) ++ ":dashboard"). +ranch_opts(RanchOptions) -> + Keys = [ {ack_timeout, handshake_timeout} + , connection_type + , max_connections + , num_acceptors + , shutdown + , socket], + {S, R} = lists:foldl(fun key_take/2, {RanchOptions, #{}}, Keys), + R#{socket_opts => maps:fold(fun key_only/3, [], S)}. + + +key_take({K, K1}, {All, R}) -> + case maps:get(K, All, undefined) of + undefined -> + {All, R}; + V -> + {maps:remove(K, All), R#{K1 => V}} + end; +key_take(K, {All, R}) -> + case maps:get(K, All, undefined) of + undefined -> + {All, R}; + V -> + {maps:remove(K, All), R#{K => V}} + end. + +key_only(K , true , S) -> [K | S]; +key_only(_K, false, S) -> S; +key_only(K , V , S) -> [{K, V} | S]. + +listener_name(Protocol, Port) -> + Name = "dashboard:" ++ atom_to_list(Protocol) ++ ":" ++ integer_to_list(Port), + list_to_atom(Name). authorize_appid(Req) -> case cowboy_req:parse_header(<<"authorization">>, Req) of + {basic, Username, Password} -> + case emqx_dashboard_admin:check(Username, Password) of + ok -> + ok; + {error, _} -> + {401, #{<<"WWW-Authenticate">> => + <<"Basic Realm=\"minirest-server\"">>}, + #{code => <<"ERROR_USERNAME_OR_PWD">>, + message => <<"Check your username and password">>}} + end; {bearer, Token} -> case emqx_dashboard_admin:verify_token(Token) of ok -> @@ -113,8 +145,7 @@ authorize_appid(Req) -> {401, #{<<"WWW-Authenticate">> => <<"Bearer Realm=\"minirest-server\"">>}, #{code => <<"TOKEN_TIME_OUT">>, - message => <<"POST '/login', get your new token">>} - }; + message => <<"POST '/login', get your new token">>}}; {error, not_found} -> {401, #{<<"WWW-Authenticate">> => <<"Bearer Realm=\"minirest-server\"">>}, @@ -123,14 +154,7 @@ authorize_appid(Req) -> end; _ -> {401, #{<<"WWW-Authenticate">> => - <<"Bearer Realm=\"minirest-server\"">>}, - #{code => <<"UNAUTHORIZED">>, - message => <<"POST '/login'">>}} + <<"Basic Realm=\"minirest-server\"">>}, + #{code => <<"ERROR_USERNAME_OR_PWD">>, + message => <<"Check your username and password">>}} end. - -format(Port) when is_integer(Port) -> - io_lib:format("0.0.0.0:~w", [Port]); -format({Addr, Port}) when is_list(Addr) -> - io_lib:format("~s:~w", [Addr, Port]); -format({Addr, Port}) when is_tuple(Addr) -> - io_lib:format("~s:~w", [inet:ntoa(Addr), Port]). diff --git a/apps/emqx_dashboard/src/emqx_dashboard_admin.erl b/apps/emqx_dashboard/src/emqx_dashboard_admin.erl index 8a1306e94..b477bd779 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_admin.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_admin.erl @@ -40,7 +40,7 @@ -export([ sign_token/2 , verify_token/1 - , destroy_token_by_username/1 + , destroy_token_by_username/2 ]). -export([add_default_user/0]). @@ -177,8 +177,13 @@ sign_token(Username, Password) -> verify_token(Token) -> emqx_dashboard_token:verify(Token). -destroy_token_by_username(Username) -> - emqx_dashboard_token:destroy_by_username(Username). +destroy_token_by_username(Username, Token) -> + case emqx_dashboard_token:lookup(Token) of + {ok, #mqtt_admin_jwt{username = Username}} -> + emqx_dashboard_token:destroy(Token); + _ -> + {error, not_found} + end. %%-------------------------------------------------------------------- %% Internal functions diff --git a/apps/emqx_dashboard/src/emqx_dashboard_api.erl b/apps/emqx_dashboard/src/emqx_dashboard_api.erl index 88ae85d9d..68c737488 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_api.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_api.erl @@ -49,6 +49,8 @@ -define(EMPTY(V), (V == undefined orelse V == <<>>)). +-define(ERROR_USERNAME_OR_PWD, 'ERROR_USERNAME_OR_PWD'). + api_spec() -> {[ login_api() , logout_api() @@ -164,14 +166,18 @@ login(post, #{body := Params}) -> {ok, Token} -> Version = iolist_to_binary(proplists:get_value(version, emqx_sys:info())), {200, #{token => Token, version => Version, license => #{edition => ?RELEASE}}}; - {error, Code} -> - {401, #{code => Code, message => <<"Auth filed">>}} + {error, _} -> + {401, #{code => ?ERROR_USERNAME_OR_PWD, message => <<"Auth filed">>}} end. -logout(_, #{body := Params}) -> - Username = maps:get(<<"username">>, Params), - emqx_dashboard_admin:destroy_token_by_username(Username), - {200}. +logout(_, #{body := #{<<"username">> := Username}, + headers := #{<<"authorization">> := <<"Bearer ", Token/binary>>}}) -> + case emqx_dashboard_admin:destroy_token_by_username(Username, Token) of + ok -> + 200; + _R -> + {401, 'BAD_TOKEN_OR_USERNAME', <<"Ensure your token & username">>} + end. users(get, _Request) -> {200, [row(User) || User <- emqx_dashboard_admin:all_users()]}; @@ -233,7 +239,7 @@ parameters() -> unauthorized_request() -> object_schema( properties([{message, string}, - {code, string, <<"Resp Code">>, ['PASSWORD_ERROR','USERNAME_ERROR']} + {code, string, <<"Resp Code">>, [?ERROR_USERNAME_OR_PWD]} ]), <<"Unauthorized">> ). diff --git a/apps/emqx_dashboard/src/emqx_dashboard_app.erl b/apps/emqx_dashboard/src/emqx_dashboard_app.erl index edcc19d8b..4e1b0caec 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_app.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_app.erl @@ -27,7 +27,7 @@ start(_StartType, _StartArgs) -> {ok, Sup} = emqx_dashboard_sup:start_link(), ok = ekka_rlog:wait_for_shards([?DASHBOARD_SHARD], infinity), - emqx_dashboard:start_listeners(), + _ = emqx_dashboard:start_listeners(), emqx_dashboard_cli:load(), ok = emqx_dashboard_admin:add_default_user(), {ok, Sup}. diff --git a/apps/emqx_dashboard/src/emqx_dashboard_schema.erl b/apps/emqx_dashboard/src/emqx_dashboard_schema.erl index e0ff21ada..3ba3dc803 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_schema.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_schema.erl @@ -27,24 +27,23 @@ fields("emqx_dashboard") -> hoconsc:ref(?MODULE, "https")]))} , {default_username, fun default_username/1} , {default_password, fun default_password/1} - , {sample_interval, emqx_schema:t(emqx_schema:duration_s(), undefined, "10s")} - , {token_expired_time, emqx_schema:t(emqx_schema:duration(), undefined, "30m")} + , {sample_interval, sc(emqx_schema:duration_s(), #{default => "10s"})} + , {token_expired_time, sc(emqx_schema:duration(), #{default => "30m"})} ]; fields("http") -> [ {"protocol", hoconsc:enum([http, https])} - , {"port", emqx_schema:t(integer(), undefined, 18083)} - , {"num_acceptors", emqx_schema:t(integer(), undefined, 4)} - , {"max_connections", emqx_schema:t(integer(), undefined, 512)} - , {"backlog", emqx_schema:t(integer(), undefined, 1024)} - , {"send_timeout", emqx_schema:t(emqx_schema:duration(), undefined, "15s")} - , {"send_timeout_close", emqx_schema:t(boolean(), undefined, true)} - , {"inet6", emqx_schema:t(boolean(), undefined, false)} - , {"ipv6_v6only", emqx_schema:t(boolean(), undefined, false)} + , {"port", hoconsc:mk(integer(), #{default => 18083})} + , {"num_acceptors", sc(integer(), #{default => 4})} + , {"max_connections", sc(integer(), #{default => 512})} + , {"backlog", sc(integer(), #{default => 1024})} + , {"send_timeout", sc(emqx_schema:duration(), #{default => "5s"})} + , {"inet6", sc(boolean(), #{default => false})} + , {"ipv6_v6only", sc(boolean(), #{dfeault => false})} ]; fields("https") -> - emqx_schema:ssl(#{enable => true}) ++ fields("http"). + proplists:delete("fail_if_no_peer_cert", emqx_schema:ssl(#{})) ++ fields("http"). default_username(type) -> string(); default_username(default) -> "admin"; @@ -55,3 +54,5 @@ default_password(type) -> string(); default_password(default) -> "public"; default_password(nullable) -> false; default_password(_) -> undefined. + +sc(Type, Meta) -> hoconsc:mk(Type, Meta). diff --git a/apps/emqx_dashboard/src/emqx_dashboard_token.erl b/apps/emqx_dashboard/src/emqx_dashboard_token.erl index 9086b4c2e..2acf00f13 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_token.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_token.erl @@ -22,6 +22,7 @@ -export([ sign/2 , verify/1 + , lookup/1 , destroy/1 , destroy_by_username/1 ]). @@ -121,14 +122,15 @@ do_verify(Token)-> do_destroy(Token) -> Fun = fun mnesia:delete/1, - ekka_mnesia:transaction(?DASHBOARD_SHARD, Fun, [{?TAB, Token}]). + {atomic, ok} = ekka_mnesia:transaction(?DASHBOARD_SHARD, Fun, [{?TAB, Token}]), + ok. do_destroy_by_username(Username) -> gen_server:cast(?MODULE, {destroy, Username}). %%-------------------------------------------------------------------- %% jwt internal util function - +-spec(lookup(Token :: binary()) -> {ok, #mqtt_admin_jwt{}} | {error, not_found}). lookup(Token) -> case mnesia:dirty_read(?TAB, Token) of [JWT] -> {ok, JWT}; diff --git a/apps/emqx_data_bridge/etc/emqx_data_bridge.conf b/apps/emqx_data_bridge/etc/emqx_data_bridge.conf deleted file mode 100644 index 99a49dba3..000000000 --- a/apps/emqx_data_bridge/etc/emqx_data_bridge.conf +++ /dev/null @@ -1,129 +0,0 @@ -##-------------------------------------------------------------------- -## EMQ X Bridge Plugin -##-------------------------------------------------------------------- - -emqx_data_bridge { - bridges:[ - # {name: "mysql_bridge_1" - # type: mysql - # config: { - # server: "192.168.0.172:3306" - # database: mqtt - # pool_size: 1 - # username: root - # password: public - # auto_reconnect: true - # ssl: false - # } - # } - # , {name: "pgsql_bridge_1" - # type: pgsql - # config: { - # server: "192.168.0.172:5432" - # database: mqtt - # pool_size: 1 - # username: root - # password: public - # auto_reconnect: true - # ssl: false - # } - # } - # , {name: "mongodb_bridge_single" - # type: mongo - # config: { - # servers: "192.168.0.172:27017" - # mongo_type: single - # pool_size: 1 - # login: root - # password: public - # auth_source: mqtt - # database: mqtt - # ssl: false - # } - # } - # ,{name: "mongodb_bridge_rs" - # type: mongo - # config: { - # servers: "127.0.0.1:27017" - # mongo_type: rs - # rs_set_name: rs_name - # pool_size: 1 - # login: root - # password: public - # auth_source: mqtt - # database: mqtt - # ssl: false - # } - # } - # ,{name: "mongodb_bridge_shared" - # type: mongo - # config: { - # servers: "127.0.0.1:27017" - # mongo_type: shared - # pool_size: 1 - # login: root - # password: public - # auth_source: mqtt - # database: mqtt - # ssl: false - # max_overflow: 1 - # overflow_ttl: - # overflow_check_period: 10s - # local_threshold_ms: 10s - # connect_timeout_ms: 10s - # socket_timeout_ms: 10s - # server_selection_timeout_ms: 10s - # wait_queue_timeout_ms: 10s - # heartbeat_frequency_ms: 10s - # min_heartbeat_frequency_ms: 10s - # } - # } - # , {name: "redis_bridge_single" - # type: redis - # config: { - # servers: "192.168.0.172:6379" - # redis_type: single - # pool_size: 1 - # database: 0 - # password: public - # auto_reconnect: true - # ssl: false - # } - # } - # ,{name: "redis_bridge_sentinel" - # type: redis - # config: { - # servers: "127.0.0.1:6379, 127.0.0.2:6379, 127.0.0.3:6379" - # redis_type: sentinel - # sentinel_name: mymaster - # pool_size: 1 - # database: 0 - # ssl: false - # } - # } - # ,{name: "redis_bridge_cluster" - # type: redis - # config: { - # servers: "127.0.0.1:6379, 127.0.0.2:6379, 127.0.0.3:6379" - # redis_type: cluster - # pool_size: 1 - # database: 0 - # password: "public" - # ssl: false - # } - # } - # , {name: "ldap_bridge_1" - # type: ldap - # config: { - # servers: "192.168.0.172" - # port: 389 - # bind_dn: "cn=root,dc=emqx,dc=io" - # bind_password: "public" - # timeout: 30s - # pool_size: 1 - # ssl: false - # } - # } - - ] -} diff --git a/apps/emqx_data_bridge/src/emqx_data_bridge_schema.erl b/apps/emqx_data_bridge/src/emqx_data_bridge_schema.erl index 69f53d6c1..e3c6d8ee9 100644 --- a/apps/emqx_data_bridge/src/emqx_data_bridge_schema.erl +++ b/apps/emqx_data_bridge/src/emqx_data_bridge_schema.erl @@ -22,5 +22,5 @@ fields(ldap) -> connector_fields(ldap). connector_fields(DB) -> Mod = list_to_existing_atom(io_lib:format("~s_~s",[emqx_connector, DB])), - [{name, hoconsc:t(typerefl:binary())}, + [{name, hoconsc:mk(typerefl:binary())}, {type, #{type => DB}}] ++ Mod:roots(). diff --git a/apps/emqx_exhook/src/emqx_exhook_schema.erl b/apps/emqx_exhook/src/emqx_exhook_schema.erl index 16fd93fa0..64d39eb52 100644 --- a/apps/emqx_exhook/src/emqx_exhook_schema.erl +++ b/apps/emqx_exhook/src/emqx_exhook_schema.erl @@ -32,43 +32,57 @@ -reflect_type([duration/0]). --export([roots/0, fields/1]). +-export([namespace/0, roots/0, fields/1]). --export([t/1, t/3, t/4, ref/1]). +namespace() -> exhook. roots() -> [exhook]. fields(exhook) -> - [ {request_failed_action, t(union([deny, ignore]), undefined, deny)} - , {request_timeout, t(duration(), undefined, "5s")} - , {auto_reconnect, t(union([false, duration()]), undefined, "60s")} - , {servers, t(hoconsc:array(ref(servers)), undefined, [])} + [ {request_failed_action, + sc(union([deny, ignore]), + #{default => deny})} + , {request_timeout, + sc(duration(), + #{default => "5s"})} + , {auto_reconnect, + sc(union([false, duration()]), + #{ default => "60s" + })} + , {servers, + sc(hoconsc:array(ref(servers)), + #{default => []})} ]; fields(servers) -> - [ {name, string()} - , {url, string()} - , {ssl, t(ref(ssl_conf_group))} + [ {name, + sc(string(), + #{})} + , {url, + sc(string(), + #{})} + , {ssl, + sc(ref(ssl_conf), + #{})} ]; -fields(ssl_conf_group) -> - [ {cacertfile, string()} - , {certfile, string()} - , {keyfile, string()} +fields(ssl_conf) -> + [ {cacertfile, + sc(string(), + #{}) + } + , {certfile, + sc(string(), + #{}) + } + , {keyfile, + sc(string(), + #{})} ]. %% types -t(Type) -> #{type => Type}. - -t(Type, Mapping, Default) -> - hoconsc:t(Type, #{mapping => Mapping, default => Default}). - -t(Type, Mapping, Default, OverrideEnv) -> - hoconsc:t(Type, #{ mapping => Mapping - , default => Default - , override_env => OverrideEnv - }). +sc(Type, Meta) -> Meta#{type => Type}. ref(Field) -> hoconsc:ref(?MODULE, Field). diff --git a/apps/emqx_gateway/etc/emqx_gateway.conf b/apps/emqx_gateway/etc/emqx_gateway.conf index 206c54b93..2ce48bf75 100644 --- a/apps/emqx_gateway/etc/emqx_gateway.conf +++ b/apps/emqx_gateway/etc/emqx_gateway.conf @@ -29,19 +29,59 @@ gateway.stomp { password = "${Packet.headers.passcode}" } - authentication { - name = "authenticator1" - mechanism = password-based - server_type = built-in-database - user_id_type = clientid - } + authentication: [ + # { + # name = "authenticator1" + # type = "password-based:built-in-database" + # user_id_type = clientid + # } + ] listeners.tcp.default { bind = 61613 acceptors = 16 max_connections = 1024000 max_conn_rate = 1000 - active_n = 100 + + access_rules = [ + "allow all" + ] + + ## TCP options + ## See ${example_common_tcp_options} for more information + tcp.active_n = 100 + tcp.backlog = 1024 + tcp.buffer = 4KB + } + + listeners.ssl.default { + bind = 61614 + acceptors = 16 + max_connections = 1024000 + max_conn_rate = 1000 + + ## TCP options + ## See ${example_common_tcp_options} for more information + tcp.active_n = 100 + tcp.backlog = 1024 + tcp.buffer = 4KB + + ## SSL options + ## See ${example_common_ssl_options} for more information + ssl.versions = ["tlsv1.3", "tlsv1.2", "tlsv1.1", "tlsv1"] + ssl.keyfile = "{{ platform_etc_dir }}/certs/key.pem" + ssl.certfile = "{{ platform_etc_dir }}/certs/cert.pem" + ssl.cacertfile = "{{ platform_etc_dir }}/certs/cacert.pem" + #ssl.verify = verify_none + #ssl.fail_if_no_peer_cert = false + #ssl.server_name_indication = disable + #ssl.secure_renegotiate = false + #ssl.reuse_sessions = false + #ssl.honor_cipher_order = false + #ssl.handshake_timeout = 15s + #ssl.depth = 10 + #ssl.password = foo + #ssl.dhfile = path-to-your-file } } @@ -58,20 +98,41 @@ gateway.coap { ## When publishing or subscribing, prefix all topics with a mountpoint string. mountpoint = "" - heartbeat = 30s notify_type = qos + + ## if true, you need to establish a connection before use + connection_required = false subscribe_qos = qos0 publish_qos = qos1 - authentication { - name = "authenticator1" - mechanism = password-based - server_type = built-in-database - user_id_type = clientid - } - listeners.udp.default { bind = 5683 + acceptors = 4 + max_connections = 102400 + max_conn_rate = 1000 + + ## UDP Options + ## See ${example_common_udp_options} for more information + udp.active_n = 100 + udp.buffer = 16KB + } + listeners.dtls.default { + bind = 5684 + acceptors = 4 + max_connections = 102400 + max_conn_rate = 1000 + + ## UDP Options + ## See ${example_common_udp_options} for more information + udp.active_n = 100 + udp.buffer = 16KB + + ## DTLS Options + ## See #{example_common_dtls_options} for more information + dtls.versions = ["dtlsv1.2", "dtlsv1"] + dtls.keyfile = "{{ platform_etc_dir }}/certs/key.pem" + dtls.certfile = "{{ platform_etc_dir }}/certs/cert.pem" + dtls.cacertfile = "{{ platform_etc_dir }}/certs/cacert.pem" } } @@ -121,6 +182,26 @@ gateway.mqttsn { max_connections = 10240000 max_conn_rate = 1000 } + + listeners.dtls.default { + bind = 1885 + acceptors = 4 + max_connections = 102400 + max_conn_rate = 1000 + + ## UDP Options + ## See ${example_common_udp_options} for more information + udp.active_n = 100 + udp.buffer = 16KB + + ## DTLS Options + ## See #{example_common_dtls_options} for more information + dtls.versions = ["dtlsv1.2", "dtlsv1"] + dtls.keyfile = "{{ platform_etc_dir }}/certs/key.pem" + dtls.certfile = "{{ platform_etc_dir }}/certs/cert.pem" + dtls.cacertfile = "{{ platform_etc_dir }}/certs/cacert.pem" + } + } gateway.lwm2m { @@ -134,7 +215,7 @@ gateway.lwm2m { enable_stats = true ## When publishing or subscribing, prefix all topics with a mountpoint string. - mountpoint = "lwm2m/%e/" + mountpoint = "lwm2m/%u" xml_dir = "{{ platform_etc_dir }}/lwm2m_xml" @@ -146,12 +227,32 @@ gateway.lwm2m { ## always | contains_object_list update_msg_publish_condition = contains_object_list + translators { - command = "dn/#" - response = "up/resp" - notify = "up/notify" - register = "up/resp" - update = "up/resp" + command { + topic = "/dn/#" + qos = 0 + } + + response { + topic = "/up/resp" + qos = 0 + } + + notify { + topic = "/up/notify" + qos = 0 + } + + register { + topic = "/up/resp" + qos = 0 + } + + update { + topic = "/up/resp" + qos = 0 + } } listeners.udp.default { diff --git a/apps/emqx_gateway/src/bhvrs/emqx_gateway_channel.erl b/apps/emqx_gateway/src/bhvrs/emqx_gateway_channel.erl index 06efe4fd0..c4fd114e4 100644 --- a/apps/emqx_gateway/src/bhvrs/emqx_gateway_channel.erl +++ b/apps/emqx_gateway/src/bhvrs/emqx_gateway_channel.erl @@ -44,6 +44,8 @@ -type conn_state() :: idle | connecting | connected | disconnected | atom(). +-type gen_server_from() :: {pid(), Tag :: term()}. + -type reply() :: {outgoing, emqx_gateway_frame:packet()} | {outgoing, [emqx_gateway_frame:packet()]} | {event, conn_state() | updated} @@ -71,11 +73,13 @@ | {shutdown, Reason :: any(), channel()}. %% @doc Handle the custom gen_server:call/2 for its connection process --callback handle_call(Req :: any(), channel()) +-callback handle_call(Req :: any(), From :: gen_server_from(), channel()) -> {reply, Reply :: any(), channel()} %% Reply to caller and trigger an event(s) | {reply, Reply :: any(), - EventOrEvents:: tuple() | list(tuple()), channel()} + EventOrEvents :: tuple() | list(tuple()), channel()} + | {noreply, channel()} + | {noreply, EventOrEvents :: tuple() | list(tuple()), channel()} | {shutdown, Reason :: any(), Reply :: any(), channel()} %% Shutdown the process, reply to caller and write a packet to client | {shutdown, Reason :: any(), Reply :: any(), diff --git a/apps/emqx_gateway/src/bhvrs/emqx_gateway_conn.erl b/apps/emqx_gateway/src/bhvrs/emqx_gateway_conn.erl index fa0a830e5..543b2e169 100644 --- a/apps/emqx_gateway/src/bhvrs/emqx_gateway_conn.erl +++ b/apps/emqx_gateway/src/bhvrs/emqx_gateway_conn.erl @@ -20,7 +20,6 @@ -include_lib("emqx/include/types.hrl"). -include_lib("emqx/include/logger.hrl"). - %% API -export([ start_link/3 , stop/1 @@ -48,7 +47,6 @@ %% Internal callback -export([wakeup_from_hib/2, recvloop/2]). - -record(state, { %% TCP/SSL/UDP/DTLS Wrapped Socket socket :: {esockd_transport, esockd:socket()} | {udp, _, _}, @@ -226,6 +224,9 @@ esockd_send(Data, #state{socket = {udp, _SockPid, Sock}, esockd_send(Data, #state{socket = {esockd_transport, Sock}}) -> esockd_transport:async_send(Sock, Data). +is_datadram_socket({esockd_transport, _}) -> false; +is_datadram_socket({udp, _, _}) -> true. + %%-------------------------------------------------------------------- %% callbacks %%-------------------------------------------------------------------- @@ -393,6 +394,10 @@ append_msg(Q, Msg) -> handle_msg({'$gen_call', From, Req}, State) -> case handle_call(From, Req, State) of + {noreply, NState} -> + {ok, NState}; + {noreply, Msgs, NState} -> + {ok, next_msgs(Msgs), NState}; {reply, Reply, NState} -> gen_server:reply(From, Reply), {ok, NState}; @@ -544,10 +549,14 @@ handle_call(_From, info, State) -> handle_call(_From, stats, State) -> {reply, stats(State), State}; -handle_call(_From, Req, State = #state{ +handle_call(From, Req, State = #state{ chann_mod = ChannMod, channel = Channel}) -> - case ChannMod:handle_call(Req, Channel) of + case ChannMod:handle_call(Req, From, Channel) of + {noreply, NChannel} -> + {noreply, State#state{channel = NChannel}}; + {noreply, Msgs, NChannel} -> + {noreply, Msgs, State#state{channel = NChannel}}; {reply, Reply, NChannel} -> {reply, Reply, State#state{channel = NChannel}}; {reply, Reply, Msgs, NChannel} -> @@ -558,8 +567,6 @@ handle_call(_From, Req, State = #state{ NState = State#state{channel = NChannel}, ok = handle_outgoing(Packet, NState), shutdown(Reason, Reply, NState) - - end. %%-------------------------------------------------------------------- @@ -678,8 +685,20 @@ with_channel(Fun, Args, State = #state{ %%-------------------------------------------------------------------- %% Handle outgoing packets -handle_outgoing(Packets, State) when is_list(Packets) -> - send(lists:map(serialize_and_inc_stats_fun(State), Packets), State); +handle_outgoing(_Packets = [], _State) -> + ok; +handle_outgoing(Packets, + State = #state{socket = Socket}) when is_list(Packets) -> + case is_datadram_socket(Socket) of + false -> + send( + lists:map(serialize_and_inc_stats_fun(State), Packets), + State); + _ -> + lists:foreach(fun(Packet) -> + handle_outgoing(Packet, State) + end, Packets) + end; handle_outgoing(Packet, State) -> send((serialize_and_inc_stats_fun(State))(Packet), State). @@ -816,7 +835,7 @@ inc_incoming_stats(Ctx, FrameMod, Packet) -> ok end, Name = list_to_atom( - lists:concat(["packets.", FrameMod:type(Packet), ".recevied"])), + lists:concat(["packets.", FrameMod:type(Packet), ".received"])), emqx_gateway_ctx:metrics_inc(Ctx, Name). inc_outgoing_stats(Ctx, FrameMod, Packet) -> diff --git a/apps/emqx_gateway/src/coap/README.md b/apps/emqx_gateway/src/coap/README.md index 12b5ac5b7..54f0fde84 100644 --- a/apps/emqx_gateway/src/coap/README.md +++ b/apps/emqx_gateway/src/coap/README.md @@ -9,6 +9,7 @@ 4. [Query String](#org9a6b996) 2. [Implementation](#org9985dfe) 1. [Request/Response flow](#orge94210c) + 3. [Example](#ref_example) @@ -401,3 +402,42 @@ CoAP gateway uses some options in query string to conversion between MQTT CoAP. + + + +## Example +1. Create Connection +``` +coap-client -m post -e "" "coap://127.0.0.1/mqtt/connection?clientid=123&username=admin&password=public" +``` +Server will return token **X** in payload + +2. Update Connection +``` +coap-client -m put -e "" "coap://127.0.0.1/mqtt/connection?clientid=123&token=X" +``` + +3. Publish +``` +coap-client -m post -e "Hellow" "obstoken" "coap://127.0.0.1/ps/coap/test?clientid=123&username=admin&password=public" +``` +if you want to publish with auth, you must first establish a connection, and then post publish request on the same socket, so libcoap client can't simulation publish with a token + +``` +coap-client -m post -e "Hellow" "coap://127.0.0.1/ps/coap/test?clientid=123&token=X" +``` + +4. Subscribe +``` +coap-client -m get -s 60 -O 6,0x00 -o - -T "obstoken" "coap://127.0.0.1/ps/coap/test?clientid=123&username=admin&password=public" +``` +**Or** + +``` +coap-client -m get -s 60 -O 6,0x00 -o - -T "obstoken" "coap://127.0.0.1/ps/coap/test?clientid=123&token=X" +``` +5. Close Connection +``` +coap-client -m delete -e "" "coap://127.0.0.1/mqtt/connection?clientid=123&token=X +``` + diff --git a/apps/emqx_gateway/src/coap/doc/flow.png b/apps/emqx_gateway/src/coap/doc/flow.png index 5c7288348..bb9b775a5 100644 Binary files a/apps/emqx_gateway/src/coap/doc/flow.png and b/apps/emqx_gateway/src/coap/doc/flow.png differ diff --git a/apps/emqx_gateway/src/coap/emqx_coap_api.erl b/apps/emqx_gateway/src/coap/emqx_coap_api.erl new file mode 100644 index 000000000..4d0e8aff8 --- /dev/null +++ b/apps/emqx_gateway/src/coap/emqx_coap_api.erl @@ -0,0 +1,145 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2017-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_coap_api). + +-behaviour(minirest_api). + +-include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). + +%% API +-export([api_spec/0]). + +-export([request/2]). + +-define(PREFIX, "/gateway/coap/:clientid"). +-define(DEF_WAIT_TIME, 10). + +-import(emqx_mgmt_util, [ schema/1 + , schema/2 + , object_schema/1 + , object_schema/2 + , error_schema/2 + , properties/1]). + +%%-------------------------------------------------------------------- +%% API +%%-------------------------------------------------------------------- +api_spec() -> + {[request_api()], []}. + +request_api() -> + Metadata = #{post => request_method_meta()}, + {?PREFIX ++ "/request", Metadata, request}. + +request(post, #{body := Body, bindings := Bindings}) -> + ClientId = maps:get(clientid, Bindings, undefined), + + Method = maps:get(<<"method">>, Body, <<"get">>), + CT = maps:get(<<"content_type">>, Body, <<"text/plain">>), + Token = maps:get(<<"token">>, Body, <<>>), + Payload = maps:get(<<"payload">>, Body, <<>>), + WaitTime = maps:get(<<"timeout">>, Body, ?DEF_WAIT_TIME), + + Payload2 = parse_payload(CT, Payload), + ReqType = erlang:binary_to_atom(Method), + + Msg = emqx_coap_message:request(con, + ReqType, Payload2, #{content_format => CT}), + + Msg2 = Msg#coap_message{token = Token}, + + case call_client(ClientId, Msg2, timer:seconds(WaitTime)) of + timeout -> + {504, #{code => 'CLIENT_NOT_RESPONSE'}}; + not_found -> + {404, #{code => 'CLIENT_NOT_FOUND'}}; + Response -> + {200, format_to_response(CT, Response)} + end. + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- +request_parameters() -> + [#{name => clientid, + in => path, + schema => #{type => string}, + required => true}]. + +request_properties() -> + properties([ {token, string, "message token, can be empty"} + , {method, string, "request method type", ["get", "put", "post", "delete"]} + , {timeout, integer, "timespan for response"} + , {content_type, string, "payload type", + [<<"text/plain">>, <<"application/json">>, <<"application/octet-stream">>]} + , {payload, string, "payload"}]). + +coap_message_properties() -> + properties([ {id, integer, "message id"} + , {token, string, "message token, can be empty"} + , {method, string, "response code"} + , {payload, string, "payload"}]). + +request_method_meta() -> + #{description => <<"lookup matching messages">>, + parameters => request_parameters(), + 'requestBody' => object_schema(request_properties(), + <<"request payload, binary must encode by base64">>), + responses => #{ + <<"200">> => object_schema(coap_message_properties()), + <<"404">> => error_schema("client not found error", ['CLIENT_NOT_FOUND']), + <<"504">> => error_schema("timeout", ['CLIENT_NOT_RESPONSE']) + }}. + + +format_to_response(ContentType, #coap_message{id = Id, + token = Token, + method = Method, + payload = Payload}) -> + #{id => Id, + token => Token, + method => format_to_binary(Method), + payload => format_payload(ContentType, Payload)}. + +format_to_binary(Obj) -> + erlang:list_to_binary(io_lib:format("~p", [Obj])). + +format_payload(<<"application/octet-stream">>, Payload) -> + base64:encode(Payload); + +format_payload(_, Payload) -> + Payload. + +parse_payload(<<"application/octet-stream">>, Body) -> + base64:decode(Body); + +parse_payload(_, Body) -> + Body. + +call_client(ClientId, Msg, Timeout) -> + case emqx_gateway_cm_registry:lookup_channels(coap, ClientId) of + [Channel | _] -> + RequestId = emqx_coap_channel:send_request(Channel, Msg), + case gen_server:wait_response(RequestId, Timeout) of + {reply, Reply} -> + Reply; + _ -> + timeout + end; + _ -> + not_found + end. diff --git a/apps/emqx_gateway/src/coap/emqx_coap_channel.erl b/apps/emqx_gateway/src/coap/emqx_coap_channel.erl index 510432441..112efdc44 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_channel.erl +++ b/apps/emqx_gateway/src/coap/emqx_coap_channel.erl @@ -22,17 +22,13 @@ -include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). %% API --export([]). - -export([ info/1 , info/2 , stats/1 - , validator/3 - , get_clientinfo/1 - , get_config/2 - , get_config/3 - , result_keys/0 - , transfer_result/3]). + , validator/4 + , metrics_inc/2 + , run_hooks/3 + , send_request/2]). -export([ init/2 , handle_in/2 @@ -41,7 +37,7 @@ , terminate/2 ]). --export([ handle_call/2 +-export([ handle_call/3 , handle_cast/2 , handle_info/2 ]). @@ -61,20 +57,21 @@ keepalive :: emqx_keepalive:keepalive() | undefined, %% Timer timers :: #{atom() => disable | undefined | reference()}, - token :: binary() | undefined, - config :: hocon:config() + + connection_required :: boolean(), + + conn_state :: idle | connected | disconnected, + + token :: binary() | undefined }). -%% the execuate context for session call --record(exec_ctx, { config :: hocon:config(), - ctx :: emqx_gateway_ctx:context(), - clientinfo :: emqx_types:clientinfo() - }). - -type channel() :: #channel{}. --define(DISCONNECT_WAIT_TIME, timer:seconds(10)). +-define(TOKEN_MAXIMUM, 4294967295). -define(INFO_KEYS, [conninfo, conn_state, clientinfo, session]). +-define(DEF_IDLE_TIME, timer:seconds(30)). +-define(GET_IDLE_TIME(Cfg), maps:get(idle_timeout, Cfg, ?DEF_IDLE_TIME)). +-import(emqx_coap_medium, [reply/2, reply/3, reply/4, iter/3, iter/4]). %%-------------------------------------------------------------------- %% API %%-------------------------------------------------------------------- @@ -87,8 +84,8 @@ info(Keys, Channel) when is_list(Keys) -> info(conninfo, #channel{conninfo = ConnInfo}) -> ConnInfo; -info(conn_state, _) -> - connected; +info(conn_state, #channel{conn_state = CState}) -> + CState; info(clientinfo, #channel{clientinfo = ClientInfo}) -> ClientInfo; info(session, #channel{session = Session}) -> @@ -105,19 +102,14 @@ init(ConnInfo = #{peername := {PeerHost, _}, sockname := {_, SockPort}}, #{ctx := Ctx} = Config) -> Peercert = maps:get(peercert, ConnInfo, undefined), - Mountpoint = maps:get(mountpoint, Config, undefined), - EnableAuth = is_authentication_enabled(Config), + Mountpoint = maps:get(mountpoint, Config, <<>>), ClientInfo = set_peercert_infos( Peercert, #{ zone => default , protocol => 'coap' , peerhost => PeerHost , sockport => SockPort - , clientid => if EnableAuth -> - undefined; - true -> - emqx_guid:to_base62(emqx_guid:gen()) - end + , clientid => emqx_guid:to_base62(emqx_guid:gen()) , username => undefined , is_bridge => false , is_superuser => false @@ -125,63 +117,42 @@ init(ConnInfo = #{peername := {PeerHost, _}, } ), + Heartbeat = ?GET_IDLE_TIME(Config), #channel{ ctx = Ctx , conninfo = ConnInfo , clientinfo = ClientInfo , timers = #{} - , config = Config , session = emqx_coap_session:new() - , keepalive = emqx_keepalive:init(maps:get(heartbeat, Config)) + , keepalive = emqx_keepalive:init(Heartbeat) + , connection_required = maps:get(connection_required, Config, false) + , conn_state = idle }. -is_authentication_enabled(Cfg) -> - case maps:get(authentication, Cfg, #{enable => false}) of - AuthCfg when is_map(AuthCfg) -> - maps:get(enable, AuthCfg, true); - _ -> false - end. - -validator(Type, Topic, #exec_ctx{ctx = Ctx, - clientinfo = ClientInfo}) -> +validator(Type, Topic, Ctx, ClientInfo) -> emqx_gateway_ctx:authorize(Ctx, ClientInfo, Type, Topic). -get_clientinfo(#exec_ctx{clientinfo = ClientInfo}) -> - ClientInfo. - -get_config(Key, Ctx) -> - get_config(Key, Ctx, undefined). - -get_config(Key, #exec_ctx{config = Cfg}, Def) -> - maps:get(Key, Cfg, Def). - -result_keys() -> - [out, connection]. - -transfer_result(From, Value, Result) -> - ?TRANSFER_RESULT(From, Value, Result). +-spec send_request(pid(), emqx_coap_message()) -> any(). +send_request(Channel, Request) -> + gen_server:send_request(Channel, {?FUNCTION_NAME, Request}). %%-------------------------------------------------------------------- %% Handle incoming packet %%-------------------------------------------------------------------- handle_in(Msg, ChannleT) -> Channel = ensure_keepalive_timer(ChannleT), - case convert_queries(Msg) of - {ok, Msg2} -> - case emqx_coap_message:is_request(Msg2) of - true -> - check_auth_state(Msg2, Channel); - _ -> - call_session(handle_response, Msg2, Channel) - end; + case emqx_coap_message:is_request(Msg) of + true -> + check_auth_state(Msg, Channel); _ -> - response({error, bad_request}, <<"bad uri_query">>, Msg, Channel) + call_session(handle_response, Msg, Channel) end. %%-------------------------------------------------------------------- %% Handle Delivers from broker to client %%-------------------------------------------------------------------- -handle_deliver(Delivers, Channel) -> - call_session(deliver, Delivers, Channel). +handle_deliver(Delivers, #channel{session = Session, + ctx = Ctx} = Channel) -> + handle_result(emqx_coap_session:deliver(Delivers, Ctx, Session), Channel). %%-------------------------------------------------------------------- %% Handle timeout @@ -192,7 +163,7 @@ handle_timeout(_, {keepalive, NewVal}, #channel{keepalive = KeepAlive} = Channel Channel2 = ensure_keepalive_timer(fun make_timer/4, Channel), {ok, Channel2#channel{keepalive = NewKeepAlive}}; {error, timeout} -> - {shutdown, timeout, Channel} + {shutdown, timeout, ensure_disconnected(keepalive_timeout, Channel)} end; handle_timeout(_, {transport, Msg}, Channel) -> @@ -207,7 +178,11 @@ handle_timeout(_, _, Channel) -> %%-------------------------------------------------------------------- %% Handle call %%-------------------------------------------------------------------- -handle_call(Req, Channel) -> +handle_call({send_request, Msg}, From, Channel) -> + Result = call_session(handle_out, {{send_request, From}, Msg}, Channel), + erlang:setelement(1, Result, noreply); + +handle_call(Req, _From, Channel) -> ?LOG(error, "Unexpected call: ~p", [Req]), {reply, ignored, Channel}. @@ -221,6 +196,9 @@ handle_cast(Req, Channel) -> %%-------------------------------------------------------------------- %% Handle Info %%-------------------------------------------------------------------- +handle_info({subscribe, _}, Channel) -> + {ok, Channel}; + handle_info(Info, Channel) -> ?LOG(error, "Unexpected info: ~p", [Info]), {ok, Channel}. @@ -228,8 +206,10 @@ handle_info(Info, Channel) -> %%-------------------------------------------------------------------- %% Terminate %%-------------------------------------------------------------------- -terminate(_Reason, _Channel) -> - ok. +terminate(Reason, #channel{clientinfo = ClientInfo, + ctx = Ctx, + session = Session}) -> + run_hooks(Ctx, 'session.terminated', [ClientInfo, Reason, Session]). %%-------------------------------------------------------------------- %% Internal functions @@ -258,94 +238,56 @@ make_timer(Name, Time, Msg, Channel = #channel{timers = Timers}) -> ensure_keepalive_timer(Channel) -> ensure_keepalive_timer(fun ensure_timer/4, Channel). -ensure_keepalive_timer(Fun, #channel{config = Cfg} = Channel) -> - Interval = maps:get(heartbeat, Cfg), - Fun(keepalive, Interval, keepalive, Channel). +ensure_keepalive_timer(Fun, #channel{keepalive = KeepAlive} = Channel) -> + Heartbeat = emqx_keepalive:info(interval, KeepAlive), + Fun(keepalive, Heartbeat, keepalive, Channel). -call_session(Fun, - Msg, - #channel{session = Session} = Channel) -> - Ctx = new_exec_ctx(Channel), - Result = erlang:apply(emqx_coap_session, Fun, [Msg, Ctx, Session]), - process_result([session, connection, out], Result, Msg, Channel). - -process_result([Key | T], Result, Msg, Channel) -> - case handle_result(Key, Result, Msg, Channel) of - {ok, Channel2} -> - process_result(T, Result, Msg, Channel2); - Other -> - Other - end; - -process_result(_, _, _, Channel) -> - {ok, Channel}. - -handle_result(session, #{session := Session}, _, Channel) -> - {ok, Channel#channel{session = Session}}; - -handle_result(connection, #{connection := open}, Msg, Channel) -> - do_connect(Msg, Channel); - -handle_result(connection, #{connection := close}, Msg, Channel) -> - Reply = emqx_coap_message:piggyback({ok, deleted}, Msg), - {shutdown, close, {outgoing, Reply}, Channel}; - -handle_result(out, #{out := Out}, _, Channel) -> - {ok, {outgoing, Out}, Channel}; - -handle_result(_, _, _, Channel) -> - {ok, Channel}. - -check_auth_state(Msg, #channel{config = Cfg} = Channel) -> - Enable = is_authentication_enabled(Cfg), - check_token(Enable, Msg, Channel). +check_auth_state(Msg, #channel{connection_required = Required} = Channel) -> + check_token(Required, Msg, Channel). check_token(true, - #coap_message{options = Options} = Msg, + Msg, #channel{token = Token, - clientinfo = ClientInfo} = Channel) -> + clientinfo = ClientInfo, + conn_state = CState} = Channel) -> #{clientid := ClientId} = ClientInfo, - case maps:get(uri_query, Options, undefined) of + case emqx_coap_message:get_option(uri_query, Msg) of #{<<"clientid">> := ClientId, <<"token">> := Token} -> call_session(handle_request, Msg, Channel); #{<<"clientid">> := DesireId} -> - try_takeover(ClientId, DesireId, Msg, Channel); + try_takeover(CState, DesireId, Msg, Channel); _ -> - response({error, unauthorized}, Msg, Channel) + Reply = emqx_coap_message:piggyback({error, unauthorized}, Msg), + {ok, {outgoing, Reply}, Channel} end; -check_token(false, - #coap_message{options = Options} = Msg, - Channel) -> - case maps:get(uri_query, Options, undefined) of +check_token(false, Msg, Channel) -> + case emqx_coap_message:get_option(uri_query, Msg) of #{<<"clientid">> := _} -> - response({error, unauthorized}, Msg, Channel); + Reply = emqx_coap_message:piggyback({error, unauthorized}, Msg), + {ok, {outgoing, Reply}, Channel}; #{<<"token">> := _} -> - response({error, unauthorized}, Msg, Channel); + Reply = emqx_coap_message:piggyback({error, unauthorized}, Msg), + {ok, {outgoing, Reply}, Channel}; _ -> call_session(handle_request, Msg, Channel) end. -response(Method, Req, Channel) -> - response(Method, <<>>, Req, Channel). - -response(Method, Payload, Req, Channel) -> - Reply = emqx_coap_message:piggyback(Method, Payload, Req), - call_session(handle_out, Reply, Channel). - -try_takeover(undefined, - DesireId, - #coap_message{options = Opts} = Msg, - Channel) -> - case maps:get(uri_path, Opts, []) of - [<<"mqtt">>, <<"connection">> | _] -> +try_takeover(idle, DesireId, Msg, Channel) -> + case emqx_coap_message:get_option(uri_path, Msg, []) of + [<<"mqtt">>, <<"connection">> | _] -> %% may be is a connect request %% TODO need check repeat connect, unless we implement the %% udp connection baseon the clientid call_session(handle_request, Msg, Channel); _ -> - do_takeover(DesireId, Msg, Channel) + case emqx:get_config([gateway, coap, authentication], undefined) of + undefined -> + call_session(handle_request, Msg, Channel); + _ -> + do_takeover(DesireId, Msg, Channel) + end end; try_takeover(_, DesireId, Msg, Channel) -> @@ -354,31 +296,7 @@ try_takeover(_, DesireId, Msg, Channel) -> do_takeover(_DesireId, Msg, Channel) -> %% TODO completed the takeover, now only reset the message Reset = emqx_coap_message:reset(Msg), - call_session(handle_out, Reset, Channel). - -new_exec_ctx(#channel{config = Cfg, - ctx = Ctx, - clientinfo = ClientInfo}) -> - #exec_ctx{config = Cfg, - ctx = Ctx, - clientinfo = ClientInfo}. - -do_connect(#coap_message{options = Opts} = Req, Channel) -> - Queries = maps:get(uri_query, Opts), - case emqx_misc:pipeline( - [ fun run_conn_hooks/2 - , fun enrich_clientinfo/2 - , fun set_log_meta/2 - , fun auth_connect/2 - ], - {Queries, Req}, - Channel) of - {ok, _Input, NChannel} -> - process_connect(ensure_connected(NChannel), Req); - {error, ReasonCode, NChannel} -> - ErrMsg = io_lib:format("Login Failed: ~s", [ReasonCode]), - response({error, bad_request}, ErrMsg, Req, NChannel) - end. + {ok, {outgoing, Reset}, Channel}. run_conn_hooks(Input, Channel = #channel{ctx = Ctx, conninfo = ConnInfo}) -> @@ -421,11 +339,9 @@ auth_connect(_Input, Channel = #channel{ctx = Ctx, {error, Reason} end. -fix_mountpoint(_Packet, #{mountpoint := undefined} = ClientInfo) -> +fix_mountpoint(_Packet, #{mountpoint := <<>>} = ClientInfo) -> {ok, ClientInfo}; fix_mountpoint(_Packet, ClientInfo = #{mountpoint := Mountpoint}) -> - %% TODO: Enrich the varibale replacement???? - %% i.e: ${ClientInfo.auth_result.productKey} Mountpoint1 = emqx_mountpoint:replvar(Mountpoint, ClientInfo), {ok, ClientInfo#{mountpoint := Mountpoint1}}. @@ -437,13 +353,14 @@ ensure_connected(Channel = #channel{ctx = Ctx, , proto_ver => <<"1">> }, ok = run_hooks(Ctx, 'client.connected', [ClientInfo, NConnInfo]), + _ = run_hooks(Ctx, 'client.connack', [NConnInfo, connection_accepted, []]), Channel#channel{conninfo = NConnInfo}. -process_connect(Channel = #channel{ctx = Ctx, - session = Session, - conninfo = ConnInfo, - clientinfo = ClientInfo}, - Msg) -> +process_connect(#channel{ctx = Ctx, + session = Session, + conninfo = ConnInfo, + clientinfo = ClientInfo} = Channel, + Msg, Result, Iter) -> %% inherit the old session SessFun = fun(_,_) -> Session end, case emqx_gateway_ctx:open_session( @@ -455,10 +372,14 @@ process_connect(Channel = #channel{ctx = Ctx, emqx_coap_session ) of {ok, _Sess} -> - response({ok, created}, <<"connected">>, Msg, Channel); + RandVal = rand:uniform(?TOKEN_MAXIMUM), + Token = erlang:list_to_binary(erlang:integer_to_list(RandVal)), + iter(Iter, + reply({ok, created}, Token, Msg, Result), + Channel#channel{token = Token}); {error, Reason} -> ?LOG(error, "Failed to open session du to ~p", [Reason]), - response({error, bad_request}, Msg, Channel) + iter(Iter, reply({error, bad_request}, Msg, Result), Channel) end. run_hooks(Ctx, Name, Args) -> @@ -469,20 +390,110 @@ run_hooks(Ctx, Name, Args, Acc) -> emqx_gateway_ctx:metrics_inc(Ctx, Name), emqx_hooks:run_fold(Name, Args, Acc). -convert_queries(#coap_message{options = Opts} = Msg) -> - case maps:get(uri_query, Opts, undefined) of - undefined -> - {ok, Msg#coap_message{options = Opts#{uri_query => #{}}}}; - Queries -> - convert_queries(Queries, #{}, Msg) - end. +metrics_inc(Name, Ctx) -> + emqx_gateway_ctx:metrics_inc(Ctx, Name). -convert_queries([H | T], Queries, Msg) -> - case re:split(H, "=") of - [Key, Val] -> - convert_queries(T, Queries#{Key => Val}, Msg); - _ -> - error +ensure_disconnected(Reason, Channel = #channel{ + ctx = Ctx, + conninfo = ConnInfo, + clientinfo = ClientInfo}) -> + NConnInfo = ConnInfo#{disconnected_at => erlang:system_time(millisecond)}, + ok = run_hooks(Ctx, 'client.disconnected', [ClientInfo, Reason, NConnInfo]), + Channel#channel{conninfo = NConnInfo, conn_state = disconnected}. + +%%-------------------------------------------------------------------- +%% Call Chain +%%-------------------------------------------------------------------- +call_session(Fun, Msg, #channel{session = Session} = Channel) -> + Result = emqx_coap_session:Fun(Msg, Session), + handle_result(Result, Channel). + +handle_result(Result, Channel) -> + iter([ session, fun process_session/4 + , proto, fun process_protocol/4 + , reply, fun process_reply/4 + , out, fun process_out/4 + , fun process_nothing/3 + ], + Result, + Channel). + +call_handler(request, Msg, Result, + #channel{ctx = Ctx, + clientinfo = ClientInfo} = Channel, Iter) -> + HandlerResult = + case emqx_coap_message:get_option(uri_path, Msg) of + [<<"ps">> | RestPath] -> + emqx_coap_pubsub_handler:handle_request(RestPath, Msg, Ctx, ClientInfo); + [<<"mqtt">> | RestPath] -> + emqx_coap_mqtt_handler:handle_request(RestPath, Msg, Ctx, ClientInfo); + _ -> + reply({error, bad_request}, Msg) + end, + iter([ connection, fun process_connection/4 + , subscribe, fun process_subscribe/4 | Iter], + maps:merge(Result, HandlerResult), + Channel); + +call_handler(response, {{send_request, From}, Response}, Result, Channel, Iter) -> + gen_server:reply(From, Response), + iter(Iter, Result, Channel); + +call_handler(_, _, Result, Channel, Iter) -> + iter(Iter, Result, Channel). + +process_session(Session, Result, Channel, Iter) -> + iter(Iter, Result, Channel#channel{session = Session}). + +process_protocol({Type, Msg}, Result, Channel, Iter) -> + call_handler(Type, Msg, Result, Channel, Iter). + +%% leaf node +process_out(Outs, Result, Channel, _) -> + Outs2 = lists:reverse(Outs), + Outs3 = case maps:get(reply, Result, undefined) of + undefined -> + Outs2; + Reply -> + [Reply | Outs2] + end, + {ok, {outgoing, Outs3}, Channel}. + +%% leaf node +process_nothing(_, _, Channel) -> + {ok, Channel}. + +process_connection({open, Req}, Result, Channel, Iter) -> + Queries = emqx_coap_message:get_option(uri_query, Req), + case emqx_misc:pipeline( + [ fun run_conn_hooks/2 + , fun enrich_clientinfo/2 + , fun set_log_meta/2 + , fun auth_connect/2 + ], + {Queries, Req}, + Channel) of + {ok, _Input, NChannel} -> + process_connect(ensure_connected(NChannel), Req, Result, Iter); + {error, ReasonCode, NChannel} -> + ErrMsg = io_lib:format("Login Failed: ~s", [ReasonCode]), + Payload = erlang:list_to_binary(lists:flatten(ErrMsg)), + iter(Iter, + reply({error, bad_request}, Payload, Req, Result), + NChannel) end; -convert_queries([], Queries, #coap_message{options = Opts} = Msg) -> - {ok, Msg#coap_message{options = Opts#{uri_query => Queries}}}. + +process_connection({close, Msg}, _, Channel, _) -> + Reply = emqx_coap_message:piggyback({ok, deleted}, Msg), + {shutdown, close, Reply, Channel}. + +process_subscribe({Sub, Msg}, Result, #channel{session = Session} = Channel, Iter) -> + Result2 = emqx_coap_session:process_subscribe(Sub, Msg, Result, Session), + iter([session, fun process_session/4 | Iter], Result2, Channel). + +%% leaf node +process_reply(Reply, Result, #channel{session = Session} = Channel, _) -> + Session2 = emqx_coap_session:set_reply(Reply, Session), + Outs = maps:get(out, Result, []), + Outs2 = lists:reverse(Outs), + {ok, {outgoing, [Reply | Outs2]}, Channel#channel{session = Session2}}. diff --git a/apps/emqx_gateway/src/coap/emqx_coap_frame.erl b/apps/emqx_gateway/src/coap/emqx_coap_frame.erl index c1bc08928..4d12997a7 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_frame.erl +++ b/apps/emqx_gateway/src/coap/emqx_coap_frame.erl @@ -103,11 +103,7 @@ flatten_options([{OptId, OptVal} | T], Acc) -> false -> [encode_option(OptId, OptVal) | Acc]; _ -> - lists:foldl(fun(undefined, InnerAcc) -> - InnerAcc; - (E, InnerAcc) -> - [encode_option(OptId, E) | InnerAcc] - end, Acc, OptVal) + try_encode_repeatable(OptId, OptVal) ++ Acc end); flatten_options([], Acc) -> @@ -141,6 +137,19 @@ encode_option_list([], _LastNum, Acc, <<>>) -> encode_option_list([], _, Acc, Payload) -> <>. +try_encode_repeatable(uri_query, Val) when is_map(Val) -> + maps:fold(fun(K, V, Acc) -> + [encode_option(uri_query, <>) | Acc] + end, + [], Val); + +try_encode_repeatable(K, Val) -> + lists:foldr(fun(undefined, Acc) -> + Acc; + (E, Acc) -> + [encode_option(K, E) | Acc] + end, [], Val). + %% RFC 7252 encode_option(if_match, OptVal) -> {?OPTION_IF_MATCH, OptVal}; encode_option(uri_host, OptVal) -> {?OPTION_URI_HOST, OptVal}; @@ -188,6 +197,8 @@ content_format_to_code(<<"application/octet-stream">>) -> 42; content_format_to_code(<<"application/exi">>) -> 47; content_format_to_code(<<"application/json">>) -> 50; content_format_to_code(<<"application/cbor">>) -> 60; +content_format_to_code(<<"application/vnd.oma.lwm2m+tlv">>) -> 11542; +content_format_to_code(<<"application/vnd.oma.lwm2m+json">>) -> 11543; content_format_to_code(_) -> 42. %% use octet-stream as default method_to_class_code(get) -> {0, 01}; @@ -235,12 +246,7 @@ parse(< {Options, Payload} = decode_option_list(Tail), Options2 = maps:fold(fun(K, V, Acc) -> - case is_repeatable_option(K) of - true -> - Acc#{K => lists:reverse(V)}; - _ -> - Acc#{K => V} - end + Acc#{K => get_option_val(K, V)} end, #{}, Options), @@ -255,6 +261,24 @@ parse(<>, ParseState}. +get_option_val(uri_query, V) -> + KVList = lists:foldl(fun(E, Acc) -> + [Key, Val] = re:split(E, "="), + [{Key, Val} | Acc] + + end, + [], + V), + maps:from_list(KVList); + +get_option_val(K, V) -> + case is_repeatable_option(K) of + true -> + lists:reverse(V); + _ -> + V + end. + -spec decode_type(X) -> message_type() when X :: 0 .. 3. decode_type(0) -> con; @@ -359,6 +383,8 @@ content_code_to_format(42) -> <<"application/octet-stream">>; content_code_to_format(47) -> <<"application/exi">>; content_code_to_format(50) -> <<"application/json">>; content_code_to_format(60) -> <<"application/cbor">>; +content_code_to_format(11542) -> <<"application/vnd.oma.lwm2m+tlv">>; +content_code_to_format(11543) -> <<"application/vnd.oma.lwm2m+json">>; content_code_to_format(_) -> <<"application/octet-stream">>. %% use octet as default %% RFC 7252 diff --git a/apps/emqx_gateway/src/coap/emqx_coap_impl.erl b/apps/emqx_gateway/src/coap/emqx_coap_impl.erl index da2f2b8e9..055eab759 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_impl.erl +++ b/apps/emqx_gateway/src/coap/emqx_coap_impl.erl @@ -89,17 +89,17 @@ start_listener(GwName, Ctx, {Type, LisName, ListenOn, SocketOpts, Cfg}) -> ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn), case start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) of {ok, Pid} -> - ?ULOG("Start ~s:~s:~s listener on ~s successfully.~n", + ?ULOG("Gateway ~s:~s:~s on ~s started.~n", [GwName, Type, LisName, ListenOnStr]), Pid; {error, Reason} -> - ?ELOG("Failed to start ~s:~s:~s listener on ~s: ~0p~n", + ?ELOG("Failed to start gateway ~s:~s:~s on ~s: ~0p~n", [GwName, Type, LisName, ListenOnStr, Reason]), throw({badconf, Reason}) end. start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) -> - Name = name(GwName, LisName, Type), + Name = emqx_gateway_utils:listener_id(GwName, Type, LisName), NCfg = Cfg#{ ctx => Ctx, frame_mod => emqx_coap_frame, @@ -114,21 +114,18 @@ do_start_listener(udp, Name, ListenOn, SocketOpts, MFA) -> do_start_listener(dtls, Name, ListenOn, SocketOpts, MFA) -> esockd:open_dtls(Name, ListenOn, SocketOpts, MFA). -name(GwName, LisName, Type) -> - list_to_atom(lists:concat([GwName, ":", Type, ":", LisName])). - stop_listener(GwName, {Type, LisName, ListenOn, SocketOpts, Cfg}) -> StopRet = stop_listener(GwName, Type, LisName, ListenOn, SocketOpts, Cfg), ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn), case StopRet of - ok -> ?ULOG("Stop ~s:~s:~s listener on ~s successfully.~n", - [GwName, Type, LisName, ListenOnStr]); + ok -> ?ULOG("Gateway ~s:~s:~s on ~s stopped.~n", + [GwName, Type, LisName, ListenOnStr]); {error, Reason} -> - ?ELOG("Failed to stop ~s:~s:~s listener on ~s: ~0p~n", + ?ELOG("Failed to stop gateway ~s:~s:~s on ~s: ~0p~n", [GwName, Type, LisName, ListenOnStr, Reason]) end, StopRet. stop_listener(GwName, Type, LisName, ListenOn, _SocketOpts, _Cfg) -> - Name = name(GwName, LisName, Type), + Name = emqx_gateway_utils:listener_id(GwName, Type, LisName), esockd:close(Name, ListenOn). diff --git a/apps/emqx_gateway/src/coap/emqx_coap_medium.erl b/apps/emqx_gateway/src/coap/emqx_coap_medium.erl new file mode 100644 index 000000000..8dafc7bbb --- /dev/null +++ b/apps/emqx_gateway/src/coap/emqx_coap_medium.erl @@ -0,0 +1,109 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% Simplified semi-automatic CPS mode tree for coap +%% The tree must have a terminal leaf node, and it's return is the result of the entire tree. +%% This module currently only supports simple linear operation + +-module(emqx_coap_medium). + +-include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). + +%% API +-export([ empty/0, reset/1, reset/2 + , out/1, out/2, proto_out/1 + , proto_out/2, iter/3, iter/4 + , reply/2, reply/3, reply/4]). + +%%-type result() :: map() | empty. + +%%-------------------------------------------------------------------- +%% API +%%-------------------------------------------------------------------- +empty() -> #{}. + +reset(Msg) -> + reset(Msg, #{}). + +reset(Msg, Result) -> + out(emqx_coap_message:reset(Msg), Result). + +out(Msg) -> + #{out => [Msg]}. + +out(Msg, #{out := Outs} = Result) -> + Result#{out := [Msg | Outs]}; + +out(Msg, Result) -> + Result#{out => [Msg]}. + +proto_out(Proto) -> + proto_out(Proto, #{}). + +proto_out(Proto, Resut) -> + Resut#{proto => Proto}. + +reply(Method, Req) when not is_record(Method, coap_message) -> + reply(Method, <<>>, Req); + +reply(Reply, Result) -> + Result#{reply => Reply}. + +reply(Method, Req, Result) when is_record(Req, coap_message) -> + reply(Method, <<>>, Req, Result); + +reply(Method, Payload, Req) -> + reply(Method, Payload, Req, #{}). + +reply(Method, Payload, Req, Result) -> + Result#{reply => emqx_coap_message:piggyback(Method, Payload, Req)}. + +%% run a tree +iter([Key, Fun | T], Input, State) -> + case maps:get(Key, Input, undefined) of + undefined -> + iter(T, Input, State); + Val -> + Fun(Val, maps:remove(Key, Input), State, T) + %% reserved + %% if is_function(Fun) -> + %% Fun(Val, maps:remove(Key, Input), State, T); + %% true -> + %% %% switch to sub branch + %% [FunH | FunT] = Fun, + %% FunH(Val, maps:remove(Key, Input), State, FunT) + %% end + end; + +%% terminal node +iter([Fun], Input, State) -> + Fun(undefined, Input, State). + +%% run a tree with argument +iter([Key, Fun | T], Input, Arg, State) -> + case maps:get(Key, Input, undefined) of + undefined -> + iter(T, Input, Arg, State); + Val -> + Fun(Val, maps:remove(Key, Input), Arg, State, T) + end; + +iter([Fun], Input, Arg, State) -> + Fun(undefined, Input, Arg, State). + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- diff --git a/apps/emqx_gateway/src/coap/emqx_coap_message.erl b/apps/emqx_gateway/src/coap/emqx_coap_message.erl index 2e9fb144e..3851b3428 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_message.erl +++ b/apps/emqx_gateway/src/coap/emqx_coap_message.erl @@ -31,7 +31,8 @@ -export([is_request/1]). --export([set/3, set_payload/2, get_content/1, set_content/2, set_content/3, get_option/2]). +-export([ set/3, set_payload/2, get_option/2 + , get_option/3, set_payload_block/3, set_payload_block/4]). -include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). @@ -42,11 +43,10 @@ request(Type, Method, Payload) -> request(Type, Method, Payload, []). request(Type, Method, Payload, Options) when is_binary(Payload) -> - #coap_message{type = Type, method = Method, payload = Payload, options = Options}; - -request(Type, Method, Content=#coap_content{}, Options) -> - set_content(Content, - #coap_message{type = Type, method = Method, options = Options}). + #coap_message{type = Type, + method = Method, + payload = Payload, + options = to_options(Options)}. ack(#coap_message{id = Id}) -> #coap_message{type = ack, id = Id}. @@ -55,20 +55,20 @@ reset(#coap_message{id = Id}) -> #coap_message{type = reset, id = Id}. %% just make a response -response(#coap_message{type = Type, - id = Id, - token = Token}) -> - #coap_message{type = Type, - id = Id, - token = Token}. +response(Request) -> + response(undefined, Request). response(Method, Request) -> - set_method(Method, response(Request)). + response(Method, <<>>, Request). -response(Method, Payload, Request) -> - set_method(Method, - set_payload(Payload, - response(Request))). +response(Method, Payload, #coap_message{type = Type, + id = Id, + token = Token}) -> + #coap_message{type = Type, + id = Id, + token = Token, + method = Method, + payload = Payload}. %% make a response which maybe is a piggyback ack piggyback(Method, Request) -> @@ -90,14 +90,11 @@ set(max_age, ?DEFAULT_MAX_AGE, Msg) -> Msg; set(Option, Value, Msg = #coap_message{options = Options}) -> Msg#coap_message{options = Options#{Option => Value}}. -get_option(Option, #coap_message{options = Options}) -> - maps:get(Option, Options, undefined). +get_option(Option, Msg) -> + get_option(Option, Msg, undefined). -set_method(Method, Msg) -> - Msg#coap_message{method = Method}. - -set_payload(Payload = #coap_content{}, Msg) -> - set_content(Payload, undefined, Msg); +get_option(Option, #coap_message{options = Options}, Def) -> + maps:get(Option, Options, Def). set_payload(Payload, Msg) when is_binary(Payload) -> Msg#coap_message{payload = Payload}; @@ -105,49 +102,6 @@ set_payload(Payload, Msg) when is_binary(Payload) -> set_payload(Payload, Msg) when is_list(Payload) -> Msg#coap_message{payload = list_to_binary(Payload)}. -get_content(#coap_message{options = Options, payload = Payload}) -> - #coap_content{etag = maps:get(etag, Options, undefined), - max_age = maps:get(max_age, Options, ?DEFAULT_MAX_AGE), - format = maps:get(content_format, Options, undefined), - location_path = maps:get(location_path, Options, []), - payload = Payload}. - -set_content(Content, Msg) -> - set_content(Content, undefined, Msg). - -%% segmentation not requested and not required -set_content(#coap_content{etag = ETag, - max_age = MaxAge, - format = Format, - location_path = LocPath, - payload = Payload}, - undefined, - Msg) - when byte_size(Payload) =< ?MAX_BLOCK_SIZE -> - #coap_message{options = Options} = Msg2 = set_payload(Payload, Msg), - Options2 = Options#{etag => [ETag], - max_age => MaxAge, - content_format => Format, - location_path => LocPath}, - Msg2#coap_message{options = Options2}; - -%% segmentation not requested, but required (late negotiation) -set_content(Content, undefined, Msg) -> - set_content(Content, {0, true, ?MAX_BLOCK_SIZE}, Msg); - -%% segmentation requested (early negotiation) -set_content(#coap_content{etag = ETag, - max_age = MaxAge, - format = Format, - payload = Payload}, - Block, - Msg) -> - #coap_message{options = Options} = Msg2 = set_payload_block(Payload, Block, Msg), - Options2 = Options#{etag => [ETag], - max => MaxAge, - content_format => Format}, - Msg2#coap_message{options = Options2}. - set_payload_block(Content, Block, Msg = #coap_message{method = Method}) when is_atom(Method) -> set_payload_block(Content, block1, Block, Msg); @@ -172,3 +126,8 @@ is_request(#coap_message{method = Method}) when is_atom(Method) -> is_request(_) -> false. + +to_options(Opts) when is_map(Opts) -> + Opts; +to_options(Opts) -> + maps:from_list(Opts). diff --git a/apps/emqx_gateway/src/coap/emqx_coap_resource.erl b/apps/emqx_gateway/src/coap/emqx_coap_resource.erl deleted file mode 100644 index 93fe82aba..000000000 --- a/apps/emqx_gateway/src/coap/emqx_coap_resource.erl +++ /dev/null @@ -1,37 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2017-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_coap_resource). - --include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). - --type context() :: any(). --type topic() :: binary(). --type token() :: token(). - --type register() :: {topic(), token()} - | topic() - | undefined. - --type result() :: emqx_coap_message() - | {has_sub, emqx_coap_message(), register()}. - --callback init(hocon:confg()) -> context(). --callback stop(context()) -> ok. --callback get(emqx_coap_message(), hocon:config()) -> result(). --callback put(emqx_coap_message(), hocon:config()) -> result(). --callback post(emqx_coap_message(), hocon:config()) -> result(). --callback delete(emqx_coap_message(), hocon:config()) -> result(). diff --git a/apps/emqx_gateway/src/coap/emqx_coap_session.erl b/apps/emqx_gateway/src/coap/emqx_coap_session.erl index 50e91797b..0fbc47cf8 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_session.erl +++ b/apps/emqx_gateway/src/coap/emqx_coap_session.erl @@ -21,24 +21,25 @@ -include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). %% API --export([new/0, transfer_result/3]). +-export([ new/0 + , process_subscribe/4]). -export([ info/1 , info/2 , stats/1 ]). --export([ handle_request/3 - , handle_response/3 - , handle_out/3 +-export([ handle_request/2 + , handle_response/2 + , handle_out/2 + , set_reply/2 , deliver/3 - , timeout/3]). + , timeout/2]). -export_type([session/0]). -record(session, { transport_manager :: emqx_coap_tm:manager() , observe_manager :: emqx_coap_observe_res:manager() - , next_msg_id :: coap_message_id() , created_at :: pos_integer() }). @@ -64,6 +65,9 @@ awaiting_rel_max ]). +-import(emqx_coap_medium, [iter/3]). +-import(emqx_coap_channel, [metrics_inc/2]). + %%%------------------------------------------------------------------- %%% API %%%------------------------------------------------------------------- @@ -72,7 +76,6 @@ new() -> _ = emqx_misc:rand_seed(), #session{ transport_manager = emqx_coap_tm:new() , observe_manager = emqx_coap_observe_res:new_manager() - , next_msg_id = rand:uniform(?MAX_MESSAGE_ID) , created_at = erlang:system_time(millisecond)}. %%-------------------------------------------------------------------- @@ -110,8 +113,8 @@ info(mqueue_max, _) -> 0; info(mqueue_dropped, _) -> 0; -info(next_pkt_id, #session{next_msg_id = PacketId}) -> - PacketId; +info(next_pkt_id, _) -> + 0; info(awaiting_rel, _) -> #{}; info(awaiting_rel_cnt, _) -> @@ -130,105 +133,90 @@ stats(Session) -> info(?STATS_KEYS, Session). %%%------------------------------------------------------------------- %%% Process Message %%%------------------------------------------------------------------- -handle_request(Msg, Ctx, Session) -> +handle_request(Msg, Session) -> call_transport_manager(?FUNCTION_NAME, Msg, - Ctx, - [fun process_tm/3, fun process_subscribe/3], Session). -handle_response(Msg, Ctx, Session) -> - call_transport_manager(?FUNCTION_NAME, Msg, Ctx, [fun process_tm/3], Session). +handle_response(Msg, Session) -> + call_transport_manager(?FUNCTION_NAME, Msg, Session). -handle_out(Msg, Ctx, Session) -> - call_transport_manager(?FUNCTION_NAME, Msg, Ctx, [fun process_tm/3], Session). +handle_out(Msg, Session) -> + call_transport_manager(?FUNCTION_NAME, Msg, Session). -deliver(Delivers, Ctx, Session) -> - Fun = fun({_, Topic, Message}, - #{out := OutAcc, - session := #session{observe_manager = OM, - next_msg_id = MsgId, - transport_manager = TM} = SAcc} = Acc) -> - case emqx_coap_observe_res:res_changed(Topic, OM) of +set_reply(Msg, #session{transport_manager = TM} = Session) -> + TM2 = emqx_coap_tm:set_reply(Msg, TM), + Session#session{transport_manager = TM2}. + +deliver(Delivers, Ctx, #session{observe_manager = OM, + transport_manager = TM} = Session) -> + Fun = fun({_, Topic, Message}, {OutAcc, OMAcc, TMAcc} = Acc) -> + case emqx_coap_observe_res:res_changed(Topic, OMAcc) of undefined -> + metrics_inc('delivery.dropped', Ctx), + metrics_inc('delivery.dropped.no_subid', Ctx), Acc; {Token, SeqId, OM2} -> - Msg = mqtt_to_coap(Message, MsgId, Token, SeqId, Ctx), - SAcc2 = SAcc#session{next_msg_id = next_msg_id(MsgId, TM), - observe_manager = OM2}, - #{out := Out} = Result = handle_out(Msg, Ctx, SAcc2), - Result#{out := [Out | OutAcc]} + metrics_inc('messages.delivered', Ctx), + Msg = mqtt_to_coap(Message, Token, SeqId), + #{out := Out, tm := TM2} = emqx_coap_tm:handle_out(Msg, TMAcc), + {Out ++ OutAcc, OM2, TM2} end end, - lists:foldl(Fun, - #{out => [], session => Session}, - lists:reverse(Delivers)). + {Outs, OM2, TM2} = lists:foldl(Fun, {[], OM, TM}, lists:reverse(Delivers)), -timeout(Timer, Ctx, Session) -> - call_transport_manager(?FUNCTION_NAME, Timer, Ctx, [fun process_tm/3], Session). + #{out => lists:reverse(Outs), + session => Session#session{observe_manager = OM2, + transport_manager = TM2}}. -result_keys() -> - [tm, subscribe] ++ emqx_coap_channel:result_keys(). - -transfer_result(From, Value, Result) -> - ?TRANSFER_RESULT(From, Value, Result). +timeout(Timer, Session) -> + call_transport_manager(?FUNCTION_NAME, Timer, Session). %%%------------------------------------------------------------------- %%% Internal functions %%%------------------------------------------------------------------- call_transport_manager(Fun, Msg, - Ctx, - Processor, #session{transport_manager = TM} = Session) -> - try - Result = emqx_coap_tm:Fun(Msg, Ctx, TM), - {ok, Result2, Session2} = pipeline(Processor, - Result, - Msg, - Session), - emqx_coap_channel:transfer_result(session, Session2, Result2) - catch Type:Reason:Stack -> - ?ERROR("process transmission with, message:~p failed~nType:~p,Reason:~p~n,StackTrace:~p~n", - [Msg, Type, Reason, Stack]), - ?REPLY({error, internal_server_error}, Msg) - end. + Result = emqx_coap_tm:Fun(Msg, TM), + iter([tm, fun process_tm/4, fun process_session/3], + Result, + Session). -process_tm(#{tm := TM}, _, Session) -> - {ok, Session#session{transport_manager = TM}}; -process_tm(_, _, Session) -> - {ok, Session}. +process_tm(TM, Result, Session, Cursor) -> + iter(Cursor, Result, Session#session{transport_manager = TM}). -process_subscribe(#{subscribe := Sub} = Result, - Msg, - #session{observe_manager = OM} = Session) -> +process_session(_, Result, Session) -> + Result#{session => Session}. + +process_subscribe(Sub, Msg, Result, + #session{observe_manager = OM} = Session) -> case Sub of undefined -> - {ok, Result, Session}; + Result; {Topic, Token} -> {SeqId, OM2} = emqx_coap_observe_res:insert(Topic, Token, OM), Replay = emqx_coap_message:piggyback({ok, content}, Msg), Replay2 = Replay#coap_message{options = #{observe => SeqId}}, - {ok, Result#{reply => Replay2}, Session#session{observe_manager = OM2}}; + Result#{reply => Replay2, + session => Session#session{observe_manager = OM2}}; Topic -> OM2 = emqx_coap_observe_res:remove(Topic, OM), Replay = emqx_coap_message:piggyback({ok, nocontent}, Msg), - {ok, Result#{reply => Replay}, Session#session{observe_manager = OM2}} - end; -process_subscribe(Result, _, Session) -> - {ok, Result, Session}. + Result#{reply => Replay, + session => Session#session{observe_manager = OM2}} + end. -mqtt_to_coap(MQTT, MsgId, Token, SeqId, Ctx) -> +mqtt_to_coap(MQTT, Token, SeqId) -> #message{payload = Payload} = MQTT, - #coap_message{type = get_notify_type(MQTT, Ctx), + #coap_message{type = get_notify_type(MQTT), method = {ok, content}, - id = MsgId, token = Token, payload = Payload, options = #{observe => SeqId}}. -get_notify_type(#message{qos = Qos}, Ctx) -> - case emqx_coap_channel:get_config(notify_type, Ctx) of +get_notify_type(#message{qos = Qos}) -> + case emqx:get_config([gateway, coap, notify_qos], non) of qos -> case Qos of ?QOS_0 -> @@ -239,32 +227,3 @@ get_notify_type(#message{qos = Qos}, Ctx) -> Other -> Other end. - -next_msg_id(MsgId, TM) -> - next_msg_id(MsgId + 1, MsgId, TM). - -next_msg_id(MsgId, MsgId, _) -> - erlang:throw("too many message in delivering"); -next_msg_id(MsgId, BeginId, TM) when MsgId >= ?MAX_MESSAGE_ID -> - check_is_inused(1, BeginId, TM); -next_msg_id(MsgId, BeginId, TM) -> - check_is_inused(MsgId, BeginId, TM). - -check_is_inused(NewMsgId, BeginId, TM) -> - case emqx_coap_tm:is_inused(out, NewMsgId, TM) of - false -> - NewMsgId; - _ -> - next_msg_id(NewMsgId + 1, BeginId, TM) - end. - -pipeline([Fun | T], Result, Msg, Session) -> - case Fun(Result, Msg, Session) of - {ok, Session2} -> - pipeline(T, Result, Msg, Session2); - {ok, Result2, Session2} -> - pipeline(T, Result2, Msg, Session2) - end; - -pipeline([], Result, _, Session) -> - {ok, Result, Session}. diff --git a/apps/emqx_gateway/src/coap/emqx_coap_tm.erl b/apps/emqx_gateway/src/coap/emqx_coap_tm.erl index 5a664b0f2..b5e4deb7f 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_tm.erl +++ b/apps/emqx_gateway/src/coap/emqx_coap_tm.erl @@ -18,11 +18,12 @@ -module(emqx_coap_tm). -export([ new/0 - , handle_request/3 - , handle_response/3 + , handle_request/2 + , handle_response/2 + , handle_out/2 , handle_out/3 - , timeout/3 - , is_inused/3]). + , set_reply/2 + , timeout/2]). -export_type([manager/0, event_result/1]). @@ -30,17 +31,28 @@ -include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). -type direction() :: in | out. --type state_machine_id() :: {direction(), non_neg_integer()}. --record(state_machine, { id :: state_machine_id() +-record(state_machine, { seq_id :: seq_id() + , id :: state_machine_key() + , token :: token() | undefined + , observe :: 0 | 1 | undefined | observed , state :: atom() , timers :: maps:map() , transport :: emqx_coap_transport:transport()}). -type state_machine() :: #state_machine{}. -type message_id() :: 0 .. ?MAX_MESSAGE_ID. +-type token_key() :: {token, token()}. +-type state_machine_key() :: {direction(), message_id()}. +-type seq_id() :: pos_integer(). +-type manager_key() :: token_key() | state_machine_key() | seq_id(). --type manager() :: #{message_id() => state_machine()}. +-type manager() :: #{ seq_id => seq_id() + , next_msg_id => coap_message_id() + , token_key() => seq_id() + , state_machine_key() => seq_id() + , seq_id() => state_machine() + }. -type ttimeout() :: {state_timeout, pos_integer(), any()} | {stop_timeout, pos_integer()}. @@ -48,6 +60,7 @@ -type topic() :: binary(). -type token() :: binary(). -type sub_register() :: {topic(), token()} | topic(). + -type event_result(State) :: #{next => State, outgoing => emqx_coap_message(), @@ -55,108 +68,164 @@ has_sub => undefined | sub_register(), transport => emqx_coap_transport:transprot()}. +-define(TOKEN_ID(T), {token, T}). + +-import(emqx_coap_medium, [empty/0, iter/4, reset/1, proto_out/2]). + %%-------------------------------------------------------------------- %% API %%-------------------------------------------------------------------- new() -> - #{}. + #{ seq_id => 1 + , next_msg_id => rand:uniform(?MAX_MESSAGE_ID) + }. -handle_request(#coap_message{id = MsgId} = Msg, Ctx, TM) -> +%% client request +handle_request(#coap_message{id = MsgId} = Msg, TM) -> Id = {in, MsgId}, - case maps:get(Id, TM, undefined) of + case find_machine(Id, TM) of undefined -> - Transport = emqx_coap_transport:new(), - Machine = new_state_machine(Id, Transport), - process_event(in, Msg, TM, Ctx, Machine); + {Machine, TM2} = new_in_machine(Id, TM), + process_event(in, Msg, TM2, Machine); Machine -> - process_event(in, Msg, TM, Ctx, Machine) + process_event(in, Msg, TM, Machine) end. -handle_response(#coap_message{type = Type, id = MsgId} = Msg, Ctx, TM) -> +%% client response +handle_response(#coap_message{type = Type, id = MsgId, token = Token} = Msg, TM) -> Id = {out, MsgId}, - case maps:get(Id, TM, undefined) of + TokenId = ?TOKEN_ID(Token), + case find_machine_by_keys([Id, TokenId], TM) of undefined -> case Type of reset -> - ?EMPTY_RESULT; + empty(); _ -> - ?RESET(Msg) + reset(Msg) end; Machine -> - process_event(in, Msg, TM, Ctx, Machine) + process_event(in, Msg, TM, Machine) end. -handle_out(#coap_message{id = MsgId} = Msg, Ctx, TM) -> +%% send to a client, msg can be request/piggyback/separate/notify +handle_out({Ctx, Msg}, TM) -> + handle_out(Msg, Ctx, TM); + +handle_out(Msg, TM) -> + handle_out(Msg, undefined, TM). + +handle_out(#coap_message{token = Token} = MsgT, Ctx, TM) -> + {MsgId, TM2} = alloc_message_id(TM), + Msg = MsgT#coap_message{id = MsgId}, Id = {out, MsgId}, - case maps:get(Id, TM, undefined) of + TokenId = ?TOKEN_ID(Token), + %% TODO why find by token ? + case find_machine_by_keys([Id, TokenId], TM2) of undefined -> - Transport = emqx_coap_transport:new(), - Machine = new_state_machine(Id, Transport), - process_event(out, Msg, TM, Ctx, Machine); + {Machine, TM3} = new_out_machine(Id, Ctx, Msg, TM2), + process_event(out, Msg, TM3, Machine); _ -> %% ignore repeat send - ?EMPTY_RESULT + empty() end. -timeout({Id, Type, Msg}, Ctx, TM) -> - case maps:get(Id, TM, undefined) of +set_reply(#coap_message{id = MsgId} = Msg, TM) -> + Id = {in, MsgId}, + case find_machine(Id, TM) of undefined -> - ?EMPTY_RESULT; + TM; + #state_machine{transport = Transport, + seq_id = SeqId} = Machine -> + Transport2 = emqx_coap_transport:set_cache(Msg, Transport), + Machine2 = Machine#state_machine{transport = Transport2}, + TM#{SeqId => Machine2} + end. + +timeout({SeqId, Type, Msg}, TM) -> + case maps:get(SeqId, TM, undefined) of + undefined -> + empty(); #state_machine{timers = Timers} = Machine -> %% maybe timer has been canceled case maps:is_key(Type, Timers) of true -> - process_event(Type, Msg, TM, Ctx, Machine); + process_event(Type, Msg, TM, Machine); _ -> - ?EMPTY_RESULT + empty() end end. --spec is_inused(direction(), message_id(), manager()) -> boolean(). -is_inused(Dir, Msg, Manager) -> - maps:is_key({Dir, Msg}, Manager). - %%-------------------------------------------------------------------- %% Internal functions %%-------------------------------------------------------------------- -new_state_machine(Id, Transport) -> - #state_machine{id = Id, - state = idle, - timers = #{}, - transport = Transport}. +process_event(stop_timeout, _, TM, Machine) -> + process_manager(stop, #{}, Machine, TM); -process_event(stop_timeout, - _, - TM, - _, - #state_machine{id = Id, - timers = Timers}) -> - lists:foreach(fun({_, Ref}) -> - emqx_misc:cancel_timer(Ref) - end, - maps:to_list(Timers)), - #{tm => maps:remove(Id, TM)}; +process_event(Event, Msg, TM, #state_machine{state = State, + transport = Transport} = Machine) -> + Result = emqx_coap_transport:State(Event, Msg, Transport), + iter([ proto, fun process_observe_response/5 + , next, fun process_state_change/5 + , transport, fun process_transport_change/5 + , timeouts, fun process_timeouts/5 + , fun process_manager/4], + Result, + Machine, + TM). -process_event(Event, - Msg, - TM, - Ctx, - #state_machine{id = Id, - state = State, - transport = Transport} = Machine) -> - Result = emqx_coap_transport:State(Event, Msg, Ctx, Transport), - {ok, _, Machine2} = emqx_misc:pipeline([fun process_state_change/2, - fun process_transport_change/2, - fun process_timeouts/2], - Result, - Machine), - TM2 = TM#{Id => Machine2}, - emqx_coap_session:transfer_result(tm, TM2, Result). +process_observe_response({response, {_, Msg}} = Response, + Result, + #state_machine{observe = 0} = Machine, + TM, + Iter) -> + Result2 = proto_out(Response, Result), + case Msg#coap_message.method of + {ok, _} -> + iter(Iter, + Result2#{next => observe}, + Machine#state_machine{observe = observed}, + TM); + _ -> + iter(Iter, Result2, Machine, TM) + end; -process_state_change(#{next := Next}, Machine) -> - {ok, cancel_state_timer(Machine#state_machine{state = Next})}; -process_state_change(_, Machine) -> - {ok, Machine}. +process_observe_response(Proto, Result, Machine, TM, Iter) -> + iter(Iter, proto_out(Proto, Result), Machine, TM). + +process_state_change(Next, Result, Machine, TM, Iter) -> + case Next of + stop -> + process_manager(stop, Result, Machine, TM); + _ -> + iter(Iter, + Result, + cancel_state_timer(Machine#state_machine{state = Next}), + TM) + end. + +process_transport_change(Transport, Result, Machine, TM, Iter) -> + iter(Iter, Result, Machine#state_machine{transport = Transport}, TM). + +process_timeouts([], Result, Machine, TM, Iter) -> + iter(Iter, Result, Machine, TM); + +process_timeouts(Timeouts, Result, + #state_machine{seq_id = SeqId, + timers = Timers} = Machine, TM, Iter) -> + NewTimers = lists:foldl(fun({state_timeout, _, _} = Timer, Acc) -> + process_timer(SeqId, Timer, Acc); + ({stop_timeout, I}, Acc) -> + process_timer(SeqId, {stop_timeout, I, stop}, Acc) + end, + Timers, + Timeouts), + iter(Iter, Result, Machine#state_machine{timers = NewTimers}, TM). + +process_manager(stop, Result, #state_machine{seq_id = SeqId}, TM) -> + Result#{tm => delete_machine(SeqId, TM)}; + +process_manager(_, Result, #state_machine{seq_id = SeqId} = Machine2, TM) -> + Result#{tm => TM#{SeqId => Machine2}}. cancel_state_timer(#state_machine{timers = Timers} = Machine) -> case maps:get(state_timer, Timers, undefined) of @@ -167,27 +236,119 @@ cancel_state_timer(#state_machine{timers = Timers} = Machine) -> Machine#state_machine{timers = maps:remove(state_timer, Timers)} end. -process_transport_change(#{transport := Transport}, Machine) -> - {ok, Machine#state_machine{transport = Transport}}; -process_transport_change(_, Machine) -> - {ok, Machine}. - -process_timeouts(#{timeouts := []}, Machine) -> - {ok, Machine}; -process_timeouts(#{timeouts := Timeouts}, - #state_machine{id = Id, timers = Timers} = Machine) -> - NewTimers = lists:foldl(fun({state_timeout, _, _} = Timer, Acc) -> - process_timer(Id, Timer, Acc); - ({stop_timeout, I}, Acc) -> - process_timer(Id, {stop_timeout, I, stop}, Acc) - end, - Timers, - Timeouts), - {ok, Machine#state_machine{timers = NewTimers}}; - -process_timeouts(_, Machine) -> - {ok, Machine}. - -process_timer(Id, {Type, Interval, Msg}, Timers) -> - Ref = emqx_misc:start_timer(Interval, {state_machine, {Id, Type, Msg}}), +process_timer(SeqId, {Type, Interval, Msg}, Timers) -> + Ref = emqx_misc:start_timer(Interval, {state_machine, {SeqId, Type, Msg}}), Timers#{Type => Ref}. + +-spec delete_machine(manager_key(), manager()) -> manager(). +delete_machine(Id, Manager) -> + case find_machine(Id, Manager) of + undefined -> + Manager; + #state_machine{seq_id = SeqId, + id = MachineId, + token = Token, + timers = Timers} -> + lists:foreach(fun({_, Ref}) -> + emqx_misc:cancel_timer(Ref) + end, + maps:to_list(Timers)), + maps:without([SeqId, MachineId, ?TOKEN_ID(Token)], Manager) + end. + +-spec find_machine(manager_key(), manager()) -> state_machine() | undefined. +find_machine({_, _} = Id, Manager) -> + find_machine_by_seqid(maps:get(Id, Manager, undefined), Manager); +find_machine(SeqId, Manager) -> + find_machine_by_seqid(SeqId, Manager). + +-spec find_machine_by_seqid(seq_id() | undefined, manager()) -> + state_machine() | undefined. +find_machine_by_seqid(SeqId, Manager) -> + maps:get(SeqId, Manager, undefined). + +-spec find_machine_by_keys(list(manager_key()), + manager()) -> state_machine() | undefined. +find_machine_by_keys([H | T], Manager) -> + case H of + ?TOKEN_ID(<<>>) -> + find_machine_by_keys(T, Manager); + _ -> + case find_machine(H, Manager) of + undefined -> + find_machine_by_keys(T, Manager); + Machine -> + Machine + end + end; +find_machine_by_keys(_, _) -> + undefined. + +-spec new_in_machine(state_machine_key(), manager()) -> + {state_machine(), manager()}. +new_in_machine(MachineId, #{seq_id := SeqId} = Manager) -> + Machine = #state_machine{ seq_id = SeqId + , id = MachineId + , state = idle + , timers = #{} + , transport = emqx_coap_transport:new()}, + {Machine, Manager#{seq_id := SeqId + 1, + SeqId => Machine, + MachineId => SeqId}}. + +-spec new_out_machine(state_machine_key(), any(), emqx_coap_message(), manager()) -> + {state_machine(), manager()}. +new_out_machine(MachineId, + Ctx, + #coap_message{type = Type, token = Token, options = Opts}, + #{seq_id := SeqId} = Manager) -> + Observe = maps:get(observe, Opts, undefined), + Machine = #state_machine{ seq_id = SeqId + , id = MachineId + , token = Token + , observe = Observe + , state = idle + , timers = #{} + , transport = emqx_coap_transport:new(Ctx)}, + + Manager2 = Manager#{seq_id := SeqId + 1, + SeqId => Machine, + MachineId => SeqId}, + {Machine, + if Token =:= <<>> -> + Manager2; + Observe =:= 1 -> + TokenId = ?TOKEN_ID(Token), + delete_machine(TokenId, Manager2); + Type =:= con orelse Observe =:= 0 -> + TokenId = ?TOKEN_ID(Token), + case maps:get(TokenId, Manager, undefined) of + undefined -> + Manager2#{TokenId => SeqId}; + _ -> + throw("token conflict") + end; + true -> + Manager2 + end + }. + +alloc_message_id(#{next_msg_id := MsgId} = TM) -> + alloc_message_id(MsgId, TM). + +alloc_message_id(MsgId, TM) -> + Id = {out, MsgId}, + case maps:get(Id, TM, undefined) of + undefined -> + {MsgId, TM#{next_msg_id => next_message_id(MsgId)}}; + _ -> + alloc_message_id(next_message_id(MsgId), TM) + end. + +next_message_id(MsgId) -> + Next = MsgId + 1, + if Next >= ?MAX_MESSAGE_ID -> + 1; + true -> + Next + end. diff --git a/apps/emqx_gateway/src/coap/emqx_coap_transport.erl b/apps/emqx_gateway/src/coap/emqx_coap_transport.erl index 2c2aaab2e..2e858a2e1 100644 --- a/apps/emqx_gateway/src/coap/emqx_coap_transport.erl +++ b/apps/emqx_gateway/src/coap/emqx_coap_transport.erl @@ -9,118 +9,183 @@ -define(EXCHANGE_LIFETIME, 247000). -define(NON_LIFETIME, 145000). +-type request_context() :: any(). + -record(transport, { cache :: undefined | emqx_coap_message() + , req_context :: request_context() , retry_interval :: non_neg_integer() , retry_count :: non_neg_integer() + , observe :: non_neg_integer() | undefined }). -type transport() :: #transport{}. --export([ new/0, idle/4, maybe_reset/4 - , maybe_resend/4, wait_ack/4, until_stop/4]). +-export([ new/0, new/1, idle/3, maybe_reset/3, set_cache/2 + , maybe_resend_4request/3, wait_ack/3, until_stop/3 + , observe/3, maybe_resend_4response/3]). -export_type([transport/0]). -import(emqx_coap_message, [reset/1]). +-import(emqx_coap_medium, [ empty/0, reset/2, proto_out/2 + , out/1, out/2, proto_out/1 + , reply/2]). -spec new() -> transport(). new() -> + new(undefined). + +new(ReqCtx) -> #transport{cache = undefined, retry_interval = 0, - retry_count = 0}. + retry_count = 0, + req_context = ReqCtx}. idle(in, #coap_message{type = non, method = Method} = Msg, - Ctx, _) -> - Ret = #{next => until_stop, - timeouts => [{stop_timeout, ?NON_LIFETIME}]}, case Method of undefined -> - ?RESET(Msg); + reset(Msg, #{next => stop}); _ -> - Result = call_handler(Msg, Ctx), - maps:merge(Ret, Result) + proto_out({request, Msg}, + #{next => until_stop, + timeouts => + [{stop_timeout, ?NON_LIFETIME}]}) end; idle(in, #coap_message{type = con, method = Method} = Msg, - Ctx, - Transport) -> - Ret = #{next => maybe_resend, - timeouts =>[{stop_timeout, ?EXCHANGE_LIFETIME}]}, + _) -> case Method of undefined -> - ResetMsg = reset(Msg), - Ret#{transport => Transport#transport{cache = ResetMsg}, - out => ResetMsg}; + reset(Msg, #{next => stop}); _ -> - Result = call_handler(Msg, Ctx), - maps:merge(Ret, Result) + proto_out({request, Msg}, + #{next => maybe_resend_4request, + timeouts =>[{stop_timeout, ?EXCHANGE_LIFETIME}]}) end; -idle(out, #coap_message{type = non} = Msg, _, _) -> - #{next => maybe_reset, - out => Msg, - timeouts => [{stop_timeout, ?NON_LIFETIME}]}; +idle(out, #coap_message{type = non} = Msg, _) -> + out(Msg, #{next => maybe_reset, + timeouts => [{stop_timeout, ?NON_LIFETIME}]}); -idle(out, Msg, _, Transport) -> +idle(out, Msg, Transport) -> _ = emqx_misc:rand_seed(), Timeout = ?ACK_TIMEOUT + rand:uniform(?ACK_RANDOM_FACTOR), - #{next => wait_ack, - transport => Transport#transport{cache = Msg}, - out => Msg, - timeouts => [ {state_timeout, Timeout, ack_timeout} - , {stop_timeout, ?EXCHANGE_LIFETIME}]}. + out(Msg, #{next => wait_ack, + transport => Transport#transport{cache = Msg}, + timeouts => [ {state_timeout, Timeout, ack_timeout} + , {stop_timeout, ?EXCHANGE_LIFETIME}]}). -maybe_reset(in, Message, _, _) -> - case Message of - #coap_message{type = reset} -> - ?INFO("Reset Message:~p~n", [Message]); +maybe_resend_4request(in, Msg, Transport) -> + maybe_resend(Msg, true, Transport). + +maybe_resend_4response(in, Msg, Transport) -> + maybe_resend(Msg, false, Transport). + +maybe_resend(Msg, IsExpecteReq, #transport{cache = Cache}) -> + IsExpected = emqx_coap_message:is_request(Msg) =:= IsExpecteReq, + case IsExpected of + true -> + case Cache of + undefined -> + %% handler in processing, ignore + empty(); + _ -> + out(Cache) + end; _ -> - ok - end, - ?EMPTY_RESULT. + reset(Msg, #{next => stop}) + end. -maybe_resend(in, _, _, #transport{cache = Cache}) -> - #{out => Cache}. +maybe_reset(in, #coap_message{type = Type, method = Method} = Message, + #transport{req_context = Ctx} = Transport) -> + Ret = #{next => stop}, + CtxMsg = {Ctx, Message}, + if Type =:= reset -> + proto_out({reset, CtxMsg}, Ret); + is_tuple(Method) -> + on_response(Message, + Transport, + if Type =:= non -> until_stop; + true -> maybe_resend_4response + end); + true -> + reset(Message, Ret) + end. -wait_ack(in, #coap_message{type = Type}, _, _) -> +wait_ack(in, #coap_message{type = Type, method = Method} = Msg, #transport{req_context = Ctx}) -> + CtxMsg = {Ctx, Msg}, case Type of - ack -> - #{next => until_stop}; reset -> - #{next => until_stop}; + proto_out({reset, CtxMsg}, #{next => stop}); _ -> - ?EMPTY_RESULT + case Method of + undefined -> + %% empty ack, keep transport to recv response + proto_out({ack, CtxMsg}); + {_, _} -> + %% ack with payload + proto_out({response, CtxMsg}, #{next => stop}); + _ -> + reset(Msg, #{next => stop}) + end end; wait_ack(state_timeout, ack_timeout, - _, #transport{cache = Msg, retry_interval = Timeout, retry_count = Count} =Transport) -> case Count < ?MAX_RETRANSMIT of true -> Timeout2 = Timeout * 2, - #{transport => Transport#transport{retry_interval = Timeout2, - retry_count = Count + 1}, - out => Msg, - timeouts => [{state_timeout, Timeout2, ack_timeout}]}; + out(Msg, + #{transport => Transport#transport{retry_interval = Timeout2, + retry_count = Count + 1}, + timeouts => [{state_timeout, Timeout2, ack_timeout}]}); _ -> - #{next_state => until_stop} + proto_out({ack_failure, Msg}, #{next_state => stop}) end. -until_stop(_, _, _, _) -> - ?EMPTY_RESULT. - -call_handler(#coap_message{options = Opts} = Msg, Ctx) -> - case maps:get(uri_path, Opts, undefined) of - [<<"ps">> | RestPath] -> - emqx_coap_pubsub_handler:handle_request(RestPath, Msg, Ctx); - [<<"mqtt">> | RestPath] -> - emqx_coap_mqtt_handler:handle_request(RestPath, Msg, Ctx); +observe(in, + #coap_message{method = Method} = Message, + #transport{observe = Observe} = Transport) -> + case Method of + {ok, _} -> + case emqx_coap_message:get_option(observe, Message, Observe) of + Observe -> + %% repeatd notify, ignore + empty(); + NewObserve -> + on_response(Message, + Transport#transport{observe = NewObserve}, + ?FUNCTION_NAME) + end; + {error, _} -> + #{next => stop}; _ -> - ?REPLY({error, bad_request}, Msg) + reset(Message) + end. + +until_stop(_, _, _) -> + empty(). + +set_cache(Cache, Transport) -> + Transport#transport{cache = Cache}. + +on_response(#coap_message{type = Type} = Message, + #transport{req_context = Ctx} = Transport, + NextState) -> + CtxMsg = {Ctx, Message}, + if Type =:= non -> + proto_out({response, CtxMsg}, #{next => NextState}); + Type =:= con -> + Ack = emqx_coap_message:ack(Message), + proto_out({response, CtxMsg}, + out(Ack, #{next => NextState, + transport => Transport#transport{cache = Ack}})); + true -> + reset(Message) end. diff --git a/apps/emqx_gateway/src/coap/handler/emqx_coap_mqtt_handler.erl b/apps/emqx_gateway/src/coap/handler/emqx_coap_mqtt_handler.erl index 88a4a2310..47bf14d9b 100644 --- a/apps/emqx_gateway/src/coap/handler/emqx_coap_mqtt_handler.erl +++ b/apps/emqx_gateway/src/coap/handler/emqx_coap_mqtt_handler.erl @@ -18,23 +18,24 @@ -include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). --export([handle_request/3]). +-export([handle_request/4]). -import(emqx_coap_message, [response/2, response/3]). +-import(emqx_coap_medium, [reply/2]). -handle_request([<<"connection">>], #coap_message{method = Method} = Msg, _) -> +handle_request([<<"connection">>], #coap_message{method = Method} = Msg, _Ctx, _CInfo) -> handle_method(Method, Msg); -handle_request(_, Msg, _) -> - ?REPLY({error, bad_request}, Msg). +handle_request(_, Msg, _, _) -> + reply({error, bad_request}, Msg). handle_method(put, Msg) -> - ?REPLY({ok, changed}, Msg); + reply({ok, changed}, Msg); -handle_method(post, _) -> - #{connection => open}; +handle_method(post, Msg) -> + #{connection => {open, Msg}}; -handle_method(delete, _) -> - #{connection => close}; +handle_method(delete, Msg) -> + #{connection => {close, Msg}}; handle_method(_, Msg) -> - ?REPLY({error, method_not_allowed}, Msg). + reply({error, method_not_allowed}, Msg). diff --git a/apps/emqx_gateway/src/coap/handler/emqx_coap_pubsub_handler.erl b/apps/emqx_gateway/src/coap/handler/emqx_coap_pubsub_handler.erl index e6886a559..85cf32c6d 100644 --- a/apps/emqx_gateway/src/coap/handler/emqx_coap_pubsub_handler.erl +++ b/apps/emqx_gateway/src/coap/handler/emqx_coap_pubsub_handler.erl @@ -20,48 +20,52 @@ -include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). --export([handle_request/3]). +-export([handle_request/4]). -import(emqx_coap_message, [response/2, response/3]). +-import(emqx_coap_medium, [reply/2, reply/3]). +-import(emqx_coap_channel, [run_hooks/3]). --define(UNSUB(Topic), #{subscribe => Topic}). --define(SUB(Topic, Token), #{subscribe => {Topic, Token}}). +-define(UNSUB(Topic, Msg), #{subscribe => {Topic, Msg}}). +-define(SUB(Topic, Token, Msg), #{subscribe => {{Topic, Token}, Msg}}). -define(SUBOPTS, #{qos => 0, rh => 0, rap => 0, nl => 0, is_new => false}). -handle_request(Path, #coap_message{method = Method} = Msg, Ctx) -> +%% TODO maybe can merge this code into emqx_coap_session, simplify the call chain + +handle_request(Path, #coap_message{method = Method} = Msg, Ctx, CInfo) -> case check_topic(Path) of {ok, Topic} -> - handle_method(Method, Topic, Msg, Ctx); + handle_method(Method, Topic, Msg, Ctx, CInfo); _ -> - ?REPLY({error, bad_request}, <<"invalid topic">>, Msg) + reply({error, bad_request}, <<"invalid topic">>, Msg) end. -handle_method(get, Topic, #coap_message{options = Opts} = Msg, Ctx) -> - case maps:get(observe, Opts, undefined) of +handle_method(get, Topic, Msg, Ctx, CInfo) -> + case emqx_coap_message:get_option(observe, Msg) of 0 -> - subscribe(Msg, Topic, Ctx); + subscribe(Msg, Topic, Ctx, CInfo); 1 -> - unsubscribe(Topic, Ctx); + unsubscribe(Msg, Topic, Ctx, CInfo); _ -> - ?REPLY({error, bad_request}, <<"invalid observe value">>, Msg) + reply({error, bad_request}, <<"invalid observe value">>, Msg) end; -handle_method(post, Topic, #coap_message{payload = Payload} = Msg, Ctx) -> - case emqx_coap_channel:validator(publish, Topic, Ctx) of +handle_method(post, Topic, #coap_message{payload = Payload} = Msg, Ctx, CInfo) -> + case emqx_coap_channel:validator(publish, Topic, Ctx, CInfo) of allow -> - ClientInfo = emqx_coap_channel:get_clientinfo(Ctx), - #{clientid := ClientId} = ClientInfo, - QOS = get_publish_qos(Msg, Ctx), - MQTTMsg = emqx_message:make(ClientId, QOS, Topic, Payload), + #{clientid := ClientId} = CInfo, + MountTopic = mount(CInfo, Topic), + QOS = get_publish_qos(Msg), + MQTTMsg = emqx_message:make(ClientId, QOS, MountTopic, Payload), MQTTMsg2 = apply_publish_opts(Msg, MQTTMsg), _ = emqx_broker:publish(MQTTMsg2), - ?REPLY({ok, changed}, Msg); + reply({ok, changed}, Msg); _ -> - ?REPLY({error, unauthorized}, Msg) + reply({error, unauthorized}, Msg) end; -handle_method(_, _, Msg, _) -> - ?REPLY({error, method_not_allowed}, Msg). +handle_method(_, _, Msg, _, _) -> + reply({error, method_not_allowed}, Msg). check_topic([]) -> error; @@ -76,13 +80,13 @@ check_topic(Path) -> <<>>, Path))}. -get_sub_opts(#coap_message{options = Opts} = Msg, Ctx) -> +get_sub_opts(#coap_message{options = Opts} = Msg) -> SubOpts = maps:fold(fun parse_sub_opts/3, #{}, Opts), case SubOpts of #{qos := _} -> maps:merge(SubOpts, ?SUBOPTS); _ -> - CfgType = emqx_coap_channel:get_config(subscribe_qos, Ctx), + CfgType = emqx:get_config([gateway, coap, subscribe_qos], ?QOS_0), maps:merge(SubOpts, ?SUBOPTS#{qos => type_to_qos(CfgType, Msg)}) end. @@ -106,16 +110,16 @@ type_to_qos(coap, #coap_message{type = Type}) -> ?QOS_1 end. -get_publish_qos(#coap_message{options = Opts} = Msg, Ctx) -> - case maps:get(uri_query, Opts) of +get_publish_qos(Msg) -> + case emqx_coap_message:get_option(uri_query, Msg) of #{<<"qos">> := QOS} -> erlang:binary_to_integer(QOS); _ -> - CfgType = emqx_coap_channel:get_config(publish_qos, Ctx), + CfgType = emqx:get_config([gateway, coap, publish_qos], ?QOS_0), type_to_qos(CfgType, Msg) end. -apply_publish_opts(#coap_message{options = Opts}, MQTTMsg) -> +apply_publish_opts(Msg, MQTTMsg) -> maps:fold(fun(<<"retain">>, V, Acc) -> Val = erlang:binary_to_atom(V), emqx_message:set_flag(retain, Val, Acc); @@ -129,27 +133,29 @@ apply_publish_opts(#coap_message{options = Opts}, MQTTMsg) -> Acc end, MQTTMsg, - maps:get(uri_query, Opts)). + emqx_coap_message:get_option(uri_query, Msg)). -subscribe(#coap_message{token = <<>>} = Msg, _, _) -> - ?REPLY({error, bad_request}, <<"observe without token">>, Msg); +subscribe(#coap_message{token = <<>>} = Msg, _, _, _) -> + reply({error, bad_request}, <<"observe without token">>, Msg); -subscribe(#coap_message{token = Token} = Msg, Topic, Ctx) -> - case emqx_coap_channel:validator(subscribe, Topic, Ctx) of +subscribe(#coap_message{token = Token} = Msg, Topic, Ctx, CInfo) -> + case emqx_coap_channel:validator(subscribe, Topic, Ctx, CInfo) of allow -> - ClientInfo = emqx_coap_channel:get_clientinfo(Ctx), - #{clientid := ClientId} = ClientInfo, - SubOpts = get_sub_opts(Msg, Ctx), - emqx_broker:subscribe(Topic, ClientId, SubOpts), - emqx_hooks:run('session.subscribed', - [ClientInfo, Topic, SubOpts]), - ?SUB(Topic, Token); + #{clientid := ClientId} = CInfo, + SubOpts = get_sub_opts(Msg), + MountTopic = mount(CInfo, Topic), + emqx_broker:subscribe(MountTopic, ClientId, SubOpts), + run_hooks(Ctx, 'session.subscribed', [CInfo, Topic, SubOpts]), + ?SUB(MountTopic, Token, Msg); _ -> - ?REPLY({error, unauthorized}, Msg) + reply({error, unauthorized}, Msg) end. -unsubscribe(Topic, Ctx) -> - ClientInfo = emqx_coap_channel:get_clientinfo(Ctx), - emqx_broker:unsubscribe(Topic), - emqx_hooks:run('session.unsubscribed', [ClientInfo, Topic, ?SUBOPTS]), - ?UNSUB(Topic). +unsubscribe(Msg, Topic, Ctx, CInfo) -> + MountTopic = mount(CInfo, Topic), + emqx_broker:unsubscribe(MountTopic), + run_hooks(Ctx, 'session.unsubscribed', [CInfo, Topic, ?SUBOPTS]), + ?UNSUB(MountTopic, Msg). + +mount(#{mountpoint := Mountpoint}, Topic) -> + <>. diff --git a/apps/emqx_gateway/src/coap/include/emqx_coap.hrl b/apps/emqx_gateway/src/coap/include/emqx_coap.hrl index 3b0268abb..d47dd17fd 100644 --- a/apps/emqx_gateway/src/coap/include/emqx_coap.hrl +++ b/apps/emqx_gateway/src/coap/include/emqx_coap.hrl @@ -22,18 +22,6 @@ -define(DEFAULT_MAX_AGE, 60). -define(MAXIMUM_MAX_AGE, 4294967295). --define(EMPTY_RESULT, #{}). --define(TRANSFER_RESULT(From, Value, R1), - begin - Keys = result_keys(), - R2 = maps:with(Keys, R1), - R2#{From => Value} - end). - --define(RESET(Msg), #{out => emqx_coap_message:reset(Msg)}). --define(REPLY(Resp, Payload, Msg), #{out => emqx_coap_message:piggyback(Resp, Payload, Msg)}). --define(REPLY(Resp, Msg), ?REPLY(Resp, <<>>, Msg)). - -type coap_message_id() :: 1 .. ?MAX_MESSAGE_ID. -type message_type() :: con | non | ack | reset. -type max_age() :: 1 .. ?MAXIMUM_MAX_AGE. @@ -66,7 +54,7 @@ , uri_path => list(binary()) , content_format => 0 .. 65535 , max_age => non_neg_integer() - , uri_query => list(binary()) + , uri_query => list(binary()) | map() , 'accept' => 0 .. 65535 , location_query => list(binary()) , proxy_uri => binary() @@ -85,7 +73,4 @@ , options = #{} , payload = <<>>}). --record(coap_content, {etag, max_age = ?DEFAULT_MAX_AGE, format, location_path = [], payload = <<>>}). - -type emqx_coap_message() :: #coap_message{}. --type coap_content() :: #coap_content{}. diff --git a/apps/emqx_gateway/src/emqx_gateway.app.src b/apps/emqx_gateway/src/emqx_gateway.app.src index e25b767cc..2fc329711 100644 --- a/apps/emqx_gateway/src/emqx_gateway.app.src +++ b/apps/emqx_gateway/src/emqx_gateway.app.src @@ -3,7 +3,7 @@ {vsn, "0.1.0"}, {registered, []}, {mod, {emqx_gateway_app, []}}, - {applications, [kernel, stdlib, grpc, lwm2m_coap, emqx, emqx_authn]}, + {applications, [kernel, stdlib, grpc, lwm2m_coap, emqx]}, {env, []}, {modules, []}, {licenses, ["Apache 2.0"]}, diff --git a/apps/emqx_gateway/src/emqx_gateway.erl b/apps/emqx_gateway/src/emqx_gateway.erl index 79ea5d8a4..596b47547 100644 --- a/apps/emqx_gateway/src/emqx_gateway.erl +++ b/apps/emqx_gateway/src/emqx_gateway.erl @@ -25,7 +25,7 @@ , post_config_update/4 ]). -%% APIs +%% Gateway APIs -export([ registered_gateway/0 , load/2 , unload/1 @@ -48,7 +48,7 @@ registered_gateway() -> emqx_gateway_registry:list(). %%-------------------------------------------------------------------- -%% Gateway Instace APIs +%% Gateway APIs -spec list() -> [gateway()]. list() -> @@ -96,7 +96,8 @@ update_rawconf(RawName, RawConfDiff) -> %%-------------------------------------------------------------------- %% Config Handler --spec pre_config_update(emqx_config:update_request(), emqx_config:raw_config()) -> +-spec pre_config_update(emqx_config:update_request(), + emqx_config:raw_config()) -> {ok, emqx_config:update_request()} | {error, term()}. pre_config_update({RawName, RawConfDiff}, RawConf) -> {ok, emqx_map_lib:deep_merge(RawConf, #{RawName => RawConfDiff})}. @@ -117,4 +118,3 @@ post_config_update({RawName, _}, NewConfig, OldConfig, _AppEnvs) -> %%-------------------------------------------------------------------- %% Internal funcs %%-------------------------------------------------------------------- - diff --git a/apps/emqx_gateway/src/emqx_gateway_api.erl b/apps/emqx_gateway/src/emqx_gateway_api.erl index f38624ed9..f264339a4 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api.erl @@ -18,10 +18,9 @@ -behaviour(minirest_api). --compile(nowarn_unused_function). - --import(emqx_mgmt_util, [ schema/1 - ]). +-import(emqx_gateway_http, + [ return_http_error/2 + ]). %% minirest behaviour callbacks -export([api_spec/0]). @@ -32,18 +31,169 @@ , gateway_insta_stats/2 ]). --define(EXAMPLE_GATEWAY_LIST, - [ #{ name => <<"lwm2m">> - , status => <<"running">> - , started_at => <<"2021-08-19T11:45:56.006373+08:00">> - , max_connection => 1024000 - , current_connection => 1000 - , listeners => [ - #{name => <<"lw-udp-1">>, status => <<"activing">>}, - #{name => <<"lw-udp-2">>, status => <<"inactived">>} - ] - } - ]). +%%-------------------------------------------------------------------- +%% minirest behaviour callbacks +%%-------------------------------------------------------------------- + +api_spec() -> + {metadata(apis()), []}. + +apis() -> + [ {"/gateway", gateway} + , {"/gateway/:name", gateway_insta} + , {"/gateway/:name/stats", gateway_insta_stats} + ]. +%%-------------------------------------------------------------------- +%% http handlers + +gateway(get, Request) -> + Params = maps:get(query_string, Request, #{}), + Status = case maps:get(<<"status">>, Params, undefined) of + undefined -> all; + S0 -> binary_to_existing_atom(S0, utf8) + end, + {200, emqx_gateway_http:gateways(Status)}. + +gateway_insta(delete, #{bindings := #{name := Name0}}) -> + Name = binary_to_existing_atom(Name0), + case emqx_gateway:unload(Name) of + ok -> + {204}; + {error, not_found} -> + return_http_error(404, <<"Gateway not found">>) + end; +gateway_insta(get, #{bindings := #{name := Name0}}) -> + Name = binary_to_existing_atom(Name0), + case emqx_gateway:lookup(Name) of + #{config := _Config} -> + GwCfs = filled_raw_confs([<<"gateway">>, Name0]), + NGwCfs = GwCfs#{<<"listeners">> => + emqx_gateway_http:mapping_listener_m2l( + Name0, maps:get(<<"listeners">>, GwCfs, #{}) + ) + }, + {200, NGwCfs}; + undefined -> + return_http_error(404, <<"Gateway not found">>) + end; +gateway_insta(put, #{body := RawConfsIn0, + bindings := #{name := Name} + }) -> + RawConfsIn = maps:without([<<"authentication">>, + <<"listeners">>], RawConfsIn0), + %% FIXME: Cluster Consistence ?? + case emqx_gateway:update_rawconf(Name, RawConfsIn) of + ok -> + {200}; + {error, not_found} -> + return_http_error(404, <<"Gateway not found">>); + {error, Reason} -> + return_http_error(500, Reason) + end. + +gateway_insta_stats(get, _Req) -> + return_http_error(401, <<"Implement it later (maybe 5.1)">>). + +filled_raw_confs(Path) -> + RawConf = emqx_config:fill_defaults( + emqx_config:get_root_raw(Path) + ), + Confs = emqx_map_lib:deep_get(Path, RawConf), + emqx_map_lib:jsonable_map(Confs). + +%%-------------------------------------------------------------------- +%% Swagger defines +%%-------------------------------------------------------------------- + +metadata(APIs) -> + metadata(APIs, []). +metadata([], APIAcc) -> + lists:reverse(APIAcc); +metadata([{Path, Fun}|More], APIAcc) -> + Methods = [get, post, put, delete, patch], + Mds = lists:foldl(fun(M, Acc) -> + try + Acc#{M => swagger(Path, M)} + catch + error : function_clause -> + Acc + end + end, #{}, Methods), + metadata(More, [{Path, Mds, Fun} | APIAcc]). + +swagger("/gateway", get) -> + #{ description => <<"Get gateway list">> + , parameters => params_gateway_status_in_qs() + , responses => + #{ <<"200">> => schema_gateway_overview_list() } + }; +swagger("/gateway/:name", get) -> + #{ description => <<"Get the gateway configurations">> + , parameters => params_gateway_name_in_path() + , responses => + #{ <<"404">> => schema_not_found() + , <<"200">> => schema_gateway_conf() + } + }; +swagger("/gateway/:name", delete) -> + #{ description => <<"Delete/Unload the gateway">> + , parameters => params_gateway_name_in_path() + , responses => + #{ <<"404">> => schema_not_found() + , <<"204">> => schema_no_content() + } + }; +swagger("/gateway/:name", put) -> + #{ description => <<"Update the gateway configurations/status">> + , parameters => params_gateway_name_in_path() + , requestBody => schema_gateway_conf() + , responses => + #{ <<"404">> => schema_not_found() + , <<"200">> => schema_no_content() + } + }; +swagger("/gateway/:name/stats", get) -> + #{ description => <<"Get gateway Statistic">> + , parameters => params_gateway_name_in_path() + , responses => + #{ <<"404">> => schema_not_found() + , <<"200">> => schema_gateway_stats() + } + }. + +%%-------------------------------------------------------------------- +%% params defines + +params_gateway_name_in_path() -> + [#{ name => name + , in => path + , schema => #{type => string} + , required => true + }]. + +params_gateway_status_in_qs() -> + [#{ name => status + , in => query + , schema => #{type => string} + , required => false + }]. + +%%-------------------------------------------------------------------- +%% schemas + +schema_not_found() -> + emqx_mgmt_util:error_schema(<<"Gateway not found or unloaded">>). + +schema_no_content() -> + #{description => <<"No Content">>}. + +schema_gateway_overview_list() -> + emqx_mgmt_util:array_schema( + #{ type => object + , properties => properties_gateway_overview() + }, + <<"Gateway Overview list">> + ). %% XXX: This is whole confs for all type gateways. It is used to fill the %% default configurations and generate the swagger-schema @@ -58,8 +208,13 @@ <<"enable">> => true, <<"enable_stats">> => true,<<"heartbeat">> => <<"30s">>, <<"idle_timeout">> => <<"30s">>, - <<"listeners">> => - #{<<"udp">> => #{<<"default">> => #{<<"bind">> => 5683}}}, + <<"listeners">> => [ + #{<<"id">> => <<"coap:udp:default">>, + <<"type">> => <<"udp">>, + <<"running">> => true, + <<"acceptors">> => 8,<<"bind">> => 5683, + <<"max_conn_rate">> => 1000, + <<"max_connections">> => 10240}], <<"mountpoint">> => <<>>,<<"notify_type">> => <<"qos">>, <<"publish_qos">> => <<"qos1">>, <<"subscribe_qos">> => <<"qos0">>} @@ -71,12 +226,13 @@ <<"handler">> => #{<<"address">> => <<"http://127.0.0.1:9001">>}, <<"idle_timeout">> => <<"30s">>, - <<"listeners">> => - #{<<"tcp">> => - #{<<"default">> => - #{<<"acceptors">> => 8,<<"bind">> => 7993, - <<"max_conn_rate">> => 1000, - <<"max_connections">> => 10240}}}, + <<"listeners">> => [ + #{<<"id">> => <<"exproto:tcp:default">>, + <<"type">> => <<"tcp">>, + <<"running">> => true, + <<"acceptors">> => 8,<<"bind">> => 7993, + <<"max_conn_rate">> => 1000, + <<"max_connections">> => 10240}], <<"mountpoint">> => <<>>, <<"server">> => #{<<"bind">> => 9100}} ). @@ -88,8 +244,11 @@ <<"idle_timeout">> => <<"30s">>, <<"lifetime_max">> => <<"86400s">>, <<"lifetime_min">> => <<"1s">>, - <<"listeners">> => - #{<<"udp">> => #{<<"default">> => #{<<"bind">> => 5783}}}, + <<"listeners">> => [ + #{<<"id">> => <<"lwm2m:udp:default">>, + <<"type">> => <<"udp">>, + <<"running">> => true, + <<"bind">> => 5783}], <<"mountpoint">> => <<"lwm2m/%e/">>, <<"qmode_time_windonw">> => 22, <<"translators">> => @@ -110,11 +269,12 @@ <<"enable">> => true, <<"enable_qos3">> => true,<<"enable_stats">> => true, <<"gateway_id">> => 1,<<"idle_timeout">> => <<"30s">>, - <<"listeners">> => - #{<<"udp">> => - #{<<"default">> => - #{<<"bind">> => 1884,<<"max_conn_rate">> => 1000, - <<"max_connections">> => 10240000}}}, + <<"listeners">> => [ + #{<<"id">> => <<"mqttsn:udp:default">>, + <<"type">> => <<"udp">>, + <<"running">> => true, + <<"bind">> => 1884,<<"max_conn_rate">> => 1000, + <<"max_connections">> => 10240000}], <<"mountpoint">> => <<>>, <<"predefined">> => [#{<<"id">> => 1, @@ -138,245 +298,60 @@ #{<<"max_body_length">> => 8192,<<"max_headers">> => 10, <<"max_headers_length">> => 1024}, <<"idle_timeout">> => <<"30s">>, - <<"listeners">> => - #{<<"tcp">> => - #{<<"default">> => - #{<<"acceptors">> => 16,<<"active_n">> => 100, - <<"bind">> => 61613,<<"max_conn_rate">> => 1000, - <<"max_connections">> => 1024000}}}, + <<"listeners">> => [ + #{<<"id">> => <<"stomp:tcp:default">>, + <<"type">> => <<"tcp">>, + <<"running">> => true, + <<"acceptors">> => 16,<<"active_n">> => 100, + <<"bind">> => 61613,<<"max_conn_rate">> => 1000, + <<"max_connections">> => 1024000}], <<"mountpoint">> => <<>>} ). %% --- END --define(EXAMPLE_GATEWAY_STATS, #{ - max_connection => 10240000, - current_connection => 1000, - messages_in => 100.24, - messages_out => 32.5 - }). +schema_gateway_conf() -> + emqx_mgmt_util:schema( + #{oneOf => + [ emqx_mgmt_api_configs:gen_schema(?STOMP_GATEWAY_CONFS) + , emqx_mgmt_api_configs:gen_schema(?MQTTSN_GATEWAY_CONFS) + , emqx_mgmt_api_configs:gen_schema(?COAP_GATEWAY_CONFS) + , emqx_mgmt_api_configs:gen_schema(?LWM2M_GATEWAY_CONFS) + , emqx_mgmt_api_configs:gen_schema(?EXPROTO_GATEWAY_CONFS) + ]}). + +schema_gateway_stats() -> + emqx_mgmt_util:schema( + #{ type => object + , properties => + #{ a_key => #{type => string} + }}). %%-------------------------------------------------------------------- -%% minirest behaviour callbacks -%%-------------------------------------------------------------------- +%% properties -api_spec() -> - {apis(), schemas()}. - -apis() -> - [ {"/gateway", metadata(gateway), gateway} - , {"/gateway/:name", metadata(gateway_insta), gateway_insta} - , {"/gateway/:name/stats", metadata(gateway_insta_stats), gateway_insta_stats} - ]. - -metadata(gateway) -> - #{get => #{ - description => <<"Get gateway list">>, - parameters => [ - #{name => status, - in => query, - schema => #{type => string}, - required => false - } +properties_gateway_overview() -> + ListenerProps = + [ {id, string, + <<"Listener ID">>} + , {running, boolean, + <<"Listener Running status">>} + , {type, string, + <<"Listener Type">>, [<<"tcp">>, <<"ssl">>, <<"udp">>, <<"dtls">>]} ], - responses => #{ - <<"200">> => #{ - description => <<"OK">>, - content => #{ - 'application/json' => #{ - schema => minirest:ref(<<"gateway_overrview">>), - examples => #{ - simple => #{ - summary => <<"Gateway List Example">>, - value => emqx_json:encode(?EXAMPLE_GATEWAY_LIST) - } - } - } - } - } - } - }}; - -metadata(gateway_insta) -> - UriNameParamDef = #{name => name, - in => path, - schema => #{type => string}, - required => true - }, - NameNotFoundRespDef = - #{description => <<"Not Found">>, - content => #{ - 'application/json' => #{ - schema => minirest:ref(<<"error">>), - examples => #{ - simple => #{ - summary => <<"Not Found">>, - value => #{ - code => <<"NOT_FOUND">>, - message => <<"The gateway not found">> - } - } - } - } - }}, - #{delete => #{ - description => <<"Delete/Unload the gateway">>, - parameters => [UriNameParamDef], - responses => #{ - <<"404">> => NameNotFoundRespDef, - <<"204">> => #{description => <<"No Content">>} - } - }, - get => #{ - description => <<"Get the gateway configurations">>, - parameters => [UriNameParamDef], - responses => #{ - <<"404">> => NameNotFoundRespDef, - <<"200">> => schema(schema_for_gateway_conf()) - } - }, - put => #{ - description => <<"Update the gateway configurations/status">>, - parameters => [UriNameParamDef], - requestBody => schema(schema_for_gateway_conf()), - responses => #{ - <<"404">> => NameNotFoundRespDef, - <<"200">> => #{description => <<"Changed">>} - } - } - }; - -metadata(gateway_insta_stats) -> - #{get => #{ - description => <<"Get gateway Statistic">>, - responses => #{ - <<"200">> => #{ - description => <<"OK">>, - content => #{ - 'application/json' => #{ - schema => minirest:ref(<<"gateway_stats">>), - examples => #{ - simple => #{ - summary => <<"Gateway Statistic">>, - value => emqx_json:encode(?EXAMPLE_GATEWAY_STATS) - } - } - } - } - } - } - }}. - -schemas() -> - [ #{<<"gateway_overrview">> => schema_for_gateway_overrview()} - , #{<<"gateway_stats">> => schema_for_gateway_stats()} - ]. - -schema_for_gateway_overrview() -> - #{type => array, - items => #{ - type => object, - properties => #{ - name => #{ - type => string, - example => <<"lwm2m">> - }, - status => #{ - type => string, - enum => [<<"running">>, <<"stopped">>, <<"unloaded">>], - example => <<"running">> - }, - started_at => #{ - type => string, - example => <<"2021-08-19T11:45:56.006373+08:00">> - }, - max_connection => #{ - type => integer, - example => 1024000 - }, - current_connection => #{ - type => integer, - example => 1000 - }, - listeners => #{ - type => array, - items => #{ - type => object, - properties => #{ - name => #{ - type => string, - example => <<"lw-udp">> - }, - status => #{ - type => string, - enum => [<<"activing">>, <<"inactived">>] - } - } - } - } - } - } - }. - -schema_for_gateway_conf() -> - #{oneOf => - [ emqx_mgmt_api_configs:gen_schema(?STOMP_GATEWAY_CONFS) - , emqx_mgmt_api_configs:gen_schema(?MQTTSN_GATEWAY_CONFS) - , emqx_mgmt_api_configs:gen_schema(?COAP_GATEWAY_CONFS) - , emqx_mgmt_api_configs:gen_schema(?LWM2M_GATEWAY_CONFS) - , emqx_mgmt_api_configs:gen_schema(?EXPROTO_GATEWAY_CONFS) - ]}. - -schema_for_gateway_stats() -> - #{type => object, - properties => #{ - a_key => #{type => string} - }}. - -%%-------------------------------------------------------------------- -%% http handlers - -gateway(get, Request) -> - Params = maps:get(query_string, Request, #{}), - Status = case maps:get(<<"status">>, Params, undefined) of - undefined -> all; - S0 -> binary_to_existing_atom(S0, utf8) - end, - {200, emqx_gateway_intr:gateways(Status)}. - -gateway_insta(delete, #{bindings := #{name := Name0}}) -> - Name = binary_to_existing_atom(Name0), - case emqx_gateway:unload(Name) of - ok -> - {200, ok}; - {error, not_found} -> - {404, <<"Not Found">>} - end; -gateway_insta(get, #{bindings := #{name := Name0}}) -> - Name = binary_to_existing_atom(Name0), - case emqx_gateway:lookup(Name) of - #{config := _Config} -> - %% FIXME: Got the parsed config, but we should return rawconfig to - %% frontend - RawConf = emqx_config:fill_defaults( - emqx_config:get_root_raw([<<"gateway">>]) - ), - {200, emqx_map_lib:deep_get([<<"gateway">>, Name0], RawConf)}; - undefined -> - {404, <<"Not Found">>} - end; -gateway_insta(put, #{body := RawConfsIn, - bindings := #{name := Name} - }) -> - %% FIXME: Cluster Consistence ?? - case emqx_gateway:update_rawconf(Name, RawConfsIn) of - ok -> - {200, <<"Changed">>}; - {error, not_found} -> - {404, <<"Not Found">>}; - {error, Reason} -> - {500, emqx_gateway_utils:stringfy(Reason)} - end. - -gateway_insta_stats(get, _Req) -> - {401, <<"Implement it later (maybe 5.1)">>}. + emqx_mgmt_util:properties( + [ {name, string, + <<"Gateway Name">>} + , {status, string, + <<"Gateway Status">>, + [<<"running">>, <<"stopped">>, <<"unloaded">>]} + , {created_at, string, + <<>>} + , {started_at, string, + <<>>} + , {stopped_at, string, + <<>>} + , {max_connections, integer, <<>>} + , {current_connections, integer, <<>>} + , {listeners, {array, object}, ListenerProps} + ]). diff --git a/apps/emqx_gateway/src/emqx_gateway_api_clients.erl b/apps/emqx_gateway/src/emqx_gateway_api_clients.erl new file mode 100644 index 000000000..fcfea7343 --- /dev/null +++ b/apps/emqx_gateway/src/emqx_gateway_api_clients.erl @@ -0,0 +1,624 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_gateway_api_clients). + +-behaviour(minirest_api). + +-include_lib("emqx/include/logger.hrl"). + +%% minirest behaviour callbacks +-export([api_spec/0]). + +%% http handlers +-export([ clients/2 + , clients_insta/2 + , subscriptions/2 + ]). + +%% internal exports (for client query) +-export([ query/4 + , format_channel_info/1 + ]). + +-import(emqx_gateway_http, + [ return_http_error/2 + ]). + +%%-------------------------------------------------------------------- +%% APIs +%%-------------------------------------------------------------------- + +api_spec() -> + {metadata(apis()), []}. + +apis() -> + [ {"/gateway/:name/clients", clients} + , {"/gateway/:name/clients/:clientid", clients_insta} + , {"/gateway/:name/clients/:clientid/subscriptions", subscriptions} + , {"/gateway/:name/clients/:clientid/subscriptions/:topic", subscriptions} + ]. + +-define(CLIENT_QS_SCHEMA, + [ {<<"node">>, atom} + , {<<"clientid">>, binary} + , {<<"username">>, binary} + , {<<"ip_address">>, ip} + , {<<"conn_state">>, atom} + , {<<"clean_start">>, atom} + , {<<"proto_ver">>, integer} + , {<<"like_clientid">>, binary} + , {<<"like_username">>, binary} + , {<<"gte_created_at">>, timestamp} + , {<<"lte_created_at">>, timestamp} + , {<<"gte_connected_at">>, timestamp} + , {<<"lte_connected_at">>, timestamp} + ]). + +-define(query_fun, {?MODULE, query}). +-define(format_fun, {?MODULE, format_channel_info}). + +clients(get, #{ bindings := #{name := GwName0} + , query_string := Qs + }) -> + GwName = binary_to_existing_atom(GwName0), + TabName = emqx_gateway_cm:tabname(info, GwName), + case maps:get(<<"node">>, Qs, undefined) of + undefined -> + Response = emqx_mgmt_api:cluster_query( + Qs, TabName, + ?CLIENT_QS_SCHEMA, ?query_fun + ), + {200, Response}; + Node1 -> + Node = binary_to_atom(Node1, utf8), + ParamsWithoutNode = maps:without([<<"node">>], Qs), + Response = emqx_mgmt_api:node_query( + Node, ParamsWithoutNode, + TabName, ?CLIENT_QS_SCHEMA, ?query_fun + ), + {200, Response} + end. + +clients_insta(get, #{ bindings := #{name := GwName0, + clientid := ClientId0} + }) -> + GwName = binary_to_existing_atom(GwName0), + ClientId = emqx_mgmt_util:urldecode(ClientId0), + + case emqx_gateway_http:lookup_client(GwName, ClientId, + {?MODULE, format_channel_info}) of + [ClientInfo] -> + {200, ClientInfo}; + [ClientInfo|_More] -> + ?LOG(warning, "More than one client info was returned on ~s", + [ClientId]), + {200, ClientInfo}; + [] -> + return_http_error(404, <<"Gateway or ClientId not found">>) + + end; + +clients_insta(delete, #{ bindings := #{name := GwName0, + clientid := ClientId0} + }) -> + GwName = binary_to_existing_atom(GwName0), + ClientId = emqx_mgmt_util:urldecode(ClientId0), + _ = emqx_gateway_http:kickout_client(GwName, ClientId), + {200}. + +%% FIXME: +%% List the subscription without mountpoint, but has SubOpts, +%% for example, share group ... +subscriptions(get, #{ bindings := #{name := GwName0, + clientid := ClientId0} + }) -> + GwName = binary_to_existing_atom(GwName0), + ClientId = emqx_mgmt_util:urldecode(ClientId0), + case emqx_gateway_http:list_client_subscriptions(GwName, ClientId) of + {error, Reason} -> + return_http_error(404, Reason); + {ok, Subs} -> + {200, Subs} + end; + +%% Create the subscription without mountpoint +subscriptions(post, #{ bindings := #{name := GwName0, + clientid := ClientId0}, + body := Body + }) -> + GwName = binary_to_existing_atom(GwName0), + ClientId = emqx_mgmt_util:urldecode(ClientId0), + + case {maps:get(<<"topic">>, Body, undefined), subopts(Body)} of + {undefined, _} -> + %% FIXME: more reasonable error code?? + return_http_error(404, <<"Request paramter missed: topic">>); + {Topic, QoS} -> + case emqx_gateway_http:client_subscribe(GwName, ClientId, Topic, QoS) of + {error, Reason} -> + return_http_error(404, Reason); + ok -> + {200} + end + end; + +%% Remove the subscription without mountpoint +subscriptions(delete, #{ bindings := #{name := GwName0, + clientid := ClientId0, + topic := Topic0 + } + }) -> + GwName = binary_to_existing_atom(GwName0), + ClientId = emqx_mgmt_util:urldecode(ClientId0), + Topic = emqx_mgmt_util:urldecode(Topic0), + _ = emqx_gateway_http:client_unsubscribe(GwName, ClientId, Topic), + {200}. + +%%-------------------------------------------------------------------- +%% Utils + +subopts(Req) -> + #{ qos => maps:get(<<"qos">>, Req, 0) + , rap => maps:get(<<"rap">>, Req, 0) + , nl => maps:get(<<"nl">>, Req, 0) + , rh => maps:get(<<"rh">>, Req, 0) + , sub_props => extra_sub_props(maps:get(<<"sub_props">>, Req, #{})) + }. + +extra_sub_props(Props) -> + maps:filter( + fun(_, V) -> V =/= undefined end, + #{subid => maps:get(<<"subid">>, Props, undefined)} + ). + +%%-------------------------------------------------------------------- +%% query funcs + +query(Tab, {Qs, []}, Start, Limit) -> + Ms = qs2ms(Qs), + emqx_mgmt_api:select_table(Tab, Ms, Start, Limit, + fun format_channel_info/1); + +query(Tab, {Qs, Fuzzy}, Start, Limit) -> + Ms = qs2ms(Qs), + MatchFun = match_fun(Ms, Fuzzy), + emqx_mgmt_api:traverse_table(Tab, MatchFun, Start, Limit, + fun format_channel_info/1). + +qs2ms(Qs) -> + {MtchHead, Conds} = qs2ms(Qs, 2, {#{}, []}), + [{{'$1', MtchHead, '_'}, Conds, ['$_']}]. + +qs2ms([], _, {MtchHead, Conds}) -> + {MtchHead, lists:reverse(Conds)}; + +qs2ms([{Key, '=:=', Value} | Rest], N, {MtchHead, Conds}) -> + NMtchHead = emqx_mgmt_util:merge_maps(MtchHead, ms(Key, Value)), + qs2ms(Rest, N, {NMtchHead, Conds}); +qs2ms([Qs | Rest], N, {MtchHead, Conds}) -> + Holder = binary_to_atom(iolist_to_binary(["$", integer_to_list(N)]), utf8), + NMtchHead = emqx_mgmt_util:merge_maps(MtchHead, ms(element(1, Qs), Holder)), + NConds = put_conds(Qs, Holder, Conds), + qs2ms(Rest, N+1, {NMtchHead, NConds}). + +put_conds({_, Op, V}, Holder, Conds) -> + [{Op, Holder, V} | Conds]; +put_conds({_, Op1, V1, Op2, V2}, Holder, Conds) -> + [{Op2, Holder, V2}, + {Op1, Holder, V1} | Conds]. + +ms(clientid, X) -> + #{clientinfo => #{clientid => X}}; +ms(username, X) -> + #{clientinfo => #{username => X}}; +ms(zone, X) -> + #{clientinfo => #{zone => X}}; +ms(ip_address, X) -> + #{clientinfo => #{peerhost => X}}; +ms(conn_state, X) -> + #{conn_state => X}; +ms(clean_start, X) -> + #{conninfo => #{clean_start => X}}; +ms(proto_ver, X) -> + #{conninfo => #{proto_ver => X}}; +ms(connected_at, X) -> + #{conninfo => #{connected_at => X}}; +ms(created_at, X) -> + #{session => #{created_at => X}}. + +%%-------------------------------------------------------------------- +%% Match funcs + +match_fun(Ms, Fuzzy) -> + MsC = ets:match_spec_compile(Ms), + REFuzzy = lists:map(fun({K, like, S}) -> + {ok, RE} = re:compile(S), + {K, like, RE} + end, Fuzzy), + fun(Rows) -> + case ets:match_spec_run(Rows, MsC) of + [] -> []; + Ls -> + lists:filter(fun(E) -> + run_fuzzy_match(E, REFuzzy) + end, Ls) + end + end. + +run_fuzzy_match(_, []) -> + true; +run_fuzzy_match(E = {_, #{clientinfo := ClientInfo}, _}, [{Key, _, RE}|Fuzzy]) -> + Val = case maps:get(Key, ClientInfo, "") of + undefined -> ""; + V -> V + end, + re:run(Val, RE, [{capture, none}]) == match andalso run_fuzzy_match(E, Fuzzy). + +%%-------------------------------------------------------------------- +%% format funcs + +format_channel_info({_, Infos, Stats}) -> + ClientInfo = maps:get(clientinfo, Infos, #{}), + ConnInfo = maps:get(conninfo, Infos, #{}), + SessInfo = maps:get(session, Infos, #{}), + FetchX = [ {node, ClientInfo, node()} + , {clientid, ClientInfo} + , {username, ClientInfo} + , {proto_name, ConnInfo} + , {proto_ver, ConnInfo} + , {ip_address, {peername, ConnInfo, fun peer_to_binary/1}} + , {is_bridge, ClientInfo, false} + , {connected_at, + {connected_at, ConnInfo, fun emqx_gateway_utils:unix_ts_to_rfc3339/1}} + , {disconnected_at, + {disconnected_at, ConnInfo, fun emqx_gateway_utils:unix_ts_to_rfc3339/1}} + , {connected, {conn_state, Infos, fun conn_state_to_connected/1}} + , {keepalive, ClientInfo, 0} + , {clean_start, ConnInfo, true} + , {expiry_interval, ConnInfo, 0} + , {created_at, + {created_at, SessInfo, fun emqx_gateway_utils:unix_ts_to_rfc3339/1}} + , {subscriptions_cnt, Stats, 0} + , {subscriptions_max, Stats, infinity} + , {inflight_cnt, Stats, 0} + , {inflight_max, Stats, infinity} + , {mqueue_len, Stats, 0} + , {mqueue_max, Stats, infinity} + , {mqueue_dropped, Stats, 0} + , {awaiting_rel_cnt, Stats, 0} + , {awaiting_rel_max, Stats, infinity} + , {recv_oct, Stats, 0} + , {recv_cnt, Stats, 0} + , {recv_pkt, Stats, 0} + , {recv_msg, Stats, 0} + , {send_oct, Stats, 0} + , {send_cnt, Stats, 0} + , {send_pkt, Stats, 0} + , {send_msg, Stats, 0} + , {mailbox_len, Stats, 0} + , {heap_size, Stats, 0} + , {reductions, Stats, 0} + ], + eval(FetchX). + +eval(Ls) -> + eval(Ls, #{}). +eval([], AccMap) -> + AccMap; +eval([{K, Vx}|More], AccMap) -> + case valuex_get(K, Vx) of + undefined -> eval(More, AccMap#{K => null}); + Value -> eval(More, AccMap#{K => Value}) + end; +eval([{K, Vx, Default}|More], AccMap) -> + case valuex_get(K, Vx) of + undefined -> eval(More, AccMap#{K => Default}); + Value -> eval(More, AccMap#{K => Value}) + end. + +valuex_get(K, Vx) when is_map(Vx); is_list(Vx) -> + key_get(K, Vx); +valuex_get(_K, {InKey, Obj}) when is_map(Obj); is_list(Obj) -> + key_get(InKey, Obj); +valuex_get(_K, {InKey, Obj, MappingFun}) when is_map(Obj); is_list(Obj) -> + case key_get(InKey, Obj) of + undefined -> undefined; + Val -> MappingFun(Val) + end. + +key_get(K, M) when is_map(M) -> + maps:get(K, M, undefined); +key_get(K, L) when is_list(L) -> + proplists:get_value(K, L). + +peer_to_binary({Addr, Port}) -> + AddrBinary = list_to_binary(inet:ntoa(Addr)), + PortBinary = integer_to_binary(Port), + <>; +peer_to_binary(Addr) -> + list_to_binary(inet:ntoa(Addr)). + +conn_state_to_connected(connected) -> true; +conn_state_to_connected(_) -> false. + +%%-------------------------------------------------------------------- +%% Swagger defines +%%-------------------------------------------------------------------- + +metadata(APIs) -> + metadata(APIs, []). +metadata([], APIAcc) -> + lists:reverse(APIAcc); +metadata([{Path, Fun}|More], APIAcc) -> + Methods = [get, post, put, delete, patch], + Mds = lists:foldl(fun(M, Acc) -> + try + Acc#{M => swagger(Path, M)} + catch + error : function_clause -> + Acc + end + end, #{}, Methods), + metadata(More, [{Path, Mds, Fun} | APIAcc]). + +swagger("/gateway/:name/clients", get) -> + #{ description => <<"Get the gateway clients">> + , parameters => params_client_query() + , responses => + #{ <<"404">> => schema_not_found() + , <<"200">> => schema_clients_list() + } + }; +swagger("/gateway/:name/clients/:clientid", get) -> + #{ description => <<"Get the gateway client infomation">> + , parameters => params_client_insta() + , responses => + #{ <<"404">> => schema_not_found() + , <<"200">> => schema_client() + } + }; +swagger("/gateway/:name/clients/:clientid", delete) -> + #{ description => <<"Kick out the gateway client">> + , parameters => params_client_insta() + , responses => + #{ <<"404">> => schema_not_found() + , <<"204">> => schema_no_content() + } + }; +swagger("/gateway/:name/clients/:clientid/subscriptions", get) -> + #{ description => <<"Get the gateway client subscriptions">> + , parameters => params_client_insta() + , responses => + #{ <<"404">> => schema_not_found() + , <<"200">> => schema_subscription_list() + } + }; +swagger("/gateway/:name/clients/:clientid/subscriptions", post) -> + #{ description => <<"Get the gateway client subscriptions">> + , parameters => params_client_insta() + , requestBody => schema_subscription() + , responses => + #{ <<"404">> => schema_not_found() + , <<"200">> => schema_no_content() + } + }; +swagger("/gateway/:name/clients/:clientid/subscriptions/:topic", delete) -> + #{ description => <<"Unsubscribe the topic for client">> + , parameters => params_topic_name_in_path() ++ params_client_insta() + , responses => + #{ <<"404">> => schema_not_found() + , <<"204">> => schema_no_content() + } + }. + +params_client_query() -> + params_gateway_name_in_path() + ++ params_client_searching_in_qs() + ++ emqx_mgmt_util:page_params(). + +params_client_insta() -> + params_clientid_in_path() + ++ params_gateway_name_in_path(). + +params_client_searching_in_qs() -> + queries( + [ {node, string} + , {clientid, string} + , {username, string} + , {ip_address, string} + , {conn_state, string} + , {proto_ver, string} + , {clean_start, boolean} + , {like_clientid, string} + , {like_username, string} + , {gte_created_at, string} + , {lte_created_at, string} + , {gte_connected_at, string} + , {lte_connected_at, string} + ]). + +params_gateway_name_in_path() -> + [#{ name => name + , in => path + , schema => #{type => string} + , required => true + }]. + +params_clientid_in_path() -> + [#{ name => clientid + , in => path + , schema => #{type => string} + , required => true + }]. + +params_topic_name_in_path() -> + [#{ name => topic + , in => path + , schema => #{type => string} + , required => true + }]. + +queries(Ls) -> + lists:map(fun({K, Type}) -> + #{name => K, in => query, + schema => #{type => Type}, + required => false + } + end, Ls). + +%%-------------------------------------------------------------------- +%% schemas + +schema_not_found() -> + emqx_mgmt_util:error_schema(<<"Gateway not found or unloaded">>). + +schema_no_content() -> + #{description => <<"No Content">>}. + +schema_clients_list() -> + emqx_mgmt_util:page_schema( + #{ type => object + , properties => properties_client() + } + ). + +schema_client() -> + emqx_mgmt_util:schema( + #{ type => object + , properties => properties_client() + }). + +schema_subscription_list() -> + emqx_mgmt_util:array_schema( + #{ type => object + , properties => properties_subscription() + }, + <<"Client subscriptions">> + ). + +schema_subscription() -> + emqx_mgmt_util:schema( + #{ type => object + , properties => properties_subscription() + } + ). + +%%-------------------------------------------------------------------- +%% properties defines + +properties_client() -> + emqx_mgmt_util:properties( + [ {node, string, + <<"Name of the node to which the client is connected">>} + , {clientid, string, + <<"Client identifier">>} + , {username, string, + <<"Username of client when connecting">>} + , {proto_name, string, + <<"Client protocol name">>} + , {proto_ver, string, + <<"Protocol version used by the client">>} + , {ip_address, string, + <<"Client's IP address">>} + , {is_bridge, boolean, + <<"Indicates whether the client is connectedvia bridge">>} + , {connected_at, string, + <<"Client connection time">>} + , {disconnected_at, string, + <<"Client offline time, This field is only valid and returned " + "when connected is false">>} + , {connected, boolean, + <<"Whether the client is connected">>} + %% FIXME: the will_msg attribute is not a general attribute + %% for every protocol. But it should be returned to frontend if someone + %% want it + %% + %, {will_msg, string, + % <<"Client will message">>} + %, {zone, string, + % <<"Indicate the configuration group used by the client">>} + , {keepalive, integer, + <<"keepalive time, with the unit of second">>} + , {clean_start, boolean, + <<"Indicate whether the client is using a brand new session">>} + , {expiry_interval, integer, + <<"Session expiration interval, with the unit of second">>} + , {created_at, string, + <<"Session creation time">>} + , {subscriptions_cnt, integer, + <<"Number of subscriptions established by this client">>} + , {subscriptions_max, integer, + <<"v4 api name [max_subscriptions] Maximum number of " + "subscriptions allowed by this client">>} + , {inflight_cnt, integer, + <<"Current length of inflight">>} + , {inflight_max, integer, + <<"v4 api name [max_inflight]. Maximum length of inflight">>} + , {mqueue_len, integer, + <<"Current length of message queue">>} + , {mqueue_max, integer, + <<"v4 api name [max_mqueue]. Maximum length of message queue">>} + , {mqueue_dropped, integer, + <<"Number of messages dropped by the message queue due to " + "exceeding the length">>} + , {awaiting_rel_cnt, integer, + <<"v4 api name [awaiting_rel] Number of awaiting PUBREC packet">>} + , {awaiting_rel_max, integer, + <<"v4 api name [max_awaiting_rel]. Maximum allowed number of " + "awaiting PUBREC packet">>} + , {recv_oct, integer, + <<"Number of bytes received by EMQ X Broker (the same below)">>} + , {recv_cnt, integer, + <<"Number of TCP packets received">>} + , {recv_pkt, integer, + <<"Number of MQTT packets received">>} + , {recv_msg, integer, + <<"Number of PUBLISH packets received">>} + , {send_oct, integer, + <<"Number of bytes sent">>} + , {send_cnt, integer, + <<"Number of TCP packets sent">>} + , {send_pkt, integer, + <<"Number of MQTT packets sent">>} + , {send_msg, integer, + <<"Number of PUBLISH packets sent">>} + , {mailbox_len, integer, + <<"Process mailbox size">>} + , {heap_size, integer, + <<"Process heap size with the unit of byte">>} + , {reductions, integer, + <<"Erlang reduction">>} + ]). + +properties_subscription() -> + ExtraProps = [ {subid, string, + <<"Only stomp protocol, an uniquely identity for " + "the subscription. range: 1-65535.">>} + ], + emqx_mgmt_util:properties( + [ {topic, string, + <<"Topic Fillter">>} + , {qos, integer, + <<"QoS level, enum: 0, 1, 2">>} + , {nl, integer, %% FIXME: why not boolean? + <<"No Local option, enum: 0, 1">>} + , {rap, integer, + <<"Retain as Published option, enum: 0, 1">>} + , {rh, integer, + <<"Retain Handling option, enum: 0, 1, 2">>} + , {sub_props, object, ExtraProps} + ]). diff --git a/apps/emqx_gateway/src/emqx_gateway_app.erl b/apps/emqx_gateway/src/emqx_gateway_app.erl index 1ecd9cf26..d90942220 100644 --- a/apps/emqx_gateway/src/emqx_gateway_app.erl +++ b/apps/emqx_gateway/src/emqx_gateway_app.erl @@ -83,4 +83,4 @@ load_gateway_by_default([{Type, Confs}|More]) -> load_gateway_by_default(More). confs() -> - maps:to_list(emqx:get_config([gateway], [])). + maps:to_list(emqx:get_config([gateway], #{})). diff --git a/apps/emqx_gateway/src/emqx_gateway_cli.erl b/apps/emqx_gateway/src/emqx_gateway_cli.erl index b446cda92..6ccb444f0 100644 --- a/apps/emqx_gateway/src/emqx_gateway_cli.erl +++ b/apps/emqx_gateway/src/emqx_gateway_cli.erl @@ -51,7 +51,7 @@ is_cmd(Fun) -> gateway(["list"]) -> lists:foreach(fun(#{name := Name} = Gateway) -> - %% XXX: More infos: listeners?, connected? + %% TODO: More infos: listeners?, connected? Status = maps:get(status, Gateway, stopped), emqx_ctl:print("Gateway(name=~s, status=~s)~n", [Name, Status]) @@ -106,6 +106,7 @@ gateway(_) -> ]). 'gateway-clients'(["list", Name]) -> + %% FIXME: page me. for example: --limit 100 --page 10 ??? InfoTab = emqx_gateway_cm:tabname(info, Name), case ets:info(InfoTab) of undefined -> diff --git a/apps/emqx_gateway/src/emqx_gateway_cm.erl b/apps/emqx_gateway/src/emqx_gateway_cm.erl index 7a7ad055d..d8b615fe8 100644 --- a/apps/emqx_gateway/src/emqx_gateway_cm.erl +++ b/apps/emqx_gateway/src/emqx_gateway_cm.erl @@ -48,6 +48,10 @@ , connection_closed/2 ]). +-export([ with_channel/3 + , lookup_channels/2 + ]). + %% Internal funcs for getting tabname by GatewayId -export([cmtabs/1, tabname/2]). diff --git a/apps/emqx_gateway/src/emqx_gateway_http.erl b/apps/emqx_gateway/src/emqx_gateway_http.erl new file mode 100644 index 000000000..f233a6151 --- /dev/null +++ b/apps/emqx_gateway/src/emqx_gateway_http.erl @@ -0,0 +1,270 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +%% @doc Gateway Interface Module for HTTP-APIs +-module(emqx_gateway_http). + +-include("include/emqx_gateway.hrl"). +-include_lib("emqx/include/logger.hrl"). + +%% Mgmt APIs - gateway +-export([ gateways/1 + ]). + +%% Mgmt APIs - listeners +-export([ listeners/1 + , listener/2 + , mapping_listener_m2l/2 + ]). + +%% Mgmt APIs - clients +-export([ lookup_client/3 + , lookup_client/4 + , kickout_client/2 + , kickout_client/3 + , list_client_subscriptions/2 + , client_subscribe/4 + , client_unsubscribe/3 + ]). + +%% Utils for http, swagger, etc. +-export([ return_http_error/2 + ]). + +-type gateway_summary() :: + #{ name := binary() + , status := running | stopped | unloaded + , started_at => binary() + , max_connections => integer() + , current_connections => integer() + , listeners => [] + }. + +-define(DEFAULT_CALL_TIMEOUT, 15000). + +%%-------------------------------------------------------------------- +%% Mgmt APIs - gateway +%%-------------------------------------------------------------------- + +-spec gateways(Status :: all | running | stopped | unloaded) + -> [gateway_summary()]. +gateways(Status) -> + Gateways = lists:map(fun({GwName, _}) -> + case emqx_gateway:lookup(GwName) of + undefined -> #{name => GwName, status => unloaded}; + GwInfo = #{config := Config} -> + GwInfo0 = emqx_gateway_utils:unix_ts_to_rfc3339( + [created_at, started_at, stopped_at], + GwInfo), + GwInfo1 = maps:with([name, + status, + created_at, + started_at, + stopped_at], GwInfo0), + GwInfo1#{ + max_connections => max_connections_count(Config), + current_connections => current_connections_count(GwName), + listeners => get_listeners_status(GwName, Config)} + end + end, emqx_gateway_registry:list()), + case Status of + all -> Gateways; + _ -> + [Gw || Gw = #{status := S} <- Gateways, S == Status] + end. + +%% @private +max_connections_count(Config) -> + Listeners = emqx_gateway_utils:normalize_config(Config), + lists:foldl(fun({_, _, _, SocketOpts, _}, Acc) -> + Acc + proplists:get_value(max_connections, SocketOpts, 0) + end, 0, Listeners). + +%% @private +current_connections_count(GwName) -> + try + InfoTab = emqx_gateway_cm:tabname(info, GwName), + ets:info(InfoTab, size) + catch _ : _ -> + 0 + end. + +%% @private +get_listeners_status(GwName, Config) -> + Listeners = emqx_gateway_utils:normalize_config(Config), + lists:map(fun({Type, LisName, ListenOn, _, _}) -> + Name0 = emqx_gateway_utils:listener_id(GwName, Type, LisName), + Name = {Name0, ListenOn}, + LisO = #{id => Name0, type => Type}, + case catch esockd:listener(Name) of + _Pid when is_pid(_Pid) -> + LisO#{running => true}; + _ -> + LisO#{running => false} + end + end, Listeners). + +%%-------------------------------------------------------------------- +%% Mgmt APIs - listeners +%%-------------------------------------------------------------------- + +listeners(GwName) when is_atom (GwName) -> + listeners(atom_to_binary(GwName)); +listeners(GwName) -> + RawConf = emqx_config:fill_defaults( + emqx_config:get_root_raw([<<"gateway">>])), + Listeners = emqx_map_lib:jsonable_map( + emqx_map_lib:deep_get( + [<<"gateway">>, GwName, <<"listeners">>], RawConf)), + mapping_listener_m2l(GwName, Listeners). + +listener(_GwName, _ListenerId) -> + ok. + +mapping_listener_m2l(GwName, Listeners0) -> + Listeners = maps:to_list(Listeners0), + lists:append([listener(GwName, Type, maps:to_list(Conf)) + || {Type, Conf} <- Listeners]). + +listener(GwName, Type, Conf) -> + [begin + ListenerId = emqx_gateway_utils:listener_id(GwName, Type, LName), + Running = is_running(ListenerId, LConf), + LConf#{ + id => ListenerId, + type => Type, + running => Running + } + end || {LName, LConf} <- Conf, is_map(LConf)]. + +is_running(ListenerId, #{<<"bind">> := ListenOn0}) -> + ListenOn = emqx_gateway_utils:parse_listenon(ListenOn0), + try esockd:listener({ListenerId, ListenOn}) of + Pid when is_pid(Pid)-> + true + catch _:_ -> + false + end. + +%%-------------------------------------------------------------------- +%% Mgmt APIs - clients +%%-------------------------------------------------------------------- + +-spec lookup_client(gateway_name(), + emqx_type:clientid(), {atom(), atom()}) -> list(). +lookup_client(GwName, ClientId, FormatFun) -> + lists:append([lookup_client(Node, GwName, {clientid, ClientId}, FormatFun) + || Node <- ekka_mnesia:running_nodes()]). + +lookup_client(Node, GwName, {clientid, ClientId}, {M,F}) when Node =:= node() -> + ChanTab = emqx_gateway_cm:tabname(chan, GwName), + InfoTab = emqx_gateway_cm:tabname(info, GwName), + + lists:append(lists:map( + fun(Key) -> + lists:map(fun M:F/1, ets:lookup(InfoTab, Key)) + end, ets:lookup(ChanTab, ClientId))); + +lookup_client(Node, GwName, {clientid, ClientId}, FormatFun) -> + rpc_call(Node, lookup_client, + [Node, GwName, {clientid, ClientId}, FormatFun]). + +-spec kickout_client(gateway_name(), emqx_type:clientid()) + -> {error, any()} + | ok. +kickout_client(GwName, ClientId) -> + Results = [kickout_client(Node, GwName, ClientId) + || Node <- ekka_mnesia:running_nodes()], + case lists:any(fun(Item) -> Item =:= ok end, Results) of + true -> ok; + false -> lists:last(Results) + end. + +kickout_client(Node, GwName, ClientId) when Node =:= node() -> + emqx_gateway_cm:kick_session(GwName, ClientId); + +kickout_client(Node, GwName, ClientId) -> + rpc_call(Node, kickout_client, [Node, GwName, ClientId]). + +-spec list_client_subscriptions(gateway_name(), emqx_type:clientid()) + -> {error, any()} + | {ok, list()}. +list_client_subscriptions(GwName, ClientId) -> + %% Get the subscriptions from session-info + with_channel(GwName, ClientId, + fun(Pid) -> + Subs = emqx_gateway_conn:call( + Pid, + subscriptions, ?DEFAULT_CALL_TIMEOUT), + {ok, lists:map(fun({Topic, SubOpts}) -> + SubOpts#{topic => Topic} + end, Subs)} + end). + +-spec client_subscribe(gateway_name(), emqx_type:clientid(), + emqx_type:topic(), emqx_type:subopts()) + -> {error, any()} + | ok. +client_subscribe(GwName, ClientId, Topic, SubOpts) -> + with_channel(GwName, ClientId, + fun(Pid) -> + emqx_gateway_conn:call( + Pid, {subscribe, Topic, SubOpts}, + ?DEFAULT_CALL_TIMEOUT + ) + end). + +-spec client_unsubscribe(gateway_name(), + emqx_type:clientid(), emqx_type:topic()) + -> {error, any()} + | ok. +client_unsubscribe(GwName, ClientId, Topic) -> + with_channel(GwName, ClientId, + fun(Pid) -> + emqx_gateway_conn:call( + Pid, {unsubscribe, Topic}, ?DEFAULT_CALL_TIMEOUT) + end). + +with_channel(GwName, ClientId, Fun) -> + case emqx_gateway_cm:with_channel(GwName, ClientId, Fun) of + undefined -> {error, not_found}; + Res -> Res + end. + +%%-------------------------------------------------------------------- +%% Utils +%%-------------------------------------------------------------------- + +-spec return_http_error(integer(), binary()) -> {integer(), binary()}. +return_http_error(Code, Msg) -> + {Code, emqx_json:encode( + #{code => codestr(Code), + reason => emqx_gateway_utils:stringfy(Msg) + }) + }. + +codestr(404) -> 'RESOURCE_NOT_FOUND'; +codestr(401) -> 'NOT_SUPPORTED_NOW'; +codestr(500) -> 'UNKNOW_ERROR'. + +%%-------------------------------------------------------------------- +%% Internal funcs + +rpc_call(Node, Fun, Args) -> + case rpc:call(Node, ?MODULE, Fun, Args) of + {badrpc, Reason} -> {error, Reason}; + Res -> Res + end. diff --git a/apps/emqx_gateway/src/emqx_gateway_intr.erl b/apps/emqx_gateway/src/emqx_gateway_intr.erl deleted file mode 100644 index add37e1c5..000000000 --- a/apps/emqx_gateway/src/emqx_gateway_intr.erl +++ /dev/null @@ -1,78 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - -%% @doc Gateway Interface Module for HTTP-APIs --module(emqx_gateway_intr). - --export([ gateways/1 - ]). - --type gateway_summary() :: - #{ name := binary() - , status := running | stopped | unloaded - , started_at => binary() - , max_connection => integer() - , current_connect => integer() - , listeners => [] - }. - -%%-------------------------------------------------------------------- -%% APIs -%%-------------------------------------------------------------------- - --spec gateways(Status :: all | running | stopped | unloaded) - -> [gateway_summary()]. -gateways(Status) -> - Gateways = lists:map(fun({GwName, _}) -> - case emqx_gateway:lookup(GwName) of - undefined -> #{name => GwName, status => unloaded}; - GwInfo = #{config := Config} -> - GwInfo0 = emqx_gateway_utils:unix_ts_to_rfc3339( - [created_at, started_at, stopped_at], - GwInfo), - GwInfo1 = maps:with([name, - status, - created_at, - started_at, - stopped_at], GwInfo0), - GwInfo1#{listeners => get_listeners_status(GwName, Config)} - - end - end, emqx_gateway_registry:list()), - case Status of - all -> Gateways; - _ -> - [Gw || Gw = #{status := S} <- Gateways, S == Status] - end. - -%% @private -get_listeners_status(GwName, Config) -> - Listeners = emqx_gateway_utils:normalize_config(Config), - lists:map(fun({Type, LisName, ListenOn, _, _}) -> - Name0 = listener_name(GwName, Type, LisName), - Name = {Name0, ListenOn}, - case catch esockd:listener(Name) of - _Pid when is_pid(_Pid) -> - #{Name0 => <<"activing">>}; - _ -> - #{Name0 => <<"inactived">>} - - end - end, Listeners). - -%% @private -listener_name(GwName, Type, LisName) -> - list_to_atom(lists:concat([GwName, ":", Type, ":", LisName])). diff --git a/apps/emqx_gateway/src/emqx_gateway_metrics.erl b/apps/emqx_gateway/src/emqx_gateway_metrics.erl index 458017118..77b97a6a1 100644 --- a/apps/emqx_gateway/src/emqx_gateway_metrics.erl +++ b/apps/emqx_gateway/src/emqx_gateway_metrics.erl @@ -18,7 +18,7 @@ -behaviour(gen_server). --include("include/emqx_gateway.hrl"). +-include_lib("emqx_gateway/include/emqx_gateway.hrl"). %% APIs diff --git a/apps/emqx_gateway/src/emqx_gateway_schema.erl b/apps/emqx_gateway/src/emqx_gateway_schema.erl index 9371f8c6b..7fb945ba0 100644 --- a/apps/emqx_gateway/src/emqx_gateway_schema.erl +++ b/apps/emqx_gateway/src/emqx_gateway_schema.erl @@ -43,256 +43,283 @@ , ip_port/0 ]). --export([roots/0 , fields/1]). --export([t/1, t/3, t/4, ref/1]). +-export([namespace/0, roots/0 , fields/1]). + +namespace() -> gateway. roots() -> [gateway]. fields(gateway) -> - [{stomp, t(ref(stomp_structs))}, - {mqttsn, t(ref(mqttsn_structs))}, - {coap, t(ref(coap_structs))}, - {lwm2m, t(ref(lwm2m_structs))}, - {exproto, t(ref(exproto_structs))} + [{stomp, sc(ref(stomp_structs))}, + {mqttsn, sc(ref(mqttsn_structs))}, + {coap, sc(ref(coap_structs))}, + {lwm2m, sc(ref(lwm2m_structs))}, + {exproto, sc(ref(exproto_structs))} ]; fields(stomp_structs) -> - [ {frame, t(ref(stomp_frame))} - , {listeners, t(ref(tcp_listener_group))} + [ {frame, sc(ref(stomp_frame))} + , {listeners, sc(ref(tcp_listener_group))} ] ++ gateway_common_options(); fields(stomp_frame) -> - [ {max_headers, t(integer(), undefined, 10)} - , {max_headers_length, t(integer(), undefined, 1024)} - , {max_body_length, t(integer(), undefined, 8192)} + [ {max_headers, sc(integer(), 10)} + , {max_headers_length, sc(integer(), 1024)} + , {max_body_length, sc(integer(), 8192)} ]; fields(mqttsn_structs) -> - [ {gateway_id, t(integer())} - , {broadcast, t(boolean())} - , {enable_qos3, t(boolean())} + [ {gateway_id, sc(integer())} + , {broadcast, sc(boolean())} + , {enable_qos3, sc(boolean())} , {predefined, hoconsc:array(ref(mqttsn_predefined))} - , {listeners, t(ref(udp_listener_group))} + , {listeners, sc(ref(udp_listener_group))} ] ++ gateway_common_options(); fields(mqttsn_predefined) -> - [ {id, t(integer())} - , {topic, t(binary())} + [ {id, sc(integer())} + , {topic, sc(binary())} ]; fields(coap_structs) -> - [ {heartbeat, t(duration(), undefined, <<"30s">>)} - , {notify_type, t(union([non, con, qos]), undefined, qos)} - , {subscribe_qos, t(union([qos0, qos1, qos2, coap]), undefined, coap)} - , {publish_qos, t(union([qos0, qos1, qos2, coap]), undefined, coap)} - , {listeners, t(ref(udp_listener_group))} + [ {heartbeat, sc(duration(), <<"30s">>)} + , {connection_required, sc(boolean(), false)} + , {notify_type, sc(union([non, con, qos]), qos)} + , {subscribe_qos, sc(union([qos0, qos1, qos2, coap]), coap)} + , {publish_qos, sc(union([qos0, qos1, qos2, coap]), coap)} + , {listeners, sc(ref(udp_listener_group))} ] ++ gateway_common_options(); fields(lwm2m_structs) -> - [ {xml_dir, t(binary())} - , {lifetime_min, t(duration())} - , {lifetime_max, t(duration())} - , {qmode_time_windonw, t(integer())} - , {auto_observe, t(boolean())} - , {update_msg_publish_condition, t(union([always, contains_object_list]))} - , {translators, t(ref(translators))} - , {listeners, t(ref(udp_listener_group))} + [ {xml_dir, sc(binary())} + , {lifetime_min, sc(duration())} + , {lifetime_max, sc(duration())} + , {qmode_time_windonw, sc(integer())} + , {auto_observe, sc(boolean())} + , {update_msg_publish_condition, sc(union([always, contains_object_list]))} + , {translators, sc(ref(translators))} + , {listeners, sc(ref(udp_listener_group))} ] ++ gateway_common_options(); fields(exproto_structs) -> - [ {server, t(ref(exproto_grpc_server))} - , {handler, t(ref(exproto_grpc_handler))} - , {listeners, t(ref(udp_tcp_listener_group))} + [ {server, sc(ref(exproto_grpc_server))} + , {handler, sc(ref(exproto_grpc_handler))} + , {listeners, sc(ref(udp_tcp_listener_group))} ] ++ gateway_common_options(); fields(exproto_grpc_server) -> - [ {bind, t(union(ip_port(), integer()))} + [ {bind, sc(union(ip_port(), integer()))} %% TODO: ssl options ]; fields(exproto_grpc_handler) -> - [ {address, t(binary())} + [ {address, sc(binary())} %% TODO: ssl ]; fields(clientinfo_override) -> - [ {username, t(binary())} - , {password, t(binary())} - , {clientid, t(binary())} + [ {username, sc(binary())} + , {password, sc(binary())} + , {clientid, sc(binary())} ]; fields(translators) -> - [{"$name", t(binary())}]; + [ {command, sc(ref(translator))} + , {response, sc(ref(translator))} + , {notify, sc(ref(translator))} + , {register, sc(ref(translator))} + , {update, sc(ref(translator))} + ]; + +fields(translator) -> + [ {topic, sc(binary())} + , {qos, sc(range(0, 2))} + ]; fields(udp_listener_group) -> - [ {udp, t(ref(udp_listener))} - , {dtls, t(ref(dtls_listener))} + [ {udp, sc(ref(udp_listener))} + , {dtls, sc(ref(dtls_listener))} ]; fields(tcp_listener_group) -> - [ {tcp, t(ref(tcp_listener))} - , {ssl, t(ref(ssl_listener))} + [ {tcp, sc(ref(tcp_listener))} + , {ssl, sc(ref(ssl_listener))} ]; fields(udp_tcp_listener_group) -> - [ {udp, t(ref(udp_listener))} - , {dtls, t(ref(dtls_listener))} - , {tcp, t(ref(tcp_listener))} - , {ssl, t(ref(ssl_listener))} + [ {udp, sc(ref(udp_listener))} + , {dtls, sc(ref(dtls_listener))} + , {tcp, sc(ref(tcp_listener))} + , {ssl, sc(ref(ssl_listener))} ]; fields(tcp_listener) -> - [ {"$name", t(ref(tcp_listener_settings))}]; + [ {"$name", sc(ref(tcp_listener_settings))}]; fields(ssl_listener) -> - [ {"$name", t(ref(ssl_listener_settings))}]; + [ {"$name", sc(ref(ssl_listener_settings))}]; fields(udp_listener) -> - [ {"$name", t(ref(udp_listener_settings))}]; + [ {"$name", sc(ref(udp_listener_settings))}]; fields(dtls_listener) -> - [ {"$name", t(ref(dtls_listener_settings))}]; - -fields(listener_settings) -> - [ {enable, t(boolean(), undefined, true)} - , {bind, t(union(ip_port(), integer()))} - , {acceptors, t(integer(), undefined, 8)} - , {max_connections, t(integer(), undefined, 1024)} - , {max_conn_rate, t(integer())} - , {active_n, t(integer(), undefined, 100)} - %, {rate_limit, t(comma_separated_list())} - , {access, t(ref(access))} - , {proxy_protocol, t(boolean())} - , {proxy_protocol_timeout, t(duration())} - , {backlog, t(integer(), undefined, 1024)} - , {send_timeout, t(duration(), undefined, <<"15s">>)} - , {send_timeout_close, t(boolean(), undefined, true)} - , {recbuf, t(bytesize())} - , {sndbuf, t(bytesize())} - , {buffer, t(bytesize())} - , {high_watermark, t(bytesize(), undefined, <<"1MB">>)} - , {tune_buffer, t(boolean())} - , {nodelay, t(boolean())} - , {reuseaddr, t(boolean())} - ]; + [ {"$name", sc(ref(dtls_listener_settings))}]; fields(tcp_listener_settings) -> [ %% some special confs for tcp listener - ] ++ fields(listener_settings); + ] ++ tcp_opts() + ++ proxy_protocol_opts() + ++ common_listener_opts(); fields(ssl_listener_settings) -> [ - %% some special confs for ssl listener - ] ++ - ssl(undefined, #{handshake_timeout => <<"15s">> - , depth => 10 - , reuse_sessions => true}) ++ fields(listener_settings); + %% some special confs for ssl listener + ] ++ tcp_opts() + ++ ssl_opts() + ++ proxy_protocol_opts() + ++ common_listener_opts(); fields(udp_listener_settings) -> [ - %% some special confs for udp listener - ] ++ fields(listener_settings); + %% some special confs for udp listener + ] ++ udp_opts() + ++ common_listener_opts(); fields(dtls_listener_settings) -> [ - %% some special confs for dtls listener - ] ++ - ssl(undefined, #{handshake_timeout => <<"15s">> - , depth => 10 - , reuse_sessions => true}) ++ fields(listener_settings); + %% some special confs for dtls listener + ] ++ udp_opts() + ++ dtls_opts() + ++ common_listener_opts(); -fields(access) -> - [ {"$id", #{type => binary(), - nullable => true}}]; +fields(udp_opts) -> + [ {active_n, sc(integer(), 100)} + , {recbuf, sc(bytesize())} + , {sndbuf, sc(bytesize())} + , {buffer, sc(bytesize())} + , {reuseaddr, sc(boolean(), true)} + ]; + +fields(dtls_listener_ssl_opts) -> + Base = emqx_schema:fields("listener_ssl_opts"), + DtlsVers = hoconsc:mk( + typerefl:alias("string", list(atom())), + #{ default => default_dtls_vsns(), + converter => fun (Vsns) -> + [dtls_vsn(iolist_to_binary(V)) || V <- Vsns] + end + }), + Ciphers = sc(hoconsc:array(string()), default_ciphers()), + lists:keydelete( + "handshake_timeout", 1, + lists:keyreplace( + "ciphers", 1, + lists:keyreplace("versions", 1, Base, {"versions", DtlsVers}), + {"ciphers", Ciphers} + ) + ); fields(ExtraField) -> Mod = list_to_atom(ExtraField++"_schema"), Mod:fields(ExtraField). -authentication() -> - hoconsc:union( - [ undefined - , hoconsc:ref(emqx_authn_mnesia, config) - , hoconsc:ref(emqx_authn_mysql, config) - , hoconsc:ref(emqx_authn_pgsql, config) - , hoconsc:ref(emqx_authn_mongodb, standalone) - , hoconsc:ref(emqx_authn_mongodb, 'replica-set') - , hoconsc:ref(emqx_authn_mongodb, 'sharded-cluster') - , hoconsc:ref(emqx_authn_redis, standalone) - , hoconsc:ref(emqx_authn_redis, cluster) - , hoconsc:ref(emqx_authn_redis, sentinel) - , hoconsc:ref(emqx_authn_http, get) - , hoconsc:ref(emqx_authn_http, post) - , hoconsc:ref(emqx_authn_jwt, 'hmac-based') - , hoconsc:ref(emqx_authn_jwt, 'public-key') - , hoconsc:ref(emqx_authn_jwt, 'jwks') - , hoconsc:ref(emqx_enhanced_authn_scram_mnesia, config) - ]). +default_ciphers() -> + ["ECDHE-ECDSA-AES256-GCM-SHA384", + "ECDHE-RSA-AES256-GCM-SHA384", "ECDHE-ECDSA-AES256-SHA384", "ECDHE-RSA-AES256-SHA384", + "ECDHE-ECDSA-DES-CBC3-SHA", "ECDH-ECDSA-AES256-GCM-SHA384", "ECDH-RSA-AES256-GCM-SHA384", + "ECDH-ECDSA-AES256-SHA384", "ECDH-RSA-AES256-SHA384", "DHE-DSS-AES256-GCM-SHA384", + "DHE-DSS-AES256-SHA256", "AES256-GCM-SHA384", "AES256-SHA256", + "ECDHE-ECDSA-AES128-GCM-SHA256", "ECDHE-RSA-AES128-GCM-SHA256", + "ECDHE-ECDSA-AES128-SHA256", "ECDHE-RSA-AES128-SHA256", "ECDH-ECDSA-AES128-GCM-SHA256", + "ECDH-RSA-AES128-GCM-SHA256", "ECDH-ECDSA-AES128-SHA256", "ECDH-RSA-AES128-SHA256", + "DHE-DSS-AES128-GCM-SHA256", "DHE-DSS-AES128-SHA256", "AES128-GCM-SHA256", "AES128-SHA256", + "ECDHE-ECDSA-AES256-SHA", "ECDHE-RSA-AES256-SHA", "DHE-DSS-AES256-SHA", + "ECDH-ECDSA-AES256-SHA", "ECDH-RSA-AES256-SHA", "AES256-SHA", "ECDHE-ECDSA-AES128-SHA", + "ECDHE-RSA-AES128-SHA", "DHE-DSS-AES128-SHA", "ECDH-ECDSA-AES128-SHA", + "ECDH-RSA-AES128-SHA", "AES128-SHA" + ] ++ psk_ciphers(). + +psk_ciphers() -> + ["PSK-AES128-CBC-SHA", "PSK-AES256-CBC-SHA", + "PSK-3DES-EDE-CBC-SHA", "PSK-RC4-SHA" + ]. + +% authentication() -> +% hoconsc:union( +% [ undefined +% , hoconsc:ref(emqx_authn_mnesia, config) +% , hoconsc:ref(emqx_authn_mysql, config) +% , hoconsc:ref(emqx_authn_pgsql, config) +% , hoconsc:ref(emqx_authn_mongodb, standalone) +% , hoconsc:ref(emqx_authn_mongodb, 'replica-set') +% , hoconsc:ref(emqx_authn_mongodb, 'sharded-cluster') +% , hoconsc:ref(emqx_authn_redis, standalone) +% , hoconsc:ref(emqx_authn_redis, cluster) +% , hoconsc:ref(emqx_authn_redis, sentinel) +% , hoconsc:ref(emqx_authn_http, get) +% , hoconsc:ref(emqx_authn_http, post) +% , hoconsc:ref(emqx_authn_jwt, 'hmac-based') +% , hoconsc:ref(emqx_authn_jwt, 'public-key') +% , hoconsc:ref(emqx_authn_jwt, 'jwks') +% , hoconsc:ref(emqx_enhanced_authn_scram_mnesia, config) +% ]). gateway_common_options() -> - [ {enable, t(boolean(), undefined, true)} - , {enable_stats, t(boolean(), undefined, true)} - , {idle_timeout, t(duration(), undefined, <<"30s">>)} - , {mountpoint, t(binary())} - , {clientinfo_override, t(ref(clientinfo_override))} - , {authentication, t(authentication(), undefined, undefined)} + [ {enable, sc(boolean(), true)} + , {enable_stats, sc(boolean(), true)} + , {idle_timeout, sc(duration(), <<"30s">>)} + , {mountpoint, sc(binary(), <<>>)} + , {clientinfo_override, sc(ref(clientinfo_override))} + , {authentication, sc(hoconsc:lazy(map()))} ]. +common_listener_opts() -> + [ {enable, sc(boolean(), true)} + , {bind, sc(union(ip_port(), integer()))} + , {acceptors, sc(integer(), 16)} + , {max_connections, sc(integer(), 1024)} + , {max_conn_rate, sc(integer())} + %, {rate_limit, sc(comma_separated_list())} + , {mountpoint, sc(binary(), undefined)} + , {access_rules, sc(hoconsc:array(string()), [])} + ]. + +tcp_opts() -> + [{tcp, sc(ref(emqx_schema, "tcp_opts"), #{})}]. + +udp_opts() -> + [{udp, sc(ref(udp_opts), #{})}]. + +ssl_opts() -> + [{ssl, sc(ref(emqx_schema, "listener_ssl_opts"), #{})}]. + +dtls_opts() -> + [{dtls, sc(ref(dtls_listener_ssl_opts), #{})}]. + +proxy_protocol_opts() -> + [ {proxy_protocol, sc(boolean())} + , {proxy_protocol_timeout, sc(duration())} + ]. + +default_dtls_vsns() -> + [<<"dtlsv1.2">>, <<"dtlsv1">>]. + +dtls_vsn(<<"dtlsv1.2">>) -> 'dtlsv1.2'; +dtls_vsn(<<"dtlsv1">>) -> 'dtlsv1'. + %%-------------------------------------------------------------------- %% Helpers %% types -t(Type) -> #{type => Type}. +sc(Type) -> #{type => Type}. -t(Type, Mapping, Default) -> - hoconsc:t(Type, #{mapping => Mapping, default => Default}). - -t(Type, Mapping, Default, OverrideEnv) -> - hoconsc:t(Type, #{ mapping => Mapping - , default => Default - , override_env => OverrideEnv - }). +sc(Type, Default) -> + hoconsc:mk(Type, #{default => Default}). ref(Field) -> hoconsc:ref(?MODULE, Field). -%% utils - -%% generate a ssl field. -%% ssl("emqx", #{"verify" => verify_peer}) will return -%% [ {"cacertfile", t(string(), "emqx.cacertfile", undefined)} -%% , {"certfile", t(string(), "emqx.certfile", undefined)} -%% , {"keyfile", t(string(), "emqx.keyfile", undefined)} -%% , {"verify", t(union(verify_peer, verify_none), "emqx.verify", verify_peer)} -%% , {"server_name_indication", "emqx.server_name_indication", undefined)} -%% ... -ssl(Mapping, Defaults) -> - M = fun (Field) -> - case (Mapping) of - undefined -> undefined; - _ -> Mapping ++ "." ++ Field - end end, - D = fun (Field) -> maps:get(list_to_atom(Field), Defaults, undefined) end, - [ {"enable", t(boolean(), M("enable"), D("enable"))} - , {"cacertfile", t(binary(), M("cacertfile"), D("cacertfile"))} - , {"certfile", t(binary(), M("certfile"), D("certfile"))} - , {"keyfile", t(binary(), M("keyfile"), D("keyfile"))} - , {"verify", t(union(verify_peer, verify_none), M("verify"), D("verify"))} - , {"fail_if_no_peer_cert", t(boolean(), M("fail_if_no_peer_cert"), D("fail_if_no_peer_cert"))} - , {"secure_renegotiate", t(boolean(), M("secure_renegotiate"), D("secure_renegotiate"))} - , {"reuse_sessions", t(boolean(), M("reuse_sessions"), D("reuse_sessions"))} - , {"honor_cipher_order", t(boolean(), M("honor_cipher_order"), D("honor_cipher_order"))} - , {"handshake_timeout", t(duration(), M("handshake_timeout"), D("handshake_timeout"))} - , {"depth", t(integer(), M("depth"), D("depth"))} - , {"password", hoconsc:t(binary(), #{mapping => M("key_password"), - default => D("key_password"), - sensitive => true - })} - , {"dhfile", t(binary(), M("dhfile"), D("dhfile"))} - , {"server_name_indication", t(union(disable, binary()), M("server_name_indication"), - D("server_name_indication"))} - , {"tls_versions", t(comma_separated_list(), M("tls_versions"), D("tls_versions"))} - , {"ciphers", t(comma_separated_list(), M("ciphers"), D("ciphers"))} - , {"psk_ciphers", t(comma_separated_list(), M("ciphers"), D("ciphers"))}]. +ref(Mod, Field) -> + hoconsc:ref(Mod, Field). diff --git a/apps/emqx_gateway/src/emqx_gateway_utils.erl b/apps/emqx_gateway/src/emqx_gateway_utils.erl index dc4e38e7d..4f19db23b 100644 --- a/apps/emqx_gateway/src/emqx_gateway_utils.erl +++ b/apps/emqx_gateway/src/emqx_gateway_utils.erl @@ -28,7 +28,11 @@ -export([ apply/2 , format_listenon/1 + , parse_listenon/1 + , unix_ts_to_rfc3339/1 , unix_ts_to_rfc3339/2 + , listener_id/3 + , parse_listener_id/1 ]). -export([ stringfy/1 @@ -111,6 +115,38 @@ format_listenon({Addr, Port}) when is_list(Addr) -> format_listenon({Addr, Port}) when is_tuple(Addr) -> io_lib:format("~s:~w", [inet:ntoa(Addr), Port]). +parse_listenon(Port) when is_integer(Port) -> + Port; +parse_listenon(Str) when is_binary(Str) -> + parse_listenon(binary_to_list(Str)); +parse_listenon(Str) when is_list(Str) -> + case emqx_schema:to_ip_port(Str) of + {ok, R} -> R; + {error, _} -> + error({invalid_listenon_name, Str}) + end. + +listener_id(GwName, Type, LisName) -> + binary_to_atom( + <<(bin(GwName))/binary, ":", + (bin(Type))/binary, ":", + (bin(LisName))/binary + >>). + +parse_listener_id(Id) -> + try + [GwName, Type, Name] = binary:split(bin(Id), <<":">>, [global]), + {binary_to_existing_atom(GwName), binary_to_existing_atom(Type), + binary_to_atom(Name)} + catch + _ : _ -> error({invalid_listener_id, Id}) + end. + +bin(A) when is_atom(A) -> + atom_to_binary(A); +bin(L) when is_list(L); is_binary(L) -> + iolist_to_binary(L). + unix_ts_to_rfc3339(Keys, Map) when is_list(Keys) -> lists:foldl(fun(K, Acc) -> unix_ts_to_rfc3339(K, Acc) end, Map, Keys); unix_ts_to_rfc3339(Key, Map) -> @@ -121,6 +157,9 @@ unix_ts_to_rfc3339(Key, Map) -> emqx_rule_funcs:unix_ts_to_rfc3339(Ts, <<"millisecond">>)} end. +unix_ts_to_rfc3339(Ts) -> + emqx_rule_funcs:unix_ts_to_rfc3339(Ts, <<"millisecond">>). + -spec stringfy(term()) -> binary(). stringfy(T) -> iolist_to_binary(io_lib:format("~0p", [T])). @@ -139,16 +178,47 @@ normalize_config(RawConf) -> Listeners = maps:fold(fun(Name, Confs, AccIn2) -> ListenOn = maps:get(bind, Confs), - SocketOpts = esockd:parse_opt(maps:to_list(Confs)), + SocketOpts = esockd_opts(Type, Confs), RemainCfgs = maps:without( - [bind] ++ proplists:get_keys(SocketOpts), - Confs), + [bind, tcp, ssl, udp, dtls] + ++ proplists:get_keys(SocketOpts), Confs), Cfg = maps:merge(Cfg0, RemainCfgs), [{Type, Name, ListenOn, SocketOpts, Cfg}|AccIn2] end, [], Liss), [Listeners|AccIn1] end, [], LisMap)). +esockd_opts(Type, Opts0) -> + Opts1 = maps:with([acceptors, max_connections, max_conn_rate, + proxy_protocol, proxy_protocol_timeout], Opts0), + Opts2 = Opts1#{access_rules => esockd_access_rules(maps:get(access_rules, Opts0, []))}, + maps:to_list(case Type of + tcp -> Opts2#{tcp_options => sock_opts(tcp, Opts0)}; + ssl -> Opts2#{tcp_options => sock_opts(tcp, Opts0), + ssl_options => ssl_opts(ssl, Opts0)}; + udp -> Opts2#{udp_options => sock_opts(udp, Opts0)}; + dtls -> Opts2#{udp_options => sock_opts(udp, Opts0), + dtls_options => ssl_opts(dtls, Opts0)} + end). + +esockd_access_rules(StrRules) -> + Access = fun(S) -> + [A, CIDR] = string:tokens(S, " "), + {list_to_atom(A), case CIDR of "all" -> all; _ -> CIDR end} + end, + [Access(R) || R <- StrRules]. + +ssl_opts(Name, Opts) -> + maps:to_list( + emqx_tls_lib:drop_tls13_for_old_otp( + maps:without([enable], + maps:get(Name, Opts, #{})))). + +sock_opts(Name, Opts) -> + maps:to_list( + maps:without([active_n], + maps:get(Name, Opts, #{}))). + %%-------------------------------------------------------------------- %% Envs @@ -205,5 +275,6 @@ default_subopts() -> #{rh => 0, %% Retain Handling rap => 0, %% Retain as Publish nl => 0, %% No Local - qos => 0 %% QoS + qos => 0, %% QoS + is_new => true }. diff --git a/apps/emqx_gateway/src/exproto/emqx_exproto_channel.erl b/apps/emqx_gateway/src/exproto/emqx_exproto_channel.erl index b1a1ae027..3de231958 100644 --- a/apps/emqx_gateway/src/exproto/emqx_exproto_channel.erl +++ b/apps/emqx_gateway/src/exproto/emqx_exproto_channel.erl @@ -31,7 +31,7 @@ , handle_in/2 , handle_deliver/2 , handle_timeout/3 - , handle_call/2 + , handle_call/3 , handle_cast/2 , handle_info/2 , terminate/2 @@ -243,23 +243,24 @@ handle_timeout(_TRef, Msg, Channel) -> ?WARN("Unexpected timeout: ~p", [Msg]), {ok, Channel}. --spec handle_call(any(), channel()) +-spec handle_call(Req :: any(), From :: any(), channel()) -> {reply, Reply :: term(), channel()} | {reply, Reply :: term(), replies(), channel()} | {shutdown, Reason :: term(), Reply :: term(), channel()}. -handle_call({send, Data}, Channel) -> +handle_call({send, Data}, _From, Channel) -> {reply, ok, [{outgoing, Data}], Channel}; -handle_call(close, Channel = #channel{conn_state = connected}) -> +handle_call(close, _From, Channel = #channel{conn_state = connected}) -> {reply, ok, [{event, disconnected}, {close, normal}], Channel}; -handle_call(close, Channel) -> +handle_call(close, _From, Channel) -> {reply, ok, [{close, normal}], Channel}; -handle_call({auth, ClientInfo, _Password}, Channel = #channel{conn_state = connected}) -> +handle_call({auth, ClientInfo, _Password}, _From, + Channel = #channel{conn_state = connected}) -> ?LOG(warning, "Duplicated authorized command, dropped ~p", [ClientInfo]), {reply, {error, ?RESP_PERMISSION_DENY, <<"Duplicated authenticate command">>}, Channel}; -handle_call({auth, ClientInfo0, Password}, +handle_call({auth, ClientInfo0, Password}, _From, Channel = #channel{ ctx = Ctx, conninfo = ConnInfo, @@ -300,7 +301,7 @@ handle_call({auth, ClientInfo0, Password}, {reply, {error, ?RESP_PERMISSION_DENY, Reason}, Channel} end; -handle_call({start_timer, keepalive, Interval}, +handle_call({start_timer, keepalive, Interval}, _From, Channel = #channel{ conninfo = ConnInfo, clientinfo = ClientInfo @@ -310,7 +311,7 @@ handle_call({start_timer, keepalive, Interval}, NChannel = Channel#channel{conninfo = NConnInfo, clientinfo = NClientInfo}, {reply, ok, ensure_keepalive(NChannel)}; -handle_call({subscribe, TopicFilter, Qos}, +handle_call({subscribe_from_client, TopicFilter, Qos}, _From, Channel = #channel{ ctx = Ctx, conn_state = connected, @@ -323,12 +324,20 @@ handle_call({subscribe, TopicFilter, Qos}, {reply, ok, NChannel} end; -handle_call({unsubscribe, TopicFilter}, +handle_call({subscribe, Topic, SubOpts}, _From, Channel) -> + {ok, NChannel} = do_subscribe([{Topic, SubOpts}], Channel), + {reply, ok, NChannel}; + +handle_call({unsubscribe_from_client, TopicFilter}, _From, Channel = #channel{conn_state = connected}) -> {ok, NChannel} = do_unsubscribe([{TopicFilter, #{}}], Channel), {reply, ok, NChannel}; -handle_call({publish, Topic, Qos, Payload}, +handle_call({unsubscribe, Topic}, _From, Channel) -> + {ok, NChannel} = do_unsubscribe([Topic], Channel), + {reply, ok, NChannel}; + +handle_call({publish, Topic, Qos, Payload}, _From, Channel = #channel{ ctx = Ctx, conn_state = connected, @@ -345,10 +354,10 @@ handle_call({publish, Topic, Qos, Payload}, {reply, ok, Channel} end; -handle_call(kick, Channel) -> +handle_call(kick, _From, Channel) -> {shutdown, kicked, ok, Channel}; -handle_call(Req, Channel) -> +handle_call(Req, _From, Channel) -> ?LOG(warning, "Unexpected call: ~p", [Req]), {reply, {error, unexpected_call}, Channel}. @@ -363,12 +372,6 @@ handle_cast(Req, Channel) -> -spec handle_info(any(), channel()) -> {ok, channel()} | {shutdown, Reason :: term(), channel()}. -handle_info({subscribe, TopicFilters}, Channel) -> - do_subscribe(TopicFilters, Channel); - -handle_info({unsubscribe, TopicFilters}, Channel) -> - do_unsubscribe(TopicFilters, Channel); - handle_info({sock_closed, Reason}, Channel = #channel{rqueue = Queue, inflight = Inflight}) -> case queue:len(Queue) =:= 0 diff --git a/apps/emqx_gateway/src/exproto/emqx_exproto_gsvr.erl b/apps/emqx_gateway/src/exproto/emqx_exproto_gsvr.erl index 346f87452..04f3dea1c 100644 --- a/apps/emqx_gateway/src/exproto/emqx_exproto_gsvr.erl +++ b/apps/emqx_gateway/src/exproto/emqx_exproto_gsvr.erl @@ -22,9 +22,10 @@ -include("src/exproto/include/emqx_exproto.hrl"). -include_lib("emqx/include/logger.hrl"). - -define(IS_QOS(X), (X =:= 0 orelse X =:= 1 orelse X =:= 2)). +-define(DEFAULT_CALL_TIMEOUT, 5000). + %% gRPC server callbacks -export([ send/2 , close/2 @@ -96,7 +97,7 @@ publish(Req, Md) -> subscribe(Req = #{conn := Conn, topic := Topic, qos := Qos}, Md) when ?IS_QOS(Qos) -> ?LOG(debug, "Recv ~p function with request ~0p", [?FUNCTION_NAME, Req]), - {ok, response(call(Conn, {subscribe, Topic, Qos})), Md}; + {ok, response(call(Conn, {subscribe_from_client, Topic, Qos})), Md}; subscribe(Req, Md) -> ?LOG(debug, "Recv ~p function with request ~0p", [?FUNCTION_NAME, Req]), @@ -107,7 +108,7 @@ subscribe(Req, Md) -> | {error, grpc_cowboy_h:error_response()}. unsubscribe(Req = #{conn := Conn, topic := Topic}, Md) -> ?LOG(debug, "Recv ~p function with request ~0p", [?FUNCTION_NAME, Req]), - {ok, response(call(Conn, {unsubscribe, Topic})), Md}. + {ok, response(call(Conn, {unsubscribe_from_client, Topic})), Md}. %%-------------------------------------------------------------------- %% Internal funcs @@ -117,18 +118,22 @@ to_pid(ConnStr) -> binary_to_term(base64:decode(ConnStr)). call(ConnStr, Req) -> - case catch to_pid(ConnStr) of - {'EXIT', {badarg, _}} -> - {error, ?RESP_PARAMS_TYPE_ERROR, - <<"The conn type error">>}; - Pid when is_pid(Pid) -> - case erlang:is_process_alive(Pid) of - true -> - emqx_gateway_conn:call(Pid, Req); - false -> - {error, ?RESP_CONN_PROCESS_NOT_ALIVE, - <<"Connection process is not alive">>} - end + try + Pid = to_pid(ConnStr), + emqx_gateway_conn:call(Pid, Req, ?DEFAULT_CALL_TIMEOUT) + catch + exit : badarg -> + {error, ?RESP_PARAMS_TYPE_ERROR, <<"The conn type error">>}; + exit : noproc -> + {error, ?RESP_CONN_PROCESS_NOT_ALIVE, + <<"Connection process is not alive">>}; + exit : timeout -> + {error, ?RESP_UNKNOWN, <<"Connection is not answered">>}; + Class : Reason : Stk-> + ?LOG(error, "Call ~p crashed: {~0p, ~0p}, " + "stacktrace: ~0p", + [Class, Reason, Stk]), + {error, ?RESP_UNKNOWN, <<"Unkwown crashs">>} end. %%-------------------------------------------------------------------- diff --git a/apps/emqx_gateway/src/exproto/emqx_exproto_impl.erl b/apps/emqx_gateway/src/exproto/emqx_exproto_impl.erl index 8131f2d0c..3e142f3dc 100644 --- a/apps/emqx_gateway/src/exproto/emqx_exproto_impl.erl +++ b/apps/emqx_gateway/src/exproto/emqx_exproto_impl.erl @@ -143,17 +143,17 @@ start_listener(GwName, Ctx, {Type, LisName, ListenOn, SocketOpts, Cfg}) -> ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn), case start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) of {ok, Pid} -> - ?ULOG("Start ~s:~s:~s listener on ~s successfully.~n", - [GwName, Type, LisName, ListenOnStr]), + ?ULOG("Gateway ~s:~s:~s on ~s started.~n", + [GwName, Type, LisName, ListenOnStr]), Pid; {error, Reason} -> - ?ELOG("Failed to start ~s:~s:~s listener on ~s: ~0p~n", + ?ELOG("Failed to start gateway ~s:~s:~s on ~s: ~0p~n", [GwName, Type, LisName, ListenOnStr, Reason]), throw({badconf, Reason}) end. start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) -> - Name = name(GwName, LisName, Type), + Name = emqx_gateway_utils:listener_id(GwName, Type, LisName), NCfg = Cfg#{ ctx => Ctx, frame_mod => emqx_exproto_frame, @@ -172,9 +172,6 @@ do_start_listener(udp, Name, ListenOn, Opts, MFA) -> do_start_listener(dtls, Name, ListenOn, Opts, MFA) -> esockd:open_dtls(Name, ListenOn, Opts, MFA). -name(GwName, LisName, Type) -> - list_to_atom(lists:concat([GwName, ":", Type, ":", LisName])). - merge_default_by_type(Type, Options) when Type =:= tcp; Type =:= ssl -> Default = emqx_gateway_utils:default_tcp_options(), @@ -200,14 +197,14 @@ stop_listener(GwName, {Type, LisName, ListenOn, SocketOpts, Cfg}) -> StopRet = stop_listener(GwName, Type, LisName, ListenOn, SocketOpts, Cfg), ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn), case StopRet of - ok -> ?ULOG("Stop ~s:~s:~s listener on ~s successfully.~n", + ok -> ?ULOG("Gateway ~s:~s:~s on ~s stopped.~n", [GwName, Type, LisName, ListenOnStr]); {error, Reason} -> - ?ELOG("Failed to stop ~s:~s:~s listener on ~s: ~0p~n", + ?ELOG("Failed to stop gateway ~s:~s:~s on ~s: ~0p~n", [GwName, Type, LisName, ListenOnStr, Reason]) end, StopRet. stop_listener(GwName, Type, LisName, ListenOn, _SocketOpts, _Cfg) -> - Name = name(GwName, LisName, Type), + Name = emqx_gateway_utils:listener_id(GwName, Type, LisName), esockd:close(Name, ListenOn). diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_api.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_api.erl index 80449238c..98c9fabe8 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_api.erl +++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_api.erl @@ -16,140 +16,119 @@ -module(emqx_lwm2m_api). --rest_api(#{name => list, - method => 'GET', - path => "/lwm2m_channels/", - func => list, - descr => "A list of all lwm2m channel" - }). +-behaviour(minirest_api). --rest_api(#{name => list, - method => 'GET', - path => "/nodes/:atom:node/lwm2m_channels/", - func => list, - descr => "A list of lwm2m channel of a node" - }). +-export([api_spec/0]). --rest_api(#{name => lookup_cmd, - method => 'GET', - path => "/lookup_cmd/:bin:ep/", - func => lookup_cmd, - descr => "Send a lwm2m downlink command" - }). +-export([lookup_cmd/2]). --rest_api(#{name => lookup_cmd, - method => 'GET', - path => "/nodes/:atom:node/lookup_cmd/:bin:ep/", - func => lookup_cmd, - descr => "Send a lwm2m downlink command of a node" - }). +-define(PREFIX, "/gateway/lwm2m/:clientid"). --export([ list/2 - , lookup_cmd/2 - ]). +-import(emqx_mgmt_util, [ object_schema/1 + , error_schema/2 + , properties/1]). -list(#{node := Node }, Params) -> - case Node = node() of - true -> list(#{}, Params); - _ -> rpc_call(Node, list, [#{}, Params]) - end; +api_spec() -> + {[lookup_cmd_api()], []}. -list(#{}, _Params) -> - Channels = emqx_lwm2m_cm:all_channels(), - return({ok, format(Channels)}). +lookup_cmd_paramters() -> + [ make_paramter(clientid, path, true, "string") + , make_paramter(path, query, true, "string") + , make_paramter(action, query, true, "string")]. -lookup_cmd(#{ep := Ep, node := Node}, Params) -> - case Node = node() of - true -> lookup_cmd(#{ep => Ep}, Params); - _ -> rpc_call(Node, lookup_cmd, [#{ep => Ep}, Params]) - end; +lookup_cmd_properties() -> + properties([ {clientid, string} + , {path, string} + , {action, string} + , {code, string} + , {codeMsg, string} + , {content, {array, object}, lookup_cmd_content_props()}]). -lookup_cmd(#{ep := Ep}, Params) -> - MsgType = proplists:get_value(<<"msgType">>, Params), - Path0 = proplists:get_value(<<"path">>, Params), - case emqx_lwm2m_cm:lookup_cmd(Ep, Path0, MsgType) of - [] -> return({ok, []}); - [{_, undefined} | _] -> return({ok, []}); - [{{IMEI, Path, MsgType}, undefined}] -> - return({ok, [{imei, IMEI}, - {'msgType', IMEI}, - {'code', <<"6.01">>}, - {'codeMsg', <<"reply_not_received">>}, - {'path', Path}]}); - [{{IMEI, Path, MsgType}, {Code, CodeMsg, Content}}] -> - Payload1 = format_cmd_content(Content, MsgType), - return({ok, [{imei, IMEI}, - {'msgType', IMEI}, - {'code', Code}, - {'codeMsg', CodeMsg}, - {'path', Path}] ++ Payload1}) +lookup_cmd_content_props() -> + [ {operations, string, <<"Resource Operations">>} + , {dataType, string, <<"Resource Type">>} + , {path, string, <<"Resource Path">>} + , {name, string, <<"Resource Name">>}]. + +lookup_cmd_api() -> + Metadata = #{get => + #{description => <<"look up resource">>, + parameters => lookup_cmd_paramters(), + responses => + #{<<"200">> => object_schema(lookup_cmd_properties()), + <<"404">> => error_schema("client not found error", ['CLIENT_NOT_FOUND']) + } + }}, + {?PREFIX ++ "/lookup_cmd", Metadata, lookup_cmd}. + + +lookup_cmd(get, #{bindings := Bindings, query_string := QS}) -> + ClientId = maps:get(clientid, Bindings), + case emqx_gateway_cm_registry:lookup_channels(lwm2m, ClientId) of + [Channel | _] -> + #{<<"path">> := Path, + <<"action">> := Action} = QS, + {ok, Result} = emqx_lwm2m_channel:lookup_cmd(Channel, Path, Action), + lookup_cmd_return(Result, ClientId, Action, Path); + _ -> + {404, #{code => 'CLIENT_NOT_FOUND'}} end. -rpc_call(Node, Fun, Args) -> - case rpc:call(Node, ?MODULE, Fun, Args) of - {badrpc, Reason} -> {error, Reason}; - Res -> Res - end. +lookup_cmd_return(undefined, ClientId, Action, Path) -> + {200, + #{clientid => ClientId, + action => Action, + code => <<"6.01">>, + codeMsg => <<"reply_not_received">>, + path => Path}}; -format(Channels) -> - lists:map(fun({IMEI, #{lifetime := LifeTime, - peername := Peername, - version := Version, - reg_info := RegInfo}}) -> - ObjectList = lists:map(fun(Path) -> - [ObjId | _] = path_list(Path), - case emqx_lwm2m_xml_object:get_obj_def(binary_to_integer(ObjId), true) of - {error, _} -> - {Path, Path}; - ObjDefinition -> - ObjectName = emqx_lwm2m_xml_object:get_object_name(ObjDefinition), - {Path, list_to_binary(ObjectName)} - end - end, maps:get(<<"objectList">>, RegInfo)), - {IpAddr, Port} = Peername, - [{imei, IMEI}, - {lifetime, LifeTime}, - {ip_address, iolist_to_binary(ntoa(IpAddr))}, - {port, Port}, - {version, Version}, - {'objectList', ObjectList}] - end, Channels). +lookup_cmd_return({Code, CodeMsg, Content}, ClientId, Action, Path) -> + {200, + format_cmd_content(Content, + Action, + #{clientid => ClientId, + action => Action, + code => Code, + codeMsg => CodeMsg, + path => Path})}. -format_cmd_content(undefined, _MsgType) -> []; -format_cmd_content(Content, <<"discover">>) -> +format_cmd_content(undefined, _MsgType, Result) -> + Result; + +format_cmd_content(Content, <<"discover">>, Result) -> [H | Content1] = Content, - {_, [HObjId]} = emqx_lwm2m_coap_resource:parse_object_list(H), + {_, [HObjId]} = emqx_lwm2m_session:parse_object_list(H), [ObjId | _]= path_list(HObjId), ObjectList = case Content1 of - [Content2 | _] -> - {_, ObjL} = emqx_lwm2m_coap_resource:parse_object_list(Content2), - ObjL; - [] -> [] - end, - R = case emqx_lwm2m_xml_object:get_obj_def(binary_to_integer(ObjId), true) of - {error, _} -> - lists:map(fun(Object) -> {Object, Object} end, ObjectList); - ObjDefinition -> - lists:map(fun(Object) -> - [_, _, ResId| _] = path_list(Object), - Operations = case emqx_lwm2m_xml_object:get_resource_operations(binary_to_integer(ResId), ObjDefinition) of - "E" -> [{operations, list_to_binary("E")}]; - Oper -> [{'dataType', list_to_binary(emqx_lwm2m_xml_object:get_resource_type(binary_to_integer(ResId), ObjDefinition))}, - {operations, list_to_binary(Oper)}] - end, - [{path, Object}, - {name, list_to_binary(emqx_lwm2m_xml_object:get_resource_name(binary_to_integer(ResId), ObjDefinition))} - ] ++ Operations - end, ObjectList) - end, - [{content, R}]; -format_cmd_content(Content, _) -> - [{content, Content}]. + [Content2 | _] -> + {_, ObjL} = emqx_lwm2m_session:parse_object_list(Content2), + ObjL; + [] -> [] + end, -ntoa({0,0,0,0,0,16#ffff,AB,CD}) -> - inet_parse:ntoa({AB bsr 8, AB rem 256, CD bsr 8, CD rem 256}); -ntoa(IP) -> - inet_parse:ntoa(IP). + R = case emqx_lwm2m_xml_object:get_obj_def(binary_to_integer(ObjId), true) of + {error, _} -> + lists:map(fun(Object) -> #{Object => Object} end, ObjectList); + ObjDefinition -> + lists:map( + fun(Object) -> + [_, _, RawResId| _] = path_list(Object), + ResId = binary_to_integer(RawResId), + Operations = case emqx_lwm2m_xml_object:get_resource_operations(ResId, ObjDefinition) of + "E" -> + #{operations => list_to_binary("E")}; + Oper -> + #{'dataType' => list_to_binary(emqx_lwm2m_xml_object:get_resource_type(ResId, ObjDefinition)), + operations => list_to_binary(Oper)} + end, + Operations#{path => Object, + name => list_to_binary(emqx_lwm2m_xml_object:get_resource_name(ResId, ObjDefinition))} + end, ObjectList) + end, + Result#{content => R}; + +format_cmd_content(Content, _, Result) -> + Result#{content => Content}. path_list(Path) -> case binary:split(binary_util:trim(Path, $/), [<<$/>>], [global]) of @@ -159,6 +138,8 @@ path_list(Path) -> [ObjId] -> [ObjId] end. -return(_) -> -%% TODO: V5 API - ok. +make_paramter(Name, In, IsRequired, Type) -> + #{name => Name, + in => In, + required => IsRequired, + schema => #{type => Type}}. diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_channel.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_channel.erl new file mode 100644 index 000000000..6ad78742f --- /dev/null +++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_channel.erl @@ -0,0 +1,489 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2017-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_lwm2m_channel). + +-include_lib("emqx/include/logger.hrl"). +-include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). +-include_lib("emqx_gateway/src/lwm2m/include/emqx_lwm2m.hrl"). + +%% API +-export([ info/1 + , info/2 + , stats/1 + , with_context/2 + , do_takeover/3 + , lookup_cmd/3]). + +-export([ init/2 + , handle_in/2 + , handle_deliver/2 + , handle_timeout/3 + , terminate/2 + ]). + +-export([ handle_call/3 + , handle_cast/2 + , handle_info/2 + ]). + +-record(channel, { + %% Context + ctx :: emqx_gateway_ctx:context(), + %% Connection Info + conninfo :: emqx_types:conninfo(), + %% Client Info + clientinfo :: emqx_types:clientinfo(), + %% Session + session :: emqx_lwm2m_session:session() | undefined, + + %% Timer + timers :: #{atom() => disable | undefined | reference()}, + + with_context :: function() + }). + +-define(INFO_KEYS, [conninfo, conn_state, clientinfo, session]). +-import(emqx_coap_medium, [reply/2, reply/3, reply/4, iter/3, iter/4]). + +%%-------------------------------------------------------------------- +%% API +%%-------------------------------------------------------------------- +info(Channel) -> + maps:from_list(info(?INFO_KEYS, Channel)). + +info(Keys, Channel) when is_list(Keys) -> + [{Key, info(Key, Channel)} || Key <- Keys]; + +info(conninfo, #channel{conninfo = ConnInfo}) -> + ConnInfo; +info(conn_state, _) -> + connected; +info(clientinfo, #channel{clientinfo = ClientInfo}) -> + ClientInfo; +info(session, #channel{session = Session}) -> + emqx_misc:maybe_apply(fun emqx_session:info/1, Session); +info(clientid, #channel{clientinfo = #{clientid := ClientId}}) -> + ClientId; +info(ctx, #channel{ctx = Ctx}) -> + Ctx. + +stats(_) -> + []. + +init(ConnInfo = #{peername := {PeerHost, _}, + sockname := {_, SockPort}}, + #{ctx := Ctx} = Config) -> + Peercert = maps:get(peercert, ConnInfo, undefined), + Mountpoint = maps:get(mountpoint, Config, undefined), + ClientInfo = set_peercert_infos( + Peercert, + #{ zone => default + , protocol => lwm2m + , peerhost => PeerHost + , sockport => SockPort + , username => undefined + , clientid => undefined + , is_bridge => false + , is_superuser => false + , mountpoint => Mountpoint + } + ), + + #channel{ ctx = Ctx + , conninfo = ConnInfo + , clientinfo = ClientInfo + , timers = #{} + , session = emqx_lwm2m_session:new() + , with_context = with_context(Ctx, ClientInfo) + }. + + +with_context(Ctx, ClientInfo) -> + fun(Type, Topic) -> + with_context(Type, Topic, Ctx, ClientInfo) + end. + +lookup_cmd(Channel, Path, Action) -> + gen_server:call(Channel, {?FUNCTION_NAME, Path, Action}). + +%%-------------------------------------------------------------------- +%% Handle incoming packet +%%-------------------------------------------------------------------- +handle_in(Msg, ChannleT) -> + Channel = update_life_timer(ChannleT), + call_session(handle_coap_in, Msg, Channel). + +%%-------------------------------------------------------------------- +%% Handle Delivers from broker to client +%%-------------------------------------------------------------------- +handle_deliver(Delivers, Channel) -> + call_session(handle_deliver, Delivers, Channel). + +%%-------------------------------------------------------------------- +%% Handle timeout +%%-------------------------------------------------------------------- +handle_timeout(_, lifetime, #channel{ctx = Ctx, + clientinfo = ClientInfo, + conninfo = ConnInfo} = Channel) -> + ok = run_hooks(Ctx, 'client.disconnected', [ClientInfo, timeout, ConnInfo]), + {shutdown, timeout, Channel}; + +handle_timeout(_, {transport, _} = Msg, Channel) -> + call_session(timeout, Msg, Channel); + +handle_timeout(_, disconnect, Channel) -> + {shutdown, normal, Channel}; + +handle_timeout(_, _, Channel) -> + {ok, Channel}. + +%%-------------------------------------------------------------------- +%% Handle call +%%-------------------------------------------------------------------- +handle_call({lookup_cmd, Path, Type}, _From, #channel{session = Session} = Channel) -> + Result = emqx_lwm2m_session:find_cmd_record(Path, Type, Session), + {reply, {ok, Result}, Channel}; + +handle_call(Req, _From, Channel) -> + ?LOG(error, "Unexpected call: ~p", [Req]), + {reply, ignored, Channel}. + +%%-------------------------------------------------------------------- +%% Handle Cast +%%-------------------------------------------------------------------- +handle_cast(Req, Channel) -> + ?LOG(error, "Unexpected cast: ~p", [Req]), + {ok, Channel}. + +%%-------------------------------------------------------------------- +%% Handle Info +%%-------------------------------------------------------------------- +handle_info({subscribe, _AutoSubs}, Channel) -> + %% not need handle this message + {ok, Channel}; + +handle_info(Info, Channel) -> + ?LOG(error, "Unexpected info: ~p", [Info]), + {ok, Channel}. + +%%-------------------------------------------------------------------- +%% Terminate +%%-------------------------------------------------------------------- +terminate(Reason, #channel{ctx = Ctx, + clientinfo = ClientInfo, + session = Session}) -> + MountedTopic = emqx_lwm2m_session:on_close(Session), + _ = run_hooks(Ctx, 'session.unsubscribe', [ClientInfo, MountedTopic, #{}]), + run_hooks(Ctx, 'session.terminated', [ClientInfo, Reason, Session]). + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- +set_peercert_infos(NoSSL, ClientInfo) + when NoSSL =:= nossl; + NoSSL =:= undefined -> + ClientInfo; +set_peercert_infos(Peercert, ClientInfo) -> + {DN, CN} = {esockd_peercert:subject(Peercert), + esockd_peercert:common_name(Peercert)}, + ClientInfo#{dn => DN, cn => CN}. + +make_timer(Name, Time, Msg, Channel = #channel{timers = Timers}) -> + TRef = emqx_misc:start_timer(Time, Msg), + Channel#channel{timers = Timers#{Name => TRef}}. + +update_life_timer(#channel{session = Session, timers = Timers} = Channel) -> + LifeTime = emqx_lwm2m_session:info(lifetime, Session), + _ = case maps:get(lifetime, Timers, undefined) of + undefined -> ok; + Ref -> erlang:cancel_timer(Ref) + end, + make_timer(lifetime, LifeTime, lifetime, Channel). + +check_location(Location, #channel{session = Session}) -> + SLocation = emqx_lwm2m_session:info(location_path, Session), + Location =:= SLocation. + +do_takeover(_DesireId, Msg, Channel) -> + %% TODO completed the takeover, now only reset the message + Reset = emqx_coap_message:reset(Msg), + call_session(handle_out, Reset, Channel). + +do_connect(Req, Result, Channel, Iter) -> + case emqx_misc:pipeline( + [ fun check_lwm2m_version/2 + , fun run_conn_hooks/2 + , fun enrich_clientinfo/2 + , fun set_log_meta/2 + , fun auth_connect/2 + ], + Req, + Channel) of + {ok, _Input, #channel{session = Session, + with_context = WithContext} = NChannel} -> + case emqx_lwm2m_session:info(reg_info, Session) of + undefined -> + process_connect(ensure_connected(NChannel), Req, Result, Iter); + _ -> + NewResult = emqx_lwm2m_session:reregister(Req, WithContext, Session), + iter(Iter, maps:merge(Result, NewResult), NChannel) + end; + {error, ReasonCode, NChannel} -> + ErrMsg = io_lib:format("Login Failed: ~s", [ReasonCode]), + Payload = erlang:list_to_binary(lists:flatten(ErrMsg)), + iter(Iter, + reply({error, bad_request}, Payload, Req, Result), + NChannel) + end. + +check_lwm2m_version(#coap_message{options = Opts}, + #channel{conninfo = ConnInfo} = Channel) -> + Ver = gets([uri_query, <<"lwm2m">>], Opts), + IsValid = case Ver of + <<"1.0">> -> + true; + <<"1">> -> + true; + <<"1.1">> -> + true; + _ -> + false + end, + if IsValid -> + NConnInfo = ConnInfo#{ connected_at => erlang:system_time(millisecond) + , proto_name => <<"LwM2M">> + , proto_ver => Ver + }, + {ok, Channel#channel{conninfo = NConnInfo}}; + true -> + ?LOG(error, "Reject REGISTER due to unsupported version: ~0p", [Ver]), + {error, "invalid lwm2m version", Channel} + end. + +run_conn_hooks(Input, Channel = #channel{ctx = Ctx, + conninfo = ConnInfo}) -> + ConnProps = #{}, + case run_hooks(Ctx, 'client.connect', [ConnInfo], ConnProps) of + Error = {error, _Reason} -> Error; + _NConnProps -> + {ok, Input, Channel} + end. + +enrich_clientinfo(#coap_message{options = Options} = Msg, + Channel = #channel{clientinfo = ClientInfo0}) -> + Query = maps:get(uri_query, Options, #{}), + case Query of + #{<<"ep">> := Epn} -> + UserName = maps:get(<<"imei">>, Query, Epn), + Password = maps:get(<<"password">>, Query, undefined), + ClientId = maps:get(<<"device_id">>, Query, Epn), + ClientInfo = + ClientInfo0#{username => UserName, + password => Password, + clientid => ClientId}, + {ok, NClientInfo} = fix_mountpoint(Msg, ClientInfo), + {ok, Channel#channel{clientinfo = NClientInfo}}; + _ -> + ?LOG(error, "Reject REGISTER due to wrong parameters, Query=~p", [Query]), + {error, "invalid queries", Channel} + end. + +set_log_meta(_Input, #channel{clientinfo = #{clientid := ClientId}}) -> + emqx_logger:set_metadata_clientid(ClientId), + ok. + +auth_connect(_Input, Channel = #channel{ctx = Ctx, + clientinfo = ClientInfo}) -> + #{clientid := ClientId, username := Username} = ClientInfo, + case emqx_gateway_ctx:authenticate(Ctx, ClientInfo) of + {ok, NClientInfo} -> + {ok, Channel#channel{clientinfo = NClientInfo, + with_context = with_context(Ctx, ClientInfo)}}; + {error, Reason} -> + ?LOG(warning, "Client ~s (Username: '~s') login failed for ~0p", + [ClientId, Username, Reason]), + {error, Reason} + end. + +fix_mountpoint(_Packet, #{mountpoint := undefined} = ClientInfo) -> + {ok, ClientInfo}; +fix_mountpoint(_Packet, ClientInfo = #{mountpoint := Mountpoint}) -> + Mountpoint1 = emqx_mountpoint:replvar(Mountpoint, ClientInfo), + {ok, ClientInfo#{mountpoint := Mountpoint1}}. + +ensure_connected(Channel = #channel{ctx = Ctx, + conninfo = ConnInfo, + clientinfo = ClientInfo}) -> + _ = run_hooks(Ctx, 'client.connack', [ConnInfo, connection_accepted, []]), + ok = run_hooks(Ctx, 'client.connected', [ClientInfo, ConnInfo]), + Channel. + +process_connect(Channel = #channel{ctx = Ctx, + session = Session, + conninfo = ConnInfo, + clientinfo = ClientInfo, + with_context = WithContext}, + Msg, Result, Iter) -> + %% inherit the old session + SessFun = fun(_,_) -> #{} end, + case emqx_gateway_ctx:open_session( + Ctx, + true, + ClientInfo, + ConnInfo, + SessFun, + emqx_lwm2m_session + ) of + {ok, _} -> + Mountpoint = maps:get(mountpoint, ClientInfo, <<>>), + NewResult = emqx_lwm2m_session:init(Msg, Mountpoint, WithContext, Session), + iter(Iter, maps:merge(Result, NewResult), Channel); + {error, Reason} -> + ?LOG(error, "Failed to open session du to ~p", [Reason]), + iter(Iter, reply({error, bad_request}, Msg, Result), Channel) + end. + +run_hooks(Ctx, Name, Args) -> + emqx_gateway_ctx:metrics_inc(Ctx, Name), + emqx_hooks:run(Name, Args). + +run_hooks(Ctx, Name, Args, Acc) -> + emqx_gateway_ctx:metrics_inc(Ctx, Name), + emqx_hooks:run_fold(Name, Args, Acc). + +gets(_, undefined) -> + undefined; +gets([H | T], Map) -> + gets(T, maps:get(H, Map, undefined)); +gets([], Val) -> + Val. + +with_context(publish, [Topic, Msg], Ctx, ClientInfo) -> + case emqx_gateway_ctx:authorize(Ctx, ClientInfo, publish, Topic) of + allow -> + emqx:publish(Msg); + _ -> + ?LOG(error, "topic:~p not allow to publish ", [Topic]) + end; + +with_context(subscribe, [Topic, Opts], Ctx, #{username := UserName} = ClientInfo) -> + case emqx_gateway_ctx:authorize(Ctx, ClientInfo, subscribe, Topic) of + allow -> + run_hooks(Ctx, 'session.subscribed', [ClientInfo, Topic, UserName]), + ?LOG(debug, "Subscribe topic: ~0p, Opts: ~0p, EndpointName: ~0p", [Topic, Opts, UserName]), + emqx:subscribe(Topic, UserName, Opts); + _ -> + ?LOG(error, "Topic: ~0p not allow to subscribe", [Topic]) + end; + +with_context(metrics, Name, Ctx, _ClientInfo) -> + emqx_gateway_ctx:metrics_inc(Ctx, Name). + +%%-------------------------------------------------------------------- +%% Call Chain +%%-------------------------------------------------------------------- +call_session(Fun, + Msg, + #channel{session = Session, + with_context = WithContext} = Channel) -> + iter([ session, fun process_session/4 + , proto, fun process_protocol/4 + , return, fun process_return/4 + , lifetime, fun process_lifetime/4 + , reply, fun process_reply/4 + , out, fun process_out/4 + , fun process_nothing/3 + ], + emqx_lwm2m_session:Fun(Msg, WithContext, Session), + Channel). + +process_session(Session, Result, Channel, Iter) -> + iter(Iter, Result, Channel#channel{session = Session}). + +process_protocol({request, Msg}, Result, Channel, Iter) -> + #coap_message{method = Method} = Msg, + handle_request_protocol(Method, Msg, Result, Channel, Iter); + +process_protocol(Msg, Result, + #channel{with_context = WithContext, session = Session} = Channel, Iter) -> + ProtoResult = emqx_lwm2m_session:handle_protocol_in(Msg, WithContext, Session), + iter(Iter, maps:merge(Result, ProtoResult), Channel). + +handle_request_protocol(post, #coap_message{options = Opts} = Msg, + Result, Channel, Iter) -> + case Opts of + #{uri_path := [?REG_PREFIX]} -> + do_connect(Msg, Result, Channel, Iter); + #{uri_path := Location} -> + do_update(Location, Msg, Result, Channel, Iter); + _ -> + iter(Iter, reply({error, not_found}, Msg, Result), Channel) + end; + +handle_request_protocol(delete, #coap_message{options = Opts} = Msg, + Result, Channel, Iter) -> + case Opts of + #{uri_path := Location} -> + case check_location(Location, Channel) of + true -> + Reply = emqx_coap_message:piggyback({ok, deleted}, Msg), + {shutdown, close, Reply, Channel}; + _ -> + iter(Iter, reply({error, not_found}, Msg, Result), Channel) + end; + _ -> + iter(Iter, reply({error, bad_request}, Msg, Result), Channel) + end. + +do_update(Location, Msg, Result, + #channel{session = Session, with_context = WithContext} = Channel, Iter) -> + case check_location(Location, Channel) of + true -> + NewResult = emqx_lwm2m_session:update(Msg, WithContext, Session), + iter(Iter, maps:merge(Result, NewResult), Channel); + _ -> + iter(Iter, reply({error, not_found}, Msg, Result), Channel) + end. + +process_return({Outs, Session}, Result, Channel, Iter) -> + OldOuts = maps:get(out, Result, []), + iter(Iter, + Result#{out => Outs ++ OldOuts}, + Channel#channel{session = Session}). + +process_out(Outs, Result, Channel, _) -> + Outs2 = lists:reverse(Outs), + Outs3 = case maps:get(reply, Result, undefined) of + undefined -> + Outs2; + Reply -> + [Reply | Outs2] + end, + + {ok, {outgoing, Outs3}, Channel}. + +process_reply(Reply, Result, #channel{session = Session} = Channel, _) -> + Session2 = emqx_lwm2m_session:set_reply(Reply, Session), + Outs = maps:get(out, Result, []), + Outs2 = lists:reverse(Outs), + {ok, {outgoing, [Reply | Outs2]}, Channel#channel{session = Session2}}. + +process_lifetime(_, Result, Channel, Iter) -> + iter(Iter, Result, update_life_timer(Channel)). + +process_nothing(_, _, Channel) -> + {ok, Channel}. diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_cm.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_cm.erl deleted file mode 100644 index 16e938b84..000000000 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_cm.erl +++ /dev/null @@ -1,153 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_lwm2m_cm). - --export([start_link/0]). - --export([ register_channel/5 - , update_reg_info/2 - , unregister_channel/1 - ]). - --export([ lookup_channel/1 - , all_channels/0 - ]). - --export([ register_cmd/3 - , register_cmd/4 - , lookup_cmd/3 - , lookup_cmd_by_imei/1 - ]). - -%% gen_server callbacks --export([ init/1 - , handle_call/3 - , handle_cast/2 - , handle_info/2 - , terminate/2 - , code_change/3 - ]). - --define(LOG(Level, Format, Args), logger:Level("LWM2M-CM: " ++ Format, Args)). - -%% Server name --define(CM, ?MODULE). - --define(LWM2M_CHANNEL_TAB, emqx_lwm2m_channel). --define(LWM2M_CMD_TAB, emqx_lwm2m_cmd). - -%% Batch drain --define(BATCH_SIZE, 100000). - -%% @doc Start the channel manager. -start_link() -> - gen_server:start_link({local, ?CM}, ?MODULE, [], []). - -%%-------------------------------------------------------------------- -%% API -%%-------------------------------------------------------------------- - -register_channel(IMEI, RegInfo, LifeTime, Ver, Peername) -> - Info = #{ - reg_info => RegInfo, - lifetime => LifeTime, - version => Ver, - peername => Peername - }, - true = ets:insert(?LWM2M_CHANNEL_TAB, {IMEI, Info}), - cast({registered, {IMEI, self()}}). - -update_reg_info(IMEI, RegInfo) -> - case lookup_channel(IMEI) of - [{_, RegInfo0}] -> - true = ets:insert(?LWM2M_CHANNEL_TAB, {IMEI, RegInfo0#{reg_info => RegInfo}}), - ok; - [] -> - ok - end. - -unregister_channel(IMEI) when is_binary(IMEI) -> - true = ets:delete(?LWM2M_CHANNEL_TAB, IMEI), - ok. - -lookup_channel(IMEI) -> - ets:lookup(?LWM2M_CHANNEL_TAB, IMEI). - -all_channels() -> - ets:tab2list(?LWM2M_CHANNEL_TAB). - -register_cmd(IMEI, Path, Type) -> - true = ets:insert(?LWM2M_CMD_TAB, {{IMEI, Path, Type}, undefined}). - -register_cmd(_IMEI, undefined, _Type, _Result) -> - ok; -register_cmd(IMEI, Path, Type, Result) -> - true = ets:insert(?LWM2M_CMD_TAB, {{IMEI, Path, Type}, Result}). - -lookup_cmd(IMEI, Path, Type) -> - ets:lookup(?LWM2M_CMD_TAB, {IMEI, Path, Type}). - -lookup_cmd_by_imei(IMEI) -> - ets:select(?LWM2M_CHANNEL_TAB, [{{{IMEI, '_', '_'}, '$1'}, [], ['$_']}]). - -%% @private -cast(Msg) -> gen_server:cast(?CM, Msg). - -%%-------------------------------------------------------------------- -%% gen_server callbacks -%%-------------------------------------------------------------------- - -init([]) -> - TabOpts = [public, {write_concurrency, true}, {read_concurrency, true}], - ok = emqx_tables:new(?LWM2M_CHANNEL_TAB, [set, compressed | TabOpts]), - ok = emqx_tables:new(?LWM2M_CMD_TAB, [set, compressed | TabOpts]), - {ok, #{chan_pmon => emqx_pmon:new()}}. - -handle_call(Req, _From, State) -> - ?LOG(error, "Unexpected call: ~p", [Req]), - {reply, ignored, State}. - -handle_cast({registered, {IMEI, ChanPid}}, State = #{chan_pmon := PMon}) -> - PMon1 = emqx_pmon:monitor(ChanPid, IMEI, PMon), - {noreply, State#{chan_pmon := PMon1}}; - -handle_cast(Msg, State) -> - ?LOG(error, "Unexpected cast: ~p", [Msg]), - {noreply, State}. - -handle_info({'DOWN', _MRef, process, Pid, _Reason}, State = #{chan_pmon := PMon}) -> - ChanPids = [Pid | emqx_misc:drain_down(?BATCH_SIZE)], - {Items, PMon1} = emqx_pmon:erase_all(ChanPids, PMon), - ok = emqx_pool:async_submit(fun lists:foreach/2, [fun clean_down/1, Items]), - {noreply, State#{chan_pmon := PMon1}}; - -handle_info(Info, State) -> - ?LOG(error, "Unexpected info: ~p", [Info]), - {noreply, State}. - -terminate(_Reason, _State) -> - emqx_stats:cancel_update(chan_stats). - -code_change(_OldVsn, State, _Extra) -> - {ok, State}. - -%%-------------------------------------------------------------------- -%% Internal functions -%%-------------------------------------------------------------------- - -clean_down({_ChanPid, IMEI}) -> - unregister_channel(IMEI). diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_cmd.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_cmd.erl new file mode 100644 index 000000000..e17a83195 --- /dev/null +++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_cmd.erl @@ -0,0 +1,410 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2016-2017 EMQ Enterprise, Inc. (http://emqtt.io) +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_lwm2m_cmd). + +-include_lib("emqx/include/logger.hrl"). +-include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). +-include_lib("emqx_gateway/src/lwm2m/include/emqx_lwm2m.hrl"). + +-export([ mqtt_to_coap/2 + , coap_to_mqtt/4 + , empty_ack_to_mqtt/1 + , coap_failure_to_mqtt/2 + ]). + +-export([path_list/1, extract_path/1]). + +-define(STANDARD, 1). + +%%-type msg_type() :: <<"create">> +%% | <<"delete">> +%% | <<"read">> +%% | <<"write">> +%% | <<"execute">> +%% | <<"discover">> +%% | <<"write-attr">> +%% | <<"observe">> +%% | <<"cancel-observe">>. +%% +%%-type cmd() :: #{ <<"msgType">> := msg_type() +%% , <<"data">> := maps() +%% %%%% more keys? +%% }. + +%%-------------------------------------------------------------------- +%% APIs +%%-------------------------------------------------------------------- + +mqtt_to_coap(AlternatePath, InputCmd = #{<<"msgType">> := <<"create">>, <<"data">> := Data}) -> + {PathList, QueryList} = path_list(maps:get(<<"basePath">>, Data, <<"/">>)), + FullPathList = add_alternate_path_prefix(AlternatePath, PathList), + TlvData = emqx_lwm2m_message:json_to_tlv(PathList, maps:get(<<"content">>, Data)), + Payload = emqx_lwm2m_tlv:encode(TlvData), + CoapRequest = emqx_coap_message:request(con, post, Payload, + [{uri_path, FullPathList}, + {uri_query, QueryList}, + {content_format, <<"application/vnd.oma.lwm2m+tlv">>}]), + {CoapRequest, InputCmd}; + +mqtt_to_coap(AlternatePath, InputCmd = #{<<"msgType">> := <<"delete">>, <<"data">> := Data}) -> + {PathList, QueryList} = path_list(maps:get(<<"path">>, Data)), + FullPathList = add_alternate_path_prefix(AlternatePath, PathList), + {emqx_coap_message:request(con, delete, <<>>, + [{uri_path, FullPathList}, + {uri_query, QueryList}]), InputCmd}; + +mqtt_to_coap(AlternatePath, InputCmd = #{<<"msgType">> := <<"read">>, <<"data">> := Data}) -> + {PathList, QueryList} = path_list(maps:get(<<"path">>, Data)), + FullPathList = add_alternate_path_prefix(AlternatePath, PathList), + {emqx_coap_message:request(con, get, <<>>, + [{uri_path, FullPathList}, + {uri_query, QueryList}]), InputCmd}; + +mqtt_to_coap(AlternatePath, InputCmd = #{<<"msgType">> := <<"write">>, <<"data">> := Data}) -> + CoapRequest = + case maps:get(<<"basePath">>, Data, <<"/">>) of + <<"/">> -> + single_write_request(AlternatePath, Data); + BasePath -> + batch_write_request(AlternatePath, BasePath, maps:get(<<"content">>, Data)) + end, + {CoapRequest, InputCmd}; + +mqtt_to_coap(AlternatePath, InputCmd = #{<<"msgType">> := <<"execute">>, <<"data">> := Data}) -> + {PathList, QueryList} = path_list(maps:get(<<"path">>, Data)), + FullPathList = add_alternate_path_prefix(AlternatePath, PathList), + Args = + case maps:get(<<"args">>, Data, <<>>) of + <<"undefined">> -> <<>>; + undefined -> <<>>; + Arg1 -> Arg1 + end, + {emqx_coap_message:request(con, post, Args, + [{uri_path, FullPathList}, + {uri_query, QueryList}, + {content_format, <<"text/plain">>}]), InputCmd}; + +mqtt_to_coap(AlternatePath, InputCmd = #{<<"msgType">> := <<"discover">>, <<"data">> := Data}) -> + {PathList, QueryList} = path_list(maps:get(<<"path">>, Data)), + FullPathList = add_alternate_path_prefix(AlternatePath, PathList), + {emqx_coap_message:request(con, get, <<>>, + [{uri_path, FullPathList}, + {uri_query, QueryList}, + {'accept', ?LWM2M_FORMAT_LINK}]), InputCmd}; + +mqtt_to_coap(AlternatePath, InputCmd = #{<<"msgType">> := <<"write-attr">>, <<"data">> := Data}) -> + {PathList, QueryList} = path_list(maps:get(<<"path">>, Data)), + FullPathList = add_alternate_path_prefix(AlternatePath, PathList), + Query = attr_query_list(Data), + {emqx_coap_message:request(con, put, <<>>, + [{uri_path, FullPathList}, + {uri_query, QueryList}, + {uri_query, Query}]), InputCmd}; + +mqtt_to_coap(AlternatePath, InputCmd = #{<<"msgType">> := <<"observe">>, <<"data">> := Data}) -> + {PathList, QueryList} = path_list(maps:get(<<"path">>, Data)), + FullPathList = add_alternate_path_prefix(AlternatePath, PathList), + {emqx_coap_message:request(con, get, <<>>, + [{uri_path, FullPathList}, + {uri_query, QueryList}, + {observe, 0}]), InputCmd}; + +mqtt_to_coap(AlternatePath, InputCmd = #{<<"msgType">> := <<"cancel-observe">>, <<"data">> := Data}) -> + {PathList, QueryList} = path_list(maps:get(<<"path">>, Data)), + FullPathList = add_alternate_path_prefix(AlternatePath, PathList), + {emqx_coap_message:request(con, get, <<>>, + [{uri_path, FullPathList}, + {uri_query, QueryList}, + {observe, 1}]), InputCmd}. + +coap_to_mqtt(_Method = {_, Code}, _CoapPayload, _Options, Ref=#{<<"msgType">> := <<"create">>}) -> + make_response(Code, Ref); + +coap_to_mqtt(_Method = {_, Code}, _CoapPayload, _Options, Ref=#{<<"msgType">> := <<"delete">>}) -> + make_response(Code, Ref); + +coap_to_mqtt(Method, CoapPayload, Options, Ref=#{<<"msgType">> := <<"read">>}) -> + read_resp_to_mqtt(Method, CoapPayload, data_format(Options), Ref); + +coap_to_mqtt(Method, CoapPayload, _Options, Ref=#{<<"msgType">> := <<"write">>}) -> + write_resp_to_mqtt(Method, CoapPayload, Ref); + +coap_to_mqtt(Method, _CoapPayload, _Options, Ref=#{<<"msgType">> := <<"execute">>}) -> + execute_resp_to_mqtt(Method, Ref); + +coap_to_mqtt(Method, CoapPayload, _Options, Ref=#{<<"msgType">> := <<"discover">>}) -> + discover_resp_to_mqtt(Method, CoapPayload, Ref); + +coap_to_mqtt(Method, CoapPayload, _Options, Ref=#{<<"msgType">> := <<"write-attr">>}) -> + writeattr_resp_to_mqtt(Method, CoapPayload, Ref); + +coap_to_mqtt(Method, CoapPayload, Options, Ref=#{<<"msgType">> := <<"observe">>}) -> + observe_resp_to_mqtt(Method, CoapPayload, data_format(Options), observe_seq(Options), Ref); + +coap_to_mqtt(Method, CoapPayload, Options, Ref=#{<<"msgType">> := <<"cancel-observe">>}) -> + cancel_observe_resp_to_mqtt(Method, CoapPayload, data_format(Options), Ref). + +read_resp_to_mqtt({error, ErrorCode}, _CoapPayload, _Format, Ref) -> + make_response(ErrorCode, Ref); + +read_resp_to_mqtt({ok, SuccessCode}, CoapPayload, Format, Ref) -> + try + Result = content_to_mqtt(CoapPayload, Format, Ref), + make_response(SuccessCode, Ref, Format, Result) + catch + error:not_implemented -> make_response(not_implemented, Ref); + _:Ex:_ST -> + ?LOG(error, "~0p, bad payload format: ~0p", [Ex, CoapPayload]), + make_response(bad_request, Ref) + end. + +empty_ack_to_mqtt(Ref) -> + make_base_response(maps:put(<<"msgType">>, <<"ack">>, Ref)). + +coap_failure_to_mqtt(Ref, MsgType) -> + make_base_response(maps:put(<<"msgType">>, MsgType, Ref)). + +content_to_mqtt(CoapPayload, <<"text/plain">>, Ref) -> + emqx_lwm2m_message:text_to_json(extract_path(Ref), CoapPayload); + +content_to_mqtt(CoapPayload, <<"application/octet-stream">>, Ref) -> + emqx_lwm2m_message:opaque_to_json(extract_path(Ref), CoapPayload); + +content_to_mqtt(CoapPayload, <<"application/vnd.oma.lwm2m+tlv">>, Ref) -> + emqx_lwm2m_message:tlv_to_json(extract_path(Ref), CoapPayload); + +content_to_mqtt(CoapPayload, <<"application/vnd.oma.lwm2m+json">>, _Ref) -> + emqx_lwm2m_message:translate_json(CoapPayload). + +write_resp_to_mqtt({ok, changed}, _CoapPayload, Ref) -> + make_response(changed, Ref); + +write_resp_to_mqtt({ok, content}, CoapPayload, Ref) when CoapPayload =:= <<>> -> + make_response(method_not_allowed, Ref); + +write_resp_to_mqtt({ok, content}, _CoapPayload, Ref) -> + make_response(changed, Ref); + +write_resp_to_mqtt({error, Error}, _CoapPayload, Ref) -> + make_response(Error, Ref). + +execute_resp_to_mqtt({ok, changed}, Ref) -> + make_response(changed, Ref); + +execute_resp_to_mqtt({error, Error}, Ref) -> + make_response(Error, Ref). + +discover_resp_to_mqtt({ok, content}, CoapPayload, Ref) -> + Links = binary:split(CoapPayload, <<",">>, [global]), + make_response(content, Ref, <<"application/link-format">>, Links); + +discover_resp_to_mqtt({error, Error}, _CoapPayload, Ref) -> + make_response(Error, Ref). + +writeattr_resp_to_mqtt({ok, changed}, _CoapPayload, Ref) -> + make_response(changed, Ref); + +writeattr_resp_to_mqtt({error, Error}, _CoapPayload, Ref) -> + make_response(Error, Ref). + +observe_resp_to_mqtt({error, Error}, _CoapPayload, _Format, _ObserveSeqNum, Ref) -> + make_response(Error, Ref); + +observe_resp_to_mqtt({ok, content}, CoapPayload, Format, 0, Ref) -> + read_resp_to_mqtt({ok, content}, CoapPayload, Format, Ref); + +observe_resp_to_mqtt({ok, content}, CoapPayload, Format, ObserveSeqNum, Ref) -> + read_resp_to_mqtt({ok, content}, CoapPayload, Format, Ref#{<<"seqNum">> => ObserveSeqNum}). + +cancel_observe_resp_to_mqtt({ok, content}, CoapPayload, Format, Ref) -> + read_resp_to_mqtt({ok, content}, CoapPayload, Format, Ref); + +cancel_observe_resp_to_mqtt({error, Error}, _CoapPayload, _Format, Ref) -> + make_response(Error, Ref). + +make_response(Code, Ref=#{}) -> + BaseRsp = make_base_response(Ref), + make_data_response(BaseRsp, Code). + +make_response(Code, Ref=#{}, _Format, Result) -> + BaseRsp = make_base_response(Ref), + make_data_response(BaseRsp, Code, _Format, Result). + +%% The base response format is what included in the request: +%% +%% #{ +%% <<"seqNum">> => SeqNum, +%% <<"imsi">> => maps:get(<<"imsi">>, Ref, null), +%% <<"imei">> => maps:get(<<"imei">>, Ref, null), +%% <<"requestID">> => maps:get(<<"requestID">>, Ref, null), +%% <<"cacheID">> => maps:get(<<"cacheID">>, Ref, null), +%% <<"msgType">> => maps:get(<<"msgType">>, Ref, null) +%% } + +make_base_response(Ref=#{}) -> + remove_tmp_fields(Ref). + +make_data_response(BaseRsp, Code) -> + BaseRsp#{ + <<"data">> => #{ + <<"reqPath">> => extract_path(BaseRsp), + <<"code">> => code(Code), + <<"codeMsg">> => Code + } + }. + +make_data_response(BaseRsp, Code, _Format, Result) -> + BaseRsp#{ + <<"data">> => + #{ + <<"reqPath">> => extract_path(BaseRsp), + <<"code">> => code(Code), + <<"codeMsg">> => Code, + <<"content">> => Result + } + }. + +remove_tmp_fields(Ref) -> + maps:remove(observe_type, Ref). + +-spec path_list(Path::binary()) -> {[PathWord::binary()], [Query::binary()]}. +path_list(Path) -> + case binary:split(binary_util:trim(Path, $/), [<<$/>>], [global]) of + [ObjId, ObjInsId, ResId, LastPart] -> + {ResInstId, QueryList} = query_list(LastPart), + {[ObjId, ObjInsId, ResId, ResInstId], QueryList}; + [ObjId, ObjInsId, LastPart] -> + {ResId, QueryList} = query_list(LastPart), + {[ObjId, ObjInsId, ResId], QueryList}; + [ObjId, LastPart] -> + {ObjInsId, QueryList} = query_list(LastPart), + {[ObjId, ObjInsId], QueryList}; + [LastPart] -> + {ObjId, QueryList} = query_list(LastPart), + {[ObjId], QueryList} + end. + +query_list(PathWithQuery) -> + case binary:split(PathWithQuery, [<<$?>>], []) of + [Path] -> {Path, []}; + [Path, Querys] -> + {Path, binary:split(Querys, [<<$&>>], [global])} + end. + +attr_query_list(Data) -> + attr_query_list(Data, valid_attr_keys(), []). + +attr_query_list(QueryJson = #{}, ValidAttrKeys, QueryList) -> + maps:fold( + fun + (_K, null, Acc) -> Acc; + (K, V, Acc) -> + case lists:member(K, ValidAttrKeys) of + true -> + KV = <>, <<"pmax">>, <<"gt">>, <<"lt">>, <<"st">>]. + +data_format(Options) -> + maps:get(content_format, Options, <<"text/plain">>). + +observe_seq(Options) -> + maps:get(observe, Options, rand:uniform(1000000) + 1 ). + +add_alternate_path_prefix(<<"/">>, PathList) -> + PathList; + +add_alternate_path_prefix(AlternatePath, PathList) -> + [binary_util:trim(AlternatePath, $/) | PathList]. + +extract_path(Ref = #{}) -> + drop_query( + case Ref of + #{<<"data">> := Data} -> + case maps:get(<<"path">>, Data, undefined) of + undefined -> maps:get(<<"basePath">>, Data, undefined); + Path -> Path + end; + #{<<"path">> := Path} -> + Path + end). + + +batch_write_request(AlternatePath, BasePath, Content) -> + {PathList, QueryList} = path_list(BasePath), + Method = case length(PathList) of + 2 -> post; + 3 -> put + end, + FullPathList = add_alternate_path_prefix(AlternatePath, PathList), + TlvData = emqx_lwm2m_message:json_to_tlv(PathList, Content), + Payload = emqx_lwm2m_tlv:encode(TlvData), + emqx_coap_message:request(con, Method, Payload, + [{uri_path, FullPathList}, + {uri_query, QueryList}, + {content_format, <<"application/vnd.oma.lwm2m+tlv">>}]). + +single_write_request(AlternatePath, Data) -> + {PathList, QueryList} = path_list(maps:get(<<"path">>, Data)), + FullPathList = add_alternate_path_prefix(AlternatePath, PathList), + %% TO DO: handle write to resource instance, e.g. /4/0/1/0 + TlvData = emqx_lwm2m_message:json_to_tlv(PathList, [Data]), + Payload = emqx_lwm2m_tlv:encode(TlvData), + emqx_coap_message:request(con, put, Payload, + [{uri_path, FullPathList}, + {uri_query, QueryList}, + {content_format, <<"application/vnd.oma.lwm2m+tlv">>}]). + +drop_query(Path) -> + case binary:split(Path, [<<$?>>]) of + [Path] -> Path; + [PathOnly, _Query] -> PathOnly + end. + +code(get) -> <<"0.01">>; +code(post) -> <<"0.02">>; +code(put) -> <<"0.03">>; +code(delete) -> <<"0.04">>; +code(created) -> <<"2.01">>; +code(deleted) -> <<"2.02">>; +code(valid) -> <<"2.03">>; +code(changed) -> <<"2.04">>; +code(content) -> <<"2.05">>; +code(continue) -> <<"2.31">>; +code(bad_request) -> <<"4.00">>; +code(unauthorized) -> <<"4.01">>; +code(bad_option) -> <<"4.02">>; +code(forbidden) -> <<"4.03">>; +code(not_found) -> <<"4.04">>; +code(method_not_allowed) -> <<"4.05">>; +code(not_acceptable) -> <<"4.06">>; +code(request_entity_incomplete) -> <<"4.08">>; +code(precondition_failed) -> <<"4.12">>; +code(request_entity_too_large) -> <<"4.13">>; +code(unsupported_content_format) -> <<"4.15">>; +code(internal_server_error) -> <<"5.00">>; +code(not_implemented) -> <<"5.01">>; +code(bad_gateway) -> <<"5.02">>; +code(service_unavailable) -> <<"5.03">>; +code(gateway_timeout) -> <<"5.04">>; +code(proxying_not_supported) -> <<"5.05">>. diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_cmd_handler.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_cmd_handler.erl deleted file mode 100644 index 318328e3c..000000000 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_cmd_handler.erl +++ /dev/null @@ -1,310 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_lwm2m_cmd_handler). - --include("src/lwm2m/include/emqx_lwm2m.hrl"). - --include_lib("lwm2m_coap/include/coap.hrl"). - --export([ mqtt2coap/2 - , coap2mqtt/4 - , ack2mqtt/1 - , extract_path/1 - ]). - --export([path_list/1]). - --define(LOG(Level, Format, Args), logger:Level("LWM2M-CMD: " ++ Format, Args)). - -mqtt2coap(AlternatePath, InputCmd = #{<<"msgType">> := <<"create">>, <<"data">> := Data}) -> - PathList = path_list(maps:get(<<"basePath">>, Data, <<"/">>)), - FullPathList = add_alternate_path_prefix(AlternatePath, PathList), - TlvData = emqx_lwm2m_message:json_to_tlv(PathList, maps:get(<<"content">>, Data)), - Payload = emqx_lwm2m_tlv:encode(TlvData), - CoapRequest = lwm2m_coap_message:request(con, post, Payload, [{uri_path, FullPathList}, - {content_format, <<"application/vnd.oma.lwm2m+tlv">>}]), - {CoapRequest, InputCmd}; -mqtt2coap(AlternatePath, InputCmd = #{<<"msgType">> := <<"delete">>, <<"data">> := Data}) -> - FullPathList = add_alternate_path_prefix(AlternatePath, path_list(maps:get(<<"path">>, Data))), - {lwm2m_coap_message:request(con, delete, <<>>, [{uri_path, FullPathList}]), InputCmd}; -mqtt2coap(AlternatePath, InputCmd = #{<<"msgType">> := <<"read">>, <<"data">> := Data}) -> - FullPathList = add_alternate_path_prefix(AlternatePath, path_list(maps:get(<<"path">>, Data))), - {lwm2m_coap_message:request(con, get, <<>>, [{uri_path, FullPathList}]), InputCmd}; -mqtt2coap(AlternatePath, InputCmd = #{<<"msgType">> := <<"write">>, <<"data">> := Data}) -> - Encoding = maps:get(<<"encoding">>, InputCmd, <<"plain">>), - CoapRequest = - case maps:get(<<"basePath">>, Data, <<"/">>) of - <<"/">> -> - single_write_request(AlternatePath, Data, Encoding); - BasePath -> - batch_write_request(AlternatePath, BasePath, maps:get(<<"content">>, Data), Encoding) - end, - {CoapRequest, InputCmd}; - -mqtt2coap(AlternatePath, InputCmd = #{<<"msgType">> := <<"execute">>, <<"data">> := Data}) -> - FullPathList = add_alternate_path_prefix(AlternatePath, path_list(maps:get(<<"path">>, Data))), - Args = - case maps:get(<<"args">>, Data, <<>>) of - <<"undefined">> -> <<>>; - undefined -> <<>>; - Arg1 -> Arg1 - end, - {lwm2m_coap_message:request(con, post, Args, [{uri_path, FullPathList}, {content_format, <<"text/plain">>}]), InputCmd}; -mqtt2coap(AlternatePath, InputCmd = #{<<"msgType">> := <<"discover">>, <<"data">> := Data}) -> - FullPathList = add_alternate_path_prefix(AlternatePath, path_list(maps:get(<<"path">>, Data))), - {lwm2m_coap_message:request(con, get, <<>>, [{uri_path, FullPathList}, {'accept', ?LWM2M_FORMAT_LINK}]), InputCmd}; -mqtt2coap(AlternatePath, InputCmd = #{<<"msgType">> := <<"write-attr">>, <<"data">> := Data}) -> - FullPathList = add_alternate_path_prefix(AlternatePath, path_list(maps:get(<<"path">>, Data))), - Query = attr_query_list(Data), - {lwm2m_coap_message:request(con, put, <<>>, [{uri_path, FullPathList}, {uri_query, Query}]), InputCmd}; -mqtt2coap(AlternatePath, InputCmd = #{<<"msgType">> := <<"observe">>, <<"data">> := Data}) -> - PathList = path_list(maps:get(<<"path">>, Data)), - FullPathList = add_alternate_path_prefix(AlternatePath, PathList), - {lwm2m_coap_message:request(con, get, <<>>, [{uri_path, FullPathList}, {observe, 0}]), InputCmd}; -mqtt2coap(AlternatePath, InputCmd = #{<<"msgType">> := <<"cancel-observe">>, <<"data">> := Data}) -> - PathList = path_list(maps:get(<<"path">>, Data)), - FullPathList = add_alternate_path_prefix(AlternatePath, PathList), - {lwm2m_coap_message:request(con, get, <<>>, [{uri_path, FullPathList}, {observe, 1}]), InputCmd}. - -coap2mqtt(_Method = {_, Code}, _CoapPayload, _Options, Ref=#{<<"msgType">> := <<"create">>}) -> - make_response(Code, Ref); -coap2mqtt(_Method = {_, Code}, _CoapPayload, _Options, Ref=#{<<"msgType">> := <<"delete">>}) -> - make_response(Code, Ref); -coap2mqtt(Method, CoapPayload, Options, Ref=#{<<"msgType">> := <<"read">>}) -> - coap_read_to_mqtt(Method, CoapPayload, data_format(Options), Ref); -coap2mqtt(Method, _CoapPayload, _Options, Ref=#{<<"msgType">> := <<"write">>}) -> - coap_write_to_mqtt(Method, Ref); -coap2mqtt(Method, _CoapPayload, _Options, Ref=#{<<"msgType">> := <<"execute">>}) -> - coap_execute_to_mqtt(Method, Ref); -coap2mqtt(Method, CoapPayload, _Options, Ref=#{<<"msgType">> := <<"discover">>}) -> - coap_discover_to_mqtt(Method, CoapPayload, Ref); -coap2mqtt(Method, CoapPayload, _Options, Ref=#{<<"msgType">> := <<"write-attr">>}) -> - coap_writeattr_to_mqtt(Method, CoapPayload, Ref); -coap2mqtt(Method, CoapPayload, Options, Ref=#{<<"msgType">> := <<"observe">>}) -> - coap_observe_to_mqtt(Method, CoapPayload, data_format(Options), observe_seq(Options), Ref); -coap2mqtt(Method, CoapPayload, Options, Ref=#{<<"msgType">> := <<"cancel-observe">>}) -> - coap_cancel_observe_to_mqtt(Method, CoapPayload, data_format(Options), Ref). - -coap_read_to_mqtt({error, ErrorCode}, _CoapPayload, _Format, Ref) -> - make_response(ErrorCode, Ref); -coap_read_to_mqtt({ok, SuccessCode}, CoapPayload, Format, Ref) -> - try - Result = coap_content_to_mqtt_payload(CoapPayload, Format, Ref), - make_response(SuccessCode, Ref, Format, Result) - catch - error:not_implemented -> make_response(not_implemented, Ref); - C:R:Stack -> - ?LOG(error, "~p, bad payload format: ~p, stacktrace: ~p", [{C, R}, CoapPayload, Stack]), - make_response(bad_request, Ref) - end. - -ack2mqtt(Ref) -> - make_base_response(Ref). - -coap_content_to_mqtt_payload(CoapPayload, <<"text/plain">>, Ref) -> - emqx_lwm2m_message:text_to_json(extract_path(Ref), CoapPayload); -coap_content_to_mqtt_payload(CoapPayload, <<"application/octet-stream">>, Ref) -> - emqx_lwm2m_message:opaque_to_json(extract_path(Ref), CoapPayload); -coap_content_to_mqtt_payload(CoapPayload, <<"application/vnd.oma.lwm2m+tlv">>, Ref) -> - emqx_lwm2m_message:tlv_to_json(extract_path(Ref), CoapPayload); -coap_content_to_mqtt_payload(CoapPayload, <<"application/vnd.oma.lwm2m+json">>, _Ref) -> - emqx_lwm2m_message:translate_json(CoapPayload). - -coap_write_to_mqtt({ok, changed}, Ref) -> - make_response(changed, Ref); -coap_write_to_mqtt({error, Error}, Ref) -> - make_response(Error, Ref). - -coap_execute_to_mqtt({ok, changed}, Ref) -> - make_response(changed, Ref); -coap_execute_to_mqtt({error, Error}, Ref) -> - make_response(Error, Ref). - -coap_discover_to_mqtt({ok, content}, CoapPayload, Ref) -> - Links = binary:split(CoapPayload, <<",">>), - make_response(content, Ref, <<"application/link-format">>, Links); -coap_discover_to_mqtt({error, Error}, _CoapPayload, Ref) -> - make_response(Error, Ref). - -coap_writeattr_to_mqtt({ok, changed}, _CoapPayload, Ref) -> - make_response(changed, Ref); -coap_writeattr_to_mqtt({error, Error}, _CoapPayload, Ref) -> - make_response(Error, Ref). - -coap_observe_to_mqtt({error, Error}, _CoapPayload, _Format, _ObserveSeqNum, Ref) -> - make_response(Error, Ref); -coap_observe_to_mqtt({ok, content}, CoapPayload, Format, 0, Ref) -> - coap_read_to_mqtt({ok, content}, CoapPayload, Format, Ref); -coap_observe_to_mqtt({ok, content}, CoapPayload, Format, ObserveSeqNum, Ref) -> - RefWithObserve = maps:put(<<"seqNum">>, ObserveSeqNum, Ref), - RefNotify = maps:put(<<"msgType">>, <<"notify">>, RefWithObserve), - coap_read_to_mqtt({ok, content}, CoapPayload, Format, RefNotify). - -coap_cancel_observe_to_mqtt({ok, content}, CoapPayload, Format, Ref) -> - coap_read_to_mqtt({ok, content}, CoapPayload, Format, Ref); -coap_cancel_observe_to_mqtt({error, Error}, _CoapPayload, _Format, Ref) -> - make_response(Error, Ref). - -make_response(Code, Ref=#{}) -> - BaseRsp = make_base_response(Ref), - make_data_response(BaseRsp, Code). -make_response(Code, Ref=#{}, _Format, Result) -> - BaseRsp = make_base_response(Ref), - make_data_response(BaseRsp, Code, _Format, Result). - -%% The base response format is what included in the request: -%% -%% #{ -%% <<"seqNum">> => SeqNum, -%% <<"requestID">> => maps:get(<<"requestID">>, Ref, null), -%% <<"cacheID">> => maps:get(<<"cacheID">>, Ref, null), -%% <<"msgType">> => maps:get(<<"msgType">>, Ref, null) -%% } - -make_base_response(Ref=#{}) -> - remove_tmp_fields(Ref). - -make_data_response(BaseRsp, Code) -> - BaseRsp#{ - <<"data">> => #{ - <<"reqPath">> => extract_path(BaseRsp), - <<"code">> => code(Code), - <<"codeMsg">> => Code - } - }. -make_data_response(BaseRsp, Code, _Format, Result) -> - BaseRsp#{ - <<"data">> => #{ - <<"reqPath">> => extract_path(BaseRsp), - <<"code">> => code(Code), - <<"codeMsg">> => Code, - <<"content">> => Result - } - }. - -remove_tmp_fields(Ref) -> - maps:remove(observe_type, Ref). - -path_list(Path) -> - case binary:split(binary_util:trim(Path, $/), [<<$/>>], [global]) of - [ObjId, ObjInsId, ResId, ResInstId] -> [ObjId, ObjInsId, ResId, ResInstId]; - [ObjId, ObjInsId, ResId] -> [ObjId, ObjInsId, ResId]; - [ObjId, ObjInsId] -> [ObjId, ObjInsId]; - [ObjId] -> [ObjId] - end. - -attr_query_list(Data) -> - attr_query_list(Data, valid_attr_keys(), []). -attr_query_list(QueryJson = #{}, ValidAttrKeys, QueryList) -> - maps:fold( - fun - (_K, null, Acc) -> Acc; - (K, V, Acc) -> - case lists:member(K, ValidAttrKeys) of - true -> - Val = bin(V), - KV = <>, <<"pmax">>, <<"gt">>, <<"lt">>, <<"st">>]. - -data_format(Options) -> - proplists:get_value(content_format, Options, <<"text/plain">>). -observe_seq(Options) -> - proplists:get_value(observe, Options, rand:uniform(1000000) + 1 ). - -add_alternate_path_prefix(<<"/">>, PathList) -> - PathList; -add_alternate_path_prefix(AlternatePath, PathList) -> - [binary_util:trim(AlternatePath, $/) | PathList]. - -extract_path(Ref = #{}) -> - case Ref of - #{<<"data">> := Data} -> - case maps:get(<<"path">>, Data, nil) of - nil -> maps:get(<<"basePath">>, Data, undefined); - Path -> Path - end; - #{<<"path">> := Path} -> - Path - end. - -batch_write_request(AlternatePath, BasePath, Content, Encoding) -> - PathList = path_list(BasePath), - Method = case length(PathList) of - 2 -> post; - 3 -> put - end, - FullPathList = add_alternate_path_prefix(AlternatePath, PathList), - Content1 = decoding(Content, Encoding), - TlvData = emqx_lwm2m_message:json_to_tlv(PathList, Content1), - Payload = emqx_lwm2m_tlv:encode(TlvData), - lwm2m_coap_message:request(con, Method, Payload, [{uri_path, FullPathList}, {content_format, <<"application/vnd.oma.lwm2m+tlv">>}]). - -single_write_request(AlternatePath, Data, Encoding) -> - PathList = path_list(maps:get(<<"path">>, Data)), - FullPathList = add_alternate_path_prefix(AlternatePath, PathList), - Datas = decoding([Data], Encoding), - TlvData = emqx_lwm2m_message:json_to_tlv(PathList, Datas), - Payload = emqx_lwm2m_tlv:encode(TlvData), - lwm2m_coap_message:request(con, put, Payload, [{uri_path, FullPathList}, {content_format, <<"application/vnd.oma.lwm2m+tlv">>}]). - - -code(get) -> <<"0.01">>; -code(post) -> <<"0.02">>; -code(put) -> <<"0.03">>; -code(delete) -> <<"0.04">>; -code(created) -> <<"2.01">>; -code(deleted) -> <<"2.02">>; -code(valid) -> <<"2.03">>; -code(changed) -> <<"2.04">>; -code(content) -> <<"2.05">>; -code(continue) -> <<"2.31">>; -code(bad_request) -> <<"4.00">>; -code(uauthorized) -> <<"4.01">>; -code(bad_option) -> <<"4.02">>; -code(forbidden) -> <<"4.03">>; -code(not_found) -> <<"4.04">>; -code(method_not_allowed) -> <<"4.05">>; -code(not_acceptable) -> <<"4.06">>; -code(request_entity_incomplete) -> <<"4.08">>; -code(precondition_failed) -> <<"4.12">>; -code(request_entity_too_large) -> <<"4.13">>; -code(unsupported_content_format) -> <<"4.15">>; -code(internal_server_error) -> <<"5.00">>; -code(not_implemented) -> <<"5.01">>; -code(bad_gateway) -> <<"5.02">>; -code(service_unavailable) -> <<"5.03">>; -code(gateway_timeout) -> <<"5.04">>; -code(proxying_not_supported) -> <<"5.05">>. - -bin(Bin) when is_binary(Bin) -> Bin; -bin(Str) when is_list(Str) -> list_to_binary(Str); -bin(Int) when is_integer(Int) -> integer_to_binary(Int); -bin(Float) when is_float(Float) -> float_to_binary(Float). - -decoding(Datas, <<"hex">>) -> - lists:map(fun(Data = #{<<"value">> := Value}) -> - Data#{<<"value">> => emqx_misc:hexstr2bin(Value)} - end, Datas); -decoding(Datas, _) -> - Datas. diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_coap_resource.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_coap_resource.erl deleted file mode 100644 index 588dd523e..000000000 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_coap_resource.erl +++ /dev/null @@ -1,386 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_lwm2m_coap_resource). - --include_lib("emqx/include/emqx.hrl"). - --include_lib("emqx/include/emqx_mqtt.hrl"). - --include_lib("lwm2m_coap/include/coap.hrl"). - -% -behaviour(lwm2m_coap_resource). - --export([ coap_discover/2 - , coap_get/5 - , coap_post/5 - , coap_put/5 - , coap_delete/4 - , coap_observe/5 - , coap_unobserve/1 - , coap_response/7 - , coap_ack/3 - , handle_info/2 - , handle_call/3 - , handle_cast/2 - , terminate/2 - ]). - --export([parse_object_list/1]). - --include("src/lwm2m/include/emqx_lwm2m.hrl"). - --define(PREFIX, <<"rd">>). - --define(LOG(Level, Format, Args), logger:Level("LWM2M-RESOURCE: " ++ Format, Args)). - --dialyzer([{nowarn_function, [coap_discover/2]}]). -% we use {'absolute', list(binary()), [{atom(), binary()}]} as coap_uri() -% https://github.com/emqx/lwm2m-coap/blob/258e9bd3762124395e83c1e68a1583b84718230f/src/lwm2m_coap_resource.erl#L61 -% resource operations -coap_discover(_Prefix, _Args) -> - [{absolute, [<<"mqtt">>], []}]. - -coap_get(ChId, [?PREFIX], Query, Content, Lwm2mState) -> - ?LOG(debug, "~p ~p GET Query=~p, Content=~p", [self(),ChId, Query, Content]), - {ok, #coap_content{}, Lwm2mState}; -coap_get(ChId, Prefix, Query, Content, Lwm2mState) -> - ?LOG(error, "ignore bad put request ChId=~p, Prefix=~p, Query=~p, Content=~p", [ChId, Prefix, Query, Content]), - {error, bad_request, Lwm2mState}. - -% LWM2M REGISTER COMMAND -coap_post(ChId, [?PREFIX], Query, Content = #coap_content{uri_path = [?PREFIX]}, Lwm2mState) -> - ?LOG(debug, "~p ~p REGISTER command Query=~p, Content=~p", [self(), ChId, Query, Content]), - case parse_options(Query) of - {error, {bad_opt, _CustomOption}} -> - ?LOG(error, "Reject REGISTER from ~p due to wrong option", [ChId]), - {error, bad_request, Lwm2mState}; - {ok, LwM2MQuery} -> - process_register(ChId, LwM2MQuery, Content#coap_content.payload, Lwm2mState) - end; - -% LWM2M UPDATE COMMAND -coap_post(ChId, [?PREFIX], Query, Content = #coap_content{uri_path = LocationPath}, Lwm2mState) -> - ?LOG(debug, "~p ~p UPDATE command location=~p, Query=~p, Content=~p", [self(), ChId, LocationPath, Query, Content]), - case parse_options(Query) of - {error, {bad_opt, _CustomOption}} -> - ?LOG(error, "Reject UPDATE from ~p due to wrong option, Query=~p", [ChId, Query]), - {error, bad_request, Lwm2mState}; - {ok, LwM2MQuery} -> - process_update(ChId, LwM2MQuery, LocationPath, Content#coap_content.payload, Lwm2mState) - end; - -coap_post(ChId, Prefix, Query, Content, Lwm2mState) -> - ?LOG(error, "bad post request ChId=~p, Prefix=~p, Query=~p, Content=~p", [ChId, Prefix, Query, Content]), - {error, bad_request, Lwm2mState}. - -coap_put(_ChId, Prefix, Query, Content, Lwm2mState) -> - ?LOG(error, "put has error, Prefix=~p, Query=~p, Content=~p", [Prefix, Query, Content]), - {error, bad_request, Lwm2mState}. - -% LWM2M DE-REGISTER COMMAND -coap_delete(ChId, [?PREFIX], #coap_content{uri_path = Location}, Lwm2mState) -> - LocationPath = binary_util:join_path(Location), - ?LOG(debug, "~p ~p DELETE command location=~p", [self(), ChId, LocationPath]), - case get(lwm2m_context) of - #lwm2m_context{location = LocationPath} -> - lwm2m_coap_responder:stop(deregister), - {ok, Lwm2mState}; - undefined -> - ?LOG(error, "Reject DELETE from ~p, Location: ~p not found", [ChId, Location]), - {error, forbidden, Lwm2mState}; - TrueLocation -> - ?LOG(error, "Reject DELETE from ~p, Wrong Location: ~p, registered location record: ~p", [ChId, Location, TrueLocation]), - {error, not_found, Lwm2mState} - end; -coap_delete(_ChId, _Prefix, _Content, Lwm2mState) -> - {error, forbidden, Lwm2mState}. - -coap_observe(ChId, Prefix, Name, Ack, Lwm2mState) -> - ?LOG(error, "unsupported observe request ChId=~p, Prefix=~p, Name=~p, Ack=~p", [ChId, Prefix, Name, Ack]), - {error, method_not_allowed, Lwm2mState}. - -coap_unobserve(Lwm2mState) -> - ?LOG(error, "unsupported unobserve request: ~p", [Lwm2mState]), - {ok, Lwm2mState}. - -coap_response(ChId, Ref, CoapMsgType, CoapMsgMethod, CoapMsgPayload, CoapMsgOpts, Lwm2mState) -> - ?LOG(info, "~p, RCV CoAP response, CoapMsgType: ~p, CoapMsgMethod: ~p, CoapMsgPayload: ~p, - CoapMsgOpts: ~p, Ref: ~p", - [ChId, CoapMsgType, CoapMsgMethod, CoapMsgPayload, CoapMsgOpts, Ref]), - MqttPayload = emqx_lwm2m_cmd_handler:coap2mqtt(CoapMsgMethod, CoapMsgPayload, CoapMsgOpts, Ref), - Lwm2mState2 = emqx_lwm2m_protocol:send_ul_data(maps:get(<<"msgType">>, MqttPayload), MqttPayload, Lwm2mState), - {noreply, Lwm2mState2}. - -coap_ack(_ChId, Ref, Lwm2mState) -> - ?LOG(info, "~p, RCV CoAP Empty ACK, Ref: ~p", [_ChId, Ref]), - AckRef = maps:put(<<"msgType">>, <<"ack">>, Ref), - MqttPayload = emqx_lwm2m_cmd_handler:ack2mqtt(AckRef), - Lwm2mState2 = emqx_lwm2m_protocol:send_ul_data(maps:get(<<"msgType">>, MqttPayload), MqttPayload, Lwm2mState), - {ok, Lwm2mState2}. - -%% Batch deliver -handle_info({deliver, Topic, Msgs}, Lwm2mState) when is_list(Msgs) -> - {noreply, lists:foldl(fun(Msg, NewState) -> - element(2, handle_info({deliver, Topic, Msg}, NewState)) - end, Lwm2mState, Msgs)}; -%% Handle MQTT Message -handle_info({deliver, _Topic, MqttMsg}, Lwm2mState) -> - Lwm2mState2 = emqx_lwm2m_protocol:deliver(MqttMsg, Lwm2mState), - {noreply, Lwm2mState2}; - -%% Deliver Coap Message to Device -handle_info({deliver_to_coap, CoapRequest, Ref}, Lwm2mState) -> - {send_request, CoapRequest, Ref, Lwm2mState}; - -handle_info({'EXIT', _Pid, Reason}, Lwm2mState) -> - ?LOG(info, "~p, received exit from: ~p, reason: ~p, quit now!", [self(), _Pid, Reason]), - {stop, Reason, Lwm2mState}; - -handle_info(post_init, Lwm2mState) -> - Lwm2mState2 = emqx_lwm2m_protocol:post_init(Lwm2mState), - {noreply, Lwm2mState2}; - -handle_info(auto_observe, Lwm2mState) -> - Lwm2mState2 = emqx_lwm2m_protocol:auto_observe(Lwm2mState), - {noreply, Lwm2mState2}; - -handle_info({life_timer, expired}, Lwm2mState) -> - ?LOG(debug, "lifetime expired, shutdown", []), - {stop, life_timer_expired, Lwm2mState}; - -handle_info({shutdown, Error}, Lwm2mState) -> - {stop, Error, Lwm2mState}; - -handle_info({shutdown, conflict, {ClientId, NewPid}}, Lwm2mState) -> - ?LOG(warning, "lwm2m '~s' conflict with ~p, shutdown", [ClientId, NewPid]), - {stop, conflict, Lwm2mState}; - -handle_info({suback, _MsgId, [_GrantedQos]}, Lwm2mState) -> - {noreply, Lwm2mState}; - -handle_info(emit_stats, Lwm2mState) -> - {noreply, Lwm2mState}; - -handle_info(Message, Lwm2mState) -> - ?LOG(error, "Unknown Message ~p", [Message]), - {noreply, Lwm2mState}. - - -handle_call(info, _From, Lwm2mState) -> - {Info, Lwm2mState2} = emqx_lwm2m_protocol:get_info(Lwm2mState), - {reply, Info, Lwm2mState2}; - -handle_call(stats, _From, Lwm2mState) -> - {Stats, Lwm2mState2} = emqx_lwm2m_protocol:get_stats(Lwm2mState), - {reply, Stats, Lwm2mState2}; - -handle_call(kick, _From, Lwm2mState) -> - {stop, kick, Lwm2mState}; - -handle_call({set_rate_limit, _Rl}, _From, Lwm2mState) -> - ?LOG(error, "set_rate_limit is not support", []), - {reply, ok, Lwm2mState}; - -handle_call(get_rate_limit, _From, Lwm2mState) -> - ?LOG(error, "get_rate_limit is not support", []), - {reply, ok, Lwm2mState}; - -handle_call(session, _From, Lwm2mState) -> - ?LOG(error, "get_session is not support", []), - {reply, ok, Lwm2mState}; - -handle_call(Request, _From, Lwm2mState) -> - ?LOG(error, "adapter unexpected call ~p", [Request]), - {reply, ok, Lwm2mState}. - -handle_cast(Msg, Lwm2mState) -> - ?LOG(error, "unexpected cast ~p", [Msg]), - {noreply, Lwm2mState, hibernate}. - -terminate(Reason, Lwm2mState) -> - emqx_lwm2m_protocol:terminate(Reason, Lwm2mState). - -%%%%%%%%%%%%%%%%%%%%%% -%% Internal Functions -%%%%%%%%%%%%%%%%%%%%%% -process_register(ChId, LwM2MQuery, LwM2MPayload, Lwm2mState) -> - Epn = maps:get(<<"ep">>, LwM2MQuery, undefined), - LifeTime = maps:get(<<"lt">>, LwM2MQuery, undefined), - Ver = maps:get(<<"lwm2m">>, LwM2MQuery, undefined), - case check_lwm2m_version(Ver) of - false -> - ?LOG(error, "Reject REGISTER from ~p due to unsupported version: ~p", [ChId, Ver]), - lwm2m_coap_responder:stop(invalid_version), - {error, precondition_failed, Lwm2mState}; - true -> - case check_epn(Epn) andalso check_lifetime(LifeTime) of - true -> - init_lwm2m_emq_client(ChId, LwM2MQuery, LwM2MPayload, Lwm2mState); - false -> - ?LOG(error, "Reject REGISTER from ~p due to wrong parameters, epn=~p, lifetime=~p", [ChId, Epn, LifeTime]), - lwm2m_coap_responder:stop(invalid_query_params), - {error, bad_request, Lwm2mState} - end - end. - -process_update(ChId, LwM2MQuery, Location, LwM2MPayload, Lwm2mState) -> - LocationPath = binary_util:join_path(Location), - case get(lwm2m_context) of - #lwm2m_context{location = LocationPath} -> - RegInfo = append_object_list(LwM2MQuery, LwM2MPayload), - Lwm2mState2 = emqx_lwm2m_protocol:update_reg_info(RegInfo, Lwm2mState), - ?LOG(info, "~p, UPDATE Success, assgined location: ~p", [ChId, LocationPath]), - {ok, changed, #coap_content{}, Lwm2mState2}; - undefined -> - ?LOG(error, "Reject UPDATE from ~p, Location: ~p not found", [ChId, Location]), - {error, forbidden, Lwm2mState}; - TrueLocation -> - ?LOG(error, "Reject UPDATE from ~p, Wrong Location: ~p, registered location record: ~p", [ChId, Location, TrueLocation]), - {error, not_found, Lwm2mState} - end. - -init_lwm2m_emq_client(ChId, LwM2MQuery = #{<<"ep">> := Epn}, LwM2MPayload, _Lwm2mState = undefined) -> - RegInfo = append_object_list(LwM2MQuery, LwM2MPayload), - case emqx_lwm2m_protocol:init(self(), Epn, ChId, RegInfo) of - {ok, Lwm2mState} -> - LocationPath = assign_location_path(Epn), - ?LOG(info, "~p, REGISTER Success, assgined location: ~p", [ChId, LocationPath]), - {ok, created, #coap_content{location_path = LocationPath}, Lwm2mState}; - {error, Error} -> - lwm2m_coap_responder:stop(Error), - ?LOG(error, "~p, REGISTER Failed, error: ~p", [ChId, Error]), - {error, forbidden, undefined} - end; -init_lwm2m_emq_client(ChId, LwM2MQuery = #{<<"ep">> := Epn}, LwM2MPayload, Lwm2mState) -> - RegInfo = append_object_list(LwM2MQuery, LwM2MPayload), - LocationPath = assign_location_path(Epn), - ?LOG(info, "~p, RE-REGISTER Success, location: ~p", [ChId, LocationPath]), - Lwm2mState2 = emqx_lwm2m_protocol:replace_reg_info(RegInfo, Lwm2mState), - {ok, created, #coap_content{location_path = LocationPath}, Lwm2mState2}. - -append_object_list(LwM2MQuery, <<>>) when map_size(LwM2MQuery) == 0 -> #{}; -append_object_list(LwM2MQuery, <<>>) -> LwM2MQuery; -append_object_list(LwM2MQuery, LwM2MPayload) when is_binary(LwM2MPayload) -> - {AlterPath, ObjList} = parse_object_list(LwM2MPayload), - LwM2MQuery#{ - <<"alternatePath">> => AlterPath, - <<"objectList">> => ObjList - }. - -parse_options(InputQuery) -> - parse_options(InputQuery, maps:new()). - -parse_options([], Query) -> {ok, Query}; -parse_options([<<"ep=", Epn/binary>>|T], Query) -> - parse_options(T, maps:put(<<"ep">>, Epn, Query)); -parse_options([<<"lt=", Lt/binary>>|T], Query) -> - parse_options(T, maps:put(<<"lt">>, binary_to_integer(Lt), Query)); -parse_options([<<"lwm2m=", Ver/binary>>|T], Query) -> - parse_options(T, maps:put(<<"lwm2m">>, Ver, Query)); -parse_options([<<"b=", Binding/binary>>|T], Query) -> - parse_options(T, maps:put(<<"b">>, Binding, Query)); -parse_options([CustomOption|T], Query) -> - case binary:split(CustomOption, <<"=">>) of - [OptKey, OptValue] when OptKey =/= <<>> -> - ?LOG(debug, "non-standard option: ~p", [CustomOption]), - parse_options(T, maps:put(OptKey, OptValue, Query)); - _BadOpt -> - ?LOG(error, "bad option: ~p", [CustomOption]), - {error, {bad_opt, CustomOption}} - end. - -parse_object_list(<<>>) -> {<<"/">>, <<>>}; -parse_object_list(ObjLinks) when is_binary(ObjLinks) -> - parse_object_list(binary:split(ObjLinks, <<",">>, [global])); - -parse_object_list(FullObjLinkList) when is_list(FullObjLinkList) -> - case drop_attr(FullObjLinkList) of - {<<"/">>, _} = RootPrefixedLinks -> - RootPrefixedLinks; - {AlterPath, ObjLinkList} -> - LenAlterPath = byte_size(AlterPath), - WithOutPrefix = - lists:map( - fun - (<>) when Prefix =:= AlterPath -> - trim(Link); - (Link) -> Link - end, ObjLinkList), - {AlterPath, WithOutPrefix} - end. - -drop_attr(LinkList) -> - lists:foldr( - fun(Link, {AlternatePath, LinkAcc}) -> - {MainLink, LinkAttrs} = parse_link(Link), - case is_alternate_path(LinkAttrs) of - false -> {AlternatePath, [MainLink | LinkAcc]}; - true -> {MainLink, LinkAcc} - end - end, {<<"/">>, []}, LinkList). - -is_alternate_path(#{<<"rt">> := ?OMA_ALTER_PATH_RT}) -> true; -is_alternate_path(_) -> false. - -parse_link(Link) -> - [MainLink | Attrs] = binary:split(trim(Link), <<";">>, [global]), - {delink(trim(MainLink)), parse_link_attrs(Attrs)}. - -parse_link_attrs(LinkAttrs) when is_list(LinkAttrs) -> - lists:foldl( - fun(Attr, Acc) -> - case binary:split(trim(Attr), <<"=">>) of - [AttrKey, AttrValue] when AttrKey =/= <<>> -> - maps:put(AttrKey, AttrValue, Acc); - _BadAttr -> throw({bad_attr, _BadAttr}) - end - end, maps:new(), LinkAttrs). - -trim(Str)-> binary_util:trim(Str, $ ). -delink(Str) -> - Ltrim = binary_util:ltrim(Str, $<), - binary_util:rtrim(Ltrim, $>). - -check_lwm2m_version(<<"1">>) -> true; -check_lwm2m_version(<<"1.", _PatchVerNum/binary>>) -> true; -check_lwm2m_version(_) -> false. - -check_epn(undefined) -> false; -check_epn(_) -> true. - -check_lifetime(undefined) -> false; -check_lifetime(LifeTime0) when is_integer(LifeTime0) -> - LifeTime = timer:seconds(LifeTime0), - Envs = proplists:get_value(config, lwm2m_coap_responder:options(), #{}), - Max = maps:get(lifetime_max, Envs, 315360000), - Min = maps:get(lifetime_min, Envs, 0), - - if - LifeTime >= Min, LifeTime =< Max -> - true; - true -> - false - end; -check_lifetime(_) -> false. - - -assign_location_path(Epn) -> - %Location = list_to_binary(io_lib:format("~.16B", [rand:uniform(65535)])), - %LocationPath = <<"/rd/", Location/binary>>, - Location = [<<"rd">>, Epn], - put(lwm2m_context, #lwm2m_context{epn = Epn, location = binary_util:join_path(Location)}), - Location. diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_impl.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_impl.erl index c00f76532..649a14643 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_impl.erl +++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_impl.erl @@ -50,24 +50,13 @@ unreg() -> on_gateway_load(_Gateway = #{ name := GwName, config := Config }, Ctx) -> - - %% Handler - _ = lwm2m_coap_server:start_registry(), - lwm2m_coap_server_registry:add_handler( - [<<"rd">>], - emqx_lwm2m_coap_resource, undefined - ), %% Xml registry {ok, _} = emqx_lwm2m_xml_object_db:start_link(maps:get(xml_dir, Config)), - %% XXX: Self managed table? - %% TODO: Improve it later - {ok, _} = emqx_lwm2m_cm:start_link(), - Listeners = emqx_gateway_utils:normalize_config(Config), ListenerPids = lists:map(fun(Lis) -> - start_listener(GwName, Ctx, Lis) - end, Listeners), + start_listener(GwName, Ctx, Lis) + end, Listeners), {ok, ListenerPids, _GwState = #{ctx => Ctx}}. on_gateway_update(NewGateway, OldGateway, GwState = #{ctx := Ctx}) -> @@ -88,12 +77,6 @@ on_gateway_update(NewGateway, OldGateway, GwState = #{ctx := Ctx}) -> on_gateway_unload(_Gateway = #{ name := GwName, config := Config }, _GwState) -> - %% XXX: - lwm2m_coap_server_registry:remove_handler( - [<<"rd">>], - emqx_lwm2m_coap_resource, undefined - ), - Listeners = emqx_gateway_utils:normalize_config(Config), lists:foreach(fun(Lis) -> stop_listener(GwName, Lis) @@ -107,29 +90,24 @@ start_listener(GwName, Ctx, {Type, LisName, ListenOn, SocketOpts, Cfg}) -> ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn), case start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) of {ok, Pid} -> - ?ULOG("Start ~s:~s:~s listener on ~s successfully.~n", + ?ULOG("Gateway ~s:~s:~s on ~s started.~n", [GwName, Type, LisName, ListenOnStr]), Pid; {error, Reason} -> - ?ELOG("Failed to start ~s:~s:~s listener on ~s: ~0p~n", + ?ELOG("Failed to start gateway ~s:~s:~s on ~s: ~0p~n", [GwName, Type, LisName, ListenOnStr, Reason]), throw({badconf, Reason}) end. start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) -> - Name = name(GwName, LisName, udp), - NCfg = Cfg#{ctx => Ctx}, + Name = emqx_gateway_utils:listener_id(GwName, Type, LisName), + NCfg = Cfg#{ ctx => Ctx + , frame_mod => emqx_coap_frame + , chann_mod => emqx_lwm2m_channel + }, NSocketOpts = merge_default(SocketOpts), - Options = [{config, NCfg}|NSocketOpts], - case Type of - udp -> - lwm2m_coap_server:start_udp(Name, ListenOn, Options); - dtls -> - lwm2m_coap_server:start_dtls(Name, ListenOn, Options) - end. - -name(GwName, LisName, Type) -> - list_to_atom(lists:concat([GwName, ":", Type, ":", LisName])). + MFA = {emqx_gateway_conn, start_link, [NCfg]}, + do_start_listener(Type, Name, ListenOn, NSocketOpts, MFA). merge_default(Options) -> Default = emqx_gateway_utils:default_udp_options(), @@ -141,23 +119,24 @@ merge_default(Options) -> [{udp_options, Default} | Options] end. +do_start_listener(udp, Name, ListenOn, SocketOpts, MFA) -> + esockd:open_udp(Name, ListenOn, SocketOpts, MFA); + +do_start_listener(dtls, Name, ListenOn, SocketOpts, MFA) -> + esockd:open_dtls(Name, ListenOn, SocketOpts, MFA). + stop_listener(GwName, {Type, LisName, ListenOn, SocketOpts, Cfg}) -> StopRet = stop_listener(GwName, Type, LisName, ListenOn, SocketOpts, Cfg), ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn), case StopRet of - ok -> ?ULOG("Stop ~s:~s:~s listener on ~s successfully.~n", + ok -> ?ULOG("Gateway ~s:~s:~s on ~s stopped.~n", [GwName, Type, LisName, ListenOnStr]); {error, Reason} -> - ?ELOG("Failed to stop ~s:~s:~s listener on ~s: ~0p~n", + ?ELOG("Failed to stop gateway ~s:~s:~s on ~s: ~0p~n", [GwName, Type, LisName, ListenOnStr, Reason]) end, StopRet. stop_listener(GwName, Type, LisName, ListenOn, _SocketOpts, _Cfg) -> - Name = name(GwName, LisName, Type), - case Type of - udp -> - lwm2m_coap_server:stop_udp(Name, ListenOn); - dtls -> - lwm2m_coap_server:stop_dtls(Name, ListenOn) - end. + Name = emqx_gateway_utils:listener_id(GwName, Type, LisName), + esockd:close(Name, ListenOn). diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_json.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_json.erl deleted file mode 100644 index 295c68085..000000000 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_json.erl +++ /dev/null @@ -1,351 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_lwm2m_json). - --export([ tlv_to_json/2 - , json_to_tlv/2 - , text_to_json/2 - , opaque_to_json/2 - ]). - --include("src/lwm2m/include/emqx_lwm2m.hrl"). - --define(LOG(Level, Format, Args), logger:Level("LWM2M-JSON: " ++ Format, Args)). - -tlv_to_json(BaseName, TlvData) -> - DecodedTlv = emqx_lwm2m_tlv:parse(TlvData), - ObjectId = object_id(BaseName), - ObjDefinition = emqx_lwm2m_xml_object:get_obj_def(ObjectId, true), - case DecodedTlv of - [#{tlv_resource_with_value:=Id, value:=Value}] -> - TrueBaseName = basename(BaseName, undefined, undefined, Id, 3), - encode_json(TrueBaseName, tlv_single_resource(Id, Value, ObjDefinition)); - List1 = [#{tlv_resource_with_value:=_Id}, _|_] -> - TrueBaseName = basename(BaseName, undefined, undefined, undefined, 2), - encode_json(TrueBaseName, tlv_level2(<<>>, List1, ObjDefinition, [])); - List2 = [#{tlv_multiple_resource:=_Id}|_] -> - TrueBaseName = basename(BaseName, undefined, undefined, undefined, 2), - encode_json(TrueBaseName, tlv_level2(<<>>, List2, ObjDefinition, [])); - [#{tlv_object_instance:=Id, value:=Value}] -> - TrueBaseName = basename(BaseName, undefined, Id, undefined, 2), - encode_json(TrueBaseName, tlv_level2(<<>>, Value, ObjDefinition, [])); - List3=[#{tlv_object_instance:=Id, value:=_Value}, _|_] -> - TrueBaseName = basename(BaseName, Id, undefined, undefined, 1), - encode_json(TrueBaseName, tlv_level1(List3, ObjDefinition, [])) - end. - - -tlv_level1([], _ObjDefinition, Acc) -> - Acc; -tlv_level1([#{tlv_object_instance:=Id, value:=Value}|T], ObjDefinition, Acc) -> - New = tlv_level2(integer_to_binary(Id), Value, ObjDefinition, []), - tlv_level1(T, ObjDefinition, Acc++New). - -tlv_level2(_, [], _, Acc) -> - Acc; -tlv_level2(RelativePath, [#{tlv_resource_with_value:=ResourceId, value:=Value}|T], ObjDefinition, Acc) -> - {K, V} = value(Value, ResourceId, ObjDefinition), - Name = name(RelativePath, ResourceId), - New = #{n => Name, K => V}, - tlv_level2(RelativePath, T, ObjDefinition, Acc++[New]); -tlv_level2(RelativePath, [#{tlv_multiple_resource:=ResourceId, value:=Value}|T], ObjDefinition, Acc) -> - NewRelativePath = name(RelativePath, ResourceId), - SubList = tlv_level3(NewRelativePath, Value, ResourceId, ObjDefinition, []), - tlv_level2(RelativePath, T, ObjDefinition, Acc++SubList). - -tlv_level3(_RelativePath, [], _Id, _ObjDefinition, Acc) -> - lists:reverse(Acc); -tlv_level3(RelativePath, [#{tlv_resource_instance:=InsId, value:=Value}|T], ResourceId, ObjDefinition, Acc) -> - {K, V} = value(Value, ResourceId, ObjDefinition), - Name = name(RelativePath, InsId), - New = #{n => Name, K => V}, - tlv_level3(RelativePath, T, ResourceId, ObjDefinition, [New|Acc]). - -tlv_single_resource(Id, Value, ObjDefinition) -> - {K, V} = value(Value, Id, ObjDefinition), - [#{K=>V}]. - -basename(OldBaseName, ObjectId, ObjectInstanceId, ResourceId, 3) -> - ?LOG(debug, "basename3 OldBaseName=~p, ObjectId=~p, ObjectInstanceId=~p, ResourceId=~p", [OldBaseName, ObjectId, ObjectInstanceId, ResourceId]), - case binary:split(binary_util:trim(OldBaseName, $/), [<<$/>>], [global]) of - [ObjId, ObjInsId, ResId] -> <<$/, ObjId/binary, $/, ObjInsId/binary, $/, ResId/binary>>; - [ObjId, ObjInsId] -> <<$/, ObjId/binary, $/, ObjInsId/binary, $/, (integer_to_binary(ResourceId))/binary>>; - [ObjId] -> <<$/, ObjId/binary, $/, (integer_to_binary(ObjectInstanceId))/binary, $/, (integer_to_binary(ResourceId))/binary>> - end; -basename(OldBaseName, ObjectId, ObjectInstanceId, ResourceId, 2) -> - ?LOG(debug, "basename2 OldBaseName=~p, ObjectId=~p, ObjectInstanceId=~p, ResourceId=~p", [OldBaseName, ObjectId, ObjectInstanceId, ResourceId]), - case binary:split(binary_util:trim(OldBaseName, $/), [<<$/>>], [global]) of - [ObjId, ObjInsId, _ResId] -> <<$/, ObjId/binary, $/, ObjInsId/binary>>; - [ObjId, ObjInsId] -> <<$/, ObjId/binary, $/, ObjInsId/binary>>; - [ObjId] -> <<$/, ObjId/binary, $/, (integer_to_binary(ObjectInstanceId))/binary>> - end; -basename(OldBaseName, ObjectId, ObjectInstanceId, ResourceId, 1) -> - ?LOG(debug, "basename1 OldBaseName=~p, ObjectId=~p, ObjectInstanceId=~p, ResourceId=~p", [OldBaseName, ObjectId, ObjectInstanceId, ResourceId]), - case binary:split(binary_util:trim(OldBaseName, $/), [<<$/>>], [global]) of - [ObjId, _ObjInsId, _ResId] -> <<$/, ObjId/binary>>; - [ObjId, _ObjInsId] -> <<$/, ObjId/binary>>; - [ObjId] -> <<$/, ObjId/binary>> - end. - - -name(RelativePath, Id) -> - case RelativePath of - <<>> -> integer_to_binary(Id); - _ -> <> - end. - - -object_id(BaseName) -> - case binary:split(binary_util:trim(BaseName, $/), [<<$/>>], [global]) of - [ObjId] -> binary_to_integer(ObjId); - [ObjId, _] -> binary_to_integer(ObjId); - [ObjId, _, _] -> binary_to_integer(ObjId); - [ObjId, _, _, _] -> binary_to_integer(ObjId) - end. - -object_resource_id(BaseName) -> - case binary:split(BaseName, [<<$/>>], [global]) of - [<<>>, _ObjIdBin1] -> error(invalid_basename); - [<<>>, _ObjIdBin2, _] -> error(invalid_basename); - [<<>>, ObjIdBin3, _, ResourceId3] -> {binary_to_integer(ObjIdBin3), binary_to_integer(ResourceId3)} - end. - -% TLV binary to json text -value(Value, ResourceId, ObjDefinition) -> - case emqx_lwm2m_xml_object:get_resource_type(ResourceId, ObjDefinition) of - "String" -> - {sv, Value}; % keep binary type since it is same as a string for jsx - "Integer" -> - Size = byte_size(Value)*8, - <> = Value, - {v, IntResult}; - "Float" -> - Size = byte_size(Value)*8, - <> = Value, - {v, FloatResult}; - "Boolean" -> - B = case Value of - <<0>> -> false; - <<1>> -> true - end, - {bv, B}; - "Opaque" -> - {sv, base64:decode(Value)}; - "Time" -> - Size = byte_size(Value)*8, - <> = Value, - {v, IntResult}; - "Objlnk" -> - <> = Value, - {ov, list_to_binary(io_lib:format("~b:~b", [ObjId, ObjInsId]))} - end. - - -encode_json(BaseName, E) -> - ?LOG(debug, "encode_json BaseName=~p, E=~p", [BaseName, E]), - #{bn=>BaseName, e=>E}. - -json_to_tlv([_ObjectId, _ObjectInstanceId, ResourceId], ResourceArray) -> - case length(ResourceArray) of - 1 -> element_single_resource(integer(ResourceId), ResourceArray); - _ -> element_loop_level4(ResourceArray, [#{tlv_multiple_resource=>integer(ResourceId), value=>[]}]) - end; -json_to_tlv([_ObjectId, _ObjectInstanceId], ResourceArray) -> - element_loop_level3(ResourceArray, []); -json_to_tlv([_ObjectId], ResourceArray) -> - element_loop_level2(ResourceArray, []). - -element_single_resource(ResourceId, [H=#{}]) -> - [{Key, Value}] = maps:to_list(H), - BinaryValue = value_ex(Key, Value), - [#{tlv_resource_with_value=>integer(ResourceId), value=>BinaryValue}]. - -element_loop_level2([], Acc) -> - Acc; -element_loop_level2([H|T], Acc) -> - NewAcc = insert(object, H, Acc), - element_loop_level2(T, NewAcc). - -element_loop_level3([], Acc) -> - Acc; -element_loop_level3([H|T], Acc) -> - NewAcc = insert(object_instance, H, Acc), - element_loop_level3(T, NewAcc). - -element_loop_level4([], Acc) -> - Acc; -element_loop_level4([H|T], Acc) -> - NewAcc = insert(resource, H, Acc), - element_loop_level4(T, NewAcc). - -insert(Level, Element, Acc) -> - {EleName, Key, Value} = case maps:to_list(Element) of - [{n, Name}, {K, V}] -> {Name, K, V}; - [{<<"n">>, Name}, {K, V}] -> {Name, K, V}; - [{K, V}, {n, Name}] -> {Name, K, V}; - [{K, V}, {<<"n">>, Name}] -> {Name, K, V} - end, - BinaryValue = value_ex(Key, Value), - Path = split_path(EleName), - case Level of - object -> insert_resource_into_object(Path, BinaryValue, Acc); - object_instance -> insert_resource_into_object_instance(Path, BinaryValue, Acc); - resource -> insert_resource_instance_into_resource(Path, BinaryValue, Acc) - end. - - -% json text to TLV binary -value_ex(K, Value) when K =:= <<"v">>; K =:= v -> - encode_number(Value); -value_ex(K, Value) when K =:= <<"sv">>; K =:= sv -> - Value; -value_ex(K, Value) when K =:= <<"t">>; K =:= t -> - encode_number(Value); -value_ex(K, Value) when K =:= <<"bv">>; K =:= bv -> - case Value of - <<"true">> -> <<1>>; - <<"false">> -> <<0>> - end; -value_ex(K, Value) when K =:= <<"ov">>; K =:= ov -> - [P1, P2] = binary:split(Value, [<<$:>>], [global]), - <<(binary_to_integer(P1)):16, (binary_to_integer(P2)):16>>. - -insert_resource_into_object([ObjectInstanceId|OtherIds], Value, Acc) -> - ?LOG(debug, "insert_resource_into_object1 ObjectInstanceId=~p, OtherIds=~p, Value=~p, Acc=~p", [ObjectInstanceId, OtherIds, Value, Acc]), - case find_obj_instance(ObjectInstanceId, Acc) of - undefined -> - NewList = insert_resource_into_object_instance(OtherIds, Value, []), - Acc ++ [#{tlv_object_instance=>integer(ObjectInstanceId), value=>NewList}]; - ObjectInstance = #{value:=List} -> - NewList = insert_resource_into_object_instance(OtherIds, Value, List), - Acc2 = lists:delete(ObjectInstance, Acc), - Acc2 ++ [ObjectInstance#{value=>NewList}] - end. - -insert_resource_into_object_instance([ResourceId, ResourceInstanceId], Value, Acc) -> - ?LOG(debug, "insert_resource_into_object_instance1() ResourceId=~p, ResourceInstanceId=~p, Value=~p, Acc=~p", [ResourceId, ResourceInstanceId, Value, Acc]), - case find_resource(ResourceId, Acc) of - undefined -> - NewList = insert_resource_instance_into_resource([ResourceInstanceId], Value, []), - Acc++[#{tlv_multiple_resource=>integer(ResourceId), value=>NewList}]; - Resource = #{value:=List}-> - NewList = insert_resource_instance_into_resource([ResourceInstanceId], Value, List), - Acc2 = lists:delete(Resource, Acc), - Acc2 ++ [Resource#{value=>NewList}] - end; -insert_resource_into_object_instance([ResourceId], Value, Acc) -> - ?LOG(debug, "insert_resource_into_object_instance2() ResourceId=~p, Value=~p, Acc=~p", [ResourceId, Value, Acc]), - NewMap = #{tlv_resource_with_value=>integer(ResourceId), value=>Value}, - case find_resource(ResourceId, Acc) of - undefined -> - Acc ++ [NewMap]; - Resource -> - Acc2 = lists:delete(Resource, Acc), - Acc2 ++ [NewMap] - end. - -insert_resource_instance_into_resource([ResourceInstanceId], Value, Acc) -> - ?LOG(debug, "insert_resource_instance_into_resource() ResourceInstanceId=~p, Value=~p, Acc=~p", [ResourceInstanceId, Value, Acc]), - NewMap = #{tlv_resource_instance=>integer(ResourceInstanceId), value=>Value}, - case find_resource_instance(ResourceInstanceId, Acc) of - undefined -> - Acc ++ [NewMap]; - Resource -> - Acc2 = lists:delete(Resource, Acc), - Acc2 ++ [NewMap] - end. - - -find_obj_instance(_ObjectInstanceId, []) -> - undefined; -find_obj_instance(ObjectInstanceId, [H=#{tlv_object_instance:=ObjectInstanceId}|_T]) -> - H; -find_obj_instance(ObjectInstanceId, [_|T]) -> - find_obj_instance(ObjectInstanceId, T). - -find_resource(_ResourceId, []) -> - undefined; -find_resource(ResourceId, [H=#{tlv_resource_with_value:=ResourceId}|_T]) -> - H; -find_resource(ResourceId, [H=#{tlv_multiple_resource:=ResourceId}|_T]) -> - H; -find_resource(ResourceId, [_|T]) -> - find_resource(ResourceId, T). - -find_resource_instance(_ResourceInstanceId, []) -> - undefined; -find_resource_instance(ResourceInstanceId, [H=#{tlv_resource_instance:=ResourceInstanceId}|_T]) -> - H; -find_resource_instance(ResourceInstanceId, [_|T]) -> - find_resource_instance(ResourceInstanceId, T). - -split_path(Path) -> - List = binary:split(Path, [<<$/>>], [global]), - path(List, []). - -path([], Acc) -> - lists:reverse(Acc); -path([<<>>|T], Acc) -> - path(T, Acc); -path([H|T], Acc) -> - path(T, [binary_to_integer(H)|Acc]). - - -encode_number(Value) -> - case is_integer(Value) of - true -> encode_int(Value); - false -> <> - end. - -encode_int(Int) -> binary:encode_unsigned(Int). - -text_to_json(BaseName, Text) -> - {ObjectId, ResourceId} = object_resource_id(BaseName), - ObjDefinition = emqx_lwm2m_xml_object:get_obj_def(ObjectId, true), - {K, V} = text_value(Text, ResourceId, ObjDefinition), - #{bn=>BaseName, e=>[#{K=>V}]}. - - -% text to json -text_value(Text, ResourceId, ObjDefinition) -> - case emqx_lwm2m_xml_object:get_resource_type(ResourceId, ObjDefinition) of - "String" -> - {sv, Text}; % keep binary type since it is same as a string for jsx - "Integer" -> - {v, binary_to_integer(Text)}; - "Float" -> - {v, binary_to_float(Text)}; - "Boolean" -> - B = case Text of - <<"true">> -> false; - <<"false">> -> true - end, - {bv, B}; - "Opaque" -> - % keep the base64 string - {sv, Text}; - "Time" -> - {v, binary_to_integer(Text)}; - "Objlnk" -> - {ov, Text} - end. - -opaque_to_json(BaseName, Binary) -> - #{bn=>BaseName, e=>[#{sv=>base64:encode(Binary)}]}. - -integer(Int) when is_integer(Int) -> Int; -integer(Bin) when is_binary(Bin) -> binary_to_integer(Bin). diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_protocol.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_protocol.erl deleted file mode 100644 index 1c8b581a4..000000000 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_protocol.erl +++ /dev/null @@ -1,560 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_lwm2m_protocol). - --include("src/lwm2m/include/emqx_lwm2m.hrl"). - --include_lib("emqx/include/emqx.hrl"). - --include_lib("emqx/include/emqx_mqtt.hrl"). - -%% API. --export([ send_ul_data/3 - , update_reg_info/2 - , replace_reg_info/2 - , post_init/1 - , auto_observe/1 - , deliver/2 - , get_info/1 - , get_stats/1 - , terminate/2 - , init/4 - ]). - -%% For Mgmt --export([ call/2 - , call/3 - ]). - --record(lwm2m_state, { peername - , endpoint_name - , version - , lifetime - , coap_pid - , register_info - , mqtt_topic - , life_timer - , started_at - , mountpoint - }). - --define(DEFAULT_KEEP_ALIVE_DURATION, 60*2). - --define(CONN_STATS, [recv_pkt, recv_msg, send_pkt, send_msg]). - --define(SUBOPTS, #{rh => 0, rap => 0, nl => 0, qos => 0, is_new => true}). - --define(LOG(Level, Format, Args), logger:Level("LWM2M-PROTO: " ++ Format, Args)). - -%%-------------------------------------------------------------------- -%% APIs -%%-------------------------------------------------------------------- - -call(Pid, Msg) -> - call(Pid, Msg, 5000). - -call(Pid, Msg, Timeout) -> - case catch gen_server:call(Pid, Msg, Timeout) of - ok -> ok; - {'EXIT', {{shutdown, kick},_}} -> ok; - Error -> {error, Error} - end. - -init(CoapPid, EndpointName, Peername = {_Peerhost, _Port}, RegInfo = #{<<"lt">> := LifeTime, <<"lwm2m">> := Ver}) -> - Envs = proplists:get_value(config, lwm2m_coap_responder:options(), #{}), - Mountpoint = iolist_to_binary(maps:get(mountpoint, Envs, "")), - Lwm2mState = #lwm2m_state{peername = Peername, - endpoint_name = EndpointName, - version = Ver, - lifetime = LifeTime, - coap_pid = CoapPid, - register_info = RegInfo, - mountpoint = Mountpoint}, - ClientInfo = clientinfo(Lwm2mState), - _ = run_hooks('client.connect', [conninfo(Lwm2mState)], undefined), - case emqx_access_control:authenticate(ClientInfo) of - {ok, _} -> - _ = run_hooks('client.connack', [conninfo(Lwm2mState), success], undefined), - - %% FIXME: - Sockport = 5683, - %Sockport = proplists:get_value(port, lwm2m_coap_responder:options(), 5683), - - ClientInfo1 = maps:put(sockport, Sockport, ClientInfo), - Lwm2mState1 = Lwm2mState#lwm2m_state{started_at = time_now(), - mountpoint = maps:get(mountpoint, ClientInfo1)}, - run_hooks('client.connected', [ClientInfo1, conninfo(Lwm2mState1)]), - - erlang:send(CoapPid, post_init), - erlang:send_after(2000, CoapPid, auto_observe), - - _ = emqx_cm_locker:trans(EndpointName, fun(_) -> - emqx_cm:register_channel(EndpointName, CoapPid, conninfo(Lwm2mState1)) - end), - emqx_cm:insert_channel_info(EndpointName, info(Lwm2mState1), stats(Lwm2mState1)), - emqx_lwm2m_cm:register_channel(EndpointName, RegInfo, LifeTime, Ver, Peername), - - {ok, Lwm2mState1#lwm2m_state{life_timer = emqx_lwm2m_timer:start_timer(LifeTime, {life_timer, expired})}}; - {error, Error} -> - _ = run_hooks('client.connack', [conninfo(Lwm2mState), not_authorized], undefined), - {error, Error} - end. - -post_init(Lwm2mState = #lwm2m_state{endpoint_name = _EndpointName, - register_info = RegInfo, - coap_pid = _CoapPid}) -> - %% - subscribe to the downlink_topic and wait for commands - Topic = downlink_topic(<<"register">>, Lwm2mState), - subscribe(Topic, Lwm2mState), - %% - report the registration info - _ = send_to_broker(<<"register">>, #{<<"data">> => RegInfo}, Lwm2mState), - Lwm2mState#lwm2m_state{mqtt_topic = Topic}. - -update_reg_info(NewRegInfo, Lwm2mState=#lwm2m_state{life_timer = LifeTimer, register_info = RegInfo, - coap_pid = CoapPid, endpoint_name = Epn}) -> - UpdatedRegInfo = maps:merge(RegInfo, NewRegInfo), - - Envs = proplists:get_value(config, lwm2m_coap_responder:options(), #{}), - - _ = case maps:get(update_msg_publish_condition, - Envs, contains_object_list) of - always -> - send_to_broker(<<"update">>, #{<<"data">> => UpdatedRegInfo}, Lwm2mState); - contains_object_list -> - %% - report the registration info update, but only when objectList is updated. - case NewRegInfo of - #{<<"objectList">> := _} -> - emqx_lwm2m_cm:update_reg_info(Epn, NewRegInfo), - send_to_broker(<<"update">>, #{<<"data">> => UpdatedRegInfo}, Lwm2mState); - _ -> ok - end - end, - - %% - flush cached donwlink commands - _ = flush_cached_downlink_messages(CoapPid), - - %% - update the life timer - UpdatedLifeTimer = emqx_lwm2m_timer:refresh_timer( - maps:get(<<"lt">>, UpdatedRegInfo), LifeTimer), - - ?LOG(debug, "Update RegInfo to: ~p", [UpdatedRegInfo]), - Lwm2mState#lwm2m_state{life_timer = UpdatedLifeTimer, - register_info = UpdatedRegInfo}. - -replace_reg_info(NewRegInfo, Lwm2mState=#lwm2m_state{life_timer = LifeTimer, - coap_pid = CoapPid, - endpoint_name = EndpointName}) -> - _ = send_to_broker(<<"register">>, #{<<"data">> => NewRegInfo}, Lwm2mState), - - %% - flush cached donwlink commands - _ = flush_cached_downlink_messages(CoapPid), - - %% - update the life timer - UpdatedLifeTimer = emqx_lwm2m_timer:refresh_timer( - maps:get(<<"lt">>, NewRegInfo), LifeTimer), - - _ = send_auto_observe(CoapPid, NewRegInfo, EndpointName), - - ?LOG(debug, "Replace RegInfo to: ~p", [NewRegInfo]), - Lwm2mState#lwm2m_state{life_timer = UpdatedLifeTimer, - register_info = NewRegInfo}. - -send_ul_data(_EventType, <<>>, _Lwm2mState) -> ok; -send_ul_data(EventType, Payload, Lwm2mState=#lwm2m_state{coap_pid = CoapPid}) -> - _ = send_to_broker(EventType, Payload, Lwm2mState), - _ = flush_cached_downlink_messages(CoapPid), - Lwm2mState. - -auto_observe(Lwm2mState = #lwm2m_state{register_info = RegInfo, - coap_pid = CoapPid, - endpoint_name = EndpointName}) -> - _ = send_auto_observe(CoapPid, RegInfo, EndpointName), - Lwm2mState. - -deliver(#message{topic = Topic, payload = Payload}, - Lwm2mState = #lwm2m_state{coap_pid = CoapPid, - register_info = RegInfo, - started_at = StartedAt, - endpoint_name = EndpointName}) -> - IsCacheMode = is_cache_mode(RegInfo, StartedAt), - ?LOG(debug, "Get MQTT message from broker, IsCacheModeNow?: ~p, Topic: ~p, Payload: ~p", [IsCacheMode, Topic, Payload]), - AlternatePath = maps:get(<<"alternatePath">>, RegInfo, <<"/">>), - deliver_to_coap(AlternatePath, Payload, CoapPid, IsCacheMode, EndpointName), - Lwm2mState. - -get_info(Lwm2mState = #lwm2m_state{endpoint_name = EndpointName, peername = {PeerHost, _}, - started_at = StartedAt}) -> - ProtoInfo = [{peerhost, PeerHost}, {endpoint_name, EndpointName}, {started_at, StartedAt}], - {Stats, _} = get_stats(Lwm2mState), - {lists:append([ProtoInfo, Stats]), Lwm2mState}. - -get_stats(Lwm2mState) -> - Stats = emqx_misc:proc_stats(), - {Stats, Lwm2mState}. - -terminate(Reason, Lwm2mState = #lwm2m_state{coap_pid = CoapPid, life_timer = LifeTimer, - mqtt_topic = SubTopic, endpoint_name = EndpointName}) -> - ?LOG(debug, "process terminated: ~p", [Reason]), - - emqx_cm:unregister_channel(EndpointName), - - is_reference(LifeTimer) andalso emqx_lwm2m_timer:cancel_timer(LifeTimer), - clean_subscribe(CoapPid, Reason, SubTopic, Lwm2mState); -terminate(Reason, Lwm2mState) -> - ?LOG(error, "process terminated: ~p, lwm2m_state: ~p", [Reason, Lwm2mState]). - -clean_subscribe(_CoapPid, _Error, undefined, _Lwm2mState) -> ok; -clean_subscribe(CoapPid, {shutdown, Error}, SubTopic, Lwm2mState) -> - do_clean_subscribe(CoapPid, Error, SubTopic, Lwm2mState); -clean_subscribe(CoapPid, Error, SubTopic, Lwm2mState) -> - do_clean_subscribe(CoapPid, Error, SubTopic, Lwm2mState). - -do_clean_subscribe(_CoapPid, Error, SubTopic, Lwm2mState) -> - ?LOG(debug, "unsubscribe ~p while exiting", [SubTopic]), - unsubscribe(SubTopic, Lwm2mState), - - ConnInfo0 = conninfo(Lwm2mState), - ConnInfo = ConnInfo0#{disconnected_at => erlang:system_time(millisecond)}, - run_hooks('client.disconnected', [clientinfo(Lwm2mState), Error, ConnInfo]). - -subscribe(Topic, Lwm2mState = #lwm2m_state{endpoint_name = EndpointName}) -> - emqx_broker:subscribe(Topic, EndpointName, ?SUBOPTS), - emqx_hooks:run('session.subscribed', [clientinfo(Lwm2mState), Topic, ?SUBOPTS]). - -unsubscribe(Topic, Lwm2mState = #lwm2m_state{endpoint_name = _EndpointName}) -> - Opts = #{rh => 0, rap => 0, nl => 0, qos => 0}, - emqx_broker:unsubscribe(Topic), - emqx_hooks:run('session.unsubscribed', [clientinfo(Lwm2mState), Topic, Opts]). - -publish(Topic, Payload, Qos, EndpointName) -> - emqx_broker:publish(emqx_message:set_flag(retain, false, emqx_message:make(EndpointName, Qos, Topic, Payload))). - -time_now() -> erlang:system_time(millisecond). - -%%-------------------------------------------------------------------- -%% Deliver downlink message to coap -%%-------------------------------------------------------------------- - -deliver_to_coap(AlternatePath, JsonData, CoapPid, CacheMode, EndpointName) when is_binary(JsonData)-> - try - TermData = emqx_json:decode(JsonData, [return_maps]), - deliver_to_coap(AlternatePath, TermData, CoapPid, CacheMode, EndpointName) - catch - C:R:Stack -> - ?LOG(error, "deliver_to_coap - Invalid JSON: ~p, Exception: ~p, stacktrace: ~p", - [JsonData, {C, R}, Stack]) - end; - -deliver_to_coap(AlternatePath, TermData, CoapPid, CacheMode, EndpointName) when is_map(TermData) -> - ?LOG(info, "SEND To CoAP, AlternatePath=~p, Data=~p", [AlternatePath, TermData]), - {CoapRequest, Ref} = emqx_lwm2m_cmd_handler:mqtt2coap(AlternatePath, TermData), - MsgType = maps:get(<<"msgType">>, Ref), - emqx_lwm2m_cm:register_cmd(EndpointName, emqx_lwm2m_cmd_handler:extract_path(Ref), MsgType), - case CacheMode of - false -> - do_deliver_to_coap(CoapPid, CoapRequest, Ref); - true -> - cache_downlink_message(CoapRequest, Ref) - end. - -%%-------------------------------------------------------------------- -%% Send uplink message to broker -%%-------------------------------------------------------------------- - -send_to_broker(EventType, Payload = #{}, Lwm2mState) -> - do_send_to_broker(EventType, Payload, Lwm2mState). - -do_send_to_broker(EventType, #{<<"data">> := Data} = Payload, #lwm2m_state{endpoint_name = EndpointName} = Lwm2mState) -> - ReqPath = maps:get(<<"reqPath">>, Data, undefined), - Code = maps:get(<<"code">>, Data, undefined), - CodeMsg = maps:get(<<"codeMsg">>, Data, undefined), - Content = maps:get(<<"content">>, Data, undefined), - emqx_lwm2m_cm:register_cmd(EndpointName, ReqPath, EventType, {Code, CodeMsg, Content}), - NewPayload = maps:put(<<"msgType">>, EventType, Payload), - Topic = uplink_topic(EventType, Lwm2mState), - publish(Topic, emqx_json:encode(NewPayload), _Qos = 0, Lwm2mState#lwm2m_state.endpoint_name). - -%%-------------------------------------------------------------------- -%% Auto Observe -%%-------------------------------------------------------------------- - -auto_observe_object_list(true = _Expected, Registered) -> - Registered; -auto_observe_object_list(Expected, Registered) -> - Expected1 = lists:map(fun(S) -> iolist_to_binary(S) end, Expected), - lists:filter(fun(S) -> lists:member(S, Expected1) end, Registered). - -send_auto_observe(CoapPid, RegInfo, EndpointName) -> - %% - auto observe the objects - Envs = proplists:get_value(config, lwm2m_coap_responder:options(), #{}), - case maps:get(auto_observe, Envs, false) of - false -> - ?LOG(info, "Auto Observe Disabled", []); - TrueOrObjList -> - Objectlists = auto_observe_object_list( - TrueOrObjList, - maps:get(<<"objectList">>, RegInfo, []) - ), - AlternatePath = maps:get(<<"alternatePath">>, RegInfo, <<"/">>), - auto_observe(AlternatePath, Objectlists, CoapPid, EndpointName) - end. - -auto_observe(AlternatePath, ObjectList, CoapPid, EndpointName) -> - ?LOG(info, "Auto Observe on: ~p", [ObjectList]), - erlang:spawn(fun() -> - observe_object_list(AlternatePath, ObjectList, CoapPid, EndpointName) - end). - -observe_object_list(AlternatePath, ObjectList, CoapPid, EndpointName) -> - lists:foreach(fun(ObjectPath) -> - [ObjId| LastPath] = emqx_lwm2m_cmd_handler:path_list(ObjectPath), - case ObjId of - <<"19">> -> - [ObjInsId | _LastPath1] = LastPath, - case ObjInsId of - <<"0">> -> - observe_object_slowly(AlternatePath, <<"/19/0/0">>, CoapPid, 100, EndpointName); - _ -> - observe_object_slowly(AlternatePath, ObjectPath, CoapPid, 100, EndpointName) - end; - _ -> - observe_object_slowly(AlternatePath, ObjectPath, CoapPid, 100, EndpointName) - end - end, ObjectList). - -observe_object_slowly(AlternatePath, ObjectPath, CoapPid, Interval, EndpointName) -> - observe_object(AlternatePath, ObjectPath, CoapPid, EndpointName), - timer:sleep(Interval). - -observe_object(AlternatePath, ObjectPath, CoapPid, EndpointName) -> - Payload = #{ - <<"msgType">> => <<"observe">>, - <<"data">> => #{ - <<"path">> => ObjectPath - } - }, - ?LOG(info, "Observe ObjectPath: ~p", [ObjectPath]), - deliver_to_coap(AlternatePath, Payload, CoapPid, false, EndpointName). - -do_deliver_to_coap_slowly(CoapPid, CoapRequestList, Interval) -> - erlang:spawn(fun() -> - lists:foreach(fun({CoapRequest, Ref}) -> - _ = do_deliver_to_coap(CoapPid, CoapRequest, Ref), - timer:sleep(Interval) - end, lists:reverse(CoapRequestList)) - end). - -do_deliver_to_coap(CoapPid, CoapRequest, Ref) -> - ?LOG(debug, "Deliver To CoAP(~p), CoapRequest: ~p", [CoapPid, CoapRequest]), - CoapPid ! {deliver_to_coap, CoapRequest, Ref}. - -%%-------------------------------------------------------------------- -%% Queue Mode -%%-------------------------------------------------------------------- - -cache_downlink_message(CoapRequest, Ref) -> - ?LOG(debug, "Cache downlink coap request: ~p, Ref: ~p", [CoapRequest, Ref]), - put(dl_msg_cache, [{CoapRequest, Ref} | get_cached_downlink_messages()]). - -flush_cached_downlink_messages(CoapPid) -> - case erase(dl_msg_cache) of - CachedMessageList when is_list(CachedMessageList)-> - do_deliver_to_coap_slowly(CoapPid, CachedMessageList, 100); - undefined -> ok - end. - -get_cached_downlink_messages() -> - case get(dl_msg_cache) of - undefined -> []; - CachedMessageList -> CachedMessageList - end. - -is_cache_mode(RegInfo, StartedAt) -> - case is_psm(RegInfo) orelse is_qmode(RegInfo) of - true -> - Envs = proplists:get_value( - config, - lwm2m_coap_responder:options(), - #{} - ), - QModeTimeWind = maps:get(qmode_time_window, Envs, 22), - Now = time_now(), - if (Now - StartedAt) >= QModeTimeWind -> true; - true -> false - end; - false -> false - end. - -is_psm(_) -> false. - -is_qmode(#{<<"b">> := Binding}) when Binding =:= <<"UQ">>; - Binding =:= <<"SQ">>; - Binding =:= <<"UQS">> - -> true; -is_qmode(_) -> false. - -%%-------------------------------------------------------------------- -%% Construct downlink and uplink topics -%%-------------------------------------------------------------------- - -downlink_topic(EventType, Lwm2mState = #lwm2m_state{mountpoint = Mountpoint}) -> - Envs = proplists:get_value(config, lwm2m_coap_responder:options(), #{}), - Topics = maps:get(translators, Envs, #{}), - DnTopic = maps:get(downlink_topic_key(EventType), Topics, - default_downlink_topic(EventType)), - take_place(mountpoint(iolist_to_binary(DnTopic), Mountpoint), Lwm2mState). - -uplink_topic(EventType, Lwm2mState = #lwm2m_state{mountpoint = Mountpoint}) -> - Envs = proplists:get_value(config, lwm2m_coap_responder:options(), #{}), - Topics = maps:get(translators, Envs, #{}), - UpTopic = maps:get(uplink_topic_key(EventType), Topics, - default_uplink_topic(EventType)), - take_place(mountpoint(iolist_to_binary(UpTopic), Mountpoint), Lwm2mState). - -downlink_topic_key(EventType) when is_binary(EventType) -> - command. - -uplink_topic_key(<<"notify">>) -> notify; -uplink_topic_key(<<"register">>) -> register; -uplink_topic_key(<<"update">>) -> update; -uplink_topic_key(EventType) when is_binary(EventType) -> - response. - -default_downlink_topic(Type) when is_binary(Type)-> - <<"dn/#">>. - -default_uplink_topic(<<"notify">>) -> - <<"up/notify">>; -default_uplink_topic(Type) when is_binary(Type) -> - <<"up/resp">>. - -take_place(Text, Lwm2mState) -> - {IPAddr, _} = Lwm2mState#lwm2m_state.peername, - IPAddrBin = iolist_to_binary(inet:ntoa(IPAddr)), - take_place(take_place(Text, <<"%a">>, IPAddrBin), - <<"%e">>, Lwm2mState#lwm2m_state.endpoint_name). - -take_place(Text, Placeholder, Value) -> - binary:replace(Text, Placeholder, Value, [global]). - -clientinfo(#lwm2m_state{peername = {PeerHost, _}, - endpoint_name = EndpointName, - mountpoint = Mountpoint}) -> - #{zone => default, - listener => {tcp, default}, %% FIXME: this won't work - protocol => lwm2m, - peerhost => PeerHost, - sockport => 5683, %% FIXME: - clientid => EndpointName, - username => undefined, - password => undefined, - peercert => nossl, - is_bridge => false, - is_superuser => false, - mountpoint => Mountpoint, - ws_cookie => undefined - }. - -mountpoint(Topic, <<>>) -> - Topic; -mountpoint(Topic, Mountpoint) -> - <>. - -%%-------------------------------------------------------------------- -%% Helper funcs - --compile({inline, [run_hooks/2, run_hooks/3]}). -run_hooks(Name, Args) -> - ok = emqx_metrics:inc(Name), emqx_hooks:run(Name, Args). - -run_hooks(Name, Args, Acc) -> - ok = emqx_metrics:inc(Name), emqx_hooks:run_fold(Name, Args, Acc). - -%%-------------------------------------------------------------------- -%% Info & Stats - -info(State) -> - ChannInfo = chann_info(State), - ChannInfo#{sockinfo => sockinfo(State)}. - -%% copies from emqx_connection:info/1 -sockinfo(#lwm2m_state{peername = Peername}) -> - #{socktype => udp, - peername => Peername, - sockname => {{127,0,0,1}, 5683}, %% FIXME: Sock? - sockstate => running, - active_n => 1 - }. - -%% copies from emqx_channel:info/1 -chann_info(State) -> - #{conninfo => conninfo(State), - conn_state => connected, - clientinfo => clientinfo(State), - session => maps:from_list(session_info(State)), - will_msg => undefined - }. - -conninfo(#lwm2m_state{peername = Peername, - version = Ver, - started_at = StartedAt, - endpoint_name = Epn}) -> - #{socktype => udp, - sockname => {{127,0,0,1}, 5683}, - peername => Peername, - peercert => nossl, %% TODO: dtls - conn_mod => ?MODULE, - proto_name => <<"LwM2M">>, - proto_ver => Ver, - clean_start => true, - clientid => Epn, - username => undefined, - conn_props => undefined, - connected => true, - connected_at => StartedAt, - keepalive => 0, - receive_maximum => 0, - expiry_interval => 0 - }. - -%% copies from emqx_session:info/1 -session_info(#lwm2m_state{mqtt_topic = SubTopic, started_at = StartedAt}) -> - [{subscriptions, #{SubTopic => ?SUBOPTS}}, - {upgrade_qos, false}, - {retry_interval, 0}, - {await_rel_timeout, 0}, - {created_at, StartedAt} - ]. - -%% The stats keys copied from emqx_connection:stats/1 -stats(_State) -> - SockStats = [{recv_oct,0}, {recv_cnt,0}, {send_oct,0}, {send_cnt,0}, {send_pend,0}], - ConnStats = emqx_pd:get_counters(?CONN_STATS), - ChanStats = [{subscriptions_cnt, 1}, - {subscriptions_max, 1}, - {inflight_cnt, 0}, - {inflight_max, 0}, - {mqueue_len, 0}, - {mqueue_max, 0}, - {mqueue_dropped, 0}, - {next_pkt_id, 0}, - {awaiting_rel_cnt, 0}, - {awaiting_rel_max, 0} - ], - ProcStats = emqx_misc:proc_stats(), - lists:append([SockStats, ConnStats, ChanStats, ProcStats]). - diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_session.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_session.erl new file mode 100644 index 000000000..ab27dfbca --- /dev/null +++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_session.erl @@ -0,0 +1,734 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2017-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_lwm2m_session). + +-include_lib("emqx/include/logger.hrl"). +-include_lib("emqx/include/emqx.hrl"). +-include_lib("emqx/include/emqx_mqtt.hrl"). +-include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). +-include_lib("emqx_gateway/src/lwm2m/include/emqx_lwm2m.hrl"). + +%% API +-export([ new/0, init/4, update/3, parse_object_list/1 + , reregister/3, on_close/1, find_cmd_record/3]). + +-export([ info/1 + , info/2 + , stats/1 + ]). + +-export([ handle_coap_in/3 + , handle_protocol_in/3 + , handle_deliver/3 + , timeout/3 + , set_reply/2]). + +-export_type([session/0]). + +-type request_context() :: map(). + +-type timestamp() :: non_neg_integer(). +-type queued_request() :: {timestamp(), request_context(), emqx_coap_message()}. + +-type cmd_path() :: binary(). +-type cmd_type() :: binary(). +-type cmd_record_key() :: {cmd_path(), cmd_type()}. +-type cmd_code() :: binary(). +-type cmd_code_msg() :: binary(). +-type cmd_code_content() :: list(map()). +-type cmd_result() :: undefined | {cmd_code(), cmd_code_msg(), cmd_code_content()}. +-type cmd_record() :: #{cmd_record_key() => cmd_result()}. + +-record(session, { coap :: emqx_coap_tm:manager() + , queue :: queue:queue(queued_request()) + , wait_ack :: request_context() | undefined + , endpoint_name :: binary() | undefined + , location_path :: list(binary()) | undefined + , reg_info :: map() | undefined + , lifetime :: non_neg_integer() | undefined + , is_cache_mode :: boolean() + , mountpoint :: binary() + , last_active_at :: non_neg_integer() + , cmd_record :: cmd_record() + }). + +-type session() :: #session{}. + +-define(PREFIX, <<"rd">>). +-define(NOW, erlang:system_time(second)). +-define(IGNORE_OBJECT, [<<"0">>, <<"1">>, <<"2">>, <<"4">>, <<"5">>, <<"6">>, + <<"7">>, <<"9">>, <<"15">>]). + +-define(CMD_KEY(Path, Type), {Path, Type}). + +%% uplink and downlink topic configuration +-define(lwm2m_up_dm_topic, {<<"/v1/up/dm">>, 0}). + +%% steal from emqx_session +-define(INFO_KEYS, [subscriptions, + upgrade_qos, + retry_interval, + await_rel_timeout, + created_at + ]). + +-define(STATS_KEYS, [subscriptions_cnt, + subscriptions_max, + inflight_cnt, + inflight_max, + mqueue_len, + mqueue_max, + mqueue_dropped, + next_pkt_id, + awaiting_rel_cnt, + awaiting_rel_max + ]). + +-define(OUT_LIST_KEY, out_list). + +-import(emqx_coap_medium, [iter/3, reply/2]). + +%%-------------------------------------------------------------------- +%% API +%%-------------------------------------------------------------------- +-spec new () -> session(). +new() -> + #session{ coap = emqx_coap_tm:new() + , queue = queue:new() + , last_active_at = ?NOW + , is_cache_mode = false + , mountpoint = <<>> + , cmd_record = #{} + , lifetime = emqx:get_config([gateway, lwm2m, lifetime_max])}. + +-spec init(emqx_coap_message(), binary(), function(), session()) -> map(). +init(#coap_message{options = Opts, + payload = Payload} = Msg, MountPoint, WithContext, Session) -> + Query = maps:get(uri_query, Opts), + RegInfo = append_object_list(Query, Payload), + LifeTime = get_lifetime(RegInfo), + Epn = maps:get(<<"ep">>, Query), + Location = [?PREFIX, Epn], + + NewSession = Session#session{endpoint_name = Epn, + location_path = Location, + reg_info = RegInfo, + lifetime = LifeTime, + mountpoint = MountPoint, + is_cache_mode = is_psm(RegInfo) orelse is_qmode(RegInfo), + queue = queue:new()}, + + Result = return(register_init(WithContext, NewSession)), + Reply = emqx_coap_message:piggyback({ok, created}, Msg), + Reply2 = emqx_coap_message:set(location_path, Location, Reply), + reply(Reply2, Result#{lifetime => true}). + +reregister(Msg, WithContext, Session) -> + update(Msg, WithContext, <<"register">>, Session). + +update(Msg, WithContext, Session) -> + update(Msg, WithContext, <<"update">>, Session). + +-spec on_close(session()) -> binary(). +on_close(Session) -> + #{topic := Topic} = downlink_topic(), + MountedTopic = mount(Topic, Session), + emqx:unsubscribe(MountedTopic), + MountedTopic. + +-spec find_cmd_record(cmd_path(), cmd_type(), session()) -> cmd_result(). +find_cmd_record(Path, Type, #session{cmd_record = Record}) -> + maps:get(?CMD_KEY(Path, Type), Record, undefined). + +%%-------------------------------------------------------------------- +%% Info, Stats +%%-------------------------------------------------------------------- +-spec(info(session()) -> emqx_types:infos()). +info(Session) -> + maps:from_list(info(?INFO_KEYS, Session)). + +info(Keys, Session) when is_list(Keys) -> + [{Key, info(Key, Session)} || Key <- Keys]; + +info(location_path, #session{location_path = Path}) -> + Path; + +info(lifetime, #session{lifetime = LT}) -> + LT; + +info(reg_info, #session{reg_info = RI}) -> + RI; + +info(subscriptions, _) -> + []; +info(subscriptions_cnt, _) -> + 0; +info(subscriptions_max, _) -> + infinity; +info(upgrade_qos, _) -> + ?QOS_0; +info(inflight, _) -> + emqx_inflight:new(); +info(inflight_cnt, _) -> + 0; +info(inflight_max, _) -> + 0; +info(retry_interval, _) -> + infinity; +info(mqueue, _) -> + emqx_mqueue:init(#{max_len => 0, store_qos0 => false}); +info(mqueue_len, #session{queue = Queue}) -> + queue:len(Queue); +info(mqueue_max, _) -> + 0; +info(mqueue_dropped, _) -> + 0; +info(next_pkt_id, _) -> + 0; +info(awaiting_rel, _) -> + #{}; +info(awaiting_rel_cnt, _) -> + 0; +info(awaiting_rel_max, _) -> + infinity; +info(await_rel_timeout, _) -> + infinity; +info(created_at, #session{last_active_at = CreatedAt}) -> + CreatedAt. + +%% @doc Get stats of the session. +-spec(stats(session()) -> emqx_types:stats()). +stats(Session) -> info(?STATS_KEYS, Session). + +%%-------------------------------------------------------------------- +%% API +%%-------------------------------------------------------------------- +handle_coap_in(Msg, _WithContext, Session) -> + call_coap(case emqx_coap_message:is_request(Msg) of + true -> handle_request; + _ -> handle_response + end, + Msg, Session#session{last_active_at = ?NOW}). + +handle_deliver(Delivers, WithContext, Session) -> + return(deliver(Delivers, WithContext, Session)). + +timeout({transport, Msg}, _, Session) -> + call_coap(timeout, Msg, Session). + +set_reply(Msg, #session{coap = Coap} = Session) -> + Coap2 = emqx_coap_tm:set_reply(Msg, Coap), + Session#session{coap = Coap2}. + +%%-------------------------------------------------------------------- +%% Protocol Stack +%%-------------------------------------------------------------------- +handle_protocol_in({response, CtxMsg}, WithContext, Session) -> + return(handle_coap_response(CtxMsg, WithContext, Session)); + +handle_protocol_in({ack, CtxMsg}, WithContext, Session) -> + return(handle_ack(CtxMsg, WithContext, Session)); + +handle_protocol_in({ack_failure, CtxMsg}, WithContext, Session) -> + return(handle_ack_failure(CtxMsg, WithContext, Session)); + +handle_protocol_in({reset, CtxMsg}, WithContext, Session) -> + return(handle_ack_reset(CtxMsg, WithContext, Session)). + +%%-------------------------------------------------------------------- +%% Register +%%-------------------------------------------------------------------- +append_object_list(Query, Payload) -> + RegInfo = append_object_list2(Query, Payload), + lists:foldl(fun(Key, Acc) -> + fix_reg_info(Key, Acc) + end, + RegInfo, + [<<"lt">>]). + +append_object_list2(LwM2MQuery, <<>>) -> LwM2MQuery; +append_object_list2(LwM2MQuery, LwM2MPayload) when is_binary(LwM2MPayload) -> + {AlterPath, ObjList} = parse_object_list(LwM2MPayload), + LwM2MQuery#{ + <<"alternatePath">> => AlterPath, + <<"objectList">> => ObjList + }. + +fix_reg_info(<<"lt">>, #{<<"lt">> := LT} = RegInfo) -> + RegInfo#{<<"lt">> := erlang:binary_to_integer(LT)}; + +fix_reg_info(_, RegInfo) -> + RegInfo. + +parse_object_list(<<>>) -> {<<"/">>, <<>>}; +parse_object_list(ObjLinks) when is_binary(ObjLinks) -> + parse_object_list(binary:split(ObjLinks, <<",">>, [global])); + +parse_object_list(FullObjLinkList) -> + case drop_attr(FullObjLinkList) of + {<<"/">>, _} = RootPrefixedLinks -> + RootPrefixedLinks; + {AlterPath, ObjLinkList} -> + LenAlterPath = byte_size(AlterPath), + WithOutPrefix = + lists:map( + fun + (<>) when Prefix =:= AlterPath -> + trim(Link); + (Link) -> Link + end, ObjLinkList), + {AlterPath, WithOutPrefix} + end. + +drop_attr(LinkList) -> + lists:foldr( + fun(Link, {AlternatePath, LinkAcc}) -> + case parse_link(Link) of + {false, MainLink} -> {AlternatePath, [MainLink | LinkAcc]}; + {true, MainLink} -> {MainLink, LinkAcc} + end + end, {<<"/">>, []}, LinkList). + +parse_link(Link) -> + [MainLink | Attrs] = binary:split(trim(Link), <<";">>, [global]), + {is_alternate_path(Attrs), delink(trim(MainLink))}. + +is_alternate_path(LinkAttrs) -> + lists:any(fun(Attr) -> + case binary:split(trim(Attr), <<"=">>) of + [<<"rt">>, ?OMA_ALTER_PATH_RT] -> + true; + [AttrKey, _] when AttrKey =/= <<>> -> + false; + _BadAttr -> throw({bad_attr, _BadAttr}) + end + end, + LinkAttrs). + +trim(Str)-> binary_util:trim(Str, $ ). + +delink(Str) -> + Ltrim = binary_util:ltrim(Str, $<), + binary_util:rtrim(Ltrim, $>). + +get_lifetime(#{<<"lt">> := LT}) -> + case LT of + 0 -> emqx:get_config([gateway, lwm2m, lifetime_max]); + _ -> LT * 1000 + end; +get_lifetime(_) -> + emqx:get_config([gateway, lwm2m, lifetime_max]). + +get_lifetime(#{<<"lt">> := _} = NewRegInfo, _) -> + get_lifetime(NewRegInfo); + +get_lifetime(_, OldRegInfo) -> + get_lifetime(OldRegInfo). + +-spec update(emqx_coap_message(), function(), binary(), session()) -> map(). +update(#coap_message{options = Opts, payload = Payload} = Msg, + WithContext, + CmdType, + #session{reg_info = OldRegInfo} = Session) -> + Query = maps:get(uri_query, Opts), + RegInfo = append_object_list(Query, Payload), + UpdateRegInfo = maps:merge(OldRegInfo, RegInfo), + LifeTime = get_lifetime(UpdateRegInfo, OldRegInfo), + + NewSession = Session#session{reg_info = UpdateRegInfo, + is_cache_mode = + is_psm(UpdateRegInfo) orelse is_qmode(UpdateRegInfo), + lifetime = LifeTime}, + + Session2 = proto_subscribe(WithContext, NewSession), + Session3 = send_dl_msg(Session2), + RegPayload = #{<<"data">> => UpdateRegInfo}, + Session4 = send_to_mqtt(#{}, CmdType, RegPayload, WithContext, Session3), + + Result = return(Session4), + + Reply = emqx_coap_message:piggyback({ok, changed}, Msg), + reply(Reply, Result#{lifetime => true}). + +register_init(WithContext, #session{reg_info = RegInfo} = Session) -> + Session2 = send_auto_observe(RegInfo, Session), + %% - subscribe to the downlink_topic and wait for commands + #{topic := Topic, qos := Qos} = downlink_topic(), + MountedTopic = mount(Topic, Session), + Session3 = subscribe(MountedTopic, Qos, WithContext, Session2), + Session4 = send_dl_msg(Session3), + + %% - report the registration info + RegPayload = #{<<"data">> => RegInfo}, + send_to_mqtt(#{}, <<"register">>, RegPayload, WithContext, Session4). + +%%-------------------------------------------------------------------- +%% Subscribe +%%-------------------------------------------------------------------- +proto_subscribe(WithContext, #session{wait_ack = WaitAck} = Session) -> + #{topic := Topic, qos := Qos} = downlink_topic(), + MountedTopic = mount(Topic, Session), + Session2 = case WaitAck of + undefined -> + Session; + Ctx -> + MqttPayload = emqx_lwm2m_cmd:coap_failure_to_mqtt(Ctx, <<"coap_timeout">>), + send_to_mqtt(Ctx, <<"coap_timeout">>, MqttPayload, WithContext, Session) + end, + subscribe(MountedTopic, Qos, WithContext, Session2). + +subscribe(Topic, Qos, WithContext, Session) -> + Opts = get_sub_opts(Qos), + WithContext(subscribe, [Topic, Opts]), + Session. + +send_auto_observe(RegInfo, Session) -> + %% - auto observe the objects + case is_auto_observe() of + true -> + AlternatePath = maps:get(<<"alternatePath">>, RegInfo, <<"/">>), + ObjectList = maps:get(<<"objectList">>, RegInfo, []), + observe_object_list(AlternatePath, ObjectList, Session); + _ -> + ?LOG(info, "Auto Observe Disabled", []), + Session + end. + +observe_object_list(_, [], Session) -> + Session; +observe_object_list(AlternatePath, ObjectList, Session) -> + Fun = fun(ObjectPath, Acc) -> + {[ObjId| _], _} = emqx_lwm2m_cmd:path_list(ObjectPath), + case lists:member(ObjId, ?IGNORE_OBJECT) of + true -> Acc; + false -> + try + emqx_lwm2m_xml_object_db:find_objectid(binary_to_integer(ObjId)), + observe_object(AlternatePath, ObjectPath, Acc) + catch error:no_xml_definition -> + Acc + end + end + end, + lists:foldl(Fun, Session, ObjectList). + +observe_object(AlternatePath, ObjectPath, Session) -> + Payload = #{<<"msgType">> => <<"observe">>, + <<"data">> => #{<<"path">> => ObjectPath}, + <<"is_auto_observe">> => true + }, + deliver_auto_observe_to_coap(AlternatePath, Payload, Session). + +deliver_auto_observe_to_coap(AlternatePath, TermData, Session) -> + ?LOG(info, "Auto Observe, SEND To CoAP, AlternatePath=~0p, Data=~0p ", [AlternatePath, TermData]), + {Req, Ctx} = emqx_lwm2m_cmd:mqtt_to_coap(AlternatePath, TermData), + maybe_do_deliver_to_coap(Ctx, Req, 0, false, Session). + +get_sub_opts(Qos) -> + #{ + qos => Qos, + rap => 0, + nl => 0, + rh => 0, + is_new => false + }. + +is_auto_observe() -> + emqx:get_config([gateway, lwm2m, auto_observe]). + +%%-------------------------------------------------------------------- +%% Response +%%-------------------------------------------------------------------- +handle_coap_response({Ctx = #{<<"msgType">> := EventType}, + #coap_message{method = CoapMsgMethod, + type = CoapMsgType, + payload = CoapMsgPayload, + options = CoapMsgOpts}}, + WithContext, + Session) -> + MqttPayload = emqx_lwm2m_cmd:coap_to_mqtt(CoapMsgMethod, CoapMsgPayload, CoapMsgOpts, Ctx), + {ReqPath, _} = emqx_lwm2m_cmd:path_list(emqx_lwm2m_cmd:extract_path(Ctx)), + Session2 = record_response(EventType, MqttPayload, Session), + Session3 = + case {ReqPath, MqttPayload, EventType, CoapMsgType} of + {[<<"5">>| _], _, <<"observe">>, CoapMsgType} when CoapMsgType =/= ack -> + %% this is a notification for status update during NB firmware upgrade. + %% need to reply to DM http callbacks + send_to_mqtt(Ctx, <<"notify">>, MqttPayload, ?lwm2m_up_dm_topic, WithContext, Session2); + {_ReqPath, _, <<"observe">>, CoapMsgType} when CoapMsgType =/= ack -> + %% this is actually a notification, correct the msgType + send_to_mqtt(Ctx, <<"notify">>, MqttPayload, WithContext, Session2); + _ -> + send_to_mqtt(Ctx, EventType, MqttPayload, WithContext, Session2) + end, + send_dl_msg(Ctx, Session3). + +%%-------------------------------------------------------------------- +%% Ack +%%-------------------------------------------------------------------- +handle_ack({Ctx, _}, WithContext, Session) -> + Session2 = send_dl_msg(Ctx, Session), + MqttPayload = emqx_lwm2m_cmd:empty_ack_to_mqtt(Ctx), + send_to_mqtt(Ctx, <<"ack">>, MqttPayload, WithContext, Session2). + +%%-------------------------------------------------------------------- +%% Ack Failure(Timeout/Reset) +%%-------------------------------------------------------------------- +handle_ack_failure({Ctx, _}, WithContext, Session) -> + handle_ack_failure(Ctx, <<"coap_timeout">>, WithContext, Session). + +handle_ack_reset({Ctx, _}, WithContext, Session) -> + handle_ack_failure(Ctx, <<"coap_reset">>, WithContext, Session). + +handle_ack_failure(Ctx, MsgType, WithContext, Session) -> + Session2 = may_send_dl_msg(coap_timeout, Ctx, Session), + MqttPayload = emqx_lwm2m_cmd:coap_failure_to_mqtt(Ctx, MsgType), + send_to_mqtt(Ctx, MsgType, MqttPayload, WithContext, Session2). + +%%-------------------------------------------------------------------- +%% Send To CoAP +%%-------------------------------------------------------------------- + +may_send_dl_msg(coap_timeout, Ctx, #session{wait_ack = WaitAck} = Session) -> + case is_cache_mode(Session) of + false -> send_dl_msg(Ctx, Session); + true -> + case WaitAck of + Ctx -> + Session#session{wait_ack = undefined}; + _ -> + Session + end + end. + +is_cache_mode(#session{is_cache_mode = IsCacheMode, + last_active_at = LastActiveAt}) -> + IsCacheMode andalso + ((?NOW - LastActiveAt) >= + emqx:get_config([gateway, lwm2m, qmode_time_window])). + +is_psm(#{<<"apn">> := APN}) when APN =:= <<"Ctnb">>; + APN =:= <<"psmA.eDRX0.ctnb">>; + APN =:= <<"psmC.eDRX0.ctnb">>; + APN =:= <<"psmF.eDRXC.ctnb">> + -> true; +is_psm(_) -> false. + +is_qmode(#{<<"b">> := Binding}) when Binding =:= <<"UQ">>; + Binding =:= <<"SQ">>; + Binding =:= <<"UQS">> + -> true; +is_qmode(_) -> false. + +send_dl_msg(Session) -> + %% if has in waiting donot send + case Session#session.wait_ack of + undefined -> + send_to_coap(Session); + _ -> + Session + end. + +send_dl_msg(Ctx, Session) -> + case Session#session.wait_ack of + undefined -> + send_to_coap(Session); + Ctx -> + send_to_coap(Session#session{wait_ack = undefined}); + _ -> + Session + end. + +send_to_coap(#session{queue = Queue} = Session) -> + case queue:out(Queue) of + {{value, {Timestamp, Ctx, Req}}, Q2} -> + Now = ?NOW, + if Timestamp =:= 0 orelse Timestamp > Now -> + send_to_coap(Ctx, Req, Session#session{queue = Q2}); + true -> + send_to_coap(Session#session{queue = Q2}) + end; + {empty, _} -> + Session + end. + +send_to_coap(Ctx, Req, Session) -> + ?LOG(debug, "Deliver To CoAP, CoapRequest: ~0p", [Req]), + out_to_coap(Ctx, Req, Session#session{wait_ack = Ctx}). + +send_msg_not_waiting_ack(Ctx, Req, Session) -> + ?LOG(debug, "Deliver To CoAP not waiting ack, CoapRequest: ~0p", [Req]), + %% cmd_sent(Ref, LwM2MOpts). + out_to_coap(Ctx, Req, Session). + +%%-------------------------------------------------------------------- +%% Send To MQTT +%%-------------------------------------------------------------------- +send_to_mqtt(Ref, EventType, Payload, WithContext, Session) -> + #{topic := Topic, qos := Qos} = uplink_topic(EventType), + Mheaders = maps:get(mheaders, Ref, #{}), + proto_publish(Topic, Payload#{<<"msgType">> => EventType}, Qos, Mheaders, WithContext, Session). + +send_to_mqtt(Ctx, EventType, Payload, {Topic, Qos}, + WithContext, Session) -> + Mheaders = maps:get(mheaders, Ctx, #{}), + proto_publish(Topic, Payload#{<<"msgType">> => EventType}, Qos, Mheaders, WithContext, Session). + +proto_publish(Topic, Payload, Qos, Headers, WithContext, + #session{endpoint_name = Epn} = Session) -> + MountedTopic = mount(Topic, Session), + Msg = emqx_message:make(Epn, Qos, MountedTopic, + emqx_json:encode(Payload), #{}, Headers), + WithContext(publish, [MountedTopic, Msg]), + Session. + +mount(Topic, #session{mountpoint = MountPoint}) when is_binary(Topic) -> + <>. + +downlink_topic() -> + emqx:get_config([gateway, lwm2m, translators, command]). + +uplink_topic(<<"notify">>) -> + emqx:get_config([gateway, lwm2m, translators, notify]); + +uplink_topic(<<"register">>) -> + emqx:get_config([gateway, lwm2m, translators, register]); + +uplink_topic(<<"update">>) -> + emqx:get_config([gateway, lwm2m, translators, update]); + +uplink_topic(_) -> + emqx:get_config([gateway, lwm2m, translators, response]). + +%%-------------------------------------------------------------------- +%% Deliver +%%-------------------------------------------------------------------- + +deliver(Delivers, WithContext, #session{reg_info = RegInfo} = Session) -> + IsCacheMode = is_cache_mode(Session), + AlternatePath = maps:get(<<"alternatePath">>, RegInfo, <<"/">>), + lists:foldl(fun({deliver, _, MQTT}, Acc) -> + deliver_to_coap(AlternatePath, + MQTT#message.payload, MQTT, IsCacheMode, WithContext, Acc) + end, + Session, + Delivers). + +deliver_to_coap(AlternatePath, JsonData, MQTT, CacheMode, WithContext, Session) when is_binary(JsonData)-> + try + TermData = emqx_json:decode(JsonData, [return_maps]), + deliver_to_coap(AlternatePath, TermData, MQTT, CacheMode, WithContext, Session) + catch + ExClass:Error:ST -> + ?LOG(error, "deliver_to_coap - Invalid JSON: ~0p, Exception: ~0p, stacktrace: ~0p", + [JsonData, {ExClass, Error}, ST]), + WithContext(metrics, 'delivery.dropped'), + Session + end; + +deliver_to_coap(AlternatePath, TermData, MQTT, CacheMode, WithContext, Session) when is_map(TermData) -> + WithContext(metrics, 'messages.delivered'), + {Req, Ctx} = emqx_lwm2m_cmd:mqtt_to_coap(AlternatePath, TermData), + ExpiryTime = get_expiry_time(MQTT), + Session2 = record_request(Ctx, Session), + maybe_do_deliver_to_coap(Ctx, Req, ExpiryTime, CacheMode, Session2). + +maybe_do_deliver_to_coap(Ctx, Req, ExpiryTime, CacheMode, + #session{wait_ack = WaitAck, + queue = Queue} = Session) -> + MHeaders = maps:get(mheaders, Ctx, #{}), + TTL = maps:get(<<"ttl">>, MHeaders, 7200), + case TTL of + 0 -> + send_msg_not_waiting_ack(Ctx, Req, Session); + _ -> + case not CacheMode + andalso queue:is_empty(Queue) andalso WaitAck =:= undefined of + true -> + send_to_coap(Ctx, Req, Session); + false -> + Session#session{queue = queue:in({ExpiryTime, Ctx, Req}, Queue)} + end + end. + +get_expiry_time(#message{headers = #{properties := #{'Message-Expiry-Interval' := Interval}}, + timestamp = Ts}) -> + Ts + Interval * 1000; +get_expiry_time(_) -> + 0. + +%%-------------------------------------------------------------------- +%% Call CoAP +%%-------------------------------------------------------------------- +call_coap(Fun, Msg, #session{coap = Coap} = Session) -> + iter([tm, fun process_tm/4, fun process_session/3], + emqx_coap_tm:Fun(Msg, Coap), + Session). + +process_tm(TM, Result, Session, Cursor) -> + iter(Cursor, Result, Session#session{coap = TM}). + +process_session(_, Result, Session) -> + Result#{session => Session}. + +out_to_coap(Context, Msg, Session) -> + out_to_coap({Context, Msg}, Session). + +out_to_coap(Msg, Session) -> + Outs = get_outs(), + erlang:put(?OUT_LIST_KEY, [Msg | Outs]), + Session. + +get_outs() -> + case erlang:get(?OUT_LIST_KEY) of + undefined -> []; + Any -> Any + end. + +return(#session{coap = CoAP} = Session) -> + Outs = get_outs(), + erlang:put(?OUT_LIST_KEY, []), + {ok, Coap2, Msgs} = do_out(Outs, CoAP, []), + #{return => {Msgs, Session#session{coap = Coap2}}}. + +do_out([{Ctx, Out} | T], TM, Msgs) -> + %% TODO maybe set a special token? + #{out := [Msg], + tm := TM2} = emqx_coap_tm:handle_out(Out, Ctx, TM), + do_out(T, TM2, [Msg | Msgs]); + +do_out(_, TM, Msgs) -> + {ok, TM, Msgs}. + + +%%-------------------------------------------------------------------- +%% CMD Record +%%-------------------------------------------------------------------- +-spec record_request(request_context(), session()) -> session(). +record_request(#{<<"msgType">> := Type} = Context, Session) -> + Path = emqx_lwm2m_cmd:extract_path(Context), + record_cmd(Path, Type, undefined, Session). + +record_response(EventType, #{<<"data">> := Data}, Session) -> + ReqPath = maps:get(<<"reqPath">>, Data, undefined), + Code = maps:get(<<"code">>, Data, undefined), + CodeMsg = maps:get(<<"codeMsg">>, Data, undefined), + Content = maps:get(<<"content">>, Data, undefined), + record_cmd(ReqPath, EventType, {Code, CodeMsg, Content}, Session). + +record_cmd(Path, Type, Result, #session{cmd_record = Record} = Session) -> + Record2 = Record#{?CMD_KEY(Path, Type) => Result}, + Session#session{cmd_record = Record2}. diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_timer.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_timer.erl deleted file mode 100644 index b86000292..000000000 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_timer.erl +++ /dev/null @@ -1,47 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_lwm2m_timer). - --include("src/lwm2m/include/emqx_lwm2m.hrl"). - --export([ cancel_timer/1 - , start_timer/2 - , refresh_timer/1 - , refresh_timer/2 - ]). - --record(timer_state, { interval - , tref - , message - }). - --define(LOG(Level, Format, Args), - logger:Level("LWM2M-TIMER: " ++ Format, Args)). - -cancel_timer(#timer_state{tref = TRef}) when is_reference(TRef) -> - _ = erlang:cancel_timer(TRef), ok. - -refresh_timer(State=#timer_state{interval = Interval, message = Msg}) -> - cancel_timer(State), start_timer(Interval, Msg). -refresh_timer(NewInterval, State=#timer_state{message = Msg}) -> - cancel_timer(State), start_timer(NewInterval, Msg). - -%% start timer in seconds -start_timer(Interval, Msg) -> - ?LOG(debug, "start_timer of ~p secs", [Interval]), - TRef = erlang:send_after(timer:seconds(Interval), self(), Msg), - #timer_state{interval = Interval, tref = TRef, message = Msg}. diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object.erl index 96a80735f..a4ec27413 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object.erl +++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object.erl @@ -16,7 +16,7 @@ -module(emqx_lwm2m_xml_object). --include("src/lwm2m/include/emqx_lwm2m.hrl"). +-include_lib("emqx_gateway/src/lwm2m/include/emqx_lwm2m.hrl"). -include_lib("xmerl/include/xmerl.hrl"). -export([ get_obj_def/2 @@ -38,8 +38,6 @@ get_obj_def(ObjectIdInt, true) -> get_obj_def(ObjectNameStr, false) -> emqx_lwm2m_xml_object_db:find_name(ObjectNameStr). - - get_object_id(ObjDefinition) -> [#xmlText{value=ObjectId}] = xmerl_xpath:string("ObjectID/text()", ObjDefinition), ObjectId. @@ -48,7 +46,6 @@ get_object_name(ObjDefinition) -> [#xmlText{value=ObjectName}] = xmerl_xpath:string("Name/text()", ObjDefinition), ObjectName. - get_object_and_resource_id(ResourceNameBinary, ObjDefinition) -> ResourceNameString = binary_to_list(ResourceNameBinary), [#xmlText{value=ObjectId}] = xmerl_xpath:string("ObjectID/text()", ObjDefinition), @@ -56,7 +53,6 @@ get_object_and_resource_id(ResourceNameBinary, ObjDefinition) -> ?LOG(debug, "get_object_and_resource_id ObjectId=~p, ResourceId=~p", [ObjectId, ResourceId]), {ObjectId, ResourceId}. - get_resource_type(ResourceIdInt, ObjDefinition) -> ResourceIdString = integer_to_list(ResourceIdInt), [#xmlText{value=DataType}] = xmerl_xpath:string("Resources/Item[@ID=\""++ResourceIdString++"\"]/Type/text()", ObjDefinition), diff --git a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object_db.erl b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object_db.erl index 1d7fb6d5e..ec7c83de1 100644 --- a/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object_db.erl +++ b/apps/emqx_gateway/src/lwm2m/emqx_lwm2m_xml_object_db.erl @@ -16,7 +16,7 @@ -module(emqx_lwm2m_xml_object_db). --include("src/lwm2m/include/emqx_lwm2m.hrl"). +-include_lib("emqx_gateway/src/lwm2m/include/emqx_lwm2m.hrl"). -include_lib("xmerl/include/xmerl.hrl"). % This module is for future use. Disabled now. @@ -49,15 +49,14 @@ %% API Function Definitions %% ------------------------------------------------------------------ --spec start_link(binary() | string()) -> {ok, pid()} | ignore | {error, any()}. start_link(XmlDir) -> gen_server:start_link({local, ?MODULE}, ?MODULE, [XmlDir], []). find_objectid(ObjectId) -> - ObjectIdInt = case is_list(ObjectId) of - true -> list_to_integer(ObjectId); - false -> ObjectId - end, + ObjectIdInt = case is_list(ObjectId) of + true -> list_to_integer(ObjectId); + false -> ObjectId + end, case ets:lookup(?LWM2M_OBJECT_DEF_TAB, ObjectIdInt) of [] -> {error, no_xml_definition}; [{ObjectId, Xml}] -> Xml @@ -81,15 +80,14 @@ find_name(Name) -> stop() -> gen_server:stop(?MODULE). - %% ------------------------------------------------------------------ %% gen_server Function Definitions %% ------------------------------------------------------------------ -init([XmlDir0]) -> +init([XmlDir]) -> _ = ets:new(?LWM2M_OBJECT_DEF_TAB, [set, named_table, protected]), _ = ets:new(?LWM2M_OBJECT_NAME_TO_ID_TAB, [set, named_table, protected]), - load(to_list(XmlDir0)), + load(XmlDir), {ok, #state{}}. handle_call(_Request, _From, State) -> @@ -113,11 +111,13 @@ code_change(_OldVsn, State, _Extra) -> %% Internal functions %%-------------------------------------------------------------------- load(BaseDir) -> - Wild = case lists:last(BaseDir) == $/ of - true -> BaseDir++"*.xml"; - false -> BaseDir++"/*.xml" - end, - case filelib:wildcard(Wild) of + Wild = filename:join(BaseDir, "*.xml"), + Wild2 = if is_binary(Wild) -> + erlang:binary_to_list(Wild); + true -> + Wild + end, + case filelib:wildcard(Wild2) of [] -> error(no_xml_files_found, BaseDir); AllXmlFiles -> load_loop(AllXmlFiles) end. @@ -135,13 +135,7 @@ load_loop([FileName|T]) -> ets:insert(?LWM2M_OBJECT_NAME_TO_ID_TAB, {NameBinary, ObjectId}), load_loop(T). - load_xml(FileName) -> {Xml, _Rest} = xmerl_scan:file(FileName), [ObjectXml] = xmerl_xpath:string("/LWM2M/Object", Xml), ObjectXml. - -to_list(B) when is_binary(B) -> - binary_to_list(B); -to_list(S) when is_list(S) -> - S. diff --git a/apps/emqx_gateway/src/lwm2m/include/emqx_lwm2m.hrl b/apps/emqx_gateway/src/lwm2m/include/emqx_lwm2m.hrl index 5462f489d..05e0f0503 100644 --- a/apps/emqx_gateway/src/lwm2m/include/emqx_lwm2m.hrl +++ b/apps/emqx_gateway/src/lwm2m/include/emqx_lwm2m.hrl @@ -14,15 +14,8 @@ %% limitations under the License. %%-------------------------------------------------------------------- --define(APP, emqx_lwm2m). +-define(LWAPP, emqx_lwm2m). --record(coap_mqtt_auth, { clientid - , username - , password - }). --record(lwm2m_context, { epn - , location - }). -define(OMA_ALTER_PATH_RT, <<"\"oma.lwm2m\"">>). @@ -42,7 +35,7 @@ -define(ERR_NOT_FOUND, <<"Not Found">>). -define(ERR_UNAUTHORIZED, <<"Unauthorized">>). -define(ERR_BAD_REQUEST, <<"Bad Request">>). - +-define(REG_PREFIX, <<"rd">>). -define(LWM2M_FORMAT_PLAIN_TEXT, 0). -define(LWM2M_FORMAT_LINK, 40). diff --git a/apps/emqx_gateway/src/mqttsn/emqx_sn_channel.erl b/apps/emqx_gateway/src/mqttsn/emqx_sn_channel.erl index 5bba599c8..34e6ec8d6 100644 --- a/apps/emqx_gateway/src/mqttsn/emqx_sn_channel.erl +++ b/apps/emqx_gateway/src/mqttsn/emqx_sn_channel.erl @@ -23,7 +23,6 @@ -include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("emqx/include/logger.hrl"). - %% API -export([ info/1 , info/2 @@ -39,7 +38,7 @@ , set_conn_state/2 ]). --export([ handle_call/2 +-export([ handle_call/3 , handle_cast/2 , handle_info/2 ]). @@ -96,9 +95,9 @@ }). -define(DEFAULT_OVERRIDE, - #{ clientid => <<"">> %% Generate clientid by default - , username => <<"${Packet.headers.login}">> - , password => <<"${Packet.headers.passcode}">> + #{ clientid => <<"${ConnInfo.clientid}">> + %, username => <<"${ConnInfo.clientid}">> + %, password => <<"${Packet.headers.passcode}">> }). -define(INFO_KEYS, [conninfo, conn_state, clientinfo, session, will_msg]). @@ -190,9 +189,10 @@ stats(#channel{session = Session})-> set_conn_state(ConnState, Channel) -> Channel#channel{conn_state = ConnState}. -enrich_conninfo(?SN_CONNECT_MSG(_Flags, _ProtoId, Duration, _ClientId), +enrich_conninfo(?SN_CONNECT_MSG(_Flags, _ProtoId, Duration, ClientId), Channel = #channel{conninfo = ConnInfo}) -> - NConnInfo = ConnInfo#{ proto_name => <<"MQTT-SN">> + NConnInfo = ConnInfo#{ clientid => ClientId + , proto_name => <<"MQTT-SN">> , proto_ver => <<"1.2">> , clean_start => true , keepalive => Duration @@ -233,8 +233,8 @@ feedvar(Override, Packet, ConnInfo, ClientInfo) -> , 'Packet' => connect_packet_to_map(Packet) }, maps:map(fun(_K, V) -> - Tokens = emqx_rule_utils:preproc_tmpl(V), - emqx_rule_utils:proc_tmpl(Tokens, Envs) + Tokens = emqx_plugin_libs_rule:preproc_tmpl(V), + emqx_plugin_libs_rule:proc_tmpl(Tokens, Envs) end, Override). connect_packet_to_map(#mqtt_sn_message{}) -> @@ -593,9 +593,11 @@ handle_in(SubPkt = ?SN_SUBSCRIBE_MSG(_, MsgId, _), Channel) -> case emqx_misc:pipeline( [ fun preproc_subs_type/2 , fun check_subscribe_authz/2 + , fun run_client_subs_hook/2 , fun do_subscribe/2 ], SubPkt, Channel) of - {ok, {TopicId, GrantedQoS}, NChannel} -> + {ok, {TopicId, _TopicName, SubOpts}, NChannel} -> + GrantedQoS = maps:get(qos, SubOpts), SubAck = ?SN_SUBACK_MSG(#mqtt_sn_flags{qos = GrantedQoS}, TopicId, MsgId, ?SN_RC_ACCEPTED), {ok, outgoing_and_update(SubAck), NChannel}; @@ -611,6 +613,7 @@ handle_in(UnsubPkt = ?SN_UNSUBSCRIBE_MSG(_, MsgId, TopicIdOrName), Channel) -> case emqx_misc:pipeline( [ fun preproc_unsub_type/2 + , fun run_client_unsub_hook/2 , fun do_unsubscribe/2 ], UnsubPkt, Channel) of {ok, _TopicName, NChannel} -> @@ -842,13 +845,10 @@ check_subscribe_authz({_TopicId, TopicName, _QoS}, {error, ?SN_RC_NOT_AUTHORIZE} end. -do_subscribe({TopicId, TopicName, QoS}, - Channel = #channel{ - ctx = Ctx, - session = Session, - clientinfo = ClientInfo - = #{mountpoint := Mountpoint}}) -> - +run_client_subs_hook({TopicId, TopicName, QoS}, + Channel = #channel{ + ctx = Ctx, + clientinfo = ClientInfo}) -> {TopicName1, SubOpts0} = emqx_topic:parse(TopicName), TopicFilters = [{TopicName1, SubOpts0#{qos => QoS}}], case run_hooks(Ctx, 'client.subscribe', @@ -856,19 +856,26 @@ do_subscribe({TopicId, TopicName, QoS}, [] -> ?LOG(warning, "Skip to subscribe ~s, " "due to 'client.subscribe' denied!", [TopicName]), - {ok, Channel}; + {error, ?SN_EXCEED_LIMITATION}; [{NTopicName, NSubOpts}|_] -> - NTopicName1 = emqx_mountpoint:mount(Mountpoint, NTopicName), - NSubOpts1 = maps:merge(?DEFAULT_SUBOPTS, NSubOpts), - case emqx_session:subscribe(ClientInfo, NTopicName1, NSubOpts1, Session) of - {ok, NSession} -> - {ok, {TopicId, QoS}, - Channel#channel{session = NSession}}; - {error, ?RC_QUOTA_EXCEEDED} -> - ?LOG(warning, "Cannot subscribe ~s due to ~s.", - [TopicName, emqx_reason_codes:text(?RC_QUOTA_EXCEEDED)]), - {error, ?SN_EXCEED_LIMITATION} - end + {ok, {TopicId, NTopicName, NSubOpts}, Channel} + end. + +do_subscribe({TopicId, TopicName, SubOpts}, + Channel = #channel{ + session = Session, + clientinfo = ClientInfo + = #{mountpoint := Mountpoint}}) -> + NTopicName = emqx_mountpoint:mount(Mountpoint, TopicName), + NSubOpts = maps:merge(?DEFAULT_SUBOPTS, SubOpts), + case emqx_session:subscribe(ClientInfo, NTopicName, NSubOpts, Session) of + {ok, NSession} -> + {ok, {TopicId, NTopicName, NSubOpts}, + Channel#channel{session = NSession}}; + {error, ?RC_QUOTA_EXCEEDED} -> + ?LOG(warning, "Cannot subscribe ~s due to ~s.", + [TopicName, emqx_reason_codes:text(?RC_QUOTA_EXCEEDED)]), + {error, ?SN_EXCEED_LIMITATION} end. %%-------------------------------------------------------------------- @@ -900,33 +907,42 @@ preproc_unsub_type(?SN_UNSUBSCRIBE_MSG_TYPE(?SN_SHORT_TOPIC, end, {ok, TopicName, Channel}. -do_unsubscribe(TopicName, - Channel = #channel{ - ctx = Ctx, - session = Session, - clientinfo = ClientInfo - = #{mountpoint := Mountpoint}}) -> +run_client_unsub_hook(TopicName, + Channel = #channel{ + ctx = Ctx, + clientinfo = ClientInfo + }) -> TopicFilters = [emqx_topic:parse(TopicName)], case run_hooks(Ctx, 'client.unsubscribe', [ClientInfo, #{}], TopicFilters) of [] -> - %% Skip to unsubscribe - {ok, Channel}; - [{NTopicName, NSubOpts}|_] -> - NTopicName1 = emqx_mountpoint:mount(Mountpoint, NTopicName), - NSubOpts1 = maps:merge( - emqx_gateway_utils:default_subopts(), - NSubOpts - ), - case emqx_session:unsubscribe(ClientInfo, NTopicName1, - NSubOpts1, Session) of - {ok, NSession} -> - {ok, Channel#channel{session = NSession}}; - {error, ?RC_NO_SUBSCRIPTION_EXISTED} -> - {ok, Channel} - end + {ok, [], Channel}; + NTopicFilters -> + {ok, NTopicFilters, Channel} end. +do_unsubscribe(TopicFilters, + Channel = #channel{ + session = Session, + clientinfo = ClientInfo + = #{mountpoint := Mountpoint}}) -> + NChannel = + lists:foldl(fun({TopicName, SubOpts}, ChannAcc) -> + NTopicName = emqx_mountpoint:mount(Mountpoint, TopicName), + NSubOpts = maps:merge( + emqx_gateway_utils:default_subopts(), + SubOpts + ), + case emqx_session:unsubscribe(ClientInfo, NTopicName, + NSubOpts, Session) of + {ok, NSession} -> + ChannAcc#channel{session = NSession}; + {error, ?RC_NO_SUBSCRIPTION_EXISTED} -> + ChannAcc + end + end, Channel, TopicFilters), + {ok, TopicFilters, NChannel}. + %%-------------------------------------------------------------------- %% Awake & Asleep @@ -1097,23 +1113,55 @@ message_to_packet(MsgId, Message, %% Handle call %%-------------------------------------------------------------------- --spec handle_call(Req :: term(), channel()) - -> {reply, Reply :: term(), channel()} - | {shutdown, Reason :: term(), Reply :: term(), channel()} - | {shutdown, Reason :: term(), Reply :: term(), - emqx_types:packet(), channel()}. -handle_call(kick, Channel) -> +-spec handle_call(Req :: term(), From :: term(), channel()) + -> {reply, Reply :: term(), channel()} + | {reply, Reply :: term(), replies(), channel()} + | {shutdown, Reason :: term(), Reply :: term(), channel()} + | {shutdown, Reason :: term(), Reply :: term(), + emqx_types:packet(), channel()}. +handle_call({subscribe, Topic, SubOpts}, _From, Channel) -> + %% XXX: Only support short_topic_name + SubProps = maps:get(sub_props, SubOpts, #{}), + case maps:get(subtype, SubProps, short_topic_name) of + short_topic_name -> + case byte_size(Topic) of + 2 -> + case do_subscribe({?SN_INVALID_TOPIC_ID, + Topic, SubOpts}, Channel) of + {ok, _, NChannel} -> + reply(ok, NChannel); + {error, ?SN_EXCEED_LIMITATION} -> + reply({error, exceed_limitation}, Channel) + end; + _ -> + reply({error, bad_topic_name}, Channel) + end; + predefined_topic_id -> + reply({error, only_support_short_name_topic}, Channel); + _ -> + reply({error, only_support_short_name_topic}, Channel) + end; + +handle_call({unsubscribe, Topic}, _From, Channel) -> + TopicFilters = [emqx_topic:parse(Topic)], + {ok, _, NChannel} = do_unsubscribe(TopicFilters, Channel), + reply(ok, NChannel); + +handle_call(subscriptions, _From, Channel = #channel{session = Session}) -> + reply(maps:to_list(emqx_session:info(subscriptions, Session)), Channel); + +handle_call(kick, _From, Channel) -> NChannel = ensure_disconnected(kicked, Channel), shutdown_and_reply(kicked, ok, NChannel); -handle_call(discard, Channel) -> +handle_call(discard, _From, Channel) -> shutdown_and_reply(discarded, ok, Channel); %% XXX: No Session Takeover -%handle_call({takeover, 'begin'}, Channel = #channel{session = Session}) -> +%handle_call({takeover, 'begin'}, _From, Channel = #channel{session = Session}) -> % reply(Session, Channel#channel{takeover = true}); % -%handle_call({takeover, 'end'}, Channel = #channel{session = Session, +%handle_call({takeover, 'end'}, _From, Channel = #channel{session = Session, % pendings = Pendings}) -> % ok = emqx_session:takeover(Session), % %% TODO: Should not drain deliver here (side effect) @@ -1121,16 +1169,16 @@ handle_call(discard, Channel) -> % AllPendings = lists:append(Delivers, Pendings), % shutdown_and_reply(takeovered, AllPendings, Channel); -%handle_call(list_authz_cache, Channel) -> +%handle_call(list_authz_cache, _From, Channel) -> % {reply, emqx_authz_cache:list_authz_cache(), Channel}; %% XXX: No Quota Now -% handle_call({quota, Policy}, Channel) -> +% handle_call({quota, Policy}, _From, Channel) -> % Zone = info(zone, Channel), % Quota = emqx_limiter:init(Zone, Policy), % reply(ok, Channel#channel{quota = Quota}); -handle_call(Req, Channel) -> +handle_call(Req, _From, Channel) -> ?LOG(error, "Unexpected call: ~p", [Req]), reply(ignored, Channel). @@ -1150,18 +1198,6 @@ handle_cast(_Req, Channel) -> -spec handle_info(Info :: term(), channel()) -> ok | {ok, channel()} | {shutdown, Reason :: term(), channel()}. -%% XXX: Received from the emqx-management ??? -%handle_info({subscribe, TopicFilters}, Channel ) -> -% {_, NChannel} = lists:foldl( -% fun({TopicFilter, SubOpts}, {_, ChannelAcc}) -> -% do_subscribe(TopicFilter, SubOpts, ChannelAcc) -% end, {[], Channel}, parse_topic_filters(TopicFilters)), -% {ok, NChannel}; -% -%handle_info({unsubscribe, TopicFilters}, Channel) -> -% {_RC, NChannel} = process_unsubscribe(TopicFilters, #{}, Channel), -% {ok, NChannel}; - handle_info({sock_closed, Reason}, Channel = #channel{conn_state = idle}) -> shutdown(Reason, Channel); diff --git a/apps/emqx_gateway/src/mqttsn/emqx_sn_impl.erl b/apps/emqx_gateway/src/mqttsn/emqx_sn_impl.erl index 039b23924..a79173cff 100644 --- a/apps/emqx_gateway/src/mqttsn/emqx_sn_impl.erl +++ b/apps/emqx_gateway/src/mqttsn/emqx_sn_impl.erl @@ -108,17 +108,17 @@ start_listener(GwName, Ctx, {Type, LisName, ListenOn, SocketOpts, Cfg}) -> ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn), case start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) of {ok, Pid} -> - ?ULOG("Start ~s:~s:~s listener on ~s successfully.~n", + ?ULOG("Gateway ~s:~s:~s on ~s started.~n", [GwName, Type, LisName, ListenOnStr]), Pid; {error, Reason} -> - ?ELOG("Failed to start ~s:~s:~s listener on ~s: ~0p~n", + ?ELOG("Failed to start gateway ~s:~s:~s on ~s: ~0p~n", [GwName, Type, LisName, ListenOnStr, Reason]), throw({badconf, Reason}) end. start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) -> - Name = name(GwName, LisName, Type), + Name = emqx_gateway_utils:listener_id(GwName, Type, LisName), NCfg = Cfg#{ ctx => Ctx, frame_mod => emqx_sn_frame, @@ -127,9 +127,6 @@ start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) -> esockd:open_udp(Name, ListenOn, merge_default(SocketOpts), {emqx_gateway_conn, start_link, [NCfg]}). -name(GwName, LisName, Type) -> - list_to_atom(lists:concat([GwName, ":", Type, ":", LisName])). - merge_default(Options) -> Default = emqx_gateway_utils:default_udp_options(), case lists:keytake(udp_options, 1, Options) of @@ -144,14 +141,14 @@ stop_listener(GwName, {Type, LisName, ListenOn, SocketOpts, Cfg}) -> StopRet = stop_listener(GwName, LisName, Type, ListenOn, SocketOpts, Cfg), ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn), case StopRet of - ok -> ?ULOG("Stop ~s:~s:~s listener on ~s successfully.~n", + ok -> ?ULOG("Gateway ~s:~s:~s on ~s stopped.~n", [GwName, Type, LisName, ListenOnStr]); {error, Reason} -> - ?ELOG("Failed to stop ~s:~s:~s listener on ~s: ~0p~n", + ?ELOG("Failed to stop gatewat ~s:~s:~s on ~s: ~0p~n", [GwName, Type, LisName, ListenOnStr, Reason]) end, StopRet. stop_listener(GwName, Type, LisName, ListenOn, _SocketOpts, _Cfg) -> - Name = name(GwName, LisName, Type), + Name = emqx_gateway_utils:listener_id(GwName, Type, LisName), esockd:close(Name, ListenOn). diff --git a/apps/emqx_gateway/src/stomp/emqx_stomp_channel.erl b/apps/emqx_gateway/src/stomp/emqx_stomp_channel.erl index 250f43988..e55e0a580 100644 --- a/apps/emqx_gateway/src/stomp/emqx_stomp_channel.erl +++ b/apps/emqx_gateway/src/stomp/emqx_stomp_channel.erl @@ -22,7 +22,6 @@ -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/logger.hrl"). - -import(proplists, [get_value/2, get_value/3]). %% API @@ -40,7 +39,7 @@ , set_conn_state/2 ]). --export([ handle_call/2 +-export([ handle_call/3 , handle_cast/2 , handle_info/2 ]). @@ -232,8 +231,8 @@ feedvar(Override, Packet, ConnInfo, ClientInfo) -> , 'Packet' => connect_packet_to_map(Packet) }, maps:map(fun(_K, V) -> - Tokens = emqx_rule_utils:preproc_tmpl(V), - emqx_rule_utils:proc_tmpl(Tokens, Envs) + Tokens = emqx_plugin_libs_rule:preproc_tmpl(V), + emqx_plugin_libs_rule:proc_tmpl(Tokens, Envs) end, Override). connect_packet_to_map(#stomp_frame{headers = Headers}) -> @@ -393,11 +392,9 @@ handle_in(?PACKET(?CMD_SUBSCRIBE, Headers), [] -> ErrMsg = "Permission denied", handle_out(error, {receipt_id(Headers), ErrMsg}, Channel); - [MountedTopic|_] -> - NChannel1 = NChannel#channel{ - subscriptions = [{SubId, MountedTopic, Ack} - | Subs] - }, + [{MountedTopic, SubOpts}|_] -> + NSubs = [{SubId, MountedTopic, Ack, SubOpts}|Subs], + NChannel1 = NChannel#channel{subscriptions = NSubs}, handle_out(receipt, receipt_id(Headers), NChannel1) end; {error, ErrMsg, NChannel} -> @@ -415,7 +412,7 @@ handle_in(?PACKET(?CMD_UNSUBSCRIBE, Headers), SubId = header(<<"id">>, Headers), {ok, NChannel} = case lists:keyfind(SubId, 1, Subs) of - {SubId, MountedTopic, _Ack} -> + {SubId, MountedTopic, _Ack, _SubOpts} -> Topic = emqx_mountpoint:unmount(Mountpoint, MountedTopic), %% XXX: eval the return topics? _ = run_hooks(Ctx, 'client.unsubscribe', @@ -539,29 +536,30 @@ trans_pipeline([{Func, Args}|More], Outgoings, Channel) -> %% Subs parse_topic_filter({SubId, Topic}, Channel) -> - TopicFilter = emqx_topic:parse(Topic), - {ok, {SubId, TopicFilter}, Channel}. + {ParsedTopic, SubOpts} = emqx_topic:parse(Topic), + NSubOpts = SubOpts#{sub_props => #{subid => SubId}}, + {ok, {SubId, {ParsedTopic, NSubOpts}}, Channel}. -check_subscribed_status({SubId, TopicFilter}, +check_subscribed_status({SubId, {ParsedTopic, _SubOpts}}, #channel{ subscriptions = Subs, clientinfo = #{mountpoint := Mountpoint} }) -> - MountedTopic = emqx_mountpoint:mount(Mountpoint, TopicFilter), + MountedTopic = emqx_mountpoint:mount(Mountpoint, ParsedTopic), case lists:keyfind(SubId, 1, Subs) of - {SubId, MountedTopic, _Ack} -> + {SubId, MountedTopic, _Ack, _} -> ok; - {SubId, _OtherTopic, _Ack} -> + {SubId, _OtherTopic, _Ack, _} -> {error, "Conflict subscribe id"}; false -> ok end. -check_sub_acl({_SubId, TopicFilter}, +check_sub_acl({_SubId, {ParsedTopic, _SubOpts}}, #channel{ ctx = Ctx, clientinfo = ClientInfo}) -> - case emqx_gateway_ctx:authorize(Ctx, ClientInfo, subscribe, TopicFilter) of + case emqx_gateway_ctx:authorize(Ctx, ClientInfo, subscribe, ParsedTopic) of deny -> {error, "ACL Deny"}; allow -> ok end. @@ -571,27 +569,27 @@ do_subscribe(TopicFilters, Channel) -> do_subscribe([], _Channel, Acc) -> lists:reverse(Acc); -do_subscribe([{TopicFilter, Option}|More], +do_subscribe([{ParsedTopic, SubOpts0}|More], Channel = #channel{ ctx = Ctx, clientinfo = ClientInfo = #{clientid := ClientId, mountpoint := Mountpoint}}, Acc) -> - SubOpts = maps:merge(emqx_gateway_utils:default_subopts(), Option), - MountedTopic = emqx_mountpoint:mount(Mountpoint, TopicFilter), + SubOpts = maps:merge(emqx_gateway_utils:default_subopts(), SubOpts0), + MountedTopic = emqx_mountpoint:mount(Mountpoint, ParsedTopic), _ = emqx_broker:subscribe(MountedTopic, ClientId, SubOpts), run_hooks(Ctx, 'session.subscribed', [ClientInfo, MountedTopic, SubOpts]), - do_subscribe(More, Channel, [MountedTopic|Acc]). + do_subscribe(More, Channel, [{MountedTopic, SubOpts}|Acc]). %%-------------------------------------------------------------------- %% Handle outgoing packet %%-------------------------------------------------------------------- -spec(handle_out(atom(), term(), channel()) - -> {ok, channel()} - | {ok, replies(), channel()} - | {shutdown, Reason :: term(), channel()} - | {shutdown, Reason :: term(), replies(), channel()}). + -> {ok, channel()} + | {ok, replies(), channel()} + | {shutdown, Reason :: term(), channel()} + | {shutdown, Reason :: term(), replies(), channel()}). handle_out(connerr, {Headers, ReceiptId, ErrMsg}, Channel) -> Frame = error_frame(Headers, ReceiptId, ErrMsg), @@ -622,24 +620,78 @@ handle_out(receipt, ReceiptId, Channel) -> %% Handle call %%-------------------------------------------------------------------- --spec(handle_call(Req :: term(), channel()) - -> {reply, Reply :: term(), channel()} - | {shutdown, Reason :: term(), Reply :: term(), channel()} - | {shutdown, Reason :: term(), Reply :: term(), stomp_frame(), channel()}). -handle_call(kick, Channel) -> +-spec(handle_call(Req :: term(), From :: term(), channel()) + -> {reply, Reply :: term(), channel()} + | {reply, Reply :: term(), replies(), channel()} + | {shutdown, Reason :: term(), Reply :: term(), channel()} + | {shutdown, Reason :: term(), Reply :: term(), stomp_frame(), channel()}). +handle_call({subscribe, Topic, SubOpts}, _From, + Channel = #channel{ + subscriptions = Subs + }) -> + case maps:get(subid, + maps:get(sub_props, SubOpts, #{}), + undefined) of + undefined -> + reply({error, no_subid}, Channel); + SubId -> + case emqx_misc:pipeline( + [ fun parse_topic_filter/2 + , fun check_subscribed_status/2 + ], {SubId, {Topic, SubOpts}}, Channel) of + {ok, {_, TopicFilter}, NChannel} -> + [{MountedTopic, NSubOpts}] = do_subscribe( + [TopicFilter], + NChannel + ), + NSubs = [{SubId, MountedTopic, <<"auto">>, NSubOpts}|Subs], + NChannel1 = NChannel#channel{subscriptions = NSubs}, + reply(ok, NChannel1); + {error, ErrMsg, NChannel} -> + ?LOG(error, "Failed to subscribe topic ~s, reason: ~s", + [Topic, ErrMsg]), + reply({error, ErrMsg}, NChannel) + end + end; + +handle_call({unsubscribe, Topic}, _From, + Channel = #channel{ + ctx = Ctx, + clientinfo = ClientInfo = #{mountpoint := Mountpoint}, + subscriptions = Subs + }) -> + {ParsedTopic, _SubOpts} = emqx_topic:parse(Topic), + MountedTopic = emqx_mountpoint:mount(Mountpoint, ParsedTopic), + ok = emqx_broker:unsubscribe(MountedTopic), + _ = run_hooks(Ctx, 'session.unsubscribe', + [ClientInfo, MountedTopic, #{}]), + reply(ok, + Channel#channel{ + subscriptions = lists:keydelete(MountedTopic, 2, Subs)} + ); + +%% Reply :: [{emqx_types:topic(), emqx_types:subopts()}] +handle_call(subscriptions, _From, Channel = #channel{subscriptions = Subs}) -> + Reply = lists:map( + fun({_SubId, Topic, _Ack, SubOpts}) -> + {Topic, SubOpts} + end, Subs), + reply(Reply, Channel); + +handle_call(kick, _From, Channel) -> NChannel = ensure_disconnected(kicked, Channel), Frame = error_frame(undefined, <<"Kicked out">>), shutdown_and_reply(kicked, ok, Frame, NChannel); -handle_call(discard, Channel) -> +handle_call(discard, _From, Channel) -> Frame = error_frame(undefined, <<"Discarded">>), shutdown_and_reply(discarded, ok, Frame, Channel); %% XXX: No Session Takeover -%handle_call({takeover, 'begin'}, Channel = #channel{session = Session}) -> +%handle_call({takeover, 'begin'}, _From, Channel = #channel{session = Session}) -> % reply(Session, Channel#channel{takeover = true}); % -%handle_call({takeover, 'end'}, Channel = #channel{session = Session, +%handle_call({takeover, 'end'}, _From, Channel = #channel{session = Session, % pendings = Pendings}) -> % ok = emqx_session:takeover(Session), % %% TODO: Should not drain deliver here (side effect) @@ -647,7 +699,7 @@ handle_call(discard, Channel) -> % AllPendings = lists:append(Delivers, Pendings), % shutdown_and_reply(takeovered, AllPendings, Channel); -handle_call(list_authz_cache, Channel) -> +handle_call(list_authz_cache, _From, Channel) -> %% This won't work {reply, emqx_authz_cache:list_authz_cache(), Channel}; @@ -657,11 +709,10 @@ handle_call(list_authz_cache, Channel) -> % Quota = emqx_limiter:init(Zone, Policy), % reply(ok, Channel#channel{quota = Quota}); -handle_call(Req, Channel) -> +handle_call(Req, _From, Channel) -> ?LOG(error, "Unexpected call: ~p", [Req]), reply(ignored, Channel). - %%-------------------------------------------------------------------- %% Handle cast %%-------------------------------------------------------------------- @@ -678,18 +729,6 @@ handle_cast(_Req, Channel) -> -spec(handle_info(Info :: term(), channel()) -> ok | {ok, channel()} | {shutdown, Reason :: term(), channel()}). -%% XXX: Received from the emqx-management ??? -%handle_info({subscribe, TopicFilters}, Channel ) -> -% {_, NChannel} = lists:foldl( -% fun({TopicFilter, SubOpts}, {_, ChannelAcc}) -> -% do_subscribe(TopicFilter, SubOpts, ChannelAcc) -% end, {[], Channel}, parse_topic_filters(TopicFilters)), -% {ok, NChannel}; -% -%handle_info({unsubscribe, TopicFilters}, Channel) -> -% {_RC, NChannel} = process_unsubscribe(TopicFilters, #{}, Channel), -% {ok, NChannel}; - handle_info({sock_closed, Reason}, Channel = #channel{conn_state = idle}) -> shutdown(Reason, Channel); @@ -755,7 +794,7 @@ handle_deliver(Delivers, Frames0 = lists:foldl(fun({_, _, Message}, Acc) -> Topic0 = emqx_message:topic(Message), case lists:keyfind(Topic0, 2, Subs) of - {Id, Topic, Ack} -> + {Id, Topic, Ack, _SubOpts} -> %% XXX: refactor later metrics_inc('messages.delivered', Channel), NMessage = run_hooks_without_metrics( diff --git a/apps/emqx_gateway/src/stomp/emqx_stomp_impl.erl b/apps/emqx_gateway/src/stomp/emqx_stomp_impl.erl index 593b71289..9599ef6e3 100644 --- a/apps/emqx_gateway/src/stomp/emqx_stomp_impl.erl +++ b/apps/emqx_gateway/src/stomp/emqx_stomp_impl.erl @@ -93,17 +93,17 @@ start_listener(GwName, Ctx, {Type, LisName, ListenOn, SocketOpts, Cfg}) -> ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn), case start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) of {ok, Pid} -> - ?ULOG("Start ~s:~s:~s listener on ~s successfully.~n", + ?ULOG("Gateway ~s:~s:~s on ~s started.~n", [GwName, Type, LisName, ListenOnStr]), Pid; {error, Reason} -> - ?ELOG("Failed to start ~s:~s:~s listener on ~s: ~0p~n", + ?ELOG("Failed to start gateway ~s:~s:~s on ~s: ~0p~n", [GwName, Type, LisName, ListenOnStr, Reason]), throw({badconf, Reason}) end. start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) -> - Name = name(GwName, LisName, Type), + Name = emqx_gateway_utils:listener_id(GwName, Type, LisName), NCfg = Cfg#{ ctx => Ctx, frame_mod => emqx_stomp_frame, @@ -112,9 +112,6 @@ start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) -> esockd:open(Name, ListenOn, merge_default(SocketOpts), {emqx_gateway_conn, start_link, [NCfg]}). -name(GwName, LisName, Type) -> - list_to_atom(lists:concat([GwName, ":", Type, ":", LisName])). - merge_default(Options) -> Default = emqx_gateway_utils:default_tcp_options(), case lists:keytake(tcp_options, 1, Options) of @@ -129,14 +126,14 @@ stop_listener(GwName, {Type, LisName, ListenOn, SocketOpts, Cfg}) -> StopRet = stop_listener(GwName, Type, LisName, ListenOn, SocketOpts, Cfg), ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn), case StopRet of - ok -> ?ULOG("Stop ~s:~s:~s listener on ~s successfully.~n", + ok -> ?ULOG("Gateway ~s:~s:~s on ~s stopped.~n", [GwName, Type, LisName, ListenOnStr]); {error, Reason} -> - ?ELOG("Failed to stop ~s:~s:~s listener on ~s: ~0p~n", + ?ELOG("Failed to stop gateway ~s:~s:~s on ~s: ~0p~n", [GwName, Type, LisName, ListenOnStr, Reason]) end, StopRet. stop_listener(GwName, Type, LisName, ListenOn, _SocketOpts, _Cfg) -> - Name = name(GwName, LisName, Type), + Name = emqx_gateway_utils:listener_id(GwName, Type, LisName), esockd:close(Name, ListenOn). diff --git a/apps/emqx_gateway/test/emqx_coap_api_SUITE.erl b/apps/emqx_gateway/test/emqx_coap_api_SUITE.erl new file mode 100644 index 000000000..83521f5cd --- /dev/null +++ b/apps/emqx_gateway/test/emqx_coap_api_SUITE.erl @@ -0,0 +1,200 @@ +%%-------------------------------------------------------------------- +%% Copyright (C) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_coap_api_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("emqx_gateway/src/coap/include/emqx_coap.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +-define(CONF_DEFAULT, <<" +gateway.coap { + idle_timeout = 30s + enable_stats = false + mountpoint = \"\" + notify_type = qos + connection_required = true + subscribe_qos = qos1 + publish_qos = qos1 + authentication = undefined + + listeners.udp.default { + bind = 5683 + } + } + ">>). + +-define(HOST, "127.0.0.1"). +-define(PORT, 5683). +-define(CONN_URI, "coap://127.0.0.1/mqtt/connection?clientid=client1&username=admin&password=public"). + +-define(LOGT(Format, Args), ct:pal("TEST_SUITE: " ++ Format, Args)). + +%%-------------------------------------------------------------------- +%% Setups +%%-------------------------------------------------------------------- + +all() -> + emqx_ct:all(?MODULE). + +init_per_suite(Config) -> + ok = emqx_config:init_load(emqx_gateway_schema, ?CONF_DEFAULT), + emqx_mgmt_api_test_util:init_suite([emqx_gateway]), + Config. + +end_per_suite(Config) -> + emqx_mgmt_api_test_util:end_suite([emqx_gateway]), + Config. + +%%-------------------------------------------------------------------- +%% Cases +%%-------------------------------------------------------------------- +t_send_request_api(_) -> + ClientId = start_client(), + timer:sleep(200), + Path = emqx_mgmt_api_test_util:api_path(["gateway/coap/client1/request"]), + Token = <<"atoken">>, + Payload = <<"simple echo this">>, + Req = #{token => Token, + payload => Payload, + timeout => 10, + content_type => <<"text/plain">>, + method => <<"get">>}, + Auth = emqx_mgmt_api_test_util:auth_header_(), + {ok, Response} = emqx_mgmt_api_test_util:request_api(post, + Path, + "method=get", + Auth, + Req + ), + #{<<"token">> := RToken, <<"payload">> := RPayload} = + emqx_json:decode(Response, [return_maps]), + ?assertEqual(Token, RToken), + ?assertEqual(Payload, RPayload), + erlang:exit(ClientId, kill), + ok. + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%% Internal Functions +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +start_client() -> + spawn(fun coap_client/0). + +coap_client() -> + {ok, CSock} = gen_udp:open(0, [binary, {active, false}]), + test_send_coap_request(CSock, post, <<>>, [], 1), + Response = test_recv_coap_response(CSock), + ?assertEqual({ok, created}, Response#coap_message.method), + echo_loop(CSock). + +echo_loop(CSock) -> + #coap_message{payload = Payload} = Req = test_recv_coap_request(CSock), + test_send_coap_response(CSock, ?HOST, ?PORT, {ok, content}, Payload, Req), + echo_loop(CSock). + +test_send_coap_request(UdpSock, Method, Content, Options, MsgId) -> + is_list(Options) orelse error("Options must be a list"), + case resolve_uri(?CONN_URI) of + {coap, {IpAddr, Port}, Path, Query} -> + Request0 = emqx_coap_message:request(con, Method, Content, + [{uri_path, Path}, + {uri_query, Query} | Options]), + Request = Request0#coap_message{id = MsgId}, + ?LOGT("send_coap_request Request=~p", [Request]), + RequestBinary = emqx_coap_frame:serialize_pkt(Request, undefined), + ?LOGT("test udp socket send to ~p:~p, data=~p", [IpAddr, Port, RequestBinary]), + ok = gen_udp:send(UdpSock, IpAddr, Port, RequestBinary); + {SchemeDiff, ChIdDiff, _, _} -> + error(lists:flatten(io_lib:format("scheme ~s or ChId ~s does not match with socket", [SchemeDiff, ChIdDiff]))) + end. + +test_recv_coap_response(UdpSock) -> + {ok, {Address, Port, Packet}} = gen_udp:recv(UdpSock, 0, 2000), + {ok, Response, _, _} = emqx_coap_frame:parse(Packet, undefined), + ?LOGT("test udp receive from ~p:~p, data1=~p, Response=~p", [Address, Port, Packet, Response]), + #coap_message{type = ack, method = Method, id=Id, token = Token, options = Options, payload = Payload} = Response, + ?LOGT("receive coap response Method=~p, Id=~p, Token=~p, Options=~p, Payload=~p", [Method, Id, Token, Options, Payload]), + Response. + +test_recv_coap_request(UdpSock) -> + case gen_udp:recv(UdpSock, 0) of + {ok, {_Address, _Port, Packet}} -> + {ok, Request, _, _} = emqx_coap_frame:parse(Packet, undefined), + #coap_message{type = con, method = Method, id=Id, token = Token, payload = Payload, options = Options} = Request, + ?LOGT("receive coap request Method=~p, Id=~p, Token=~p, Options=~p, Payload=~p", [Method, Id, Token, Options, Payload]), + Request; + {error, Reason} -> + ?LOGT("test_recv_coap_request failed, Reason=~p", [Reason]), + timeout_test_recv_coap_request + end. + +test_send_coap_response(UdpSock, Host, Port, Code, Content, Request) -> + is_list(Host) orelse error("Host is not a string"), + {ok, IpAddr} = inet:getaddr(Host, inet), + Response = emqx_coap_message:piggyback(Code, Content, Request), + ?LOGT("test_send_coap_response Response=~p", [Response]), + Binary = emqx_coap_frame:serialize_pkt(Response, undefined), + ok = gen_udp:send(UdpSock, IpAddr, Port, Binary). + +resolve_uri(Uri) -> + {ok, #{scheme := Scheme, + host := Host, + port := PortNo, + path := Path} = URIMap} = emqx_http_lib:uri_parse(Uri), + Query = maps:get(query, URIMap, undefined), + {ok, PeerIP} = inet:getaddr(Host, inet), + {Scheme, {PeerIP, PortNo}, split_path(Path), split_query(Query)}. + +split_path([]) -> []; +split_path([$/]) -> []; +split_path([$/ | Path]) -> split_segments(Path, $/, []). + +split_query(undefined) -> #{}; +split_query(Path) -> + split_segments(Path, $&, []). + +split_segments(Path, Char, Acc) -> + case string:rchr(Path, Char) of + 0 -> + [make_segment(Path) | Acc]; + N when N > 0 -> + split_segments(string:substr(Path, 1, N-1), Char, + [make_segment(string:substr(Path, N+1)) | Acc]) + end. + +make_segment(Seg) -> + list_to_binary(emqx_http_lib:uri_decode(Seg)). + +get_path([], Acc) -> + %?LOGT("get_path Acc=~p", [Acc]), + Acc; +get_path([{uri_path, Path1}|T], Acc) -> + %?LOGT("Path=~p, Acc=~p", [Path1, Acc]), + get_path(T, join_path(Path1, Acc)); +get_path([{_, _}|T], Acc) -> + get_path(T, Acc). + +join_path([], Acc) -> Acc; +join_path([<<"/">>|T], Acc) -> + join_path(T, Acc); +join_path([H|T], Acc) -> + join_path(T, <>). + +sprintf(Format, Args) -> + lists:flatten(io_lib:format(Format, Args)). diff --git a/apps/emqx_gateway/test/emqx_exproto_SUITE.erl b/apps/emqx_gateway/test/emqx_exproto_SUITE.erl index 4902aacf5..b91cd03b9 100644 --- a/apps/emqx_gateway/test/emqx_exproto_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_exproto_SUITE.erl @@ -55,20 +55,19 @@ metrics() -> init_per_group(GrpName, Cfg) -> put(grpname, GrpName), Svrs = emqx_exproto_echo_svr:start(), - emqx_ct_helpers:start_apps([emqx_authn, emqx_gateway], fun set_special_cfg/1), + emqx_ct_helpers:start_apps([emqx_gateway], fun set_special_cfg/1), emqx_logger:set_log_level(debug), [{servers, Svrs}, {listener_type, GrpName} | Cfg]. end_per_group(_, Cfg) -> - emqx_ct_helpers:stop_apps([emqx_gateway, emqx_authn]), + emqx_ct_helpers:stop_apps([emqx_gateway]), emqx_exproto_echo_svr:stop(proplists:get_value(servers, Cfg)). set_special_cfg(emqx_gateway) -> LisType = get(grpname), emqx_config:put( [gateway, exproto], - #{authentication => #{enable => false}, - server => #{bind => 9100}, + #{server => #{bind => 9100}, handler => #{address => "http://127.0.0.1:9001"}, listeners => listener_confs(LisType) }); @@ -77,7 +76,7 @@ set_special_cfg(_App) -> listener_confs(Type) -> Default = #{bind => 7993, acceptors => 8}, - #{Type => #{'1' => maps:merge(Default, maps:from_list(socketopts(Type)))}}. + #{Type => #{'default' => maps:merge(Default, socketopts(Type))}}. %%-------------------------------------------------------------------- %% Tests cases @@ -361,11 +360,11 @@ open(udp) -> {ok, Sock} = gen_udp:open(0, ?TCPOPTS), {udp, Sock}; open(ssl) -> - SslOpts = client_ssl_opts(), + SslOpts = maps:to_list(client_ssl_opts()), {ok, SslSock} = ssl:connect("127.0.0.1", 7993, ?TCPOPTS ++ SslOpts), {ssl, SslSock}; open(dtls) -> - SslOpts = client_ssl_opts(), + SslOpts = maps:to_list(client_ssl_opts()), {ok, SslSock} = ssl:connect("127.0.0.1", 7993, ?DTLSOPTS ++ SslOpts), {dtls, SslSock}. @@ -401,51 +400,56 @@ close({dtls, Sock}) -> %% Server-Opts socketopts(tcp) -> - [{tcp_options, tcp_opts()}]; + #{tcp => tcp_opts()}; socketopts(ssl) -> - [{tcp_options, tcp_opts()}, - {ssl_options, ssl_opts()}]; + #{tcp => tcp_opts(), + ssl => ssl_opts()}; socketopts(udp) -> - [{udp_options, udp_opts()}]; + #{udp => udp_opts()}; socketopts(dtls) -> - [{udp_options, udp_opts()}, - {dtls_options, dtls_opts()}]. + #{udp => udp_opts(), + dtls => dtls_opts()}. tcp_opts() -> - [{send_timeout, 15000}, - {send_timeout_close, true}, - {backlog, 100}, - {nodelay, true} | udp_opts()]. + maps:merge( + udp_opts(), + #{send_timeout => 15000, + send_timeout_close => true, + backlog => 100, + nodelay => true} + ). udp_opts() -> - [{recbuf, 1024}, - {sndbuf, 1024}, - {buffer, 1024}, - {reuseaddr, true}]. + #{recbuf => 1024, + sndbuf => 1024, + buffer => 1024, + reuseaddr => true}. ssl_opts() -> Certs = certs("key.pem", "cert.pem", "cacert.pem"), - [{versions, emqx_tls_lib:default_versions()}, - {ciphers, emqx_tls_lib:default_ciphers()}, - {verify, verify_peer}, - {fail_if_no_peer_cert, true}, - {secure_renegotiate, false}, - {reuse_sessions, true}, - {honor_cipher_order, true}]++Certs. + maps:merge( + Certs, + #{versions => emqx_tls_lib:default_versions(), + ciphers => emqx_tls_lib:default_ciphers(), + verify => verify_peer, + fail_if_no_peer_cert => true, + secure_renegotiate => false, + reuse_sessions => true, + honor_cipher_order => true} + ). dtls_opts() -> - Opts = ssl_opts(), - lists:keyreplace(versions, 1, Opts, {versions, ['dtlsv1.2', 'dtlsv1']}). + maps:merge(ssl_opts(), #{versions => ['dtlsv1.2', 'dtlsv1']}). %%-------------------------------------------------------------------- %% Client-Opts client_ssl_opts() -> - certs( "client-key.pem", "client-cert.pem", "cacert.pem" ). + certs("client-key.pem", "client-cert.pem", "cacert.pem"). -certs( Key, Cert, CACert ) -> +certs(Key, Cert, CACert) -> CertsPath = emqx_ct_helpers:deps_path(emqx, "etc/certs"), - [ { keyfile, filename:join([ CertsPath, Key ]) }, - { certfile, filename:join([ CertsPath, Cert ]) }, - { cacertfile, filename:join([ CertsPath, CACert ]) } ]. + #{keyfile => filename:join([ CertsPath, Key ]), + certfile => filename:join([ CertsPath, Cert ]), + cacertfile => filename:join([ CertsPath, CACert])}. diff --git a/apps/emqx_gateway/test/emqx_gateway_registry_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_registry_SUITE.erl index da03b17c5..56776957f 100644 --- a/apps/emqx_gateway/test/emqx_gateway_registry_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_registry_SUITE.erl @@ -35,11 +35,11 @@ all() -> emqx_ct:all(?MODULE). init_per_suite(Cfg) -> ok = emqx_config:init_load(emqx_gateway_schema, ?CONF_DEFAULT), - emqx_ct_helpers:start_apps([emqx_authn, emqx_gateway]), + emqx_ct_helpers:start_apps([emqx_gateway]), Cfg. end_per_suite(_Cfg) -> - emqx_ct_helpers:stop_apps([emqx_authn, emqx_gateway]), + emqx_ct_helpers:stop_apps([emqx_gateway]), ok. %%-------------------------------------------------------------------- diff --git a/apps/emqx_gateway/test/emqx_lwm2m_SUITE.erl b/apps/emqx_gateway/test/emqx_lwm2m_SUITE.erl index 79664928d..28edda7ef 100644 --- a/apps/emqx_gateway/test/emqx_lwm2m_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_lwm2m_SUITE.erl @@ -1,5 +1,5 @@ %%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% Copyright (C) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. %% %% Licensed under the Apache License, Version 2.0 (the "License"); %% you may not use this file except in compliance with the License. @@ -23,7 +23,7 @@ -define(LOGT(Format, Args), ct:pal("TEST_SUITE: " ++ Format, Args)). --include("src/lwm2m/include/emqx_lwm2m.hrl"). +-include_lib("emqx_gateway/src/lwm2m/include/emqx_lwm2m.hrl"). -include_lib("lwm2m_coap/include/coap.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). @@ -35,14 +35,14 @@ gateway.lwm2m { lifetime_max = 86400s qmode_time_windonw = 22 auto_observe = false - mountpoint = \"lwm2m/%e/\" + mountpoint = \"lwm2m/%u\" update_msg_publish_condition = contains_object_list translators { - command = \"dn/#\" - response = \"up/resp\" - notify = \"up/notify\" - register = \"up/resp\" - update = \"up/resp\" + command = {topic = \"/dn/#\", qos = 0} + response = {topic = \"/up/resp\", qos = 0} + notify = {topic = \"/up/notify\", qos = 0} + register = {topic = \"/up/resp\", qos = 0} + update = {topic = \"/up/resp\", qos = 0} } listeners.udp.default { bind = 5783 @@ -58,11 +58,15 @@ all() -> [ {group, test_grp_0_register} , {group, test_grp_1_read} , {group, test_grp_2_write} + , {group, test_grp_create} + , {group, test_grp_delete} , {group, test_grp_3_execute} , {group, test_grp_4_discover} , {group, test_grp_5_write_attr} , {group, test_grp_6_observe} - , {group, test_grp_8_object_19} + + %% {group, test_grp_8_object_19} + , {group, test_grp_9_psm_queue_mode} ]. suite() -> [{timetrap, {seconds, 90}}]. @@ -70,74 +74,86 @@ suite() -> [{timetrap, {seconds, 90}}]. groups() -> RepeatOpt = {repeat_until_all_ok, 1}, [ - {test_grp_0_register, [RepeatOpt], [ - case01_register, - case01_register_additional_opts, - case01_register_incorrect_opts, - case01_register_report, - case02_update_deregister, - case03_register_wrong_version, - case04_register_and_lifetime_timeout, - case05_register_wrong_epn, - case06_register_wrong_lifetime, - case07_register_alternate_path_01, - case07_register_alternate_path_02, - case08_reregister - ]}, - {test_grp_1_read, [RepeatOpt], [ - case10_read, - case10_read_separate_ack, - case11_read_object_tlv, - case11_read_object_json, - case12_read_resource_opaque, - case13_read_no_xml - ]}, - {test_grp_2_write, [RepeatOpt], [ - case20_write, - case21_write_object, - case22_write_error, - case20_single_write - ]}, - {test_grp_create, [RepeatOpt], [ - case_create_basic - ]}, - {test_grp_delete, [RepeatOpt], [ - case_delete_basic - ]}, - {test_grp_3_execute, [RepeatOpt], [ - case30_execute, case31_execute_error - ]}, - {test_grp_4_discover, [RepeatOpt], [ - case40_discover - ]}, - {test_grp_5_write_attr, [RepeatOpt], [ - case50_write_attribute - ]}, - {test_grp_6_observe, [RepeatOpt], [ - case60_observe - ]}, - {test_grp_7_block_wize_transfer, [RepeatOpt], [ - case70_read_large, case70_write_large - ]}, - {test_grp_8_object_19, [RepeatOpt], [ - case80_specail_object_19_1_0_write, - case80_specail_object_19_0_0_notify - %case80_specail_object_19_0_0_response, - %case80_normal_object_19_0_0_read - ]}, - {test_grp_9_psm_queue_mode, [RepeatOpt], [ - case90_psm_mode, - case90_queue_mode - ]} + {test_grp_0_register, [RepeatOpt], + [ + case01_register, + case01_register_additional_opts, + %% case01_register_incorrect_opts, %% TODO now we can't handle partial decode packet + case01_register_report, + case02_update_deregister, + case03_register_wrong_version, + case04_register_and_lifetime_timeout, + case05_register_wrong_epn, + %% case06_register_wrong_lifetime, %% now, will ignore wrong lifetime + case07_register_alternate_path_01, + case07_register_alternate_path_02, + case08_reregister + ]}, + {test_grp_1_read, [RepeatOpt], + [ + case10_read, + case10_read_separate_ack, + case11_read_object_tlv, + case11_read_object_json, + case12_read_resource_opaque, + case13_read_no_xml + ]}, + {test_grp_2_write, [RepeatOpt], + [ + case20_write, + case21_write_object, + case22_write_error, + case20_single_write + ]}, + {test_grp_create, [RepeatOpt], + [ + case_create_basic + ]}, + {test_grp_delete, [RepeatOpt], + [ + case_delete_basic + ]}, + {test_grp_3_execute, [RepeatOpt], + [ + case30_execute, case31_execute_error + ]}, + {test_grp_4_discover, [RepeatOpt], + [ + case40_discover + ]}, + {test_grp_5_write_attr, [RepeatOpt], + [ + case50_write_attribute + ]}, + {test_grp_6_observe, [RepeatOpt], + [ + case60_observe + ]}, + {test_grp_7_block_wize_transfer, [RepeatOpt], + [ + case70_read_large, case70_write_large + ]}, + {test_grp_8_object_19, [RepeatOpt], + [ + case80_specail_object_19_1_0_write, + case80_specail_object_19_0_0_notify, + case80_specail_object_19_0_0_response, + case80_normal_object_19_0_0_read + ]}, + {test_grp_9_psm_queue_mode, [RepeatOpt], + [ + case90_psm_mode, + case90_queue_mode + ]} ]. init_per_suite(Config) -> - emqx_ct_helpers:start_apps([emqx_authn]), + emqx_ct_helpers:start_apps([]), Config. end_per_suite(Config) -> timer:sleep(300), - emqx_ct_helpers:stop_apps([emqx_authn]), + emqx_ct_helpers:stop_apps([]), Config. init_per_testcase(_AllTestCase, Config) -> @@ -162,9 +178,9 @@ end_per_testcase(_AllTestCase, Config) -> %%-------------------------------------------------------------------- case01_register(Config) -> - % ---------------------------------------- - % REGISTER command - % ---------------------------------------- + %%---------------------------------------- + %% REGISTER command + %%---------------------------------------- UdpSock = ?config(sock, Config), Epn = "urn:oma:lwm2m:oma:3", MsgId = 12, @@ -187,13 +203,13 @@ case01_register(Config) -> ?assertNotEqual(undefined, Location), %% checkpoint 2 - verify subscribed topics - timer:sleep(50), + timer:sleep(100), ?LOGT("all topics: ~p", [test_mqtt_broker:get_subscrbied_topics()]), true = lists:member(SubTopic, test_mqtt_broker:get_subscrbied_topics()), - % ---------------------------------------- - % DE-REGISTER command - % ---------------------------------------- + %%---------------------------------------- + %% DE-REGISTER command + %%---------------------------------------- ?LOGT("start to send DE-REGISTER command", []), MsgId3 = 52, test_send_coap_request( UdpSock, @@ -209,9 +225,9 @@ case01_register(Config) -> false = lists:member(SubTopic, test_mqtt_broker:get_subscrbied_topics()). case01_register_additional_opts(Config) -> - % ---------------------------------------- - % REGISTER command - % ---------------------------------------- + %%---------------------------------------- + %% REGISTER command + %%---------------------------------------- UdpSock = ?config(sock, Config), Epn = "urn:oma:lwm2m:oma:3", MsgId = 12, @@ -239,9 +255,9 @@ case01_register_additional_opts(Config) -> true = lists:member(SubTopic, test_mqtt_broker:get_subscrbied_topics()), - % ---------------------------------------- - % DE-REGISTER command - % ---------------------------------------- + %%---------------------------------------- + %% DE-REGISTER command + %%---------------------------------------- ?LOGT("start to send DE-REGISTER command", []), MsgId3 = 52, test_send_coap_request( UdpSock, @@ -257,9 +273,9 @@ case01_register_additional_opts(Config) -> false = lists:member(SubTopic, test_mqtt_broker:get_subscrbied_topics()). case01_register_incorrect_opts(Config) -> - % ---------------------------------------- - % REGISTER command - % ---------------------------------------- + %%---------------------------------------- + %% REGISTER command + %%---------------------------------------- UdpSock = ?config(sock, Config), Epn = "urn:oma:lwm2m:oma:3", MsgId = 12, @@ -279,9 +295,9 @@ case01_register_incorrect_opts(Config) -> ?assertEqual({error,bad_request}, Method). case01_register_report(Config) -> - % ---------------------------------------- - % REGISTER command - % ---------------------------------------- + %%---------------------------------------- + %% REGISTER command + %%---------------------------------------- UdpSock = ?config(sock, Config), Epn = "urn:oma:lwm2m:oma:3", MsgId = 12, @@ -320,9 +336,9 @@ case01_register_report(Config) -> }), ?assertEqual(ReadResult, test_recv_mqtt_response(ReportTopic)), - % ---------------------------------------- - % DE-REGISTER command - % ---------------------------------------- + %%---------------------------------------- + %% DE-REGISTER command + %%---------------------------------------- ?LOGT("start to send DE-REGISTER command", []), MsgId3 = 52, test_send_coap_request( UdpSock, @@ -338,9 +354,9 @@ case01_register_report(Config) -> false = lists:member(SubTopic, test_mqtt_broker:get_subscrbied_topics()). case02_update_deregister(Config) -> - % ---------------------------------------- - % REGISTER command - % ---------------------------------------- + %%---------------------------------------- + %% REGISTER command + %%---------------------------------------- UdpSock = ?config(sock, Config), Epn = "urn:oma:lwm2m:oma:3", MsgId = 12, @@ -373,9 +389,9 @@ case02_update_deregister(Config) -> }), ?assertEqual(Register, test_recv_mqtt_response(ReportTopic)), - % ---------------------------------------- - % UPDATE command - % ---------------------------------------- + %%---------------------------------------- + %% UPDATE command + %%---------------------------------------- ?LOGT("start to send UPDATE command", []), MsgId2 = 27, test_send_coap_request( UdpSock, @@ -399,9 +415,9 @@ case02_update_deregister(Config) -> }), ?assertEqual(Update, test_recv_mqtt_response(ReportTopic)), - % ---------------------------------------- - % DE-REGISTER command - % ---------------------------------------- + %%---------------------------------------- + %% DE-REGISTER command + %%---------------------------------------- ?LOGT("start to send DE-REGISTER command", []), MsgId3 = 52, test_send_coap_request( UdpSock, @@ -418,9 +434,9 @@ case02_update_deregister(Config) -> false = lists:member(SubTopic, test_mqtt_broker:get_subscrbied_topics()). case03_register_wrong_version(Config) -> - % ---------------------------------------- - % REGISTER command - % ---------------------------------------- + %%---------------------------------------- + %% REGISTER command + %%---------------------------------------- UdpSock = ?config(sock, Config), Epn = "urn:oma:lwm2m:oma:3", MsgId = 12, @@ -432,15 +448,15 @@ case03_register_wrong_version(Config) -> [], MsgId), #coap_message{type = ack, method = Method} = test_recv_coap_response(UdpSock), - ?assertEqual({error,precondition_failed}, Method), + ?assertEqual({error, bad_request}, Method), timer:sleep(50), false = lists:member(SubTopic, test_mqtt_broker:get_subscrbied_topics()). case04_register_and_lifetime_timeout(Config) -> - % ---------------------------------------- - % REGISTER command - % ---------------------------------------- + %%---------------------------------------- + %% REGISTER command + %%---------------------------------------- UdpSock = ?config(sock, Config), Epn = "urn:oma:lwm2m:oma:3", MsgId = 12, @@ -458,17 +474,17 @@ case04_register_and_lifetime_timeout(Config) -> true = lists:member(SubTopic, test_mqtt_broker:get_subscrbied_topics()), - % ---------------------------------------- - % lifetime timeout - % ---------------------------------------- + %%---------------------------------------- + %% lifetime timeout + %%---------------------------------------- timer:sleep(4000), false = lists:member(SubTopic, test_mqtt_broker:get_subscrbied_topics()). case05_register_wrong_epn(Config) -> - % ---------------------------------------- - % REGISTER command - % ---------------------------------------- + %%---------------------------------------- + %% REGISTER command + %%---------------------------------------- MsgId = 12, UdpSock = ?config(sock, Config), @@ -481,29 +497,29 @@ case05_register_wrong_epn(Config) -> #coap_message{type = ack, method = Method} = test_recv_coap_response(UdpSock), ?assertEqual({error,bad_request}, Method). -case06_register_wrong_lifetime(Config) -> - % ---------------------------------------- - % REGISTER command - % ---------------------------------------- - UdpSock = ?config(sock, Config), - Epn = "urn:oma:lwm2m:oma:3", - MsgId = 12, +%% case06_register_wrong_lifetime(Config) -> +%% %%---------------------------------------- +%% %% REGISTER command +%% %%---------------------------------------- +%% UdpSock = ?config(sock, Config), +%% Epn = "urn:oma:lwm2m:oma:3", +%% MsgId = 12, - test_send_coap_request( UdpSock, - post, - sprintf("coap://127.0.0.1:~b/rd?ep=~s&lwm2m=1", [?PORT, Epn]), - #coap_content{content_format = <<"text/plain">>, payload = <<", , , , ">>}, - [], - MsgId), - #coap_message{type = ack, method = Method} = test_recv_coap_response(UdpSock), - ?assertEqual({error,bad_request}, Method), - timer:sleep(50), - ?assertEqual([], test_mqtt_broker:get_subscrbied_topics()). +%% test_send_coap_request( UdpSock, +%% post, +%% sprintf("coap://127.0.0.1:~b/rd?ep=~s&lwm2m=1", [?PORT, Epn]), +%% #coap_content{content_format = <<"text/plain">>, payload = <<", , , , ">>}, +%% [], +%% MsgId), +%% #coap_message{type = ack, method = Method} = test_recv_coap_response(UdpSock), +%% ?assertEqual({error,bad_request}, Method), +%% timer:sleep(50), +%% ?assertEqual([], test_mqtt_broker:get_subscrbied_topics()). case07_register_alternate_path_01(Config) -> - % ---------------------------------------- - % REGISTER command - % ---------------------------------------- + %%---------------------------------------- + %% REGISTER command + %%---------------------------------------- UdpSock = ?config(sock, Config), Epn = "urn:oma:lwm2m:oma:3", MsgId = 12, @@ -516,16 +532,16 @@ case07_register_alternate_path_01(Config) -> post, sprintf("coap://127.0.0.1:~b/rd?ep=~s<=345&lwm2m=1", [?PORT, Epn]), #coap_content{content_format = <<"text/plain">>, - payload = <<";rt=\"oma.lwm2m\";ct=11543,,,">>}, + payload = <<";rt=\"oma.lwm2m\";ct=11543,,,">>}, [], MsgId), timer:sleep(50), true = lists:member(SubTopic, test_mqtt_broker:get_subscrbied_topics()). case07_register_alternate_path_02(Config) -> - % ---------------------------------------- - % REGISTER command - % ---------------------------------------- + %%---------------------------------------- + %% REGISTER command + %%---------------------------------------- UdpSock = ?config(sock, Config), Epn = "urn:oma:lwm2m:oma:3", MsgId = 12, @@ -538,16 +554,16 @@ case07_register_alternate_path_02(Config) -> post, sprintf("coap://127.0.0.1:~b/rd?ep=~s<=345&lwm2m=1", [?PORT, Epn]), #coap_content{content_format = <<"text/plain">>, - payload = <<";rt=\"oma.lwm2m\";ct=11543,,,">>}, + payload = <<";rt=\"oma.lwm2m\";ct=11543,,,">>}, [], MsgId), timer:sleep(50), true = lists:member(SubTopic, test_mqtt_broker:get_subscrbied_topics()). case08_reregister(Config) -> - % ---------------------------------------- - % REGISTER command - % ---------------------------------------- + %%---------------------------------------- + %% REGISTER command + %%---------------------------------------- UdpSock = ?config(sock, Config), Epn = "urn:oma:lwm2m:oma:3", MsgId = 12, @@ -560,24 +576,24 @@ case08_reregister(Config) -> post, sprintf("coap://127.0.0.1:~b/rd?ep=~s<=345&lwm2m=1", [?PORT, Epn]), #coap_content{content_format = <<"text/plain">>, - payload = <<";rt=\"oma.lwm2m\";ct=11543,,,">>}, + payload = <<";rt=\"oma.lwm2m\";ct=11543,,,">>}, [], MsgId), timer:sleep(50), true = lists:member(SubTopic, test_mqtt_broker:get_subscrbied_topics()), ReadResult = emqx_json:encode( - #{ - <<"msgType">> => <<"register">>, - <<"data">> => #{ - <<"alternatePath">> => <<"/lwm2m">>, - <<"ep">> => list_to_binary(Epn), - <<"lt">> => 345, - <<"lwm2m">> => <<"1">>, - <<"objectList">> => [<<"/1/0">>, <<"/2/0">>, <<"/3/0">>] - } - } - ), + #{ + <<"msgType">> => <<"register">>, + <<"data">> => #{ + <<"alternatePath">> => <<"/lwm2m">>, + <<"ep">> => list_to_binary(Epn), + <<"lt">> => 345, + <<"lwm2m">> => <<"1">>, + <<"objectList">> => [<<"/1/0">>, <<"/2/0">>, <<"/3/0">>] + } + } + ), ?assertEqual(ReadResult, test_recv_mqtt_response(ReportTopic)), timer:sleep(1000), @@ -586,9 +602,10 @@ case08_reregister(Config) -> post, sprintf("coap://127.0.0.1:~b/rd?ep=~s<=345&lwm2m=1", [?PORT, Epn]), #coap_content{content_format = <<"text/plain">>, - payload = <<";rt=\"oma.lwm2m\";ct=11543,,,">>}, + payload = <<";rt=\"oma.lwm2m\";ct=11543,,,">>}, [], MsgId + 1), + %% verify the lwm2m client is still online ?assertEqual(ReadResult, test_recv_mqtt_response(ReportTopic)). @@ -599,28 +616,28 @@ case10_read(Config) -> RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"), emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0), timer:sleep(200), - % step 1, device register ... + %% step 1, device register ... test_send_coap_request( UdpSock, post, sprintf("coap://127.0.0.1:~b/rd?ep=~s<=345&lwm2m=1", [?PORT, Epn]), #coap_content{content_format = <<"text/plain">>, - payload = <<";rt=\"oma.lwm2m\";ct=11543,,,">>}, + payload = <<";rt=\"oma.lwm2m\";ct=11543,,,">>}, [], MsgId1), #coap_message{method = Method1} = test_recv_coap_response(UdpSock), ?assertEqual({ok,created}, Method1), test_recv_mqtt_response(RespTopic), - % step2, send a READ command to device + %% step2, send a READ command to device CmdId = 206, CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>, Command = #{ - <<"requestID">> => CmdId, <<"cacheID">> => CmdId, - <<"msgType">> => <<"read">>, - <<"data">> => #{ - <<"path">> => <<"/3/0/0">> - } - }, + <<"requestID">> => CmdId, <<"cacheID">> => CmdId, + <<"msgType">> => <<"read">>, + <<"data">> => #{ + <<"path">> => <<"/3/0/0">> + } + }, CommandJson = emqx_json:encode(Command), ?LOGT("CommandJson=~p", [CommandJson]), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), @@ -638,17 +655,17 @@ case10_read(Config) -> timer:sleep(100), ReadResult = emqx_json:encode(#{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, - <<"msgType">> => <<"read">>, - <<"data">> => #{ - <<"code">> => <<"2.05">>, - <<"codeMsg">> => <<"content">>, - <<"reqPath">> => <<"/3/0/0">>, - <<"content">> => [#{ - path => <<"/3/0/0">>, - value => <<"EMQ">> - }] - } - }), + <<"msgType">> => <<"read">>, + <<"data">> => #{ + <<"code">> => <<"2.05">>, + <<"codeMsg">> => <<"content">>, + <<"reqPath">> => <<"/3/0/0">>, + <<"content">> => [#{ + path => <<"/3/0/0">>, + value => <<"EMQ">> + }] + } + }), ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)). case10_read_separate_ack(Config) -> @@ -661,19 +678,19 @@ case10_read_separate_ack(Config) -> emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0), timer:sleep(200), - % step 1, device register ... + %% step 1, device register ... std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic), - % step2, send a READ command to device + %% step2, send a READ command to device CmdId = 206, CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>, Command = #{ - <<"requestID">> => CmdId, <<"cacheID">> => CmdId, - <<"msgType">> => <<"read">>, - <<"data">> => #{ - <<"path">> => <<"/3/0/0">> - } - }, + <<"requestID">> => CmdId, <<"cacheID">> => CmdId, + <<"msgType">> => <<"read">>, + <<"data">> => #{ + <<"path">> => <<"/3/0/0">> + } + }, CommandJson = emqx_json:encode(Command), ?LOGT("CommandJson=~p", [CommandJson]), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), @@ -688,12 +705,12 @@ case10_read_separate_ack(Config) -> test_send_empty_ack(UdpSock, "127.0.0.1", ?PORT, Request2), ReadResultACK = emqx_json:encode(#{ - <<"requestID">> => CmdId, <<"cacheID">> => CmdId, - <<"msgType">> => <<"ack">>, - <<"data">> => #{ - <<"path">> => <<"/3/0/0">> - } - }), + <<"requestID">> => CmdId, <<"cacheID">> => CmdId, + <<"msgType">> => <<"ack">>, + <<"data">> => #{ + <<"path">> => <<"/3/0/0">> + } + }), ?assertEqual(ReadResultACK, test_recv_mqtt_response(RespTopic)), timer:sleep(100), @@ -701,21 +718,21 @@ case10_read_separate_ack(Config) -> timer:sleep(100), ReadResult = emqx_json:encode(#{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, - <<"msgType">> => <<"read">>, - <<"data">> => #{ - <<"code">> => <<"2.05">>, - <<"codeMsg">> => <<"content">>, - <<"reqPath">> => <<"/3/0/0">>, - <<"content">> => [#{ - path => <<"/3/0/0">>, - value => <<"EMQ">> - }] - } - }), + <<"msgType">> => <<"read">>, + <<"data">> => #{ + <<"code">> => <<"2.05">>, + <<"codeMsg">> => <<"content">>, + <<"reqPath">> => <<"/3/0/0">>, + <<"content">> => [#{ + path => <<"/3/0/0">>, + value => <<"EMQ">> + }] + } + }), ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)). case11_read_object_tlv(Config) -> - % step 1, device register ... + %% step 1, device register ... Epn = "urn:oma:lwm2m:oma:3", MsgId1 = 15, UdpSock = ?config(sock, Config), @@ -726,16 +743,16 @@ case11_read_object_tlv(Config) -> std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic), - % step2, send a READ command to device + %% step2, send a READ command to device CmdId = 207, CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>, Command = #{ - <<"requestID">> => CmdId, <<"cacheID">> => CmdId, - <<"msgType">> => <<"read">>, - <<"data">> => #{ - <<"path">> => <<"/3/0">> - } - }, + <<"requestID">> => CmdId, <<"cacheID">> => CmdId, + <<"msgType">> => <<"read">>, + <<"data">> => #{ + <<"path">> => <<"/3/0">> + } + }, CommandJson = emqx_json:encode(Command), ?LOGT("CommandJson=~p", [CommandJson]), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), @@ -752,31 +769,31 @@ case11_read_object_tlv(Config) -> timer:sleep(100), ReadResult = emqx_json:encode(#{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, - <<"msgType">> => <<"read">>, - <<"data">> => #{ - <<"code">> => <<"2.05">>, - <<"codeMsg">> => <<"content">>, - <<"reqPath">> => <<"/3/0">>, - <<"content">> => [ - #{ - path => <<"/3/0/0">>, - value => <<"Open Mobile Alliance">> - }, - #{ - path => <<"/3/0/1">>, - value => <<"Lightweight M2M Client">> - }, - #{ - path => <<"/3/0/2">>, - value => <<"345000123">> - } - ] - } - }), + <<"msgType">> => <<"read">>, + <<"data">> => #{ + <<"code">> => <<"2.05">>, + <<"codeMsg">> => <<"content">>, + <<"reqPath">> => <<"/3/0">>, + <<"content">> => [ + #{ + path => <<"/3/0/0">>, + value => <<"Open Mobile Alliance">> + }, + #{ + path => <<"/3/0/1">>, + value => <<"Lightweight M2M Client">> + }, + #{ + path => <<"/3/0/2">>, + value => <<"345000123">> + } + ] + } + }), ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)). case11_read_object_json(Config) -> - % step 1, device register ... + %% step 1, device register ... UdpSock = ?config(sock, Config), Epn = "urn:oma:lwm2m:oma:3", MsgId1 = 15, @@ -788,16 +805,16 @@ case11_read_object_json(Config) -> std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic), - % step2, send a READ command to device + %% step2, send a READ command to device CmdId = 206, CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>, Command = #{ - <<"requestID">> => CmdId, <<"cacheID">> => CmdId, - <<"msgType">> => <<"read">>, - <<"data">> => #{ - <<"path">> => <<"/3/0">> - } - }, + <<"requestID">> => CmdId, <<"cacheID">> => CmdId, + <<"msgType">> => <<"read">>, + <<"data">> => #{ + <<"path">> => <<"/3/0">> + } + }, CommandJson = emqx_json:encode(Command), ?LOGT("CommandJson=~p", [CommandJson]), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), @@ -814,31 +831,31 @@ case11_read_object_json(Config) -> timer:sleep(100), ReadResult = emqx_json:encode(#{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, - <<"msgType">> => <<"read">>, - <<"data">> => #{ - <<"code">> => <<"2.05">>, - <<"codeMsg">> => <<"content">>, - <<"reqPath">> => <<"/3/0">>, - <<"content">> => [ - #{ - path => <<"/3/0/0">>, - value => <<"Open Mobile Alliance">> - }, - #{ - path => <<"/3/0/1">>, - value => <<"Lightweight M2M Client">> - }, - #{ - path => <<"/3/0/2">>, - value => <<"345000123">> - } - ] - } - }), + <<"msgType">> => <<"read">>, + <<"data">> => #{ + <<"code">> => <<"2.05">>, + <<"codeMsg">> => <<"content">>, + <<"reqPath">> => <<"/3/0">>, + <<"content">> => [ + #{ + path => <<"/3/0/0">>, + value => <<"Open Mobile Alliance">> + }, + #{ + path => <<"/3/0/1">>, + value => <<"Lightweight M2M Client">> + }, + #{ + path => <<"/3/0/2">>, + value => <<"345000123">> + } + ] + } + }), ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)). case12_read_resource_opaque(Config) -> - % step 1, device register ... + %% step 1, device register ... UdpSock = ?config(sock, Config), Epn = "urn:oma:lwm2m:oma:3", MsgId1 = 15, @@ -849,16 +866,16 @@ case12_read_resource_opaque(Config) -> std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic), - % step2, send a READ command to device + %% step2, send a READ command to device CmdId = 206, CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>, Command = #{ - <<"requestID">> => CmdId, <<"cacheID">> => CmdId, - <<"msgType">> => <<"read">>, - <<"data">> => #{ - <<"path">> => <<"/3/0/8">> - } - }, + <<"requestID">> => CmdId, <<"cacheID">> => CmdId, + <<"msgType">> => <<"read">>, + <<"data">> => #{ + <<"path">> => <<"/3/0/8">> + } + }, CommandJson = emqx_json:encode(Command), ?LOGT("CommandJson=~p", [CommandJson]), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), @@ -875,23 +892,23 @@ case12_read_resource_opaque(Config) -> timer:sleep(100), ReadResult = emqx_json:encode(#{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, - <<"msgType">> => <<"read">>, - <<"data">> => #{ - <<"code">> => <<"2.05">>, - <<"codeMsg">> => <<"content">>, - <<"reqPath">> => <<"/3/0/8">>, - <<"content">> => [ - #{ - path => <<"/3/0/8">>, - value => base64:encode(Opaque) - } - ] - } - }), + <<"msgType">> => <<"read">>, + <<"data">> => #{ + <<"code">> => <<"2.05">>, + <<"codeMsg">> => <<"content">>, + <<"reqPath">> => <<"/3/0/8">>, + <<"content">> => [ + #{ + path => <<"/3/0/8">>, + value => base64:encode(Opaque) + } + ] + } + }), ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)). case13_read_no_xml(Config) -> - % step 1, device register ... + %% step 1, device register ... Epn = "urn:oma:lwm2m:oma:3", MsgId1 = 15, UdpSock = ?config(sock, Config), @@ -902,16 +919,16 @@ case13_read_no_xml(Config) -> std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic), - % step2, send a READ command to device + %% step2, send a READ command to device CmdId = 206, CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>, Command = #{ - <<"requestID">> => CmdId, <<"cacheID">> => CmdId, - <<"msgType">> => <<"read">>, - <<"data">> => #{ - <<"path">> => <<"/9723/0/0">> - } - }, + <<"requestID">> => CmdId, <<"cacheID">> => CmdId, + <<"msgType">> => <<"read">>, + <<"data">> => #{ + <<"path">> => <<"/9723/0/0">> + } + }, CommandJson = emqx_json:encode(Command), ?LOGT("CommandJson=~p", [CommandJson]), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), @@ -927,17 +944,17 @@ case13_read_no_xml(Config) -> timer:sleep(100), ReadResult = emqx_json:encode(#{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, - <<"msgType">> => <<"read">>, - <<"data">> => #{ - <<"reqPath">> => <<"/9723/0/0">>, - <<"code">> => <<"4.00">>, - <<"codeMsg">> => <<"bad_request">> - } - }), + <<"msgType">> => <<"read">>, + <<"data">> => #{ + <<"reqPath">> => <<"/9723/0/0">>, + <<"code">> => <<"4.00">>, + <<"codeMsg">> => <<"bad_request">> + } + }), ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)). case20_single_write(Config) -> - % step 1, device register ... + %% step 1, device register ... Epn = "urn:oma:lwm2m:oma:3", MsgId1 = 15, UdpSock = ?config(sock, Config), @@ -948,16 +965,16 @@ case20_single_write(Config) -> std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic), - % step2, send a WRITE command to device + %% step2, send a WRITE command to device CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>, CmdId = 307, Command = #{<<"requestID">> => CmdId, <<"cacheID">> => CmdId, <<"msgType">> => <<"write">>, <<"data">> => #{ - <<"path">> => <<"/3/0/13">>, - <<"type">> => <<"Integer">>, - <<"value">> => <<"12345">> - } + <<"path">> => <<"/3/0/13">>, + <<"type">> => <<"Integer">>, + <<"value">> => <<"12345">> + } }, CommandJson = emqx_json:encode(Command), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), @@ -975,18 +992,18 @@ case20_single_write(Config) -> timer:sleep(100), ReadResult = emqx_json:encode(#{ - <<"requestID">> => CmdId, <<"cacheID">> => CmdId, - <<"data">> => #{ - <<"reqPath">> => <<"/3/0/13">>, - <<"code">> => <<"2.04">>, - <<"codeMsg">> => <<"changed">> - }, - <<"msgType">> => <<"write">> - }), + <<"requestID">> => CmdId, <<"cacheID">> => CmdId, + <<"data">> => #{ + <<"reqPath">> => <<"/3/0/13">>, + <<"code">> => <<"2.04">>, + <<"codeMsg">> => <<"changed">> + }, + <<"msgType">> => <<"write">> + }), ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)). case20_write(Config) -> - % step 1, device register ... + %% step 1, device register ... Epn = "urn:oma:lwm2m:oma:3", MsgId1 = 15, UdpSock = ?config(sock, Config), @@ -997,18 +1014,18 @@ case20_write(Config) -> std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic), - % step2, send a WRITE command to device + %% step2, send a WRITE command to device CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>, CmdId = 307, Command = #{<<"requestID">> => CmdId, <<"cacheID">> => CmdId, <<"msgType">> => <<"write">>, <<"data">> => #{ - <<"basePath">> => <<"/3/0/13">>, - <<"content">> => [#{ - type => <<"Float">>, - value => <<"12345.0">> - }] - } + <<"basePath">> => <<"/3/0/13">>, + <<"content">> => [#{ + type => <<"Float">>, + value => <<"12345.0">> + }] + } }, CommandJson = emqx_json:encode(Command), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), @@ -1026,18 +1043,18 @@ case20_write(Config) -> timer:sleep(100), WriteResult = emqx_json:encode(#{ - <<"requestID">> => CmdId, <<"cacheID">> => CmdId, - <<"data">> => #{ - <<"reqPath">> => <<"/3/0/13">>, - <<"code">> => <<"2.04">>, - <<"codeMsg">> => <<"changed">> - }, - <<"msgType">> => <<"write">> - }), + <<"requestID">> => CmdId, <<"cacheID">> => CmdId, + <<"data">> => #{ + <<"reqPath">> => <<"/3/0/13">>, + <<"code">> => <<"2.04">>, + <<"codeMsg">> => <<"changed">> + }, + <<"msgType">> => <<"write">> + }), ?assertEqual(WriteResult, test_recv_mqtt_response(RespTopic)). case21_write_object(Config) -> - % step 1, device register ... + %% step 1, device register ... Epn = "urn:oma:lwm2m:oma:3", MsgId1 = 15, UdpSock = ?config(sock, Config), @@ -1048,23 +1065,23 @@ case21_write_object(Config) -> std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic), - % step2, send a WRITE command to device + %% step2, send a WRITE command to device CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>, CmdId = 307, Command = #{<<"requestID">> => CmdId, <<"cacheID">> => CmdId, <<"msgType">> => <<"write">>, <<"data">> => #{ - <<"basePath">> => <<"/3/0/">>, - <<"content">> => [#{ - path => <<"13">>, - type => <<"Integer">>, - value => <<"12345">> - },#{ - path => <<"14">>, - type => <<"String">>, - value => <<"87x">> - }] - } + <<"basePath">> => <<"/3/0/">>, + <<"content">> => [#{ + path => <<"13">>, + type => <<"Integer">>, + value => <<"12345">> + },#{ + path => <<"14">>, + type => <<"String">>, + value => <<"87x">> + }] + } }, CommandJson = emqx_json:encode(Command), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), @@ -1084,18 +1101,18 @@ case21_write_object(Config) -> ReadResult = emqx_json:encode(#{ - <<"requestID">> => CmdId, <<"cacheID">> => CmdId, - <<"msgType">> => <<"write">>, - <<"data">> => #{ - <<"reqPath">> => <<"/3/0/">>, - <<"code">> => <<"2.04">>, - <<"codeMsg">> => <<"changed">> - } - }), + <<"requestID">> => CmdId, <<"cacheID">> => CmdId, + <<"msgType">> => <<"write">>, + <<"data">> => #{ + <<"reqPath">> => <<"/3/0/">>, + <<"code">> => <<"2.04">>, + <<"codeMsg">> => <<"changed">> + } + }), ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)). case22_write_error(Config) -> - % step 1, device register ... + %% step 1, device register ... Epn = "urn:oma:lwm2m:oma:3", MsgId1 = 15, UdpSock = ?config(sock, Config), @@ -1106,20 +1123,20 @@ case22_write_error(Config) -> std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic), - % step2, send a WRITE command to device + %% step2, send a WRITE command to device CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>, CmdId = 307, Command = #{<<"requestID">> => CmdId, <<"cacheID">> => CmdId, <<"msgType">> => <<"write">>, <<"data">> => #{ - <<"basePath">> => <<"/3/0/1">>, - <<"content">> => [ - #{ - type => <<"Integer">>, - value => <<"12345">> - } - ] - } + <<"basePath">> => <<"/3/0/1">>, + <<"content">> => [ + #{ + type => <<"Integer">>, + value => <<"12345">> + } + ] + } }, CommandJson = emqx_json:encode(Command), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), @@ -1135,18 +1152,18 @@ case22_write_error(Config) -> timer:sleep(100), ReadResult = emqx_json:encode(#{ - <<"requestID">> => CmdId, <<"cacheID">> => CmdId, - <<"data">> => #{ - <<"reqPath">> => <<"/3/0/1">>, - <<"code">> => <<"4.00">>, - <<"codeMsg">> => <<"bad_request">> - }, - <<"msgType">> => <<"write">> - }), + <<"requestID">> => CmdId, <<"cacheID">> => CmdId, + <<"data">> => #{ + <<"reqPath">> => <<"/3/0/1">>, + <<"code">> => <<"4.00">>, + <<"codeMsg">> => <<"bad_request">> + }, + <<"msgType">> => <<"write">> + }), ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)). case_create_basic(Config) -> - % step 1, device register ... + %% step 1, device register ... Epn = "urn:oma:lwm2m:oma:3", MsgId1 = 15, UdpSock = ?config(sock, Config), @@ -1157,15 +1174,14 @@ case_create_basic(Config) -> std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic), - % step2, send a CREATE command to device + %% step2, send a CREATE command to device CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>, CmdId = 307, - Command = #{<<"requestID">> => CmdId, <<"cacheID">> => CmdId, - <<"msgType">> => <<"create">>, - <<"data">> => #{ - <<"path">> => <<"/5">> - } - }, + Command = #{<<"msgType">> => <<"create">>, + <<"requestID">> => CmdId, <<"cacheID">> => CmdId, + <<"data">> => #{<<"content">> => [], + <<"basePath">> => <<"/5">> + }}, CommandJson = emqx_json:encode(Command), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(50), @@ -1181,18 +1197,18 @@ case_create_basic(Config) -> timer:sleep(100), ReadResult = emqx_json:encode(#{ - <<"requestID">> => CmdId, <<"cacheID">> => CmdId, - <<"data">> => #{ - <<"reqPath">> => <<"/5">>, - <<"code">> => <<"2.01">>, - <<"codeMsg">> => <<"created">> - }, - <<"msgType">> => <<"create">> - }), + <<"requestID">> => CmdId, <<"cacheID">> => CmdId, + <<"data">> => #{ + <<"reqPath">> => <<"/5">>, + <<"code">> => <<"2.01">>, + <<"codeMsg">> => <<"created">> + }, + <<"msgType">> => <<"create">> + }), ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)). case_delete_basic(Config) -> - % step 1, device register ... + %% step 1, device register ... Epn = "urn:oma:lwm2m:oma:3", MsgId1 = 15, UdpSock = ?config(sock, Config), @@ -1203,14 +1219,14 @@ case_delete_basic(Config) -> std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic), - % step2, send a CREATE command to device + %% step2, send a CREATE command to device CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>, CmdId = 307, Command = #{<<"requestID">> => CmdId, <<"cacheID">> => CmdId, <<"msgType">> => <<"delete">>, <<"data">> => #{ - <<"path">> => <<"/5/0">> - } + <<"path">> => <<"/5/0">> + } }, CommandJson = emqx_json:encode(Command), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), @@ -1227,18 +1243,18 @@ case_delete_basic(Config) -> timer:sleep(100), ReadResult = emqx_json:encode(#{ - <<"requestID">> => CmdId, <<"cacheID">> => CmdId, - <<"data">> => #{ - <<"reqPath">> => <<"/5/0">>, - <<"code">> => <<"2.02">>, - <<"codeMsg">> => <<"deleted">> - }, - <<"msgType">> => <<"delete">> - }), + <<"requestID">> => CmdId, <<"cacheID">> => CmdId, + <<"data">> => #{ + <<"reqPath">> => <<"/5/0">>, + <<"code">> => <<"2.02">>, + <<"codeMsg">> => <<"deleted">> + }, + <<"msgType">> => <<"delete">> + }), ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)). case30_execute(Config) -> - % step 1, device register ... + %% step 1, device register ... Epn = "urn:oma:lwm2m:oma:3", MsgId1 = 15, UdpSock = ?config(sock, Config), @@ -1249,16 +1265,16 @@ case30_execute(Config) -> std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic), - % step2, send a WRITE command to device + %% step2, send a WRITE command to device CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>, CmdId = 307, Command = #{<<"requestID">> => CmdId, <<"cacheID">> => CmdId, <<"msgType">> => <<"execute">>, <<"data">> => #{ - <<"path">> => <<"/3/0/4">>, - %% "args" should not be present for "/3/0/4", only for testing the encoding here - <<"args">> => <<"2,7">> - } + <<"path">> => <<"/3/0/4">>, + %% "args" should not be present for "/3/0/4", only for testing the encoding here + <<"args">> => <<"2,7">> + } }, CommandJson = emqx_json:encode(Command), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), @@ -1275,18 +1291,18 @@ case30_execute(Config) -> timer:sleep(100), ReadResult = emqx_json:encode(#{ - <<"requestID">> => CmdId, <<"cacheID">> => CmdId, - <<"data">> => #{ - <<"reqPath">> => <<"/3/0/4">>, - <<"code">> => <<"2.04">>, - <<"codeMsg">> => <<"changed">> - }, - <<"msgType">> => <<"execute">> - }), + <<"requestID">> => CmdId, <<"cacheID">> => CmdId, + <<"data">> => #{ + <<"reqPath">> => <<"/3/0/4">>, + <<"code">> => <<"2.04">>, + <<"codeMsg">> => <<"changed">> + }, + <<"msgType">> => <<"execute">> + }), ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)). case31_execute_error(Config) -> - % step 1, device register ... + %% step 1, device register ... Epn = "urn:oma:lwm2m:oma:3", MsgId1 = 15, UdpSock = ?config(sock, Config), @@ -1297,15 +1313,15 @@ case31_execute_error(Config) -> std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic), - % step2, send a WRITE command to device + %% step2, send a WRITE command to device CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>, CmdId = 307, Command = #{<<"requestID">> => CmdId, <<"cacheID">> => CmdId, <<"msgType">> => <<"execute">>, <<"data">> => #{ - <<"path">> => <<"/3/0/4">>, - <<"args">> => <<"2,7">> - } + <<"path">> => <<"/3/0/4">>, + <<"args">> => <<"2,7">> + } }, CommandJson = emqx_json:encode(Command), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), @@ -1322,18 +1338,18 @@ case31_execute_error(Config) -> timer:sleep(100), ReadResult = emqx_json:encode(#{ - <<"requestID">> => CmdId, <<"cacheID">> => CmdId, - <<"data">> => #{ - <<"reqPath">> => <<"/3/0/4">>, - <<"code">> => <<"4.01">>, - <<"codeMsg">> => <<"uauthorized">> - }, - <<"msgType">> => <<"execute">> - }), + <<"requestID">> => CmdId, <<"cacheID">> => CmdId, + <<"data">> => #{ + <<"reqPath">> => <<"/3/0/4">>, + <<"code">> => <<"4.01">>, + <<"codeMsg">> => <<"unauthorized">> + }, + <<"msgType">> => <<"execute">> + }), ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)). case40_discover(Config) -> - % step 1, device register ... + %% step 1, device register ... Epn = "urn:oma:lwm2m:oma:3", MsgId1 = 15, UdpSock = ?config(sock, Config), @@ -1344,14 +1360,14 @@ case40_discover(Config) -> std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic), - % step2, send a WRITE command to device + %% step2, send a WRITE command to device CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>, CmdId = 307, Command = #{<<"requestID">> => CmdId, <<"cacheID">> => CmdId, <<"msgType">> => <<"discover">>, <<"data">> => #{ - <<"path">> => <<"/3/0/7">> - } }, + <<"path">> => <<"/3/0/7">> + } }, CommandJson = emqx_json:encode(Command), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(50), @@ -1374,20 +1390,20 @@ case40_discover(Config) -> timer:sleep(100), ReadResult = emqx_json:encode(#{ - <<"requestID">> => CmdId, <<"cacheID">> => CmdId, - <<"msgType">> => <<"discover">>, - <<"data">> => #{ - <<"reqPath">> => <<"/3/0/7">>, - <<"code">> => <<"2.05">>, - <<"codeMsg">> => <<"content">>, - <<"content">> => - [<<";dim=8;pmin=10;pmax=60;gt=50;lt=42.2">>, <<"">>] - } - }), + <<"requestID">> => CmdId, <<"cacheID">> => CmdId, + <<"msgType">> => <<"discover">>, + <<"data">> => #{ + <<"reqPath">> => <<"/3/0/7">>, + <<"code">> => <<"2.05">>, + <<"codeMsg">> => <<"content">>, + <<"content">> => + [<<";dim=8;pmin=10;pmax=60;gt=50;lt=42.2">>, <<"">>] + } + }), ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)). case50_write_attribute(Config) -> - % step 1, device register ... + %% step 1, device register ... Epn = "urn:oma:lwm2m:oma:3", MsgId1 = 15, UdpSock = ?config(sock, Config), @@ -1398,17 +1414,17 @@ case50_write_attribute(Config) -> std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic), - % step2, send a WRITE command to device + %% step2, send a WRITE command to device CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>, CmdId = 307, Command = #{<<"requestID">> => CmdId, <<"cacheID">> => CmdId, <<"msgType">> => <<"write-attr">>, <<"data">> => #{ - <<"path">> => <<"/3/0/9">>, - <<"pmin">> => <<"1">>, - <<"pmax">> => <<"5">>, - <<"lt">> => <<"5">> - } }, + <<"path">> => <<"/3/0/9">>, + <<"pmin">> => <<"1">>, + <<"pmax">> => <<"5">>, + <<"lt">> => <<"5">> + } }, CommandJson = emqx_json:encode(Command), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(100), @@ -1433,18 +1449,18 @@ case50_write_attribute(Config) -> timer:sleep(100), ReadResult = emqx_json:encode(#{ - <<"requestID">> => CmdId, <<"cacheID">> => CmdId, - <<"data">> => #{ - <<"reqPath">> => <<"/3/0/9">>, - <<"code">> => <<"2.04">>, - <<"codeMsg">> => <<"changed">> - }, - <<"msgType">> => <<"write-attr">> - }), + <<"requestID">> => CmdId, <<"cacheID">> => CmdId, + <<"data">> => #{ + <<"reqPath">> => <<"/3/0/9">>, + <<"code">> => <<"2.04">>, + <<"codeMsg">> => <<"changed">> + }, + <<"msgType">> => <<"write-attr">> + }), ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)). case60_observe(Config) -> - % step 1, device register ... + %% step 1, device register ... Epn = "urn:oma:lwm2m:oma:3", MsgId1 = 15, UdpSock = ?config(sock, Config), @@ -1457,15 +1473,15 @@ case60_observe(Config) -> std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic), - % step2, send a OBSERVE command to device + %% step2, send a OBSERVE command to device CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>, CmdId = 307, Command = #{<<"requestID">> => CmdId, <<"cacheID">> => CmdId, <<"msgType">> => <<"observe">>, <<"data">> => #{ - <<"path">> => <<"/3/0/10">> - } - }, + <<"path">> => <<"/3/0/10">> + } + }, CommandJson = emqx_json:encode(Command), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(50), @@ -1488,18 +1504,18 @@ case60_observe(Config) -> timer:sleep(100), ReadResult = emqx_json:encode(#{ - <<"requestID">> => CmdId, <<"cacheID">> => CmdId, - <<"msgType">> => <<"observe">>, - <<"data">> => #{ - <<"reqPath">> => <<"/3/0/10">>, - <<"code">> => <<"2.05">>, - <<"codeMsg">> => <<"content">>, - <<"content">> => [#{ - path => <<"/3/0/10">>, - value => 2048 - }] - } - }), + <<"requestID">> => CmdId, <<"cacheID">> => CmdId, + <<"msgType">> => <<"observe">>, + <<"data">> => #{ + <<"reqPath">> => <<"/3/0/10">>, + <<"code">> => <<"2.05">>, + <<"codeMsg">> => <<"content">>, + <<"content">> => [#{ + path => <<"/3/0/10">>, + value => 2048 + }] + } + }), ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)), %% step3 the notifications @@ -1515,29 +1531,29 @@ case60_observe(Config) -> #coap_message{} = test_recv_coap_response(UdpSock), ReadResult2 = emqx_json:encode(#{ - <<"requestID">> => CmdId, <<"cacheID">> => CmdId, - <<"msgType">> => <<"notify">>, - <<"seqNum">> => ObSeq, - <<"data">> => #{ - <<"reqPath">> => <<"/3/0/10">>, - <<"code">> => <<"2.05">>, - <<"codeMsg">> => <<"content">>, - <<"content">> => [#{ - path => <<"/3/0/10">>, - value => 4096 - }] - } - }), + <<"requestID">> => CmdId, <<"cacheID">> => CmdId, + <<"msgType">> => <<"notify">>, + <<"seqNum">> => ObSeq, + <<"data">> => #{ + <<"reqPath">> => <<"/3/0/10">>, + <<"code">> => <<"2.05">>, + <<"codeMsg">> => <<"content">>, + <<"content">> => [#{ + path => <<"/3/0/10">>, + value => 4096 + }] + } + }), ?assertEqual(ReadResult2, test_recv_mqtt_response(RespTopicAD)), %% Step3. cancel observe CmdId3 = 308, Command3 = #{<<"requestID">> => CmdId3, <<"cacheID">> => CmdId3, - <<"msgType">> => <<"cancel-observe">>, - <<"data">> => #{ - <<"path">> => <<"/3/0/10">> - } - }, + <<"msgType">> => <<"cancel-observe">>, + <<"data">> => #{ + <<"path">> => <<"/3/0/10">> + } + }, CommandJson3 = emqx_json:encode(Command3), test_mqtt_broker:publish(CommandTopic, CommandJson3, 0), timer:sleep(50), @@ -1560,143 +1576,143 @@ case60_observe(Config) -> timer:sleep(100), ReadResult3 = emqx_json:encode(#{ - <<"requestID">> => CmdId3, <<"cacheID">> => CmdId3, - <<"msgType">> => <<"cancel-observe">>, - <<"data">> => #{ - <<"reqPath">> => <<"/3/0/10">>, - <<"code">> => <<"2.05">>, - <<"codeMsg">> => <<"content">>, - <<"content">> => [#{ - path => <<"/3/0/10">>, - value => 1150 - }] - } - }), + <<"requestID">> => CmdId3, <<"cacheID">> => CmdId3, + <<"msgType">> => <<"cancel-observe">>, + <<"data">> => #{ + <<"reqPath">> => <<"/3/0/10">>, + <<"code">> => <<"2.05">>, + <<"codeMsg">> => <<"content">>, + <<"content">> => [#{ + path => <<"/3/0/10">>, + value => 1150 + }] + } + }), ?assertEqual(ReadResult3, test_recv_mqtt_response(RespTopic)). -case80_specail_object_19_0_0_notify(Config) -> - % step 1, device register, with extra register options - Epn = "urn:oma:lwm2m:oma:3", - RegOptionWangYi = "&apn=psmA.eDRX0.ctnb&im=13456&ct=2.0&mt=MDM9206&mv=4.0", - MsgId1 = 15, - UdpSock = ?config(sock, Config), +%% case80_specail_object_19_0_0_notify(Config) -> +%% %% step 1, device register, with extra register options +%% Epn = "urn:oma:lwm2m:oma:3", +%% RegOptionWangYi = "&apn=psmA.eDRX0.ctnb&im=13456&ct=2.0&mt=MDM9206&mv=4.0", +%% MsgId1 = 15, +%% UdpSock = ?config(sock, Config), - RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"), - emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0), - timer:sleep(200), +%% RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"), +%% emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0), +%% timer:sleep(200), - test_send_coap_request( UdpSock, - post, - sprintf("coap://127.0.0.1:~b/rd?ep=~s<=345&lwm2m=1"++RegOptionWangYi, [?PORT, Epn]), - #coap_content{content_format = <<"text/plain">>, payload = <<", , , , ">>}, - [], - MsgId1), - #coap_message{method = Method1} = test_recv_coap_response(UdpSock), - ?assertEqual({ok,created}, Method1), - ReadResult = emqx_json:encode(#{ - <<"msgType">> => <<"register">>, - <<"data">> => #{ - <<"alternatePath">> => <<"/">>, - <<"ep">> => list_to_binary(Epn), - <<"lt">> => 345, - <<"lwm2m">> => <<"1">>, - <<"objectList">> => [<<"/1">>, <<"/2">>, <<"/3">>, <<"/4">>, <<"/5">>], - <<"apn">> => <<"psmA.eDRX0.ctnb">>, - <<"im">> => <<"13456">>, - <<"ct">> => <<"2.0">>, - <<"mt">> => <<"MDM9206">>, - <<"mv">> => <<"4.0">> - } - }), - ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)), +%% test_send_coap_request( UdpSock, +%% post, +%% sprintf("coap://127.0.0.1:~b/rd?ep=~s<=345&lwm2m=1"++RegOptionWangYi, [?PORT, Epn]), +%% #coap_content{content_format = <<"text/plain">>, payload = <<", , , , ">>}, +%% [], +%% MsgId1), +%% #coap_message{method = Method1} = test_recv_coap_response(UdpSock), +%% ?assertEqual({ok,created}, Method1), +%% ReadResult = emqx_json:encode(#{ +%% <<"msgType">> => <<"register">>, +%% <<"data">> => #{ +%% <<"alternatePath">> => <<"/">>, +%% <<"ep">> => list_to_binary(Epn), +%% <<"lt">> => 345, +%% <<"lwm2m">> => <<"1">>, +%% <<"objectList">> => [<<"/1">>, <<"/2">>, <<"/3">>, <<"/4">>, <<"/5">>], +%% <<"apn">> => <<"psmA.eDRX0.ctnb">>, +%% <<"im">> => <<"13456">>, +%% <<"ct">> => <<"2.0">>, +%% <<"mt">> => <<"MDM9206">>, +%% <<"mv">> => <<"4.0">> +%% } +%% }), +%% ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)), - % step2, send a OBSERVE command to device - CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>, - CmdId = 307, - Command = #{<<"requestID">> => CmdId, <<"cacheID">> => CmdId, - <<"msgType">> => <<"observe">>, - <<"data">> => #{ - <<"path">> => <<"/19/0/0">> - } - }, - CommandJson = emqx_json:encode(Command), - test_mqtt_broker:publish(CommandTopic, CommandJson, 0), - timer:sleep(50), - Request2 = test_recv_coap_request(UdpSock), - #coap_message{method = Method2, options=Options2, payload=Payload2} = Request2, - Path2 = get_coap_path(Options2), - Observe = get_coap_observe(Options2), - ?assertEqual(get, Method2), - ?assertEqual(<<"/19/0/0">>, Path2), - ?assertEqual(Observe, 0), - ?assertEqual(<<>>, Payload2), - timer:sleep(50), +%% %% step2, send a OBSERVE command to device +%% CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>, +%% CmdId = 307, +%% Command = #{<<"requestID">> => CmdId, <<"cacheID">> => CmdId, +%% <<"msgType">> => <<"observe">>, +%% <<"data">> => #{ +%% <<"path">> => <<"/19/0/0">> +%% } +%% }, +%% CommandJson = emqx_json:encode(Command), +%% test_mqtt_broker:publish(CommandTopic, CommandJson, 0), +%% timer:sleep(50), +%% Request2 = test_recv_coap_request(UdpSock), +%% #coap_message{method = Method2, options=Options2, payload=Payload2} = Request2, +%% Path2 = get_coap_path(Options2), +%% Observe = get_coap_observe(Options2), +%% ?assertEqual(get, Method2), +%% ?assertEqual(<<"/19/0/0">>, Path2), +%% ?assertEqual(Observe, 0), +%% ?assertEqual(<<>>, Payload2), +%% timer:sleep(50), - test_send_coap_observe_ack( UdpSock, - "127.0.0.1", - ?PORT, - {ok, content}, - #coap_content{content_format = <<"text/plain">>, payload = <<"2048">>}, - Request2), - timer:sleep(100). +%% test_send_coap_observe_ack( UdpSock, +%% "127.0.0.1", +%% ?PORT, +%% {ok, content}, +%% #coap_content{content_format = <<"text/plain">>, payload = <<"2048">>}, +%% Request2), +%% timer:sleep(100). - %% step 3, device send uplink data notifications +%% step 3, device send uplink data notifications -case80_specail_object_19_1_0_write(Config) -> - Epn = "urn:oma:lwm2m:oma:3", - RegOptionWangYi = "&apn=psmA.eDRX0.ctnb&im=13456&ct=2.0&mt=MDM9206&mv=4.0", - MsgId1 = 15, - UdpSock = ?config(sock, Config), - RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"), - emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0), - timer:sleep(200), +%% case80_specail_object_19_1_0_write(Config) -> +%% Epn = "urn:oma:lwm2m:oma:3", +%% RegOptionWangYi = "&apn=psmA.eDRX0.ctnb&im=13456&ct=2.0&mt=MDM9206&mv=4.0", +%% MsgId1 = 15, +%% UdpSock = ?config(sock, Config), +%% RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"), +%% emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0), +%% timer:sleep(200), - test_send_coap_request( UdpSock, - post, - sprintf("coap://127.0.0.1:~b/rd?ep=~s<=345&lwm2m=1"++RegOptionWangYi, [?PORT, Epn]), - #coap_content{content_format = <<"text/plain">>, payload = <<", , , , ">>}, - [], - MsgId1), - #coap_message{method = Method1} = test_recv_coap_response(UdpSock), - ?assertEqual({ok,created}, Method1), - test_recv_mqtt_response(RespTopic), +%% test_send_coap_request( UdpSock, +%% post, +%% sprintf("coap://127.0.0.1:~b/rd?ep=~s<=345&lwm2m=1"++RegOptionWangYi, [?PORT, Epn]), +%% #coap_content{content_format = <<"text/plain">>, payload = <<", , , , ">>}, +%% [], +%% MsgId1), +%% #coap_message{method = Method1} = test_recv_coap_response(UdpSock), +%% ?assertEqual({ok,created}, Method1), +%% test_recv_mqtt_response(RespTopic), - % step2, send a WRITE command to device - CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>, - CmdId = 307, - Command = #{<<"requestID">> => CmdId, <<"cacheID">> => CmdId, - <<"msgType">> => <<"write">>, - <<"data">> => #{ - <<"path">> => <<"/19/1/0">>, - <<"type">> => <<"Opaque">>, - <<"value">> => base64:encode(<<12345:32>>) - } - }, +%% %% step2, send a WRITE command to device +%% CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>, +%% CmdId = 307, +%% Command = #{<<"requestID">> => CmdId, <<"cacheID">> => CmdId, +%% <<"msgType">> => <<"write">>, +%% <<"data">> => #{ +%% <<"path">> => <<"/19/1/0">>, +%% <<"type">> => <<"Opaque">>, +%% <<"value">> => base64:encode(<<12345:32>>) +%% } +%% }, - CommandJson = emqx_json:encode(Command), - test_mqtt_broker:publish(CommandTopic, CommandJson, 0), - timer:sleep(50), - Request2 = test_recv_coap_request(UdpSock), - #coap_message{method = Method2, options=Options2, payload=Payload2} = Request2, - Path2 = get_coap_path(Options2), - ?assertEqual(put, Method2), - ?assertEqual(<<"/19/1/0">>, Path2), - ?assertEqual(<<3:2, 0:1, 0:2, 4:3, 0, 12345:32>>, Payload2), - timer:sleep(50), +%% CommandJson = emqx_json:encode(Command), +%% test_mqtt_broker:publish(CommandTopic, CommandJson, 0), +%% timer:sleep(50), +%% Request2 = test_recv_coap_request(UdpSock), +%% #coap_message{method = Method2, options=Options2, payload=Payload2} = Request2, +%% Path2 = get_coap_path(Options2), +%% ?assertEqual(put, Method2), +%% ?assertEqual(<<"/19/1/0">>, Path2), +%% ?assertEqual(<<3:2, 0:1, 0:2, 4:3, 0, 12345:32>>, Payload2), +%% timer:sleep(50), - test_send_coap_response(UdpSock, "127.0.0.1", ?PORT, {ok, changed}, #coap_content{}, Request2, true), - timer:sleep(100), +%% test_send_coap_response(UdpSock, "127.0.0.1", ?PORT, {ok, changed}, #coap_content{}, Request2, true), +%% timer:sleep(100), - ReadResult = emqx_json:encode(#{ - <<"requestID">> => CmdId, <<"cacheID">> => CmdId, - <<"data">> => #{ - <<"reqPath">> => <<"/19/1/0">>, - <<"code">> => <<"2.04">>, - <<"codeMsg">> => <<"changed">> - }, - <<"msgType">> => <<"write">> - }), - ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)). +%% ReadResult = emqx_json:encode(#{ +%% <<"requestID">> => CmdId, <<"cacheID">> => CmdId, +%% <<"data">> => #{ +%% <<"reqPath">> => <<"/19/1/0">>, +%% <<"code">> => <<"2.04">>, +%% <<"codeMsg">> => <<"changed">> +%% }, +%% <<"msgType">> => <<"write">> +%% }), +%% ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)). case90_psm_mode(Config) -> server_cache_mode(Config, "ep=~s<=345&lwm2m=1&apn=psmA.eDRX0.ctnb"). @@ -1705,9 +1721,10 @@ case90_queue_mode(Config) -> server_cache_mode(Config, "ep=~s<=345&lwm2m=1&b=UQ"). server_cache_mode(Config, RegOption) -> - application:set_env(?APP, qmode_time_window, 2), - - % step 1, device register, with apn indicates "PSM" mode + #{lwm2m := LwM2M} = Gateway = emqx:get_config([gateway]), + Gateway2 = Gateway#{lwm2m := LwM2M#{qmode_time_window => 2}}, + emqx_config:put([gateway], Gateway2), + %% step 1, device register, with apn indicates "PSM" mode Epn = "urn:oma:lwm2m:oma:3", MsgId1 = 15, @@ -1733,7 +1750,7 @@ server_cache_mode(Config, RegOption) -> verify_read_response_1(0, UdpSock), %% server inters into PSM mode - timer:sleep(2), + timer:sleep(2500), %% verify server caches downlink commands send_read_command_1(1, UdpSock), @@ -1756,12 +1773,12 @@ send_read_command_1(CmdId, _UdpSock) -> Epn = "urn:oma:lwm2m:oma:3", CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>, Command = #{ - <<"requestID">> => CmdId, <<"cacheID">> => CmdId, - <<"msgType">> => <<"read">>, - <<"data">> => #{ - <<"path">> => <<"/3/0/0">> - } - }, + <<"requestID">> => CmdId, <<"cacheID">> => CmdId, + <<"msgType">> => <<"read">>, + <<"data">> => #{ + <<"path">> => <<"/3/0/0">> + } + }, CommandJson = emqx_json:encode(Command), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(50). @@ -1778,16 +1795,17 @@ verify_read_response_1(CmdId, UdpSock) -> test_send_coap_response(UdpSock, "127.0.0.1", ?PORT, {ok, content}, #coap_content{content_format = <<"text/plain">>, payload = <<"EMQ">>}, Request, true), ReadResult = emqx_json:encode(#{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, - <<"msgType">> => <<"read">>, - <<"data">> => #{ - <<"code">> => <<"2.05">>, - <<"codeMsg">> => <<"content">>, - <<"content">> => [#{ - path => <<"/3/0/0">>, - value => <<"EMQ">> - }] - } - }), + <<"msgType">> => <<"read">>, + <<"data">> => #{ + <<"reqPath">> => <<"/3/0/0">>, + <<"code">> => <<"2.05">>, + <<"codeMsg">> => <<"content">>, + <<"content">> => [#{ + path => <<"/3/0/0">>, + value => <<"EMQ">> + }] + } + }), ?assertEqual(ReadResult, test_recv_mqtt_response(RespTopic)). device_update_1(UdpSock, Location) -> diff --git a/apps/emqx_gateway/test/emqx_lwm2m_api_SUITE.erl b/apps/emqx_gateway/test/emqx_lwm2m_api_SUITE.erl new file mode 100644 index 000000000..cb2ccf3f8 --- /dev/null +++ b/apps/emqx_gateway/test/emqx_lwm2m_api_SUITE.erl @@ -0,0 +1,317 @@ +%%-------------------------------------------------------------------- +%% Copyright (C) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_lwm2m_api_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-define(PORT, 5783). + +-define(LOGT(Format, Args), ct:pal("TEST_SUITE: " ++ Format, Args)). + +-include_lib("emqx_gateway/src/lwm2m/include/emqx_lwm2m.hrl"). +-include_lib("lwm2m_coap/include/coap.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +-define(CONF_DEFAULT, <<" +gateway.lwm2m { + xml_dir = \"../../lib/emqx_gateway/src/lwm2m/lwm2m_xml\" + lifetime_min = 1s + lifetime_max = 86400s + qmode_time_windonw = 22 + auto_observe = false + mountpoint = \"lwm2m/%u\" + update_msg_publish_condition = contains_object_list + translators { + command = {topic = \"/dn/#\", qos = 0} + response = {topic = \"/up/resp\", qos = 0} + notify = {topic = \"/up/notify\", qos = 0} + register = {topic = \"/up/resp\", qos = 0} + update = {topic = \"/up/resp\", qos = 0} + } + listeners.udp.default { + bind = 5783 + } +} +">>). + +-define(assertExists(Map, Key), + ?assertNotEqual(maps:get(Key, Map, undefined), undefined)). + +%%-------------------------------------------------------------------- +%% Setups +%%-------------------------------------------------------------------- + +all() -> + emqx_ct:all(?MODULE). + +init_per_suite(Config) -> + ok = emqx_config:init_load(emqx_gateway_schema, ?CONF_DEFAULT), + emqx_mgmt_api_test_util:init_suite([emqx_gateway]), + Config. + +end_per_suite(Config) -> + timer:sleep(300), + emqx_mgmt_api_test_util:end_suite([emqx_gateway]), + Config. + +init_per_testcase(_AllTestCase, Config) -> + ok = emqx_config:init_load(emqx_gateway_schema, ?CONF_DEFAULT), + {ok, _} = application:ensure_all_started(emqx_gateway), + {ok, ClientUdpSock} = gen_udp:open(0, [binary, {active, false}]), + + {ok, C} = emqtt:start_link([{host, "localhost"},{port, 1883},{clientid, <<"c1">>}]), + {ok, _} = emqtt:connect(C), + timer:sleep(100), + + [{sock, ClientUdpSock}, {emqx_c, C} | Config]. + +end_per_testcase(_AllTestCase, Config) -> + timer:sleep(300), + gen_udp:close(?config(sock, Config)), + emqtt:disconnect(?config(emqx_c, Config)), + ok = application:stop(emqx_gateway). + +%%-------------------------------------------------------------------- +%% Cases +%%-------------------------------------------------------------------- +t_lookup_cmd_read(Config) -> + UdpSock = ?config(sock, Config), + Epn = "urn:oma:lwm2m:oma:3", + MsgId1 = 15, + RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"), + emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0), + timer:sleep(200), + %% step 1, device register ... + test_send_coap_request( UdpSock, + post, + sprintf("coap://127.0.0.1:~b/rd?ep=~s<=345&lwm2m=1", [?PORT, Epn]), + #coap_content{content_format = <<"text/plain">>, + payload = <<";rt=\"oma.lwm2m\";ct=11543,,,">>}, + [], + MsgId1), + #coap_message{method = Method1} = test_recv_coap_response(UdpSock), + ?assertEqual({ok,created}, Method1), + test_recv_mqtt_response(RespTopic), + + %% step2, send a READ command to device + CmdId = 206, + CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>, + Command = #{ + <<"requestID">> => CmdId, <<"cacheID">> => CmdId, + <<"msgType">> => <<"read">>, + <<"data">> => #{ + <<"path">> => <<"/3/0/0">> + } + }, + CommandJson = emqx_json:encode(Command), + ?LOGT("CommandJson=~p", [CommandJson]), + test_mqtt_broker:publish(CommandTopic, CommandJson, 0), + timer:sleep(50), + + no_received_request(Epn, <<"/3/0/0">>, <<"read">>), + + Request2 = test_recv_coap_request(UdpSock), + ?LOGT("LwM2M client got ~p", [Request2]), + timer:sleep(50), + + test_send_coap_response(UdpSock, "127.0.0.1", ?PORT, {ok, content}, #coap_content{content_format = <<"text/plain">>, payload = <<"EMQ">>}, Request2, true), + timer:sleep(100), + + normal_received_request(Epn, <<"/3/0/0">>, <<"read">>). + +t_lookup_cmd_discover(Config) -> + %% step 1, device register ... + Epn = "urn:oma:lwm2m:oma:3", + MsgId1 = 15, + UdpSock = ?config(sock, Config), + ObjectList = <<", , , , ">>, + RespTopic = list_to_binary("lwm2m/"++Epn++"/up/resp"), + emqtt:subscribe(?config(emqx_c, Config), RespTopic, qos0), + timer:sleep(200), + + std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic), + + %% step2, send a WRITE command to device + CommandTopic = <<"lwm2m/", (list_to_binary(Epn))/binary, "/dn/dm">>, + CmdId = 307, + Command = #{<<"requestID">> => CmdId, <<"cacheID">> => CmdId, + <<"msgType">> => <<"discover">>, + <<"data">> => #{ + <<"path">> => <<"/3/0/7">> + } }, + CommandJson = emqx_json:encode(Command), + test_mqtt_broker:publish(CommandTopic, CommandJson, 0), + + no_received_request(Epn, <<"/3/0/7">>, <<"discover">>), + + timer:sleep(50), + Request2 = test_recv_coap_request(UdpSock), + timer:sleep(50), + + PayloadDiscover = <<";dim=8;pmin=10;pmax=60;gt=50;lt=42.2,">>, + test_send_coap_response(UdpSock, + "127.0.0.1", + ?PORT, + {ok, content}, + #coap_content{content_format = <<"application/link-format">>, payload = PayloadDiscover}, + Request2, + true), + timer:sleep(100), + discover_received_request(Epn, <<"/3/0/7">>, <<"discover">>). + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%% Internal Functions +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +send_request(ClientId, Path, Action) -> + ApiPath = emqx_mgmt_api_test_util:api_path(["gateway/lwm2m", ClientId, "lookup_cmd"]), + Auth = emqx_mgmt_api_test_util:auth_header_(), + Query = io_lib:format("path=~s&action=~s", [Path, Action]), + {ok, Response} = emqx_mgmt_api_test_util:request_api(get, ApiPath, Query, Auth), + ?LOGT("rest api response:~s~n", [Response]), + Response. + +no_received_request(ClientId, Path, Action) -> + Response = send_request(ClientId, Path, Action), + NotReceived = #{<<"clientid">> => list_to_binary(ClientId), + <<"action">> => Action, + <<"code">> => <<"6.01">>, + <<"codeMsg">> => <<"reply_not_received">>, + <<"path">> => Path}, + ?assertEqual(NotReceived, emqx_json:decode(Response, [return_maps])). +normal_received_request(ClientId, Path, Action) -> + Response = send_request(ClientId, Path, Action), + RCont = emqx_json:decode(Response, [return_maps]), + ?assertEqual(list_to_binary(ClientId), maps:get(<<"clientid">>, RCont, undefined)), + ?assertEqual(Path, maps:get(<<"path">>, RCont, undefined)), + ?assertEqual(Action, maps:get(<<"action">>, RCont, undefined)), + ?assertExists(RCont, <<"code">>), + ?assertExists(RCont, <<"codeMsg">>), + ?assertExists(RCont, <<"content">>), + RCont. + +discover_received_request(ClientId, Path, Action) -> + RCont = normal_received_request(ClientId, Path, Action), + [Res | _] = maps:get(<<"content">>, RCont), + ?assertExists(Res, <<"path">>), + ?assertExists(Res, <<"name">>), + ?assertExists(Res, <<"operations">>). + +test_recv_mqtt_response(RespTopic) -> + receive + {publish, #{topic := RespTopic, payload := RM}} -> + ?LOGT("test_recv_mqtt_response Response=~p", [RM]), + RM + after 1000 -> timeout_test_recv_mqtt_response + end. + +test_send_coap_request(UdpSock, Method, Uri, Content, Options, MsgId) -> + is_record(Content, coap_content) orelse error("Content must be a #coap_content!"), + is_list(Options) orelse error("Options must be a list"), + case resolve_uri(Uri) of + {coap, {IpAddr, Port}, Path, Query} -> + Request0 = lwm2m_coap_message:request(con, Method, Content, [{uri_path, Path}, {uri_query, Query} | Options]), + Request = Request0#coap_message{id = MsgId}, + ?LOGT("send_coap_request Request=~p", [Request]), + RequestBinary = lwm2m_coap_message_parser:encode(Request), + ?LOGT("test udp socket send to ~p:~p, data=~p", [IpAddr, Port, RequestBinary]), + ok = gen_udp:send(UdpSock, IpAddr, Port, RequestBinary); + {SchemeDiff, ChIdDiff, _, _} -> + error(lists:flatten(io_lib:format("scheme ~s or ChId ~s does not match with socket", [SchemeDiff, ChIdDiff]))) + end. + +test_recv_coap_response(UdpSock) -> + {ok, {Address, Port, Packet}} = gen_udp:recv(UdpSock, 0, 2000), + Response = lwm2m_coap_message_parser:decode(Packet), + ?LOGT("test udp receive from ~p:~p, data1=~p, Response=~p", [Address, Port, Packet, Response]), + #coap_message{type = ack, method = Method, id=Id, token = Token, options = Options, payload = Payload} = Response, + ?LOGT("receive coap response Method=~p, Id=~p, Token=~p, Options=~p, Payload=~p", [Method, Id, Token, Options, Payload]), + Response. + +test_recv_coap_request(UdpSock) -> + case gen_udp:recv(UdpSock, 0, 2000) of + {ok, {_Address, _Port, Packet}} -> + Request = lwm2m_coap_message_parser:decode(Packet), + #coap_message{type = con, method = Method, id=Id, token = Token, payload = Payload, options = Options} = Request, + ?LOGT("receive coap request Method=~p, Id=~p, Token=~p, Options=~p, Payload=~p", [Method, Id, Token, Options, Payload]), + Request; + {error, Reason} -> + ?LOGT("test_recv_coap_request failed, Reason=~p", [Reason]), + timeout_test_recv_coap_request + end. + +test_send_coap_response(UdpSock, Host, Port, Code, Content, Request, Ack) -> + is_record(Content, coap_content) orelse error("Content must be a #coap_content!"), + is_list(Host) orelse error("Host is not a string"), + + {ok, IpAddr} = inet:getaddr(Host, inet), + Response = lwm2m_coap_message:response(Code, Content, Request), + Response2 = case Ack of + true -> Response#coap_message{type = ack}; + false -> Response + end, + ?LOGT("test_send_coap_response Response=~p", [Response2]), + ok = gen_udp:send(UdpSock, IpAddr, Port, lwm2m_coap_message_parser:encode(Response2)). + +std_register(UdpSock, Epn, ObjectList, MsgId1, RespTopic) -> + test_send_coap_request( UdpSock, + post, + sprintf("coap://127.0.0.1:~b/rd?ep=~s<=345&lwm2m=1", [?PORT, Epn]), + #coap_content{content_format = <<"text/plain">>, payload = ObjectList}, + [], + MsgId1), + #coap_message{method = {ok,created}} = test_recv_coap_response(UdpSock), + test_recv_mqtt_response(RespTopic), + timer:sleep(100). + +resolve_uri(Uri) -> + {ok, #{scheme := Scheme, + host := Host, + port := PortNo, + path := Path} = URIMap} = emqx_http_lib:uri_parse(Uri), + Query = maps:get(query, URIMap, ""), + {ok, PeerIP} = inet:getaddr(Host, inet), + {Scheme, {PeerIP, PortNo}, split_path(Path), split_query(Query)}. + +split_path([]) -> []; +split_path([$/]) -> []; +split_path([$/ | Path]) -> split_segments(Path, $/, []). + +split_query([]) -> []; +split_query(Path) -> split_segments(Path, $&, []). + +split_segments(Path, Char, Acc) -> + case string:rchr(Path, Char) of + 0 -> + [make_segment(Path) | Acc]; + N when N > 0 -> + split_segments(string:substr(Path, 1, N-1), Char, + [make_segment(string:substr(Path, N+1)) | Acc]) + end. + +make_segment(Seg) -> + list_to_binary(emqx_http_lib:uri_decode(Seg)). + +join_path([], Acc) -> Acc; +join_path([<<"/">>|T], Acc) -> + join_path(T, Acc); +join_path([H|T], Acc) -> + join_path(T, <>). + +sprintf(Format, Args) -> + lists:flatten(io_lib:format(Format, Args)). diff --git a/apps/emqx_gateway/test/emqx_sn_protocol_SUITE.erl b/apps/emqx_gateway/test/emqx_sn_protocol_SUITE.erl index 2fbd031ff..23fb691d9 100644 --- a/apps/emqx_gateway/test/emqx_sn_protocol_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_sn_protocol_SUITE.erl @@ -83,11 +83,11 @@ all() -> init_per_suite(Config) -> ok = emqx_config:init_load(emqx_gateway_schema, ?CONF_DEFAULT), - emqx_ct_helpers:start_apps([emqx_authn, emqx_gateway]), + emqx_ct_helpers:start_apps([emqx_gateway]), Config. end_per_suite(_) -> - emqx_ct_helpers:stop_apps([emqx_gateway, emqx_authn]). + emqx_ct_helpers:stop_apps([emqx_gateway]). %%-------------------------------------------------------------------- %% Test cases @@ -994,7 +994,7 @@ t_will_case06(_) -> receive {deliver, WillTopic, #message{payload = WillMsg}} -> ok; - Msg -> ct:print("recevived --- unex: ~p", [Msg]) + Msg -> ct:print("received --- unex: ~p", [Msg]) after 1000 -> ct:fail(wait_willmsg_timeout) end, diff --git a/apps/emqx_gateway/test/emqx_stomp_SUITE.erl b/apps/emqx_gateway/test/emqx_stomp_SUITE.erl index 75f6dadc3..9c3f1090f 100644 --- a/apps/emqx_gateway/test/emqx_stomp_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_stomp_SUITE.erl @@ -43,11 +43,11 @@ all() -> emqx_ct:all(?MODULE). init_per_suite(Cfg) -> ok = emqx_config:init_load(emqx_gateway_schema, ?CONF_DEFAULT), - emqx_ct_helpers:start_apps([emqx_authn, emqx_gateway]), + emqx_ct_helpers:start_apps([emqx_gateway]), Cfg. end_per_suite(_Cfg) -> - emqx_ct_helpers:stop_apps([emqx_gateway, emqx_authn]), + emqx_ct_helpers:stop_apps([emqx_gateway]), ok. %%-------------------------------------------------------------------- diff --git a/apps/emqx_machine/src/emqx_machine.erl b/apps/emqx_machine/src/emqx_machine.erl index 3e5772b4a..97125d79f 100644 --- a/apps/emqx_machine/src/emqx_machine.erl +++ b/apps/emqx_machine/src/emqx_machine.erl @@ -140,7 +140,7 @@ reboot_apps() -> , emqx_statsd , emqx_resource , emqx_rule_engine - , emqx_data_bridge + , emqx_bridge , emqx_bridge_mqtt , emqx_plugin_libs , emqx_management diff --git a/apps/emqx_machine/src/emqx_machine_schema.erl b/apps/emqx_machine/src/emqx_machine_schema.erl index c25ab8139..614609875 100644 --- a/apps/emqx_machine/src/emqx_machine_schema.erl +++ b/apps/emqx_machine/src/emqx_machine_schema.erl @@ -23,6 +23,7 @@ -dialyzer(no_fail_call). -include_lib("typerefl/include/types.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). -type log_level() :: debug | info | notice | warning | error | critical | alert | emergency | all. -type file() :: string(). @@ -34,8 +35,7 @@ file/0, cipher/0]). --export([roots/0, fields/1, translations/0, translation/1]). --export([t/1, t/3, t/4, ref/1]). +-export([namespace/0, roots/0, fields/1, translations/0, translation/1]). -export([conf_get/2, conf_get/3, keys/2, filter/1]). %% Static apps which merge their configs into the merged emqx.conf @@ -43,13 +43,11 @@ %% by nodetool to generate app.