diff --git a/.ci/fvt_tests/http_server/rebar.config b/.ci/fvt_tests/http_server/rebar.config index 29cc392ea..314679264 100644 --- a/.ci/fvt_tests/http_server/rebar.config +++ b/.ci/fvt_tests/http_server/rebar.config @@ -1,7 +1,7 @@ {erl_opts, [debug_info]}. {deps, [ - {minirest, {git, "https://github.com/emqx/minirest.git", {tag, "0.3.5"}}} + {minirest, {git, "https://github.com/emqx/minirest.git", {tag, "0.3.6"}}} ]}. {shell, [ diff --git a/.ci/fvt_tests/relup.lux b/.ci/fvt_tests/relup.lux index 33c35cb3e..91c11d3ae 100644 --- a/.ci/fvt_tests/relup.lux +++ b/.ci/fvt_tests/relup.lux @@ -43,7 +43,7 @@ !sed -i '/emqx_telemetry/d' data/loaded_plugins !./bin/emqx start - ?EMQ X (.*) is started successfully! + ?EMQ X .* is started successfully! ?SH-PROMPT !./bin/emqx_ctl cluster join emqx@127.0.0.1 @@ -99,6 +99,10 @@ """ ?SH-PROMPT + !./bin/emqx_ctl plugins list | grep emqx_management + ?Plugin\(emqx_management.*active=true\) + ?SH-PROMPT + [shell emqx2] !echo "" > log/emqx.log.1 ?SH-PROMPT @@ -120,6 +124,10 @@ """ ?SH-PROMPT + !./bin/emqx_ctl plugins list | grep emqx_management + ?Plugin\(emqx_management.*active=true\) + ?SH-PROMPT + [shell bench] ???publish complete ??SH-PROMPT: diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 2090f722e..b983eaa67 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -83,6 +83,7 @@ jobs: - name: build env: PYTHON: python + DIAGNOSTIC: 1 run: | $env:PATH = "${{ steps.install_erlang.outputs.erlpath }}\bin;$env:PATH" @@ -168,9 +169,11 @@ jobs: - name: build run: | . $HOME/.kerl/${{ matrix.erl_otp }}/activate - make -C source ensure-rebar3 - sudo cp source/rebar3 /usr/local/bin/rebar3 - make -C source ${{ matrix.profile }}-zip + cd source + make ensure-rebar3 + sudo cp rebar3 /usr/local/bin/rebar3 + rm -rf _build/${{ matrix.profile }}/lib + make ${{ matrix.profile }}-zip - name: test run: | cd source diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index 6c9bbf04a..bf85578c5 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -38,6 +38,11 @@ jobs: run: make ${EMQX_NAME}-zip - name: build deb/rpm packages run: make ${EMQX_NAME}-pkg + - uses: actions/upload-artifact@v1 + if: failure() + with: + name: rebar3.crashdump + path: ./rebar3.crashdump - name: pakcages test run: | export CODE_PATH=$GITHUB_WORKSPACE @@ -94,6 +99,11 @@ jobs: make ensure-rebar3 sudo cp rebar3 /usr/local/bin/rebar3 make ${EMQX_NAME}-zip + - uses: actions/upload-artifact@v1 + if: failure() + with: + name: rebar3.crashdump + path: ./rebar3.crashdump - name: test run: | pkg_name=$(basename _packages/${EMQX_NAME}/emqx-*.zip) diff --git a/apps/emqx/etc/emqx.conf b/apps/emqx/etc/emqx.conf index d2b5fd11d..3387a6439 100644 --- a/apps/emqx/etc/emqx.conf +++ b/apps/emqx/etc/emqx.conf @@ -355,8 +355,8 @@ rpc.port_discovery = stateless ## Number of outgoing RPC connections. ## ## Value: Interger [0-256] -## Defaults to NumberOfCPUSchedulers / 2 when set to 0 -#rpc.tcp_client_num = 0 +## Default = 1 +#rpc.tcp_client_num = 1 ## RCP Client connect timeout. ## diff --git a/apps/emqx/include/emqx_mqtt.hrl b/apps/emqx/include/emqx_mqtt.hrl index e6e9bffe5..5dd9a317c 100644 --- a/apps/emqx/include/emqx_mqtt.hrl +++ b/apps/emqx/include/emqx_mqtt.hrl @@ -30,11 +30,13 @@ %% MQTT Protocol Version and Names %%-------------------------------------------------------------------- +-define(MQTT_SN_PROTO_V1, 1). -define(MQTT_PROTO_V3, 3). -define(MQTT_PROTO_V4, 4). -define(MQTT_PROTO_V5, 5). -define(PROTOCOL_NAMES, [ + {?MQTT_SN_PROTO_V1, <<"MQTT-SN">>}, %% XXX:Compatible with emqx-sn plug-in {?MQTT_PROTO_V3, <<"MQIsdp">>}, {?MQTT_PROTO_V4, <<"MQTT">>}, {?MQTT_PROTO_V5, <<"MQTT">>}]). diff --git a/apps/emqx/src/emqx.appup.src b/apps/emqx/src/emqx.appup.src index b51a7f3b7..4f9f00673 100644 --- a/apps/emqx/src/emqx.appup.src +++ b/apps/emqx/src/emqx.appup.src @@ -1,12 +1,31 @@ %% -*- mode: erlang -*- {VSN, - [{"4.3.2", - [{load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, + [ + {"4.3.4", + [{load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, + {load_module,emqx_app,brutal_purge,soft_purge,[]} + ]}, + {"4.3.3", + [{load_module,emqx_packet,brutal_purge,soft_purge,[]}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, + {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_cm,brutal_purge,soft_purge,[]}, + {load_module,emqx_app,brutal_purge,soft_purge,[]} + ]}, + {"4.3.2", + [{load_module,emqx_packet,brutal_purge,soft_purge,[]}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, + {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_connection,brutal_purge,soft_purge,[]}]}, + {load_module,emqx_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_cm,brutal_purge,soft_purge,[]} + ]}, {"4.3.1", - [{load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, + [{load_module,emqx_packet,brutal_purge,soft_purge,[]}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, + {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, @@ -18,7 +37,9 @@ {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}]}, {"4.3.0", - [{load_module,emqx_logger_jsonfmt,brutal_purge,soft_purge,[]}, + [{load_module,emqx_packet,brutal_purge,soft_purge,[]}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, + {load_module,emqx_logger_jsonfmt,brutal_purge,soft_purge,[]}, {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_congestion,brutal_purge,soft_purge,[]}, {load_module,emqx_connection,brutal_purge,soft_purge,[]}, @@ -34,13 +55,32 @@ {apply,{emqx_metrics,upgrade_retained_delayed_counter_type,[]}}, {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.3.2", - [{load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, + [ + {"4.3.4", + [{load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, + {load_module,emqx_app,brutal_purge,soft_purge,[]} + ]}, + {"4.3.3", + [{load_module,emqx_packet,brutal_purge,soft_purge,[]}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, + {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_cm,brutal_purge,soft_purge,[]}, + {load_module,emqx_app,brutal_purge,soft_purge,[]} + ]}, + {"4.3.2", + [{load_module,emqx_packet,brutal_purge,soft_purge,[]}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, + {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_connection,brutal_purge,soft_purge,[]}]}, + {load_module,emqx_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_cm,brutal_purge,soft_purge,[]} + ]}, {"4.3.1", - [{load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, + [{load_module,emqx_packet,brutal_purge,soft_purge,[]}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, + {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, @@ -52,7 +92,9 @@ {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}]}, {"4.3.0", - [{load_module,emqx_logger_jsonfmt,brutal_purge,soft_purge,[]}, + [{load_module,emqx_packet,brutal_purge,soft_purge,[]}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, + {load_module,emqx_logger_jsonfmt,brutal_purge,soft_purge,[]}, {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_congestion,brutal_purge,soft_purge,[]}, diff --git a/apps/emqx/src/emqx_cm.erl b/apps/emqx/src/emqx_cm.erl index b15a2ff79..6eb375aba 100644 --- a/apps/emqx/src/emqx_cm.erl +++ b/apps/emqx/src/emqx_cm.erl @@ -294,6 +294,9 @@ do_discard_session(ClientId, Pid) -> _ : {noproc, _} -> % emqx_connection: gen_server:call ?tp(debug, "session_already_gone", #{pid => Pid}), ok; + _ : {'EXIT', {noproc, _}} -> % rpc_call/3 + ?tp(debug, "session_already_gone", #{pid => Pid}), + ok; _ : {{shutdown, _}, _} -> ?tp(debug, "session_already_shutdown", #{pid => Pid}), ok; diff --git a/apps/emqx/src/emqx_shared_sub.erl b/apps/emqx/src/emqx_shared_sub.erl index 4707f63db..97aa778f3 100644 --- a/apps/emqx/src/emqx_shared_sub.erl +++ b/apps/emqx/src/emqx_shared_sub.erl @@ -336,9 +336,13 @@ handle_info({mnesia_table_event, {write, NewRecord, _}}, State = #state{pmon = P #emqx_shared_subscription{subpid = SubPid} = NewRecord, {noreply, update_stats(State#state{pmon = emqx_pmon:monitor(SubPid, PMon)})}; -handle_info({mnesia_table_event, {delete_object, OldRecord, _}}, State = #state{pmon = PMon}) -> - #emqx_shared_subscription{subpid = SubPid} = OldRecord, - {noreply, update_stats(State#state{pmon = emqx_pmon:demonitor(SubPid, PMon)})}; +%% The subscriber may have subscribed multiple topics, so we need to keep monitoring the PID until +%% it `unsubscribed` the last topic. +%% The trick is we don't demonitor the subscriber here, and (after a long time) it will eventually +%% be disconnected. +% handle_info({mnesia_table_event, {delete_object, OldRecord, _}}, State = #state{pmon = PMon}) -> +% #emqx_shared_subscription{subpid = SubPid} = OldRecord, +% {noreply, update_stats(State#state{pmon = emqx_pmon:demonitor(SubPid, PMon)})}; handle_info({mnesia_table_event, _Event}, State) -> {noreply, State}; @@ -348,8 +352,7 @@ handle_info({'DOWN', _MRef, process, SubPid, _Reason}, State = #state{pmon = PMo cleanup_down(SubPid), {noreply, update_stats(State#state{pmon = emqx_pmon:erase(SubPid, PMon)})}; -handle_info(Info, State) -> - ?LOG(error, "Unexpected info: ~p", [Info]), +handle_info(_Info, State) -> {noreply, State}. terminate(_Reason, _State) -> diff --git a/apps/emqx/src/emqx_ws_connection.erl b/apps/emqx/src/emqx_ws_connection.erl index 7bc68c271..d686e1611 100644 --- a/apps/emqx/src/emqx_ws_connection.erl +++ b/apps/emqx/src/emqx_ws_connection.erl @@ -403,7 +403,10 @@ websocket_close(Reason, State) -> terminate(Reason, _Req, #state{channel = Channel}) -> ?LOG(debug, "Terminated due to ~p", [Reason]), - emqx_channel:terminate(Reason, Channel). + emqx_channel:terminate(Reason, Channel); + +terminate(_Reason, _Req, _UnExpectedState) -> + ok. %%-------------------------------------------------------------------- %% Handle call diff --git a/apps/emqx_dashboard/src/emqx_dashboard.appup.src b/apps/emqx_dashboard/src/emqx_dashboard.appup.src new file mode 100644 index 000000000..678bd3b22 --- /dev/null +++ b/apps/emqx_dashboard/src/emqx_dashboard.appup.src @@ -0,0 +1,16 @@ +%% -*- mode: erlang -*- +{VSN, + [ {"4.3.0", + %% load all plugins + %% NOTE: this depends on the fact that emqx_dashboard is always + %% the last application gets upgraded + [ {apply, {emqx_plugins, load, []}} + ]}, + {<<".*">>, []} + ], + [ {"4.3.0", + [ {apply, {emqx_plugins, load, []}} + ]}, + {<<".*">>, []} + ] +}. diff --git a/apps/emqx_exhook/src/emqx_exhook.app.src b/apps/emqx_exhook/src/emqx_exhook.app.src index 452d2a742..e703cfe5e 100644 --- a/apps/emqx_exhook/src/emqx_exhook.app.src +++ b/apps/emqx_exhook/src/emqx_exhook.app.src @@ -1,6 +1,6 @@ {application, emqx_exhook, [{description, "EMQ X Extension for Hook"}, - {vsn, "4.3.1"}, + {vsn, "4.3.2"}, {modules, []}, {registered, []}, {mod, {emqx_exhook_app, []}}, diff --git a/apps/emqx_exhook/src/emqx_exhook.appup.src b/apps/emqx_exhook/src/emqx_exhook.appup.src index 2811c1554..26e84d88f 100644 --- a/apps/emqx_exhook/src/emqx_exhook.appup.src +++ b/apps/emqx_exhook/src/emqx_exhook.appup.src @@ -1,14 +1,22 @@ %% -*-: erlang -*- {VSN, [ + {"4.3.1", [ + {load_module, emqx_exhook_server, brutal_purge, soft_purge, []} + ]}, {"4.3.0", [ - {load_module, emqx_exhook_pb, brutal_purge, soft_purge, []} + {load_module, emqx_exhook_pb, brutal_purge, soft_purge, []}, + {load_module, emqx_exhook_server, brutal_purge, soft_purge, []} ]}, {<<".*">>, []} ], [ + {"4.3.1", [ + {load_module, emqx_exhook_server, brutal_purge, soft_purge, []} + ]}, {"4.3.0", [ - {load_module, emqx_exhook_pb, brutal_purge, soft_purge, []} + {load_module, emqx_exhook_pb, brutal_purge, soft_purge, []}, + {load_module, emqx_exhook_server, brutal_purge, soft_purge, []} ]}, {<<".*">>, []} ] diff --git a/apps/emqx_exhook/src/emqx_exhook_server.erl b/apps/emqx_exhook/src/emqx_exhook_server.erl index 848a3f59d..f4965e4ca 100644 --- a/apps/emqx_exhook/src/emqx_exhook_server.erl +++ b/apps/emqx_exhook/src/emqx_exhook_server.erl @@ -122,7 +122,7 @@ channel_opts(Opts) -> Scheme = proplists:get_value(scheme, Opts), Host = proplists:get_value(host, Opts), Port = proplists:get_value(port, Opts), - SvrAddr = lists:flatten(io_lib:format("~s://~s:~w", [Scheme, Host, Port])), + SvrAddr = format_http_uri(Scheme, Host, Port), ClientOpts = case Scheme of https -> SslOpts = lists:keydelete(ssl, 1, proplists:get_value(ssl_options, Opts, [])), @@ -133,6 +133,13 @@ channel_opts(Opts) -> end, {SvrAddr, ClientOpts}. +format_http_uri(Scheme, Host0, Port) -> + Host = case is_tuple(Host0) of + true -> inet:ntoa(Host0); + _ -> Host0 + end, + lists:flatten(io_lib:format("~s://~s:~w", [Scheme, Host, Port])). + -spec unload(server()) -> ok. unload(#server{name = Name, hookspec = HookSpecs}) -> _ = do_deinit(Name), diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m.app.src b/apps/emqx_lwm2m/src/emqx_lwm2m.app.src index 19fabc526..209499fc5 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m.app.src +++ b/apps/emqx_lwm2m/src/emqx_lwm2m.app.src @@ -1,6 +1,6 @@ {application,emqx_lwm2m, [{description,"EMQ X LwM2M Gateway"}, - {vsn, "4.3.1"}, % strict semver, bump manually! + {vsn, "4.3.2"}, % strict semver, bump manually! {modules,[]}, {registered,[emqx_lwm2m_sup]}, {applications,[kernel,stdlib,lwm2m_coap]}, diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m.appup.src b/apps/emqx_lwm2m/src/emqx_lwm2m.appup.src index fb22c843d..b6d49302f 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m.appup.src +++ b/apps/emqx_lwm2m/src/emqx_lwm2m.appup.src @@ -1,15 +1,13 @@ %% -*-: erlang -*- {VSN, [ - {"4.3.0", [ - {load_module, emqx_lwm2m_protocol, brutal_purge, soft_purge, []} - ]}, - {<<".*">>, []} + {<<"4.3.[0-1]">>, [ + {restart_application, emqx_lwm2m} + ]} ], [ - {"4.3.0", [ - {load_module, emqx_lwm2m_protocol, brutal_purge, soft_purge, []} - ]}, - {<<".*">>, []} + {<<"4.3.[0-1]">>, [ + {restart_application, emqx_lwm2m} + ]} ] }. diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_api.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_api.erl new file mode 100644 index 000000000..6018aa7c7 --- /dev/null +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_api.erl @@ -0,0 +1,162 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_lwm2m_api). + +-import(minirest, [return/1]). + +-rest_api(#{name => list, + method => 'GET', + path => "/lwm2m_channels/", + func => list, + descr => "A list of all lwm2m channel" + }). + +-rest_api(#{name => list, + method => 'GET', + path => "/nodes/:atom:node/lwm2m_channels/", + func => list, + descr => "A list of lwm2m channel of a node" + }). + +-rest_api(#{name => lookup_cmd, + method => 'GET', + path => "/lookup_cmd/:bin:ep/", + func => lookup_cmd, + descr => "Send a lwm2m downlink command" + }). + +-rest_api(#{name => lookup_cmd, + method => 'GET', + path => "/nodes/:atom:node/lookup_cmd/:bin:ep/", + func => lookup_cmd, + descr => "Send a lwm2m downlink command of a node" + }). + +-export([ list/2 + , lookup_cmd/2 + ]). + +list(#{node := Node }, Params) -> + case Node = node() of + true -> list(#{}, Params); + _ -> rpc_call(Node, list, [#{}, Params]) + end; + +list(#{}, _Params) -> + Channels = emqx_lwm2m_cm:all_channels(), + return({ok, format(Channels)}). + +lookup_cmd(#{ep := Ep, node := Node}, Params) -> + case Node = node() of + true -> lookup_cmd(#{ep => Ep}, Params); + _ -> rpc_call(Node, lookup_cmd, [#{ep => Ep}, Params]) + end; + +lookup_cmd(#{ep := Ep}, Params) -> + MsgType = proplists:get_value(<<"msgType">>, Params), + Path0 = proplists:get_value(<<"path">>, Params), + case emqx_lwm2m_cm:lookup_cmd(Ep, Path0, MsgType) of + [] -> return({ok, []}); + [{_, undefined} | _] -> return({ok, []}); + [{{IMEI, Path, MsgType}, undefined}] -> + return({ok, [{imei, IMEI}, + {'msgType', IMEI}, + {'code', <<"6.01">>}, + {'codeMsg', <<"reply_not_received">>}, + {'path', Path}]}); + [{{IMEI, Path, MsgType}, {Code, CodeMsg, Content}}] -> + Payload1 = format_cmd_content(Content, MsgType), + return({ok, [{imei, IMEI}, + {'msgType', IMEI}, + {'code', Code}, + {'codeMsg', CodeMsg}, + {'path', Path}] ++ Payload1}) + end. + +rpc_call(Node, Fun, Args) -> + case rpc:call(Node, ?MODULE, Fun, Args) of + {badrpc, Reason} -> {error, Reason}; + Res -> Res + end. + +format(Channels) -> + lists:map(fun({IMEI, #{lifetime := LifeTime, + peername := Peername, + version := Version, + reg_info := RegInfo}}) -> + ObjectList = lists:map(fun(Path) -> + [ObjId | _] = path_list(Path), + case emqx_lwm2m_xml_object:get_obj_def(binary_to_integer(ObjId), true) of + {error, _} -> + {Path, Path}; + ObjDefinition -> + ObjectName = emqx_lwm2m_xml_object:get_object_name(ObjDefinition), + {Path, list_to_binary(ObjectName)} + end + end, maps:get(<<"objectList">>, RegInfo)), + {IpAddr, Port} = Peername, + [{imei, IMEI}, + {lifetime, LifeTime}, + {ip_address, iolist_to_binary(ntoa(IpAddr))}, + {port, Port}, + {version, Version}, + {'objectList', ObjectList}] + end, Channels). + +format_cmd_content(undefined, _MsgType) -> []; +format_cmd_content(Content, <<"discover">>) -> + [H | Content1] = Content, + {_, [HObjId]} = emqx_lwm2m_coap_resource:parse_object_list(H), + [ObjId | _]= path_list(HObjId), + ObjectList = case Content1 of + [Content2 | _] -> + {_, ObjL} = emqx_lwm2m_coap_resource:parse_object_list(Content2), + ObjL; + [] -> [] + end, + R = case emqx_lwm2m_xml_object:get_obj_def(binary_to_integer(ObjId), true) of + {error, _} -> + lists:map(fun(Object) -> {Object, Object} end, ObjectList); + ObjDefinition -> + lists:map(fun(Object) -> + [_, _, ResId| _] = path_list(Object), + Operations = case emqx_lwm2m_xml_object:get_resource_operations(binary_to_integer(ResId), ObjDefinition) of + "E" -> [{operations, list_to_binary("E")}]; + Oper -> [{'dataType', list_to_binary(emqx_lwm2m_xml_object:get_resource_type(binary_to_integer(ResId), ObjDefinition))}, + {operations, list_to_binary(Oper)}] + end, + [{path, Object}, + {name, list_to_binary(emqx_lwm2m_xml_object:get_resource_name(binary_to_integer(ResId), ObjDefinition))} + ] ++ Operations + end, ObjectList) + end, + [{content, R}]; +format_cmd_content(Content, _) -> + [{content, Content}]. + +ntoa({0,0,0,0,0,16#ffff,AB,CD}) -> + inet_parse:ntoa({AB bsr 8, AB rem 256, CD bsr 8, CD rem 256}); +ntoa(IP) -> + inet_parse:ntoa(IP). + +path_list(Path) -> + case binary:split(binary_util:trim(Path, $/), [<<$/>>], [global]) of + [ObjId, ObjInsId, ResId, ResInstId] -> [ObjId, ObjInsId, ResId, ResInstId]; + [ObjId, ObjInsId, ResId] -> [ObjId, ObjInsId, ResId]; + [ObjId, ObjInsId] -> [ObjId, ObjInsId]; + [ObjId] -> [ObjId] + end. diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_cm.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_cm.erl new file mode 100644 index 000000000..16e938b84 --- /dev/null +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_cm.erl @@ -0,0 +1,153 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_lwm2m_cm). + +-export([start_link/0]). + +-export([ register_channel/5 + , update_reg_info/2 + , unregister_channel/1 + ]). + +-export([ lookup_channel/1 + , all_channels/0 + ]). + +-export([ register_cmd/3 + , register_cmd/4 + , lookup_cmd/3 + , lookup_cmd_by_imei/1 + ]). + +%% gen_server callbacks +-export([ init/1 + , handle_call/3 + , handle_cast/2 + , handle_info/2 + , terminate/2 + , code_change/3 + ]). + +-define(LOG(Level, Format, Args), logger:Level("LWM2M-CM: " ++ Format, Args)). + +%% Server name +-define(CM, ?MODULE). + +-define(LWM2M_CHANNEL_TAB, emqx_lwm2m_channel). +-define(LWM2M_CMD_TAB, emqx_lwm2m_cmd). + +%% Batch drain +-define(BATCH_SIZE, 100000). + +%% @doc Start the channel manager. +start_link() -> + gen_server:start_link({local, ?CM}, ?MODULE, [], []). + +%%-------------------------------------------------------------------- +%% API +%%-------------------------------------------------------------------- + +register_channel(IMEI, RegInfo, LifeTime, Ver, Peername) -> + Info = #{ + reg_info => RegInfo, + lifetime => LifeTime, + version => Ver, + peername => Peername + }, + true = ets:insert(?LWM2M_CHANNEL_TAB, {IMEI, Info}), + cast({registered, {IMEI, self()}}). + +update_reg_info(IMEI, RegInfo) -> + case lookup_channel(IMEI) of + [{_, RegInfo0}] -> + true = ets:insert(?LWM2M_CHANNEL_TAB, {IMEI, RegInfo0#{reg_info => RegInfo}}), + ok; + [] -> + ok + end. + +unregister_channel(IMEI) when is_binary(IMEI) -> + true = ets:delete(?LWM2M_CHANNEL_TAB, IMEI), + ok. + +lookup_channel(IMEI) -> + ets:lookup(?LWM2M_CHANNEL_TAB, IMEI). + +all_channels() -> + ets:tab2list(?LWM2M_CHANNEL_TAB). + +register_cmd(IMEI, Path, Type) -> + true = ets:insert(?LWM2M_CMD_TAB, {{IMEI, Path, Type}, undefined}). + +register_cmd(_IMEI, undefined, _Type, _Result) -> + ok; +register_cmd(IMEI, Path, Type, Result) -> + true = ets:insert(?LWM2M_CMD_TAB, {{IMEI, Path, Type}, Result}). + +lookup_cmd(IMEI, Path, Type) -> + ets:lookup(?LWM2M_CMD_TAB, {IMEI, Path, Type}). + +lookup_cmd_by_imei(IMEI) -> + ets:select(?LWM2M_CHANNEL_TAB, [{{{IMEI, '_', '_'}, '$1'}, [], ['$_']}]). + +%% @private +cast(Msg) -> gen_server:cast(?CM, Msg). + +%%-------------------------------------------------------------------- +%% gen_server callbacks +%%-------------------------------------------------------------------- + +init([]) -> + TabOpts = [public, {write_concurrency, true}, {read_concurrency, true}], + ok = emqx_tables:new(?LWM2M_CHANNEL_TAB, [set, compressed | TabOpts]), + ok = emqx_tables:new(?LWM2M_CMD_TAB, [set, compressed | TabOpts]), + {ok, #{chan_pmon => emqx_pmon:new()}}. + +handle_call(Req, _From, State) -> + ?LOG(error, "Unexpected call: ~p", [Req]), + {reply, ignored, State}. + +handle_cast({registered, {IMEI, ChanPid}}, State = #{chan_pmon := PMon}) -> + PMon1 = emqx_pmon:monitor(ChanPid, IMEI, PMon), + {noreply, State#{chan_pmon := PMon1}}; + +handle_cast(Msg, State) -> + ?LOG(error, "Unexpected cast: ~p", [Msg]), + {noreply, State}. + +handle_info({'DOWN', _MRef, process, Pid, _Reason}, State = #{chan_pmon := PMon}) -> + ChanPids = [Pid | emqx_misc:drain_down(?BATCH_SIZE)], + {Items, PMon1} = emqx_pmon:erase_all(ChanPids, PMon), + ok = emqx_pool:async_submit(fun lists:foreach/2, [fun clean_down/1, Items]), + {noreply, State#{chan_pmon := PMon1}}; + +handle_info(Info, State) -> + ?LOG(error, "Unexpected info: ~p", [Info]), + {noreply, State}. + +terminate(_Reason, _State) -> + emqx_stats:cancel_update(chan_stats). + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- + +clean_down({_ChanPid, IMEI}) -> + unregister_channel(IMEI). diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_cm_sup.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_cm_sup.erl new file mode 100644 index 000000000..b55f4f33c --- /dev/null +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_cm_sup.erl @@ -0,0 +1,41 @@ + +%%-------------------------------------------------------------------- +%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_lwm2m_cm_sup). + +-behaviour(supervisor). + +-export([start_link/0]). + +-export([init/1]). + +start_link() -> + supervisor:start_link({local, ?MODULE}, ?MODULE, []). + +init([]) -> + CM = #{id => emqx_lwm2m_cm, + start => {emqx_lwm2m_cm, start_link, []}, + restart => permanent, + shutdown => 5000, + type => worker, + modules => [emqx_lwm2m_cm]}, + SupFlags = #{strategy => one_for_one, + intensity => 100, + period => 10 + }, + {ok, {SupFlags, [CM]}}. + diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_cmd_handler.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_cmd_handler.erl index 2d208bdb4..b3251a275 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m_cmd_handler.erl +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_cmd_handler.erl @@ -23,6 +23,7 @@ -export([ mqtt2coap/2 , coap2mqtt/4 , ack2mqtt/1 + , extract_path/1 ]). -export([path_list/1]). diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_coap_resource.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_coap_resource.erl index de83643a8..f94c2bc72 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m_coap_resource.erl +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_coap_resource.erl @@ -48,11 +48,11 @@ -define(LOG(Level, Format, Args), logger:Level("LWM2M-RESOURCE: " ++ Format, Args)). -dialyzer([{nowarn_function, [coap_discover/2]}]). -% we use {'absolute', string(), [{atom(), binary()}]} as coap_uri() +% we use {'absolute', list(binary()), [{atom(), binary()}]} as coap_uri() % https://github.com/emqx/lwm2m-coap/blob/258e9bd3762124395e83c1e68a1583b84718230f/src/lwm2m_coap_resource.erl#L61 % resource operations coap_discover(_Prefix, _Args) -> - [{absolute, "mqtt", []}]. + [{absolute, [<<"mqtt">>], []}]. coap_get(ChId, [?PREFIX], Query, Content, Lwm2mState) -> ?LOG(debug, "~p ~p GET Query=~p, Content=~p", [self(),ChId, Query, Content]), diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_message.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_message.erl index 2dc9fa08a..6b8bc8d50 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m_message.erl +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_message.erl @@ -197,7 +197,10 @@ value_ex(K, Value) when K =:= <<"Integer">>; K =:= <<"Float">>; K =:= <<"Time">> value_ex(K, Value) when K =:= <<"String">> -> Value; value_ex(K, Value) when K =:= <<"Opaque">> -> - Value; + %% XXX: force to decode it with base64 + %% This may not be a good implementation, but it is + %% consistent with the treatment of Opaque in value/3 + base64:decode(Value); value_ex(K, <<"true">>) when K =:= <<"Boolean">> -> <<1>>; value_ex(K, <<"false">>) when K =:= <<"Boolean">> -> <<0>>; diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_protocol.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_protocol.erl index 55f992da6..34c72dcca 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m_protocol.erl +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_protocol.erl @@ -103,6 +103,7 @@ init(CoapPid, EndpointName, Peername = {_Peerhost, _Port}, RegInfo = #{<<"lt">> emqx_cm:register_channel(EndpointName, CoapPid, conninfo(Lwm2mState1)) end), emqx_cm:insert_channel_info(EndpointName, info(Lwm2mState1), stats(Lwm2mState1)), + emqx_lwm2m_cm:register_channel(EndpointName, RegInfo, LifeTime, Ver, Peername), {ok, Lwm2mState1#lwm2m_state{life_timer = emqx_lwm2m_timer:start_timer(LifeTime, {life_timer, expired})}}; {error, Error} -> @@ -120,10 +121,8 @@ post_init(Lwm2mState = #lwm2m_state{endpoint_name = _EndpointName, _ = send_to_broker(<<"register">>, #{<<"data">> => RegInfo}, Lwm2mState), Lwm2mState#lwm2m_state{mqtt_topic = Topic}. -update_reg_info(NewRegInfo, Lwm2mState = #lwm2m_state{ - life_timer = LifeTimer, register_info = RegInfo, - coap_pid = CoapPid}) -> - +update_reg_info(NewRegInfo, Lwm2mState=#lwm2m_state{life_timer = LifeTimer, register_info = RegInfo, + coap_pid = CoapPid, endpoint_name = Epn}) -> UpdatedRegInfo = maps:merge(RegInfo, NewRegInfo), _ = case proplists:get_value(update_msg_publish_condition, @@ -134,6 +133,7 @@ update_reg_info(NewRegInfo, Lwm2mState = #lwm2m_state{ %% - report the registration info update, but only when objectList is updated. case NewRegInfo of #{<<"objectList">> := _} -> + emqx_lwm2m_cm:update_reg_info(Epn, NewRegInfo), send_to_broker(<<"update">>, #{<<"data">> => UpdatedRegInfo}, Lwm2mState); _ -> ok end @@ -151,7 +151,8 @@ update_reg_info(NewRegInfo, Lwm2mState = #lwm2m_state{ register_info = UpdatedRegInfo}. replace_reg_info(NewRegInfo, Lwm2mState=#lwm2m_state{life_timer = LifeTimer, - coap_pid = CoapPid}) -> + coap_pid = CoapPid, + endpoint_name = EndpointName}) -> _ = send_to_broker(<<"register">>, #{<<"data">> => NewRegInfo}, Lwm2mState), %% - flush cached donwlink commands @@ -161,7 +162,7 @@ replace_reg_info(NewRegInfo, Lwm2mState=#lwm2m_state{life_timer = LifeTimer, UpdatedLifeTimer = emqx_lwm2m_timer:refresh_timer( maps:get(<<"lt">>, NewRegInfo), LifeTimer), - _ = send_auto_observe(CoapPid, NewRegInfo), + _ = send_auto_observe(CoapPid, NewRegInfo, EndpointName), ?LOG(debug, "Replace RegInfo to: ~p", [NewRegInfo]), Lwm2mState#lwm2m_state{life_timer = UpdatedLifeTimer, @@ -174,15 +175,20 @@ send_ul_data(EventType, Payload, Lwm2mState=#lwm2m_state{coap_pid = CoapPid}) -> Lwm2mState. auto_observe(Lwm2mState = #lwm2m_state{register_info = RegInfo, - coap_pid = CoapPid}) -> - _ = send_auto_observe(CoapPid, RegInfo), + coap_pid = CoapPid, + endpoint_name = EndpointName}) -> + _ = send_auto_observe(CoapPid, RegInfo, EndpointName), Lwm2mState. -deliver(#message{topic = Topic, payload = Payload}, Lwm2mState = #lwm2m_state{coap_pid = CoapPid, register_info = RegInfo, started_at = StartedAt}) -> +deliver(#message{topic = Topic, payload = Payload}, + Lwm2mState = #lwm2m_state{coap_pid = CoapPid, + register_info = RegInfo, + started_at = StartedAt, + endpoint_name = EndpointName}) -> IsCacheMode = is_cache_mode(RegInfo, StartedAt), ?LOG(debug, "Get MQTT message from broker, IsCacheModeNow?: ~p, Topic: ~p, Payload: ~p", [IsCacheMode, Topic, Payload]), AlternatePath = maps:get(<<"alternatePath">>, RegInfo, <<"/">>), - deliver_to_coap(AlternatePath, Payload, CoapPid, IsCacheMode), + deliver_to_coap(AlternatePath, Payload, CoapPid, IsCacheMode, EndpointName), Lwm2mState. get_info(Lwm2mState = #lwm2m_state{endpoint_name = EndpointName, peername = {PeerHost, _}, @@ -238,20 +244,21 @@ time_now() -> erlang:system_time(millisecond). %% Deliver downlink message to coap %%-------------------------------------------------------------------- -deliver_to_coap(AlternatePath, JsonData, CoapPid, CacheMode) when is_binary(JsonData)-> +deliver_to_coap(AlternatePath, JsonData, CoapPid, CacheMode, EndpointName) when is_binary(JsonData)-> try TermData = emqx_json:decode(JsonData, [return_maps]), - deliver_to_coap(AlternatePath, TermData, CoapPid, CacheMode) + deliver_to_coap(AlternatePath, TermData, CoapPid, CacheMode, EndpointName) catch C:R:Stack -> ?LOG(error, "deliver_to_coap - Invalid JSON: ~p, Exception: ~p, stacktrace: ~p", [JsonData, {C, R}, Stack]) end; -deliver_to_coap(AlternatePath, TermData, CoapPid, CacheMode) when is_map(TermData) -> +deliver_to_coap(AlternatePath, TermData, CoapPid, CacheMode, EndpointName) when is_map(TermData) -> ?LOG(info, "SEND To CoAP, AlternatePath=~p, Data=~p", [AlternatePath, TermData]), {CoapRequest, Ref} = emqx_lwm2m_cmd_handler:mqtt2coap(AlternatePath, TermData), - + MsgType = maps:get(<<"msgType">>, Ref), + emqx_lwm2m_cm:register_cmd(EndpointName, emqx_lwm2m_cmd_handler:extract_path(Ref), MsgType), case CacheMode of false -> do_deliver_to_coap(CoapPid, CoapRequest, Ref); @@ -266,7 +273,12 @@ deliver_to_coap(AlternatePath, TermData, CoapPid, CacheMode) when is_map(TermDat send_to_broker(EventType, Payload = #{}, Lwm2mState) -> do_send_to_broker(EventType, Payload, Lwm2mState). -do_send_to_broker(EventType, Payload, Lwm2mState) -> +do_send_to_broker(EventType, #{<<"data">> := Data} = Payload, #lwm2m_state{endpoint_name = EndpointName} = Lwm2mState) -> + ReqPath = maps:get(<<"reqPath">>, Data, undefined), + Code = maps:get(<<"code">>, Data, undefined), + CodeMsg = maps:get(<<"codeMsg">>, Data, undefined), + Content = maps:get(<<"content">>, Data, undefined), + emqx_lwm2m_cm:register_cmd(EndpointName, ReqPath, EventType, {Code, CodeMsg, Content}), NewPayload = maps:put(<<"msgType">>, EventType, Payload), Topic = uplink_topic(EventType, Lwm2mState), publish(Topic, emqx_json:encode(NewPayload), _Qos = 0, Lwm2mState#lwm2m_state.endpoint_name). @@ -281,7 +293,7 @@ auto_observe_object_list(Expected, Registered) -> Expected1 = lists:map(fun(S) -> iolist_to_binary(S) end, Expected), lists:filter(fun(S) -> lists:member(S, Expected1) end, Registered). -send_auto_observe(CoapPid, RegInfo) -> +send_auto_observe(CoapPid, RegInfo, EndpointName) -> %% - auto observe the objects case proplists:get_value(auto_observe, lwm2m_coap_responder:options(), false) of false -> @@ -292,25 +304,37 @@ send_auto_observe(CoapPid, RegInfo) -> maps:get(<<"objectList">>, RegInfo, []) ), AlternatePath = maps:get(<<"alternatePath">>, RegInfo, <<"/">>), - auto_observe(AlternatePath, Objectlists, CoapPid) + auto_observe(AlternatePath, Objectlists, CoapPid, EndpointName) end. -auto_observe(AlternatePath, ObjectList, CoapPid) -> +auto_observe(AlternatePath, ObjectList, CoapPid, EndpointName) -> ?LOG(info, "Auto Observe on: ~p", [ObjectList]), erlang:spawn(fun() -> - observe_object_list(AlternatePath, ObjectList, CoapPid) + observe_object_list(AlternatePath, ObjectList, CoapPid, EndpointName) end). -observe_object_list(AlternatePath, ObjectList, CoapPid) -> +observe_object_list(AlternatePath, ObjectList, CoapPid, EndpointName) -> lists:foreach(fun(ObjectPath) -> - observe_object_slowly(AlternatePath, ObjectPath, CoapPid, 100) + [ObjId| LastPath] = emqx_lwm2m_cmd_handler:path_list(ObjectPath), + case ObjId of + <<"19">> -> + [ObjInsId | _LastPath1] = LastPath, + case ObjInsId of + <<"0">> -> + observe_object_slowly(AlternatePath, <<"/19/0/0">>, CoapPid, 100, EndpointName); + _ -> + observe_object_slowly(AlternatePath, ObjectPath, CoapPid, 100, EndpointName) + end; + _ -> + observe_object_slowly(AlternatePath, ObjectPath, CoapPid, 100, EndpointName) + end end, ObjectList). -observe_object_slowly(AlternatePath, ObjectPath, CoapPid, Interval) -> - observe_object(AlternatePath, ObjectPath, CoapPid), +observe_object_slowly(AlternatePath, ObjectPath, CoapPid, Interval, EndpointName) -> + observe_object(AlternatePath, ObjectPath, CoapPid, EndpointName), timer:sleep(Interval). -observe_object(AlternatePath, ObjectPath, CoapPid) -> +observe_object(AlternatePath, ObjectPath, CoapPid, EndpointName) -> Payload = #{ <<"msgType">> => <<"observe">>, <<"data">> => #{ @@ -318,7 +342,7 @@ observe_object(AlternatePath, ObjectPath, CoapPid) -> } }, ?LOG(info, "Observe ObjectPath: ~p", [ObjectPath]), - deliver_to_coap(AlternatePath, Payload, CoapPid, false). + deliver_to_coap(AlternatePath, Payload, CoapPid, false, EndpointName). do_deliver_to_coap_slowly(CoapPid, CoapRequestList, Interval) -> erlang:spawn(fun() -> diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_sup.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_sup.erl index d28e00874..0546e0080 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m_sup.erl +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_sup.erl @@ -29,4 +29,11 @@ start_link() -> supervisor:start_link({local, ?MODULE}, ?MODULE, []). init(_Args) -> - {ok, { {one_for_all, 10, 3600}, [?CHILD(emqx_lwm2m_xml_object_db)] }}. + CmSup = #{id => emqx_lwm2m_cm_sup, + start => {emqx_lwm2m_cm_sup, start_link, []}, + restart => permanent, + shutdown => infinity, + type => supervisor, + modules => [emqx_lwm2m_cm_sup] + }, + {ok, { {one_for_all, 10, 3600}, [?CHILD(emqx_lwm2m_xml_object_db), CmSup] }}. diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_xml_object.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_xml_object.erl index 5931ae75e..dd9911407 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m_xml_object.erl +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_xml_object.erl @@ -21,9 +21,11 @@ -export([ get_obj_def/2 , get_object_id/1 + , get_object_name/1 , get_object_and_resource_id/2 , get_resource_type/2 , get_resource_name/2 + , get_resource_operations/2 ]). -define(LOG(Level, Format, Args), @@ -42,6 +44,10 @@ get_object_id(ObjDefinition) -> [#xmlText{value=ObjectId}] = xmerl_xpath:string("ObjectID/text()", ObjDefinition), ObjectId. +get_object_name(ObjDefinition) -> + [#xmlText{value=ObjectName}] = xmerl_xpath:string("Name/text()", ObjDefinition), + ObjectName. + get_object_and_resource_id(ResourceNameBinary, ObjDefinition) -> ResourceNameString = binary_to_list(ResourceNameBinary), @@ -60,3 +66,8 @@ get_resource_name(ResourceIdInt, ObjDefinition) -> ResourceIdString = integer_to_list(ResourceIdInt), [#xmlText{value=Name}] = xmerl_xpath:string("Resources/Item[@ID=\""++ResourceIdString++"\"]/Name/text()", ObjDefinition), Name. + +get_resource_operations(ResourceIdInt, ObjDefinition) -> + ResourceIdString = integer_to_list(ResourceIdInt), + [#xmlText{value=Operations}] = xmerl_xpath:string("Resources/Item[@ID=\""++ResourceIdString++"\"]/Operations/text()", ObjDefinition), + Operations. diff --git a/apps/emqx_lwm2m/src/emqx_lwm2m_xml_object_db.erl b/apps/emqx_lwm2m/src/emqx_lwm2m_xml_object_db.erl index b974c148c..012d0a649 100644 --- a/apps/emqx_lwm2m/src/emqx_lwm2m_xml_object_db.erl +++ b/apps/emqx_lwm2m/src/emqx_lwm2m_xml_object_db.erl @@ -58,7 +58,7 @@ find_objectid(ObjectId) -> false -> ObjectId end, case ets:lookup(?LWM2M_OBJECT_DEF_TAB, ObjectIdInt) of - [] -> error(no_xml_definition); + [] -> {error, no_xml_definition}; [{ObjectId, Xml}] -> Xml end. @@ -121,8 +121,10 @@ load(BaseDir) -> true -> BaseDir++"*.xml"; false -> BaseDir++"/*.xml" end, - AllXmlFiles = filelib:wildcard(Wild), - load_loop(AllXmlFiles). + case filelib:wildcard(Wild) of + [] -> error(no_xml_files_found, BaseDir); + AllXmlFiles -> load_loop(AllXmlFiles) + end. load_loop([]) -> ok; diff --git a/apps/emqx_lwm2m/test/emqx_lwm2m_SUITE.erl b/apps/emqx_lwm2m/test/emqx_lwm2m_SUITE.erl index 257502f2f..7f2f37466 100644 --- a/apps/emqx_lwm2m/test/emqx_lwm2m_SUITE.erl +++ b/apps/emqx_lwm2m/test/emqx_lwm2m_SUITE.erl @@ -40,6 +40,7 @@ all() -> , {group, test_grp_4_discover} , {group, test_grp_5_write_attr} , {group, test_grp_6_observe} + , {group, test_grp_8_object_19} ]. suite() -> [{timetrap, {seconds, 90}}]. @@ -98,9 +99,9 @@ groups() -> ]}, {test_grp_8_object_19, [RepeatOpt], [ case80_specail_object_19_1_0_write, - case80_specail_object_19_0_0_notify, - case80_specail_object_19_0_0_response, - case80_normal_object_19_0_0_read + case80_specail_object_19_0_0_notify + %case80_specail_object_19_0_0_response, + %case80_normal_object_19_0_0_read ]}, {test_grp_9_psm_queue_mode, [RepeatOpt], [ case90_psm_mode, @@ -1655,6 +1656,7 @@ case80_specail_object_19_1_0_write(Config) -> <<"value">> => base64:encode(<<12345:32>>) } }, + CommandJson = emqx_json:encode(Command), test_mqtt_broker:publish(CommandTopic, CommandJson, 0), timer:sleep(50), @@ -1663,7 +1665,7 @@ case80_specail_object_19_1_0_write(Config) -> Path2 = get_coap_path(Options2), ?assertEqual(put, Method2), ?assertEqual(<<"/19/1/0">>, Path2), - ?assertEqual(<<12345:32>>, Payload2), + ?assertEqual(<<3:2, 0:1, 0:2, 4:3, 0, 12345:32>>, Payload2), timer:sleep(50), test_send_coap_response(UdpSock, "127.0.0.1", ?PORT, {ok, changed}, #coap_content{}, Request2, true), @@ -1672,6 +1674,7 @@ case80_specail_object_19_1_0_write(Config) -> ReadResult = emqx_json:encode(#{ <<"requestID">> => CmdId, <<"cacheID">> => CmdId, <<"data">> => #{ + <<"reqPath">> => <<"/19/1/0">>, <<"code">> => <<"2.04">>, <<"codeMsg">> => <<"changed">> }, diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src index 83b2d7632..aebb73150 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src @@ -1,6 +1,6 @@ {application, emqx_rule_engine, [{description, "EMQ X Rule Engine"}, - {vsn, "4.3.2"}, % strict semver, bump manually! + {vsn, "4.3.3"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_rule_engine_sup, emqx_rule_registry]}, {applications, [kernel,stdlib,rulesql,getopt]}, diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src index 446e082b7..01c07c124 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.appup.src @@ -1,21 +1,37 @@ %% -*-: erlang -*- -{"4.3.2", +{"4.3.3", [ {"4.3.0", - [ {load_module, emqx_rule_funcs, brutal_purge, soft_purge, []}, - {load_module, emqx_rule_engine, brutal_purge, soft_purge, []} + [ {load_module, emqx_rule_funcs, brutal_purge, soft_purge, []} + , {load_module, emqx_rule_engine, brutal_purge, soft_purge, []} + , {load_module, emqx_rule_registry, brutal_purge, soft_purge, []} + , {apply, {emqx_stats, cancel_update, [rule_registery_stats]}} ]}, {"4.3.1", [ {load_module, emqx_rule_engine, brutal_purge, soft_purge, []} + , {load_module, emqx_rule_registry, brutal_purge, soft_purge, []} + , {apply, {emqx_stats, cancel_update, [rule_registery_stats]}} + ]}, + {"4.3.2", + [ {load_module, emqx_rule_registry, brutal_purge, soft_purge, []} + , {apply, {emqx_stats, cancel_update, [rule_registery_stats]}} ]}, {<<".*">>, []} ], [ {"4.3.0", - [ {load_module, emqx_rule_funcs, brutal_purge, soft_purge, []}, - {load_module, emqx_rule_engine, brutal_purge, soft_purge, []} + [ {load_module, emqx_rule_funcs, brutal_purge, soft_purge, []} + , {load_module, emqx_rule_engine, brutal_purge, soft_purge, []} + , {load_module, emqx_rule_registry, brutal_purge, soft_purge, []} + , {apply, {emqx_stats, cancel_update, [rule_registery_stats]}} ]}, {"4.3.1", [ {load_module, emqx_rule_engine, brutal_purge, soft_purge, []} + , {load_module, emqx_rule_registry, brutal_purge, soft_purge, []} + , {apply, {emqx_stats, cancel_update, [rule_registery_stats]}} + ]}, + {"4.3.2", + [ {load_module, emqx_rule_registry, brutal_purge, soft_purge, []} + , {apply, {emqx_stats, cancel_update, [rule_registery_stats]}} ]}, {<<".*">>, []} ] diff --git a/apps/emqx_rule_engine/src/emqx_rule_registry.erl b/apps/emqx_rule_engine/src/emqx_rule_registry.erl index 4ec5d1cb7..2d029f8e3 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_registry.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_registry.erl @@ -93,13 +93,6 @@ -define(REGISTRY, ?MODULE). -%% Statistics --define(STATS, - [ {?RULE_TAB, 'rules.count', 'rules.max'} - , {?ACTION_TAB, 'actions.count', 'actions.max'} - , {?RES_TAB, 'resources.count', 'resources.max'} - ]). - -define(T_CALL, 10000). %%------------------------------------------------------------------------------ @@ -392,8 +385,11 @@ find_rules_depends_on_resource(ResId) -> end, [], get_rules()). search_action_despends_on_resource(ResId, Actions) -> - lists:search(fun(#action_instance{args = #{<<"$resource">> := ResId0}}) -> - ResId0 =:= ResId + lists:search(fun + (#action_instance{args = #{<<"$resource">> := ResId0}}) -> + ResId0 =:= ResId; + (_) -> + false end, Actions). %%------------------------------------------------------------------------------ @@ -439,8 +435,6 @@ delete_resource_type(Type) -> %%------------------------------------------------------------------------------ init([]) -> - %% Enable stats timer - ok = emqx_stats:update_interval(rule_registery_stats, fun update_stats/0), _TableId = ets:new(?KV_TAB, [named_table, set, public, {write_concurrency, true}, {read_concurrency, true}]), {ok, #{}}. @@ -466,7 +460,7 @@ handle_info(Info, State) -> {noreply, State}. terminate(_Reason, _State) -> - emqx_stats:cancel_update(rule_registery_stats). + ok. code_change(_OldVsn, State, _Extra) -> {ok, State}. @@ -475,13 +469,6 @@ code_change(_OldVsn, State, _Extra) -> %% Private functions %%------------------------------------------------------------------------------ -update_stats() -> - lists:foreach( - fun({Tab, Stat, MaxStat}) -> - Size = mnesia:table_info(Tab, size), - emqx_stats:setstat(Stat, MaxStat, Size) - end, ?STATS). - get_all_records(Tab) -> %mnesia:dirty_match_object(Tab, mnesia:table_info(Tab, wild_pattern)). ets:tab2list(Tab). diff --git a/apps/emqx_sn/src/emqx_sn.appup.src b/apps/emqx_sn/src/emqx_sn.appup.src index a6a1a6c84..2bd6f5646 100644 --- a/apps/emqx_sn/src/emqx_sn.appup.src +++ b/apps/emqx_sn/src/emqx_sn.appup.src @@ -1,11 +1,17 @@ %% -*-: erlang -*- {VSN, [ + {"4.3.2", [ + {load_module, emqx_sn_gateway, brutal_purge, soft_purge, []} + ]}, {<<"4.3.[0-1]">>, [ {restart_application, emqx_sn} ]} ], [ + {"4.3.2", [ + {load_module, emqx_sn_gateway, brutal_purge, soft_purge, []} + ]}, {<<"4.3.[0-1]">>, [ {restart_application, emqx_sn} ]} diff --git a/apps/emqx_sn/src/emqx_sn_gateway.erl b/apps/emqx_sn/src/emqx_sn_gateway.erl index 2339961cf..1bccf0c1a 100644 --- a/apps/emqx_sn/src/emqx_sn_gateway.erl +++ b/apps/emqx_sn/src/emqx_sn_gateway.erl @@ -250,8 +250,9 @@ wait_for_will_topic(cast, {incoming, ?SN_ADVERTISE_MSG(_GwId, _Radius)}, _State) % ignore keep_state_and_data; -wait_for_will_topic(cast, {incoming, ?SN_CONNECT_MSG(Flags, _ProtoId, Duration, ClientId)}, State) -> - do_2nd_connect(Flags, Duration, ClientId, State); +wait_for_will_topic(cast, {incoming, ?SN_CONNECT_MSG(_Flags, _ProtoId, _Duration, _ClientId)}, _State) -> + ?LOG(warning, "Receive connect packet in wait_for_will_topic state", []), + keep_state_and_data; wait_for_will_topic(cast, {outgoing, Packet}, State) -> {keep_state, handle_outgoing(Packet, State)}; @@ -275,9 +276,9 @@ wait_for_will_msg(cast, {incoming, ?SN_ADVERTISE_MSG(_GwId, _Radius)}, _State) - % ignore keep_state_and_data; -%% XXX: ?? Why we will handling the 2nd CONNECT packet ?? -wait_for_will_msg(cast, {incoming, ?SN_CONNECT_MSG(Flags, _ProtoId, Duration, ClientId)}, State) -> - do_2nd_connect(Flags, Duration, ClientId, State); +wait_for_will_msg(cast, {incoming, ?SN_CONNECT_MSG(_Flags, _ProtoId, _Duration, _ClientId)}, _State) -> + ?LOG(warning, "Receive connect packet in wait_for_will_msg state", []), + keep_state_and_data; wait_for_will_msg(cast, {outgoing, Packet}, State) -> {keep_state, handle_outgoing(Packet, State)}; @@ -365,8 +366,9 @@ connected(cast, {incoming, ?SN_ADVERTISE_MSG(_GwId, _Radius)}, State) -> % ignore {keep_state, State}; -connected(cast, {incoming, ?SN_CONNECT_MSG(Flags, _ProtoId, Duration, ClientId)}, State) -> - do_2nd_connect(Flags, Duration, ClientId, State); +connected(cast, {incoming, ?SN_CONNECT_MSG(_Flags, _ProtoId, _Duration, _ClientId)}, _State) -> + ?LOG(warning, "Receive connect packet in wait_for_will_topic state", []), + keep_state_and_data; connected(cast, {outgoing, Packet}, State) -> {keep_state, handle_outgoing(Packet, State)}; @@ -826,15 +828,17 @@ do_connect(ClientId, CleanStart, WillFlag, Duration, State) -> clean_start = CleanStart, username = State#state.username, password = State#state.password, + proto_name = <<"MQTT-SN">>, keepalive = Duration, - properties = OnlyOneInflight + properties = OnlyOneInflight, + proto_ver = 1 }, case WillFlag of true -> State0 = send_message(?SN_WILLTOPICREQ_MSG(), State), NState = State0#state{connpkt = ConnPkt, - clientid = ClientId, - keepalive_interval = Duration - }, + clientid = ClientId, + keepalive_interval = Duration + }, {next_state, wait_for_will_topic, NState}; false -> NState = State#state{clientid = ClientId, @@ -843,26 +847,6 @@ do_connect(ClientId, CleanStart, WillFlag, Duration, State) -> handle_incoming(?CONNECT_PACKET(ConnPkt), NState) end. -do_2nd_connect(Flags, Duration, ClientId, State = #state{sockname = Sockname, - peername = Peername, - channel = Channel}) -> - emqx_logger:set_metadata_clientid(ClientId), - #mqtt_sn_flags{will = Will, clean_start = CleanStart} = Flags, - NChannel = case CleanStart of - true -> - emqx_channel:terminate(normal, Channel), - emqx_sn_registry:unregister_topic(ClientId), - emqx_channel:init(#{socktype => udp, - sockname => Sockname, - peername => Peername, - peercert => ?NO_PEERCERT, - conn_mod => ?MODULE - }, ?DEFAULT_CHAN_OPTIONS); - false -> Channel - end, - NState = State#state{channel = NChannel}, - do_connect(ClientId, CleanStart, Will, Duration, NState). - handle_subscribe(?SN_NORMAL_TOPIC, TopicName, QoS, MsgId, State=#state{channel = Channel}) -> ClientId = emqx_channel:info(clientid, Channel), diff --git a/apps/emqx_sn/test/emqx_sn_protocol_SUITE.erl b/apps/emqx_sn/test/emqx_sn_protocol_SUITE.erl index a953a45e7..0947bdaca 100644 --- a/apps/emqx_sn/test/emqx_sn_protocol_SUITE.erl +++ b/apps/emqx_sn/test/emqx_sn_protocol_SUITE.erl @@ -98,19 +98,6 @@ t_connect(_) -> ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), gen_udp:close(Socket). -t_do_2nd_connect(_) -> - {ok, Socket} = gen_udp:open(0, [binary]), - ClientId = ?CLIENTID, - send_connect_msg(Socket, ClientId), - ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), - timer:sleep(100), - send_connect_msg(Socket, <<"client_id_other">>), - ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)), - - send_disconnect_msg(Socket, undefined), - ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)), - gen_udp:close(Socket). - t_subscribe(_) -> Dup = 0, QoS = 0, diff --git a/apps/emqx_web_hook/src/emqx_web_hook.app.src b/apps/emqx_web_hook/src/emqx_web_hook.app.src index 97624a900..e1cfda173 100644 --- a/apps/emqx_web_hook/src/emqx_web_hook.app.src +++ b/apps/emqx_web_hook/src/emqx_web_hook.app.src @@ -1,6 +1,6 @@ {application, emqx_web_hook, [{description, "EMQ X WebHook Plugin"}, - {vsn, "4.3.1"}, % strict semver, bump manually! + {vsn, "4.3.2"}, % strict semver, bump manually! {modules, []}, {registered, [emqx_web_hook_sup]}, {applications, [kernel,stdlib,ehttpc]}, diff --git a/apps/emqx_web_hook/src/emqx_web_hook.appup.src b/apps/emqx_web_hook/src/emqx_web_hook.appup.src index b92284b64..ae6e9e1ae 100644 --- a/apps/emqx_web_hook/src/emqx_web_hook.appup.src +++ b/apps/emqx_web_hook/src/emqx_web_hook.appup.src @@ -2,14 +2,16 @@ {VSN, [ - {"4.3.0", [ - {load_module, emqx_web_hook_actions, brutal_purge, soft_purge, []} - ]}, + {<<"4.3.[0-1]">>, [ + {restart_application, emqx_web_hook}, + {apply,{emqx_rule_engine,refresh_resource,[web_hook]}} + ]}, {<<".*">>, []} ], [ - {"4.3.0", [ - {load_module, emqx_web_hook_actions, brutal_purge, soft_purge, []} + {<<"4.3.[0-1]">>, [ + {restart_application, emqx_web_hook}, + {apply,{emqx_rule_engine,refresh_resource,[web_hook]}} ]}, {<<".*">>, []} ] diff --git a/apps/emqx_web_hook/src/emqx_web_hook_actions.erl b/apps/emqx_web_hook/src/emqx_web_hook_actions.erl index f8e4d8f08..8bb1c7de0 100644 --- a/apps/emqx_web_hook/src/emqx_web_hook_actions.erl +++ b/apps/emqx_web_hook/src/emqx_web_hook_actions.erl @@ -292,7 +292,7 @@ parse_action_params(Params = #{<<"url">> := URL}) -> Headers = headers(maps:get(<<"headers">>, Params, undefined)), NHeaders = ensure_content_type_header(Headers, Method), #{method => Method, - path => path(filename:join(CommonPath, maps:get(<<"path">>, Params, <<>>))), + path => merge_path(CommonPath, maps:get(<<"path">>, Params, <<>>)), headers => NHeaders, body => maps:get(<<"body">>, Params, <<>>), request_timeout => cuttlefish_duration:parse(str(maps:get(<<"request_timeout">>, Params, <<"5s">>))), @@ -306,8 +306,16 @@ ensure_content_type_header(Headers, Method) when Method =:= post orelse Method = ensure_content_type_header(Headers, _Method) -> lists:keydelete("content-type", 1, Headers). -path(<<>>) -> <<"/">>; -path(Path) -> Path. +merge_path(CommonPath, <<>>) -> + CommonPath; +merge_path(CommonPath, Path0) -> + case emqx_http_lib:uri_parse(Path0) of + {ok, #{path := Path1, 'query' := Query}} -> + Path2 = filename:join(CommonPath, Path1), + <>; + {ok, #{path := Path1}} -> + filename:join(CommonPath, Path1) + end. method(GET) when GET == <<"GET">>; GET == <<"get">> -> get; method(POST) when POST == <<"POST">>; POST == <<"post">> -> post; diff --git a/apps/emqx_web_hook/src/emqx_web_hook_app.erl b/apps/emqx_web_hook/src/emqx_web_hook_app.erl index 492af628d..580742c47 100644 --- a/apps/emqx_web_hook/src/emqx_web_hook_app.erl +++ b/apps/emqx_web_hook/src/emqx_web_hook_app.erl @@ -42,10 +42,9 @@ stop(_State) -> translate_env() -> {ok, URL} = application:get_env(?APP, url), {ok, #{host := Host, - path := Path0, port := Port, - scheme := Scheme}} = emqx_http_lib:uri_parse(URL), - Path = path(Path0), + scheme := Scheme} = URIMap} = emqx_http_lib:uri_parse(URL), + Path = path(URIMap), PoolSize = application:get_env(?APP, pool_size, 32), MoreOpts = case Scheme of http -> @@ -89,9 +88,13 @@ translate_env() -> NHeaders = set_content_type(Headers), application:set_env(?APP, headers, NHeaders). -path("") -> +path(#{path := "", 'query' := Query}) -> + "?" ++ Query; +path(#{path := Path, 'query' := Query}) -> + Path ++ "?" ++ Query; +path(#{path := ""}) -> "/"; -path(Path) -> +path(#{path := Path}) -> Path. set_content_type(Headers) -> diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile index 4e25a001d..9e0ba9d5b 100644 --- a/deploy/docker/Dockerfile +++ b/deploy/docker/Dockerfile @@ -26,7 +26,9 @@ COPY . /emqx ARG PKG_VSN ARG EMQX_NAME=emqx -RUN cd /emqx && make $EMQX_NAME +RUN cd /emqx \ + && rm -rf _build/$EMQX_NAME/lib \ + && make $EMQX_NAME FROM $RUN_FROM diff --git a/deploy/docker/README.md b/deploy/docker/README.md index 4c89a73cd..ac494b430 100644 --- a/deploy/docker/README.md +++ b/deploy/docker/README.md @@ -83,7 +83,7 @@ These environment variables will ignore for configuration file. The list is incomplete and may changed with [etc/emqx.conf](https://github.com/emqx/emqx/blob/master/etc/emqx.conf) and plugin configuration files. But the mapping rule is similar. -If set ``EMQX_NAME`` and ``EMQX_HOST``, and unset ``EMQX_NODE__NAME``, ``EMQX_NODE__NAME=$EMQX_NAME@$EMQX_HOST``. +If set ``EMQX_NAME`` and ``EMQX_HOST``, and unset ``EMQX_NODE_NAME``, ``EMQX_NODE_NAME=$EMQX_NAME@$EMQX_HOST``. For example, set mqtt tcp port to 1883 diff --git a/priv/emqx.schema b/priv/emqx.schema new file mode 100644 index 000000000..142a9acf4 --- /dev/null +++ b/priv/emqx.schema @@ -0,0 +1,2470 @@ +%%-*- mode: erlang -*- +%% EMQ X R4.0 config mapping + +%%-------------------------------------------------------------------- +%% Cluster +%%-------------------------------------------------------------------- + +%% @doc Cluster name +{mapping, "cluster.name", "ekka.cluster_name", [ + {default, emqxcl}, + {datatype, atom} +]}. + +%% @doc Cluster discovery +{mapping, "cluster.discovery", "ekka.cluster_discovery", [ + {default, manual}, + {datatype, atom} +]}. + +%% @doc Clean down node from the cluster +{mapping, "cluster.autoclean", "ekka.cluster_autoclean", [ + {datatype, {duration, ms}} +]}. + +%% @doc Cluster autoheal +{mapping, "cluster.autoheal", "ekka.cluster_autoheal", [ + {datatype, flag}, + {default, off} +]}. + +%%-------------------------------------------------------------------- +%% Cluster by static node list + +{mapping, "cluster.static.seeds", "ekka.cluster_discovery", [ + {datatype, string} +]}. + +%%-------------------------------------------------------------------- +%% Cluster by UDP Multicast + +{mapping, "cluster.mcast.addr", "ekka.cluster_discovery", [ + {default, "239.192.0.1"}, + {datatype, string} +]}. + +{mapping, "cluster.mcast.ports", "ekka.cluster_discovery", [ + {default, "4369"}, + {datatype, string} +]}. + +{mapping, "cluster.mcast.iface", "ekka.cluster_discovery", [ + {datatype, string}, + {default, "0.0.0.0"} +]}. + +{mapping, "cluster.mcast.ttl", "ekka.cluster_discovery", [ + {datatype, integer}, + {default, 255} +]}. + +{mapping, "cluster.mcast.loop", "ekka.cluster_discovery", [ + {datatype, flag}, + {default, on} +]}. + +{mapping, "cluster.mcast.sndbuf", "ekka.cluster_discovery", [ + {datatype, bytesize}, + {default, "16KB"} +]}. + +{mapping, "cluster.mcast.recbuf", "ekka.cluster_discovery", [ + {datatype, bytesize}, + {default, "16KB"} +]}. + +{mapping, "cluster.mcast.buffer", "ekka.cluster_discovery", [ + {datatype, bytesize}, + {default, "32KB"} +]}. + +%%-------------------------------------------------------------------- +%% Cluster by DNS A Record + +{mapping, "cluster.dns.name", "ekka.cluster_discovery", [ + {datatype, string} +]}. + +%% @doc The erlang distributed protocol +{mapping, "cluster.proto_dist", "ekka.proto_dist", [ + {default, "inet_tcp"}, + {datatype, {enum, [inet_tcp, inet6_tcp, inet_tls]}}, + hidden +]}. + +{mapping, "cluster.dns.app", "ekka.cluster_discovery", [ + {datatype, string} +]}. + +%%-------------------------------------------------------------------- +%% Cluster using etcd + +{mapping, "cluster.etcd.server", "ekka.cluster_discovery", [ + {datatype, string} +]}. + +{mapping, "cluster.etcd.prefix", "ekka.cluster_discovery", [ + {datatype, string} +]}. + +{mapping, "cluster.etcd.node_ttl", "ekka.cluster_discovery", [ + {datatype, {duration, ms}}, + {default, "1m"} +]}. + +{mapping, "cluster.etcd.ssl.keyfile", "ekka.cluster_discovery", [ + {datatype, string} +]}. + +{mapping, "cluster.etcd.ssl.certfile", "ekka.cluster_discovery", [ + {datatype, string} +]}. + +{mapping, "cluster.etcd.ssl.cacertfile", "ekka.cluster_discovery", [ + {datatype, string} +]}. + +%%-------------------------------------------------------------------- +%% Cluster on K8s + +{mapping, "cluster.k8s.apiserver", "ekka.cluster_discovery", [ + {datatype, string} +]}. + +{mapping, "cluster.k8s.service_name", "ekka.cluster_discovery", [ + {datatype, string} +]}. + +{mapping, "cluster.k8s.address_type", "ekka.cluster_discovery", [ + {datatype, {enum, [ip, dns, hostname]}} +]}. + +{mapping, "cluster.k8s.app_name", "ekka.cluster_discovery", [ + {datatype, string} +]}. + +{mapping, "cluster.k8s.namespace", "ekka.cluster_discovery", [ + {datatype, string} +]}. + +{mapping, "cluster.k8s.suffix", "ekka.cluster_discovery", [ + {datatype, string}, + {default, ""} + ]}. + +{translation, "ekka.cluster_discovery", fun(Conf) -> + Strategy = cuttlefish:conf_get("cluster.discovery", Conf), + Filter = fun(Opts) -> [{K, V} || {K, V} <- Opts, V =/= undefined] end, + IpPort = fun(S) -> + [Addr, Port] = string:tokens(S, ":"), + {ok, Ip} = inet:parse_address(Addr), + {Ip, Port} + end, + Options = fun(static) -> + [{seeds, [list_to_atom(S) || S <- string:tokens(cuttlefish:conf_get("cluster.static.seeds", Conf, ""), ",")]}]; + (mcast) -> + {ok, Addr} = inet:parse_address(cuttlefish:conf_get("cluster.mcast.addr", Conf)), + {ok, Iface} = inet:parse_address(cuttlefish:conf_get("cluster.mcast.iface", Conf)), + Ports = [list_to_integer(S) || S <- string:tokens(cuttlefish:conf_get("cluster.mcast.ports", Conf), ",")], + [{addr, Addr}, {ports, Ports}, {iface, Iface}, + {ttl, cuttlefish:conf_get("cluster.mcast.ttl", Conf, 1)}, + {loop, cuttlefish:conf_get("cluster.mcast.loop", Conf, true)}]; + (dns) -> + [{name, cuttlefish:conf_get("cluster.dns.name", Conf)}, + {app, cuttlefish:conf_get("cluster.dns.app", Conf)}]; + (etcd) -> + SslOpts = fun(Conf) -> + Options = cuttlefish_variable:filter_by_prefix("cluster.etcd.ssl", Conf), + lists:map(fun({["cluster", "etcd", "ssl", Name], Value}) -> + {list_to_atom(Name), Value} + end, Options) + end, + [{server, string:tokens(cuttlefish:conf_get("cluster.etcd.server", Conf), ",")}, + {prefix, cuttlefish:conf_get("cluster.etcd.prefix", Conf, "emqcl")}, + {node_ttl, cuttlefish:conf_get("cluster.etcd.node_ttl", Conf, 60)}, + {ssl_options, SslOpts(Conf)}]; + (k8s) -> + [{apiserver, cuttlefish:conf_get("cluster.k8s.apiserver", Conf)}, + {service_name, cuttlefish:conf_get("cluster.k8s.service_name", Conf)}, + {address_type, cuttlefish:conf_get("cluster.k8s.address_type", Conf, ip)}, + {app_name, cuttlefish:conf_get("cluster.k8s.app_name", Conf)}, + {namespace, cuttlefish:conf_get("cluster.k8s.namespace", Conf)}, + {suffix, cuttlefish:conf_get("cluster.k8s.suffix", Conf, "")}]; + (manual) -> + [ ] + end, + {Strategy, Filter(Options(Strategy))} +end}. + +%%-------------------------------------------------------------------- +%% Node +%%-------------------------------------------------------------------- + +%% @doc Node name +{mapping, "node.name", "vm_args.-name", [ + {default, "emqx@127.0.0.1"}, + {override_env, "NODE_NAME"} +]}. + +%% @doc Specify SSL Options in the file if using SSL for erlang distribution +{mapping, "node.ssl_dist_optfile", "vm_args.-ssl_dist_optfile", [ + {datatype, string}, + hidden +]}. + +%% @doc Secret cookie for distributed erlang node +{mapping, "node.cookie", "vm_args.-setcookie", [ + {default, "emqxsecretcookie"}, + {override_env, "NODE_COOKIE"} +]}. + +{mapping, "node.data_dir", "emqx.data_dir", [ + {datatype, string} +]}. + +%% @doc http://erlang.org/doc/man/heart.html +{mapping, "node.heartbeat", "vm_args.-heart", [ + {datatype, flag}, + hidden +]}. + +{translation, "vm_args.-heart", fun(Conf) -> + case cuttlefish:conf_get("node.heartbeat", Conf) of + true -> ""; + false -> cuttlefish:invalid("should be 'on' or comment the line!") + end +end}. + +%% @doc More information at: http://erlang.org/doc/man/erl.html +{mapping, "node.async_threads", "vm_args.+A", [ + {datatype, integer}, + {validators, ["range:0-1024"]} +]}. + +%% @doc Erlang Process Limit +{mapping, "node.process_limit", "vm_args.+P", [ + {datatype, integer}, + hidden +]}. + +%% @doc The maximum number of concurrent ports/sockets. +%% Valid range is 1024-134217727 +{mapping, "node.max_ports", "vm_args.+Q", [ + {datatype, integer}, + {validators, ["range4ports"]}, + {override_env, "MAX_PORTS"} +]}. + +{validator, "range4ports", "must be 1024 to 134217727", + fun(X) -> X >= 1024 andalso X =< 134217727 end}. + +%% @doc http://www.erlang.org/doc/man/erl.html#%2bzdbbl +{mapping, "node.dist_buffer_size", "vm_args.+zdbbl", [ + {datatype, bytesize}, + {commented, "32MB"}, + hidden, + {validators, ["zdbbl_range"]} +]}. + +{translation, "vm_args.+zdbbl", + fun(Conf) -> + ZDBBL = cuttlefish:conf_get("node.dist_buffer_size", Conf, undefined), + case ZDBBL of + undefined -> undefined; + X when is_integer(X) -> cuttlefish_util:ceiling(X / 1024); %% Bytes to Kilobytes; + _ -> undefined + end + end}. + +{validator, "zdbbl_range", "must be between 1KB and 2097151KB", + fun(ZDBBL) -> + %% 2097151KB = 2147482624 + ZDBBL >= 1024 andalso ZDBBL =< 2147482624 + end +}. + +%% @doc Global GC Interval +{mapping, "node.global_gc_interval", "emqx.global_gc_interval", [ + {datatype, {duration, s}} +]}. + +%% @doc http://www.erlang.org/doc/man/erlang.html#system_flag-2 +{mapping, "node.fullsweep_after", "vm_args.-env ERL_FULLSWEEP_AFTER", [ + {default, 1000}, + {datatype, integer}, + hidden, + {validators, ["positive_integer"]} +]}. + +{validator, "positive_integer", "must be a positive integer", + fun(X) -> X >= 0 end}. + +%% Note: OTP R15 and earlier uses -env ERL_MAX_ETS_TABLES, +%% R16+ uses +e +%% @doc The ETS table limit +{mapping, "node.max_ets_tables", + cuttlefish:otp("R16", "vm_args.+e", "vm_args.-env ERL_MAX_ETS_TABLES"), [ + {default, 256000}, + {datatype, integer}, + hidden +]}. + +%% @doc Set the location of crash dumps +{mapping, "node.crash_dump", "vm_args.-env ERL_CRASH_DUMP", [ + {default, "{{crash_dump}}"}, + {datatype, file}, + hidden +]}. + +%% @doc http://www.erlang.org/doc/man/kernel_app.html#net_ticktime +{mapping, "node.dist_net_ticktime", "vm_args.-kernel net_ticktime", [ + {datatype, integer}, + hidden +]}. + +%% @doc http://www.erlang.org/doc/man/kernel_app.html +{mapping, "node.dist_listen_min", "kernel.inet_dist_listen_min", [ + {commented, 6369}, + {datatype, integer}, + hidden +]}. + +%% @see node.dist_listen_min +{mapping, "node.dist_listen_max", "kernel.inet_dist_listen_max", [ + {commented, 6369}, + {datatype, integer}, + hidden +]}. + +{mapping, "node.backtrace_depth", "emqx.backtrace_depth", [ + {default, 16}, + {datatype, integer} +]}. + +%%-------------------------------------------------------------------- +%% RPC +%%-------------------------------------------------------------------- + +%% RPC Mode. +{mapping, "rpc.mode", "emqx.rpc_mode", [ + {default, async}, + {datatype, {enum, [sync, async]}} +]}. + +{mapping, "rpc.async_batch_size", "gen_rpc.max_batch_size", [ + {default, 256}, + {datatype, integer} +]}. + +{mapping, "rpc.port_discovery", "gen_rpc.port_discovery", [ + {default, stateless}, + {datatype, {enum, [manual, stateless]}} +]}. + +%% RPC server port. +{mapping, "rpc.tcp_server_port", "gen_rpc.tcp_server_port", [ + {default, 5369}, + {datatype, integer} +]}. + +%% Number of tcp connections when connecting to RPC server +{mapping, "rpc.tcp_client_num", "gen_rpc.tcp_client_num", [ + {default, 0}, + {datatype, integer}, + {validators, ["range:gt_0_lt_256"]} +]}. + +{translation, "gen_rpc.tcp_client_num", fun(Conf) -> + case cuttlefish:conf_get("rpc.tcp_client_num", Conf) of + 0 -> 1; %% keep allowing 0 for backward compatibility + V -> V + end +end}. + +%% Client connect timeout +{mapping, "rpc.connect_timeout", "gen_rpc.connect_timeout", [ + {default, "5s"}, + {datatype, {duration, ms}} +]}. + +%% Client and Server send timeout +{mapping, "rpc.send_timeout", "gen_rpc.send_timeout", [ + {default, 5000}, + {datatype, {duration, ms}} +]}. + +%% Authentication timeout +{mapping, "rpc.authentication_timeout", "gen_rpc.authentication_timeout", [ + {default, 5000}, + {datatype, {duration, ms}} +]}. + +%% Default receive timeout for call() functions +{mapping, "rpc.call_receive_timeout", "gen_rpc.call_receive_timeout", [ + {default, 15000}, + {datatype, {duration, ms}} +]}. + +%% Socket keepalive configuration +{mapping, "rpc.socket_keepalive_idle", "gen_rpc.socket_keepalive_idle", [ + {default, 7200}, + {datatype, {duration, s}} +]}. + +%% Seconds between probes +{mapping, "rpc.socket_keepalive_interval", "gen_rpc.socket_keepalive_interval", [ + {default, 75}, + {datatype, {duration, s}} +]}. + +%% Probes lost to close the connection +{mapping, "rpc.socket_keepalive_count", "gen_rpc.socket_keepalive_count", [ + {default, 9}, + {datatype, integer} +]}. + +%% Size of TCP send buffer +{mapping, "rpc.socket_sndbuf", "gen_rpc.socket_sndbuf", [ + {default, "1MB"}, + {datatype, bytesize} +]}. + +%% Size of TCP receive buffer +{mapping, "rpc.socket_recbuf", "gen_rpc.socket_recbuf", [ + {default, "1MB"}, + {datatype, bytesize} +]}. + +%% Size of TCP receive buffer +{mapping, "rpc.socket_buffer", "gen_rpc.socket_buffer", [ + {default, "1MB"}, + {datatype, bytesize} +]}. + +{validator, "range:gt_0_lt_256", "must greater than 0 and less than 256", + fun(X) -> X >= 0 andalso X < 256 end +}. + +%% Force client to use server listening port, because we do no provide +%% per-node listening port manual mapping from configs. +%% i.e. all nodes in the cluster should agree to the same +%% listening port number. +{translation, "gen_rpc.tcp_client_port", fun(_, _, Conf) -> + cuttlefish:conf_get("rpc.tcp_server_port", Conf) +end}. + +%%-------------------------------------------------------------------- +%% Log +%%-------------------------------------------------------------------- + +{mapping, "log.to", "kernel.logger", [ + {default, file}, + {datatype, {enum, [file, console, both]}} +]}. + +{mapping, "log.level", "kernel.logger", [ + {default, warning}, + {datatype, {enum, [debug, info, notice, warning, error, critical, alert, emergency, all]}} +]}. + +{mapping, "log.primary_log_level", "kernel.logger_level", [ + {default, warning}, + {datatype, {enum, [debug, info, notice, warning, error, critical, alert, emergency, all]}} +]}. + +{mapping, "log.dir", "kernel.logger", [ + {default, "log"}, + {datatype, string} +]}. + +{mapping, "log.file", "kernel.logger", [ + {default, "emqx.log"}, + {datatype, file} +]}. + +{mapping, "log.chars_limit", "kernel.logger", [ + {default, -1}, + {datatype, integer} +]}. + +{mapping, "log.supervisor_reports", "kernel.logger", [ + {default, error}, + {datatype, {enum, [error, progress]}}, + hidden +]}. + +%% @doc Maximum depth in Erlang term log formatting +%% and message queue inspection. +{mapping, "log.max_depth", "kernel.error_logger_format_depth", [ + {default, 20}, + {datatype, [{enum, [unlimited]}, integer]} +]}. + +%% @doc format logs as JSON objects +{mapping, "log.formatter", "kernel.logger", [ + {default, text}, + {datatype, {enum, [text, json]}} +]}. + +%% @doc format logs in a single line. +{mapping, "log.single_line", "kernel.logger", [ + {default, true}, + {datatype, {enum, [true, false]}} +]}. + +{mapping, "log.rotation", "kernel.logger", [ + {default, on}, + {datatype, flag} +]}. + +{mapping, "log.rotation.size", "kernel.logger", [ + {default, "10MB"}, + {datatype, bytesize} +]}. + +{mapping, "log.size", "kernel.logger", [ + {default, infinity}, + {datatype, [bytesize, atom]} +]}. + +{mapping, "log.rotation.count", "kernel.logger", [ + {default, 5}, + {datatype, integer} +]}. + +{mapping, "log.$level.file", "kernel.logger", [ + {datatype, file} +]}. + +{mapping, "log.sync_mode_qlen", "kernel.logger", [ + {default, 100}, + {datatype, integer} +]}. + +{mapping, "log.drop_mode_qlen", "kernel.logger", [ + {default, 3000}, + {datatype, integer} +]}. + +{mapping, "log.flush_qlen", "kernel.logger", [ + {default, 8000}, + {datatype, integer} +]}. + +{mapping, "log.overload_kill", "kernel.logger", [ + {default, on}, + {datatype, flag} +]}. + +{mapping, "log.overload_kill_mem_size", "kernel.logger", [ + {default, "30MB"}, + {datatype, bytesize} +]}. + +{mapping, "log.overload_kill_qlen", "kernel.logger", [ + {default, 20000}, + {datatype, integer} +]}. + +{mapping, "log.overload_kill_restart_after", "kernel.logger", [ + {default, "5s"}, + {datatype, [{duration, ms}, atom]} +]}. + +{mapping, "log.burst_limit", "kernel.logger", [ + {default, "disabled"}, + {datatype, string} +]}. + +{mapping, "log.error_logger", "kernel.error_logger", [ + {default, silent}, + {datatype, {enum, [silent]}}, + hidden +]}. + +{translation, "kernel.logger_level", fun(_, _, Conf) -> + cuttlefish:conf_get("log.level", Conf) +end}. + +{translation, "kernel.logger", fun(Conf) -> + LogTo = cuttlefish:conf_get("log.to", Conf), + LogLevel = cuttlefish:conf_get("log.level", Conf), + LogType = case cuttlefish:conf_get("log.rotation", Conf) of + true -> wrap; + false -> halt + end, + CharsLimit = case cuttlefish:conf_get("log.chars_limit", Conf) of + -1 -> unlimited; + V -> V + end, + SingleLine = cuttlefish:conf_get("log.single_line", Conf), + FmtName = cuttlefish:conf_get("log.formatter", Conf), + Formatter = + case FmtName of + json -> + {emqx_logger_jsonfmt, + #{chars_limit => CharsLimit, + single_line => SingleLine + }}; + text -> + {emqx_logger_textfmt, + #{template => + [time," [",level,"] ", + {clientid, + [{peername, + [clientid,"@",peername," "], + [clientid, " "]}], + [{peername, + [peername," "], + []}]}, + msg,"\n"], + chars_limit => CharsLimit, + single_line => SingleLine + }} + end, + {BustLimitOn, {MaxBurstCount, TimeWindow}} = + case string:tokens(cuttlefish:conf_get("log.burst_limit", Conf), ", ") of + ["disabled"] -> {false, {20000, 1000}}; + [Count, Window] -> + {true, {list_to_integer(Count), + case cuttlefish_duration:parse(Window, ms) of + Secs when is_integer(Secs) -> Secs; + {error, Reason1} -> error(Reason1) + end}} + end, + FileConf = fun(Filename) -> + BasicConf = + #{type => LogType, + file => filename:join(cuttlefish:conf_get("log.dir", Conf), Filename), + max_no_files => cuttlefish:conf_get("log.rotation.count", Conf), + sync_mode_qlen => cuttlefish:conf_get("log.sync_mode_qlen", Conf), + drop_mode_qlen => cuttlefish:conf_get("log.drop_mode_qlen", Conf), + flush_qlen => cuttlefish:conf_get("log.flush_qlen", Conf), + overload_kill_enable => cuttlefish:conf_get("log.overload_kill", Conf), + overload_kill_qlen => cuttlefish:conf_get("log.overload_kill_qlen", Conf), + overload_kill_mem_size => cuttlefish:conf_get("log.overload_kill_mem_size", Conf), + overload_kill_restart_after => cuttlefish:conf_get("log.overload_kill_restart_after", Conf), + burst_limit_enable => BustLimitOn, + burst_limit_max_count => MaxBurstCount, + burst_limit_window_time => TimeWindow + }, + MaxNoBytes = case LogType of + wrap -> cuttlefish:conf_get("log.rotation.size", Conf); + halt -> cuttlefish:conf_get("log.size", Conf) + end, + BasicConf#{max_no_bytes => MaxNoBytes} + end, + + Filters = case cuttlefish:conf_get("log.supervisor_reports", Conf) of + error -> [{drop_progress_reports, {fun logger_filters:progress/2, stop}}]; + progress -> [] + end, + + %% For the default logger that outputs to console + DefaultHandler = + if LogTo =:= console orelse LogTo =:= both -> + [{handler, console, logger_std_h, + #{level => LogLevel, + config => #{type => standard_io}, + formatter => Formatter, + filters => Filters + } + }]; + true -> + [{handler, default, undefined}] + end, + + %% For the file logger + FileHandler = + if LogTo =:= file orelse LogTo =:= both -> + [{handler, file, logger_disk_log_h, + #{level => LogLevel, + config => FileConf(cuttlefish:conf_get("log.file", Conf)), + formatter => Formatter, + filesync_repeat_interval => no_repeat, + filters => Filters + }}]; + true -> [] + end, + + %% For creating additional log files for specific log levels. + AdditionalLogFiles = + lists:foldl( + fun({[_, Level, _] = K, Filename}, Acc) when LogTo =:= file; LogTo =:= both -> + case cuttlefish_variable:is_fuzzy_match(K, ["log", "$level", "file"]) of + true -> [{Level, Filename} | Acc]; + false -> Acc + end; + ({_K, _V}, Acc) -> + Acc + end, [], Conf), + AdditionalHandlers = + [{handler, list_to_atom("file_for_"++Level), logger_disk_log_h, + #{level => list_to_atom(Level), + config => FileConf(Filename), + formatter => Formatter, + filesync_repeat_interval => no_repeat}} + || {Level, Filename} <- AdditionalLogFiles], + + DefaultHandler ++ FileHandler ++ AdditionalHandlers +end}. + +%%-------------------------------------------------------------------- +%% Authentication/ACL +%%-------------------------------------------------------------------- + +%% @doc Allow anonymous authentication. +{mapping, "allow_anonymous", "emqx.allow_anonymous", [ + {default, false}, + {datatype, {enum, [true, false]}} +]}. + +%% @doc ACL nomatch. +{mapping, "acl_nomatch", "emqx.acl_nomatch", [ + {default, deny}, + {datatype, {enum, [allow, deny]}} +]}. + +%% @doc Default ACL file. +{mapping, "acl_file", "emqx.acl_file", [ + {datatype, string}, + hidden +]}. + +%% @doc Enable ACL cache for publish. +{mapping, "enable_acl_cache", "emqx.enable_acl_cache", [ + {default, on}, + {datatype, flag} +]}. + +%% @doc ACL cache time-to-live. +{mapping, "acl_cache_ttl", "emqx.acl_cache_ttl", [ + {default, "1m"}, + {datatype, {duration, ms}} +]}. + +%% @doc ACL cache size. +{mapping, "acl_cache_max_size", "emqx.acl_cache_max_size", [ + {default, 32}, + {datatype, integer}, + {validators, ["range:gt_0"]} +]}. + +%% @doc Action when acl check reject current operation +{mapping, "acl_deny_action", "emqx.acl_deny_action", [ + {default, ignore}, + {datatype, {enum, [ignore, disconnect]}} +]}. + +%% @doc Flapping detect policy +{mapping, "flapping_detect_policy", "emqx.flapping_detect_policy", [ + {datatype, string}, + {default, "30,1m,5m"} +]}. + +{translation, "emqx.flapping_detect_policy", fun(Conf) -> + Policy = cuttlefish:conf_get("flapping_detect_policy", Conf), + [Threshold, Duration, Interval] = string:tokens(Policy, ", "), + ParseDuration = fun(S, Dur) -> + case cuttlefish_duration:parse(S, Dur) of + I when is_integer(I) -> I; + {error, Reason} -> error(Reason) + end + end, + #{threshold => list_to_integer(Threshold), + duration => ParseDuration(Duration, ms), + banned_interval => ParseDuration(Interval, s) + } +end}. + +{validator, "range:gt_0", "must greater than 0", + fun(X) -> X > 0 end +}. + +%%-------------------------------------------------------------------- +%% MQTT Protocol +%%-------------------------------------------------------------------- + +%% @doc Max Packet Size Allowed, 1MB by default. +{mapping, "mqtt.max_packet_size", "emqx.max_packet_size", [ + {default, "1MB"}, + {datatype, bytesize}, + {override_env, "MAX_PACKET_SIZE"} +]}. + +%% @doc Set the Max ClientId Length Allowed. +{mapping, "mqtt.max_clientid_len", "emqx.max_clientid_len", [ + {default, 65535}, + {datatype, integer} +]}. + +%% @doc Set the Maximum topic levels. +{mapping, "mqtt.max_topic_levels", "emqx.max_topic_levels", [ + {default, 0}, + {datatype, integer} +]}. + +%% @doc Set the Maximum QoS allowed. +{mapping, "mqtt.max_qos_allowed", "emqx.max_qos_allowed", [ + {default, 2}, + {datatype, integer}, + {validators, ["range:0-2"]} +]}. + +%% @doc Set the Maximum Topic Alias. +{mapping, "mqtt.max_topic_alias", "emqx.max_topic_alias", [ + {default, 65535}, + {datatype, integer} +]}. + +%% @doc Whether the server supports MQTT retained messages. +{mapping, "mqtt.retain_available", "emqx.retain_available", [ + {default, true}, + {datatype, {enum, [true, false]}} +]}. + +%% @doc Whether the Server supports MQTT Wildcard Subscriptions. +{mapping, "mqtt.wildcard_subscription", "emqx.wildcard_subscription", [ + {default, true}, + {datatype, {enum, [true, false]}} +]}. + +%% @doc Whether the Server supports MQTT Shared Subscriptions. +{mapping, "mqtt.shared_subscription", "emqx.shared_subscription", [ + {default, true}, + {datatype, {enum, [true, false]}} +]}. + +%% @doc Whether to ignore loop delivery of messages.(for mqtt v3.1.1) +{mapping, "mqtt.ignore_loop_deliver", "emqx.ignore_loop_deliver", [ + {default, true}, + {datatype, {enum, [true, false]}} +]}. + +%% @doc Whether to parse the MQTT frame in strict mode +{mapping, "mqtt.strict_mode", "emqx.strict_mode", [ + {default, false}, + {datatype, {enum, [true, false]}} +]}. + +%% @doc Specify the response information returned to the client +{mapping, "mqtt.response_information", "emqx.response_information", [ + {datatype, string} +]}. + +%%-------------------------------------------------------------------- +%% Zones +%%-------------------------------------------------------------------- + +%% @doc Idle timeout of the MQTT connection. +{mapping, "zone.$name.idle_timeout", "emqx.zones", [ + {default, "15s"}, + {datatype, {duration, ms}} +]}. + +{mapping, "zone.$name.allow_anonymous", "emqx.zones", [ + {datatype, {enum, [true, false]}} +]}. + +{mapping, "zone.$name.acl_nomatch", "emqx.zones", [ + {datatype, {enum, [allow, deny]}} +]}. + +%% @doc Enable ACL check. +{mapping, "zone.$name.enable_acl", "emqx.zones", [ + {default, off}, + {datatype, flag} +]}. + +%% @doc Action when acl check reject current operation +{mapping, "zone.$name.acl_deny_action", "emqx.zones", [ + {default, ignore}, + {datatype, {enum, [ignore, disconnect]}} +]}. + +%% @doc Enable Ban. +{mapping, "zone.$name.enable_ban", "emqx.zones", [ + {default, off}, + {datatype, flag} +]}. + +%% @doc Enable per connection statistics. +{mapping, "zone.$name.enable_stats", "emqx.zones", [ + {default, off}, + {datatype, flag} +]}. + +%% @doc Publish limit of the MQTT connections. +{mapping, "zone.$name.publish_limit", "emqx.zones", [ + {datatype, string} +]}. + +%% @doc Max Packet Size Allowed, 64K by default. +{mapping, "zone.$name.max_packet_size", "emqx.zones", [ + {datatype, bytesize} +]}. + +%% @doc Set the Max ClientId Length Allowed. +{mapping, "zone.$name.max_clientid_len", "emqx.zones", [ + {datatype, integer} +]}. + +%% @doc Set the Maximum topic levels. +{mapping, "zone.$name.max_topic_levels", "emqx.zones", [ + {datatype, integer} +]}. + +%% @doc Set the Maximum QoS allowed. +{mapping, "zone.$name.max_qos_allowed", "emqx.zones", [ + {datatype, integer}, + {validators, ["range:0-2"]} +]}. + +%% @doc Set the Maximum topic alias. +{mapping, "zone.$name.max_topic_alias", "emqx.zones", [ + {datatype, integer} +]}. + +%% @doc Whether the server supports retained messages. +{mapping, "zone.$name.retain_available", "emqx.zones", [ + {datatype, {enum, [true, false]}} +]}. + +%% @doc Whether the Server supports Wildcard Subscriptions. +{mapping, "zone.$name.wildcard_subscription", "emqx.zones", [ + {datatype, {enum, [true, false]}} +]}. + +%% @doc Whether the Server supports Shared Subscriptions. +{mapping, "zone.$name.shared_subscription", "emqx.zones", [ + {datatype, {enum, [true, false]}} +]}. + +%% @doc Server Keepalive +{mapping, "zone.$name.server_keepalive", "emqx.zones", [ + {datatype, integer} +]}. + +%% @doc Keepalive backoff +{mapping, "zone.$name.keepalive_backoff", "emqx.zones", [ + {default, 0.75}, + {datatype, float} +]}. + +%% @doc Max Number of Subscriptions Allowed. +{mapping, "zone.$name.max_subscriptions", "emqx.zones", [ + {default, 0}, + {datatype, integer} +]}. + +%% @doc Upgrade QoS according to subscription? +{mapping, "zone.$name.upgrade_qos", "emqx.zones", [ + {default, off}, + {datatype, flag} +]}. + +%% @doc Max number of QoS 1 and 2 messages that can be “inflight” at one time. +%% 0 is equivalent to maximum allowed +{mapping, "zone.$name.max_inflight", "emqx.zones", [ + {default, 0}, + {datatype, integer}, + {validators, ["range:1-65535"]} +]}. + +%% @doc Retry interval for redelivering QoS1/2 messages. +{mapping, "zone.$name.retry_interval", "emqx.zones", [ + {default, "30s"}, + {datatype, {duration, s}} +]}. + +%% @doc Max Packets that Awaiting PUBREL, 0 means no limit +{mapping, "zone.$name.max_awaiting_rel", "emqx.zones", [ + {default, 0}, + {datatype, integer} +]}. + +%% @doc Awaiting PUBREL timeout +{mapping, "zone.$name.await_rel_timeout", "emqx.zones", [ + {default, "300s"}, + {datatype, {duration, s}} +]}. + +%% @doc Ignore loop delivery of messages +{mapping, "zone.$name.ignore_loop_deliver", "emqx.zones", [ + {datatype, {enum, [true, false]}} +]}. + +%% @doc Session Expiry Interval +{mapping, "zone.$name.session_expiry_interval", "emqx.zones", [ + {default, "2h"}, + {datatype, {duration, s}} +]}. + +%% @doc Max queue length. Enqueued messages when persistent client +%% disconnected, or inflight window is full. 0 means no limit. +{mapping, "zone.$name.max_mqueue_len", "emqx.zones", [ + {default, 1000}, + {datatype, integer} +]}. + +%% @doc Topic Priorities, comma separated topic=priority pairs, +%% where priority should be integer in range 1-255 (inclusive) +%% 1 being the lowest and 255 being the highest. +%% default value `none` to indicate no priority table, hence all +%% messages are treated equal, which means either highest ('infinity'), +%% or lowest (0) depending on mqueue_default_priority config. +{mapping, "zone.$name.mqueue_priorities", "emqx.zones", [ + {default, "none"}, + {datatype, string} +]}. + +%% @doc Default priority for topics not in priority table. +{mapping, "zone.$name.mqueue_default_priority", "emqx.zones", [ + {default, lowest}, + {datatype, {enum, [highest, lowest]}} +]}. + +%% @doc Queue Qos0 messages? +{mapping, "zone.$name.mqueue_store_qos0", "emqx.zones", [ + {default, true}, + {datatype, {enum, [true, false]}} +]}. + +{mapping, "zone.$name.enable_flapping_detect", "emqx.zones", [ + {datatype, flag}, + {default, off} +]}. + +{mapping, "zone.$name.rate_limit.conn_messages_in", "emqx.zones", [ + {datatype, string} +]}. + +{mapping, "zone.$name.rate_limit.conn_bytes_in", "emqx.zones", [ + {datatype, string} +]}. + +{mapping, "zone.$name.conn_congestion.alarm", "emqx.zones", [ + {datatype, flag}, + {default, off} +]}. + +{mapping, "zone.$name.conn_congestion.min_alarm_sustain_duration", "emqx.zones", [ + {default, "1m"}, + {datatype, {duration, ms}} +]}. + +{mapping, "zone.$name.quota.conn_messages_routing", "emqx.zones", [ + {datatype, string} +]}. + +{mapping, "zone.$name.quota.overall_messages_routing", "emqx.zones", [ + {datatype, string} +]}. + +%% @doc Force connection/session process GC after this number of +%% messages | bytes passed through. +%% Numbers delimited by `|'. Zero or negative is to disable. +{mapping, "zone.$name.force_gc_policy", "emqx.zones", [ + {datatype, string} + ]}. + +%% @doc Max message queue length and total heap size to force shutdown +%% connection/session process. +%% Message queue here is the Erlang process mailbox, but not the number +%% of queued MQTT messages of QoS 1 and 2. +%% Zero or negative is to disable. +{mapping, "zone.$name.force_shutdown_policy", "emqx.zones", [ + {default, "default"}, + {datatype, string} +]}. + +{mapping, "zone.$name.mountpoint", "emqx.zones", [ + {datatype, string} +]}. + +%% @doc Use username replace client id +{mapping, "zone.$name.use_username_as_clientid", "emqx.zones", [ + {default, false}, + {datatype, {enum, [true, false]}} +]}. + +%% @doc Whether to parse the MQTT frame in strict mode +{mapping, "zone.$name.strict_mode", "emqx.zones", [ + {default, false}, + {datatype, {enum, [true, false]}} +]}. + +%% @doc Specify the response information returned to the client +{mapping, "zone.$name.response_information", "emqx.zones", [ + {datatype, string} +]}. + +%% @doc Whether to bypass the authentication step +{mapping, "zone.$name.bypass_auth_plugins", "emqx.zones", [ + {default, false}, + {datatype, {enum, [true, false]}} +]}. + +{translation, "emqx.zones", fun(Conf) -> + Ratelimit = fun(Val) -> + [L, D] = string:tokens(Val, ", "), + Limit = case cuttlefish_bytesize:parse(L) of + Sz when is_integer(Sz) -> Sz; + {error, Reason1} -> error(Reason1) + end, + Duration = case cuttlefish_duration:parse(D, s) of + Secs when is_integer(Secs) -> Secs; + {error, Reason} -> error(Reason) + end, + {Limit, Duration} + end, + Mapping = fun(["publish_limit"], Val) -> + %% XXX: Deprecated at v4.2 + {publish_limit, Ratelimit(Val)}; + (["force_gc_policy"], Val) -> + [Count, Bytes] = string:tokens(Val, "| "), + GcPolicy = case cuttlefish_bytesize:parse(Bytes) of + {error, Reason} -> + error(Reason); + Bytes1 -> + #{bytes => Bytes1, + count => list_to_integer(Count)} + end, + {force_gc_policy, GcPolicy}; + (["force_shutdown_policy"], "default") -> + {DefaultLen, DefaultSize} = + case WordSize = erlang:system_info(wordsize) of + 8 -> % arch_64 + {10000, cuttlefish_bytesize:parse("64MB")}; + 4 -> % arch_32 + {1000, cuttlefish_bytesize:parse("32MB")} + end, + {force_shutdown_policy, #{message_queue_len => DefaultLen, + max_heap_size => DefaultSize div WordSize + }}; + (["force_shutdown_policy"], Val) -> + [Len, Siz] = string:tokens(Val, "| "), + MaxSiz = case WordSize = erlang:system_info(wordsize) of + 8 -> % arch_64 + (1 bsl 59) - 1; + 4 -> % arch_32 + (1 bsl 27) - 1 + end, + ShutdownPolicy = + case cuttlefish_bytesize:parse(Siz) of + {error, Reason} -> + error(Reason); + Siz1 when Siz1 > MaxSiz -> + cuttlefish:invalid(io_lib:format("force_shutdown_policy: heap-size ~s is too large", [Siz])); + Siz1 -> + #{message_queue_len => list_to_integer(Len), + max_heap_size => Siz1 div WordSize} + end, + {force_shutdown_policy, ShutdownPolicy}; + (["mqueue_priorities"], Val) -> + case Val of + "none" -> {mqueue_priorities, none}; % NO_PRIORITY_TABLE + _ -> + MqueuePriorities = lists:foldl(fun(T, Acc) -> + %% NOTE: space in "= " is intended + [Topic, Prio] = string:tokens(T, "= "), + P = list_to_integer(Prio), + (P < 0 orelse P > 255) andalso error({bad_priority, Topic, Prio}), + maps:put(iolist_to_binary(Topic), P, Acc) + end, #{}, string:tokens(Val, ",")), + {mqueue_priorities, MqueuePriorities} + end; + (["mountpoint"], Val) -> + {mountpoint, iolist_to_binary(Val)}; + (["response_information"], Val) -> + {response_information, iolist_to_binary(Val)}; + (["rate_limit", "conn_messages_in"], Val) -> + {ratelimit, {conn_messages_in, Ratelimit(Val)}}; + (["rate_limit", "conn_bytes_in"], Val) -> + {ratelimit, {conn_bytes_in, Ratelimit(Val)}}; + (["conn_congestion", "alarm"], Val) -> + {conn_congestion_alarm_enabled, Val}; + (["conn_congestion", "min_alarm_sustain_duration"], Val) -> + {conn_congestion_min_alarm_sustain_duration, Val}; + (["quota", "conn_messages_routing"], Val) -> + {quota, {conn_messages_routing, Ratelimit(Val)}}; + (["quota", "overall_messages_routing"], Val) -> + {quota, {overall_messages_routing, Ratelimit(Val)}}; + ([Opt], Val) -> + {list_to_atom(Opt), Val} + end, + maps:to_list( + lists:foldl( + fun({["zone", Name | Opt], Val}, Zones) -> + NVal = Mapping(Opt, Val), + maps:update_with(list_to_atom(Name), + fun(Opts) -> + case NVal of + {Key, Rl} when Key == ratelimit; + Key == quota -> + Rls = proplists:get_value(Key, Opts, []), + lists:keystore(Key, 1, Opts, {Key, [Rl|Rls]}); + _ -> + [NVal|Opts] + end + end, [NVal], Zones) + end, #{}, lists:usort(cuttlefish_variable:filter_by_prefix("zone.", Conf)))) +end}. + +%%-------------------------------------------------------------------- +%% Listeners +%%-------------------------------------------------------------------- + +%%-------------------------------------------------------------------- +%% TCP Listeners + +{mapping, "listener.tcp.$name", "emqx.listeners", [ + {datatype, [integer, ip]} +]}. + +{mapping, "listener.tcp.$name.acceptors", "emqx.listeners", [ + {default, 8}, + {datatype, integer} +]}. + +{mapping, "listener.tcp.$name.max_connections", "emqx.listeners", [ + {default, 1024}, + {datatype, integer} +]}. + +{mapping, "listener.tcp.$name.max_conn_rate", "emqx.listeners", [ + {datatype, integer} +]}. + +{mapping, "listener.tcp.$name.active_n", "emqx.listeners", [ + {default, 100}, + {datatype, integer} +]}. + +{mapping, "listener.tcp.$name.zone", "emqx.listeners", [ + {datatype, string} +]}. + +{mapping, "listener.tcp.$name.rate_limit", "emqx.listeners", [ + {default, undefined}, + {datatype, string} +]}. + +{mapping, "listener.tcp.$name.access.$id", "emqx.listeners", [ + {datatype, string} +]}. + +{mapping, "listener.tcp.$name.proxy_protocol", "emqx.listeners", [ + {datatype, flag} +]}. + +{mapping, "listener.tcp.$name.proxy_protocol_timeout", "emqx.listeners", [ + {datatype, {duration, ms}} +]}. + +%% The proxy-protocol protocol can get the certificate CN through tcp +{mapping, "listener.tcp.$name.peer_cert_as_username", "emqx.listeners", [ + {datatype, {enum, [cn]}} +]}. + +%% The proxy-protocol protocol can get the certificate CN through tcp +{mapping, "listener.tcp.$name.peer_cert_as_clientid", "emqx.listeners", [ + {datatype, {enum, [cn]}} +]}. + +{mapping, "listener.tcp.$name.backlog", "emqx.listeners", [ + {datatype, integer}, + {default, 1024} +]}. + +{mapping, "listener.tcp.$name.send_timeout", "emqx.listeners", [ + {datatype, {duration, ms}}, + {default, "15s"} +]}. + +{mapping, "listener.tcp.$name.send_timeout_close", "emqx.listeners", [ + {datatype, flag}, + {default, on} +]}. + +{mapping, "listener.tcp.$name.recbuf", "emqx.listeners", [ + {datatype, bytesize}, + hidden +]}. + +{mapping, "listener.tcp.$name.sndbuf", "emqx.listeners", [ + {datatype, bytesize}, + hidden +]}. + +{mapping, "listener.tcp.$name.buffer", "emqx.listeners", [ + {datatype, bytesize}, + hidden +]}. + +{mapping, "listener.tcp.$name.high_watermark", "emqx.listeners", [ + {datatype, bytesize}, + {default, "1MB"} + ]}. + +{mapping, "listener.tcp.$name.tune_buffer", "emqx.listeners", [ + {datatype, flag}, + hidden +]}. + +{mapping, "listener.tcp.$name.nodelay", "emqx.listeners", [ + {datatype, {enum, [true, false]}}, + hidden +]}. + +{mapping, "listener.tcp.$name.reuseaddr", "emqx.listeners", [ + {datatype, {enum, [true, false]}}, + hidden +]}. + +%%-------------------------------------------------------------------- +%% SSL Listeners + +{mapping, "listener.ssl.$name", "emqx.listeners", [ + {datatype, [integer, ip]} +]}. + +{mapping, "listener.ssl.$name.acceptors", "emqx.listeners", [ + {default, 8}, + {datatype, integer} +]}. + +{mapping, "listener.ssl.$name.max_connections", "emqx.listeners", [ + {default, 1024}, + {datatype, integer} +]}. + +{mapping, "listener.ssl.$name.max_conn_rate", "emqx.listeners", [ + {datatype, integer} +]}. + +{mapping, "listener.ssl.$name.active_n", "emqx.listeners", [ + {default, 100}, + {datatype, integer} +]}. + +{mapping, "listener.ssl.$name.zone", "emqx.listeners", [ + {datatype, string} +]}. + +{mapping, "listener.ssl.$name.rate_limit", "emqx.listeners", [ + {default, undefined}, + {datatype, string} +]}. + +{mapping, "listener.ssl.$name.access.$id", "emqx.listeners", [ + {datatype, string} +]}. + +{mapping, "listener.ssl.$name.proxy_protocol", "emqx.listeners", [ + {datatype, flag} +]}. + +{mapping, "listener.ssl.$name.proxy_protocol_timeout", "emqx.listeners", [ + {datatype, {duration, ms}} +]}. + +{mapping, "listener.ssl.$name.backlog", "emqx.listeners", [ + {default, 1024}, + {datatype, integer} +]}. + +{mapping, "listener.ssl.$name.send_timeout", "emqx.listeners", [ + {datatype, {duration, ms}}, + {default, "15s"} +]}. + +{mapping, "listener.ssl.$name.send_timeout_close", "emqx.listeners", [ + {datatype, flag}, + {default, on} +]}. + +{mapping, "listener.ssl.$name.recbuf", "emqx.listeners", [ + {datatype, bytesize}, + hidden +]}. + +{mapping, "listener.ssl.$name.sndbuf", "emqx.listeners", [ + {datatype, bytesize}, + hidden +]}. + +{mapping, "listener.ssl.$name.buffer", "emqx.listeners", [ + {datatype, bytesize}, + hidden +]}. + +{mapping, "listener.ssl.$name.high_watermark", "emqx.listeners", [ + {datatype, bytesize}, + {default, "1MB"} + ]}. + +{mapping, "listener.ssl.$name.tune_buffer", "emqx.listeners", [ + {datatype, flag}, + hidden +]}. + +{mapping, "listener.ssl.$name.nodelay", "emqx.listeners", [ + {datatype, {enum, [true, false]}}, + hidden +]}. + +{mapping, "listener.ssl.$name.reuseaddr", "emqx.listeners", [ + {datatype, {enum, [true, false]}}, + hidden +]}. + +{mapping, "listener.ssl.$name.tls_versions", "emqx.listeners", [ + {datatype, string} +]}. + +{mapping, "listener.ssl.$name.ciphers", "emqx.listeners", [ + {datatype, string} +]}. + +{mapping, "listener.ssl.$name.psk_ciphers", "emqx.listeners", [ + {datatype, string} +]}. + +{mapping, "listener.ssl.$name.handshake_timeout", "emqx.listeners", [ + {default, "15s"}, + {datatype, {duration, ms}} +]}. + +{mapping, "listener.ssl.$name.depth", "emqx.listeners", [ + {default, 10}, + {datatype, integer} +]}. + +{mapping, "listener.ssl.$name.key_password", "emqx.listeners", [ + {datatype, string} +]}. + +{mapping, "listener.ssl.$name.dhfile", "emqx.listeners", [ + {datatype, string} +]}. + +{mapping, "listener.ssl.$name.keyfile", "emqx.listeners", [ + {datatype, string} +]}. + +{mapping, "listener.ssl.$name.certfile", "emqx.listeners", [ + {datatype, string} +]}. + +{mapping, "listener.ssl.$name.cacertfile", "emqx.listeners", [ + {datatype, string} +]}. + +{mapping, "listener.ssl.$name.verify", "emqx.listeners", [ + {datatype, atom} +]}. + +{mapping, "listener.ssl.$name.fail_if_no_peer_cert", "emqx.listeners", [ + {datatype, {enum, [true, false]}} +]}. + +{mapping, "listener.ssl.$name.secure_renegotiate", "emqx.listeners", [ + {datatype, flag} +]}. + +{mapping, "listener.ssl.$name.reuse_sessions", "emqx.listeners", [ + {default, on}, + {datatype, flag} +]}. + +{mapping, "listener.ssl.$name.honor_cipher_order", "emqx.listeners", [ + {datatype, flag} +]}. + +{mapping, "listener.ssl.$name.peer_cert_as_username", "emqx.listeners", [ + {datatype, {enum, [cn, dn, crt, pem, md5]}} +]}. + +{mapping, "listener.ssl.$name.peer_cert_as_clientid", "emqx.listeners", [ + {datatype, {enum, [cn, dn, crt, pem, md5]}} +]}. + +%%-------------------------------------------------------------------- +%% MQTT/WebSocket Listeners + +{mapping, "listener.ws.$name", "emqx.listeners", [ + {datatype, [integer, ip]} +]}. + +{mapping, "listener.ws.$name.mqtt_path", "emqx.listeners", [ + {default, "/mqtt"}, + {datatype, string} +]}. + +{mapping, "listener.ws.$name.acceptors", "emqx.listeners", [ + {default, 8}, + {datatype, integer} +]}. + +{mapping, "listener.ws.$name.max_connections", "emqx.listeners", [ + {default, 1024}, + {datatype, integer} +]}. + +{mapping, "listener.ws.$name.max_conn_rate", "emqx.listeners", [ + {datatype, integer} +]}. + +{mapping, "listener.ws.$name.active_n", "emqx.listeners", [ + {default, 100}, + {datatype, integer} +]}. + +{mapping, "listener.ws.$name.zone", "emqx.listeners", [ + {datatype, string} +]}. + +{mapping, "listener.ws.$name.rate_limit", "emqx.listeners", [ + {default, undefined}, + {datatype, string} +]}. + +{mapping, "listener.ws.$name.access.$id", "emqx.listeners", [ + {datatype, string} +]}. + +{mapping, "listener.ws.$name.fail_if_no_subprotocol", "emqx.listeners", [ + {default, true}, + {datatype, {enum, [true, false]}} +]}. + +{mapping, "listener.ws.$name.supported_subprotocols", "emqx.listeners", [ + {default, "mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5"}, + {datatype, string} +]}. + +{mapping, "listener.ws.$name.proxy_address_header", "emqx.listeners", [ + {default, "X-Forwarded-For"}, + {datatype, string} +]}. + +{mapping, "listener.ws.$name.proxy_port_header", "emqx.listeners", [ + {default, "X-Forwarded-Port"}, + {datatype, string} +]}. + +{mapping, "listener.ws.$name.proxy_protocol", "emqx.listeners", [ + {datatype, flag} +]}. + +{mapping, "listener.ws.$name.proxy_protocol_timeout", "emqx.listeners", [ + {datatype, {duration, ms}} +]}. + +{mapping, "listener.ws.$name.backlog", "emqx.listeners", [ + {default, 1024}, + {datatype, integer} +]}. + +{mapping, "listener.ws.$name.send_timeout", "emqx.listeners", [ + {datatype, {duration, ms}}, + {default, "15s"} +]}. + +{mapping, "listener.ws.$name.send_timeout_close", "emqx.listeners", [ + {datatype, flag}, + {default, on} +]}. + +{mapping, "listener.ws.$name.recbuf", "emqx.listeners", [ + {datatype, bytesize}, + hidden +]}. + +{mapping, "listener.ws.$name.sndbuf", "emqx.listeners", [ + {datatype, bytesize}, + hidden +]}. + +{mapping, "listener.ws.$name.buffer", "emqx.listeners", [ + {datatype, bytesize}, + hidden +]}. + +{mapping, "listener.ws.$name.tune_buffer", "emqx.listeners", [ + {datatype, flag}, + hidden +]}. + +{mapping, "listener.ws.$name.nodelay", "emqx.listeners", [ + {datatype, {enum, [true, false]}}, + hidden +]}. + +{mapping, "listener.ws.$name.compress", "emqx.listeners", [ + {datatype, {enum, [true, false]}}, + hidden +]}. + +{mapping, "listener.ws.$name.deflate_opts.level", "emqx.listeners", [ + {datatype, {enum, [none, default, best_compression, best_speed]}}, + hidden +]}. + +{mapping, "listener.ws.$name.deflate_opts.mem_level", "emqx.listeners", [ + {datatype, integer}, + {validators, ["range:1-9"]}, + hidden +]}. + +{mapping, "listener.ws.$name.deflate_opts.strategy", "emqx.listeners", [ + {datatype, {enum, [default, filtered, huffman_only, rle]}}, + hidden +]}. + +{mapping, "listener.ws.$name.deflate_opts.server_context_takeover", "emqx.listeners", [ + {datatype, {enum, [takeover, no_takeover]}}, + hidden +]}. + +{mapping, "listener.ws.$name.deflate_opts.client_context_takeover", "emqx.listeners", [ + {datatype, {enum, [takeover, no_takeover]}}, + hidden +]}. + +{mapping, "listener.ws.$name.deflate_opts.server_max_window_bits", "emqx.listeners", [ + {datatype, integer}, + hidden +]}. + +{mapping, "listener.ws.$name.deflate_opts.client_max_window_bits", "emqx.listeners", [ + {datatype, integer}, + hidden +]}. + +{mapping, "listener.ws.$name.idle_timeout", "emqx.listeners", [ + {datatype, {duration, ms}}, + hidden +]}. + +{mapping, "listener.ws.$name.max_frame_size", "emqx.listeners", [ + {datatype, integer}, + hidden +]}. + +{mapping, "listener.ws.$name.mqtt_piggyback", "emqx.listeners", [ + {datatype, {enum, [single, multiple]}}, + {default, multiple}, + hidden +]}. + +{mapping, "listener.ws.$name.peer_cert_as_username", "emqx.listeners", [ + {datatype, {enum, [cn]}} +]}. + +{mapping, "listener.ws.$name.peer_cert_as_clientid", "emqx.listeners", [ + {datatype, {enum, [cn]}} +]}. + +{mapping, "listener.ws.$name.check_origin_enable", "emqx.listeners", [ + {datatype, {enum, [true, false]}}, + {default, false}, + hidden +]}. + +{mapping, "listener.ws.$name.allow_origin_absence", "emqx.listeners", [ + {datatype, {enum, [true, false]}}, + {default, true}, + hidden +]}. + +{mapping, "listener.ws.$name.check_origins", "emqx.listeners", [ + {datatype, string}, + hidden +]}. + +%%-------------------------------------------------------------------- +%% MQTT/WebSocket/SSL Listeners + +{mapping, "listener.wss.$name", "emqx.listeners", [ + {datatype, [integer, ip]} +]}. + +{mapping, "listener.wss.$name.mqtt_path", "emqx.listeners", [ + {default, "/mqtt"}, + {datatype, string} +]}. + +{mapping, "listener.wss.$name.acceptors", "emqx.listeners", [ + {default, 8}, + {datatype, integer} +]}. + +{mapping, "listener.wss.$name.max_connections", "emqx.listeners", [ + {default, 1024}, + {datatype, integer} +]}. + +{mapping, "listener.wss.$name.max_conn_rate", "emqx.listeners", [ + {datatype, integer} +]}. + +{mapping, "listener.wss.$name.active_n", "emqx.listeners", [ + {default, 100}, + {datatype, integer} +]}. + +{mapping, "listener.wss.$name.zone", "emqx.listeners", [ + {datatype, string} +]}. + +{mapping, "listener.wss.$name.rate_limit", "emqx.listeners", [ + {datatype, string} +]}. + +{mapping, "listener.wss.$name.fail_if_no_subprotocol", "emqx.listeners", [ + {default, true}, + {datatype, {enum, [true, false]}} +]}. + +{mapping, "listener.wss.$name.supported_subprotocols", "emqx.listeners", [ + {default, "mqtt, mqtt-v3, mqtt-v3.1.1, mqtt-v5"}, + {datatype, string} +]}. + +{mapping, "listener.wss.$name.access.$id", "emqx.listeners", [ + {datatype, string} +]}. + +{mapping, "listener.wss.$name.proxy_address_header", "emqx.listeners", [ + {default, "X-Forwarded-For"}, + {datatype, string} +]}. + +{mapping, "listener.wss.$name.proxy_port_header", "emqx.listeners", [ + {default, "X-Forwarded-Port"}, + {datatype, string} +]}. + +{mapping, "listener.wss.$name.proxy_protocol", "emqx.listeners", [ + {datatype, flag} +]}. + +{mapping, "listener.wss.$name.proxy_protocol_timeout", "emqx.listeners", [ + {datatype, {duration, ms}} +]}. + +%%{mapping, "listener.wss.$name.handshake_timeout", "emqx.listeners", [ +%% {default, "15s"}, +%% {datatype, {duration, ms}} +%%]}. + +{mapping, "listener.wss.$name.backlog", "emqx.listeners", [ + {default, 1024}, + {datatype, integer} +]}. + +{mapping, "listener.wss.$name.send_timeout", "emqx.listeners", [ + {datatype, {duration, ms}}, + {default, "15s"} +]}. + +{mapping, "listener.wss.$name.send_timeout_close", "emqx.listeners", [ + {datatype, flag}, + {default, on} +]}. + +{mapping, "listener.wss.$name.recbuf", "emqx.listeners", [ + {datatype, bytesize}, + hidden +]}. + +{mapping, "listener.wss.$name.sndbuf", "emqx.listeners", [ + {datatype, bytesize}, + hidden +]}. + +{mapping, "listener.wss.$name.buffer", "emqx.listeners", [ + {datatype, bytesize}, + hidden +]}. + +{mapping, "listener.wss.$name.tune_buffer", "emqx.listeners", [ + {datatype, flag}, + hidden +]}. + +{mapping, "listener.wss.$name.nodelay", "emqx.listeners", [ + {datatype, {enum, [true, false]}}, + hidden +]}. + +{mapping, "listener.wss.$name.tls_versions", "emqx.listeners", [ + {datatype, string} +]}. + +{mapping, "listener.wss.$name.ciphers", "emqx.listeners", [ + {datatype, string} +]}. + +{mapping, "listener.wss.$name.psk_ciphers", "emqx.listeners", [ + {datatype, string} +]}. + +{mapping, "listener.wss.$name.keyfile", "emqx.listeners", [ + {datatype, string} +]}. + +{mapping, "listener.wss.$name.certfile", "emqx.listeners", [ + {datatype, string} +]}. + +{mapping, "listener.wss.$name.cacertfile", "emqx.listeners", [ + {datatype, string} +]}. + +{mapping, "listener.wss.$name.dhfile", "emqx.listeners", [ + {datatype, string} +]}. + +{mapping, "listener.wss.$name.depth", "emqx.listeners", [ + {default, 10}, + {datatype, integer} +]}. + +{mapping, "listener.wss.$name.key_password", "emqx.listeners", [ + {datatype, string} +]}. + +{mapping, "listener.wss.$name.verify", "emqx.listeners", [ + {datatype, atom} +]}. + +{mapping, "listener.wss.$name.fail_if_no_peer_cert", "emqx.listeners", [ + {datatype, {enum, [true, false]}} +]}. + +{mapping, "listener.wss.$name.secure_renegotiate", "emqx.listeners", [ + {datatype, flag} +]}. + +{mapping, "listener.wss.$name.reuse_sessions", "emqx.listeners", [ + {default, on}, + {datatype, flag} +]}. + +{mapping, "listener.wss.$name.honor_cipher_order", "emqx.listeners", [ + {datatype, flag} +]}. + +{mapping, "listener.wss.$name.peer_cert_as_username", "emqx.listeners", [ + {datatype, {enum, [cn, dn, crt, pem, md5]}} +]}. + +{mapping, "listener.wss.$name.peer_cert_as_clientid", "emqx.listeners", [ + {datatype, {enum, [cn, dn, crt, pem, md5]}} +]}. + +{mapping, "listener.wss.$name.compress", "emqx.listeners", [ + {datatype, {enum, [true, false]}}, + hidden +]}. + +{mapping, "listener.wss.$name.deflate_opts.level", "emqx.listeners", [ + {datatype, {enum, [none, default, best_compression, best_speed]}}, + hidden +]}. + +{mapping, "listener.wss.$name.deflate_opts.mem_level", "emqx.listeners", [ + {datatype, integer}, + {validators, ["range:1-9"]}, + hidden +]}. + +{mapping, "listener.wss.$name.deflate_opts.strategy", "emqx.listeners", [ + {datatype, {enum, [default, filtered, huffman_only, rle]}}, + hidden +]}. + +{mapping, "listener.wss.$name.deflate_opts.server_context_takeover", "emqx.listeners", [ + {datatype, {enum, [takeover, no_takeover]}}, + hidden +]}. + +{mapping, "listener.wss.$name.deflate_opts.client_context_takeover", "emqx.listeners", [ + {datatype, {enum, [takeover, no_takeover]}}, + hidden +]}. + +{mapping, "listener.wss.$name.deflate_opts.server_max_window_bits", "emqx.listeners", [ + {datatype, integer}, + {validators, ["range:8-15"]}, + hidden +]}. + +{mapping, "listener.wss.$name.deflate_opts.client_max_window_bits", "emqx.listeners", [ + {datatype, integer}, + {validators, ["range:8-15"]}, + hidden +]}. + +{mapping, "listener.wss.$name.idle_timeout", "emqx.listeners", [ + {datatype, {duration, ms}}, + hidden +]}. + +{mapping, "listener.wss.$name.max_frame_size", "emqx.listeners", [ + {datatype, integer}, + hidden +]}. + +{mapping, "listener.wss.$name.mqtt_piggyback", "emqx.listeners", [ + {datatype, {enum, [single, multiple]}}, + {default, multiple}, + hidden +]}. + +{mapping, "listener.wss.$name.check_origin_enable", "emqx.listeners", [ + {datatype, {enum, [true, false]}}, + {default, false}, + hidden +]}. + +{mapping, "listener.wss.$name.allow_origin_absence", "emqx.listeners", [ + {datatype, {enum, [true, false]}}, + {default, true}, + hidden +]}. + +{mapping, "listener.wss.$name.check_origins", "emqx.listeners", [ + {datatype, string}, + hidden +]}. + +{translation, "emqx.listeners", fun(Conf) -> + + Filter = fun(Opts) -> [{K, V} || {K, V} <- Opts, V =/= undefined] end, + + Atom = fun(undefined) -> undefined; (S) -> list_to_atom(S) end, + + Access = fun(S) -> + [A, CIDR] = string:tokens(S, " "), + {list_to_atom(A), case CIDR of "all" -> all; _ -> CIDR end} + end, + + AccOpts = fun(Prefix) -> + case cuttlefish_variable:filter_by_prefix(Prefix ++ ".access", Conf) of + [] -> []; + Rules -> [{access_rules, [Access(Rule) || {_, Rule} <- Rules]}] + end + end, + + RateLimit = fun(undefined) -> + undefined; + (Val) -> + [L, D] = string:tokens(Val, ", "), + Limit = case cuttlefish_bytesize:parse(L) of + Sz when is_integer(Sz) -> Sz; + {error, Reason} -> error(Reason) + end, + Duration = case cuttlefish_duration:parse(D, s) of + Secs when is_integer(Secs) -> Secs; + {error, Reason1} -> error(Reason1) + end, + {Limit, Duration} + end, + + CheckOrigin = fun(S) -> + Origins = string:tokens(S, ","), + [ list_to_binary(string:trim(O)) || O <- Origins] + end, + + WsOpts = fun(Prefix) -> + case cuttlefish_variable:filter_by_prefix(Prefix ++ ".check_origins", Conf) of + [] -> undefined; + Rules -> + OriginList = [CheckOrigin(Rule) || {_, Rule} <- Rules], + lists:flatten(OriginList) + end + end, + + LisOpts = fun(Prefix) -> + Filter([{acceptors, cuttlefish:conf_get(Prefix ++ ".acceptors", Conf)}, + {mqtt_path, cuttlefish:conf_get(Prefix ++ ".mqtt_path", Conf, undefined)}, + {max_connections, cuttlefish:conf_get(Prefix ++ ".max_connections", Conf)}, + {max_conn_rate, cuttlefish:conf_get(Prefix ++ ".max_conn_rate", Conf, undefined)}, + {active_n, cuttlefish:conf_get(Prefix ++ ".active_n", Conf, undefined)}, + {tune_buffer, cuttlefish:conf_get(Prefix ++ ".tune_buffer", Conf, undefined)}, + {zone, Atom(cuttlefish:conf_get(Prefix ++ ".zone", Conf, undefined))}, + {rate_limit, RateLimit(cuttlefish:conf_get(Prefix ++ ".rate_limit", Conf, undefined))}, + {proxy_protocol, cuttlefish:conf_get(Prefix ++ ".proxy_protocol", Conf, undefined)}, + {proxy_address_header, list_to_binary(string:lowercase(cuttlefish:conf_get(Prefix ++ ".proxy_address_header", Conf, "")))}, + {proxy_port_header, list_to_binary(string:lowercase(cuttlefish:conf_get(Prefix ++ ".proxy_port_header", Conf, "")))}, + {proxy_protocol_timeout, cuttlefish:conf_get(Prefix ++ ".proxy_protocol_timeout", Conf, undefined)}, + {fail_if_no_subprotocol, cuttlefish:conf_get(Prefix ++ ".fail_if_no_subprotocol", Conf, undefined)}, + {supported_subprotocols, string:tokens(cuttlefish:conf_get(Prefix ++ ".supported_subprotocols", Conf, ""), ", ")}, + {peer_cert_as_username, cuttlefish:conf_get(Prefix ++ ".peer_cert_as_username", Conf, undefined)}, + {peer_cert_as_clientid, cuttlefish:conf_get(Prefix ++ ".peer_cert_as_clientid", Conf, undefined)}, + {compress, cuttlefish:conf_get(Prefix ++ ".compress", Conf, undefined)}, + {idle_timeout, cuttlefish:conf_get(Prefix ++ ".idle_timeout", Conf, undefined)}, + {max_frame_size, cuttlefish:conf_get(Prefix ++ ".max_frame_size", Conf, undefined)}, + {mqtt_piggyback, cuttlefish:conf_get(Prefix ++ ".mqtt_piggyback", Conf, undefined)}, + {check_origin_enable, cuttlefish:conf_get(Prefix ++ ".check_origin_enable", Conf, undefined)}, + {allow_origin_absence, cuttlefish:conf_get(Prefix ++ ".allow_origin_absence", Conf, undefined)}, + {check_origins, WsOpts(Prefix)} | AccOpts(Prefix)]) + end, + DeflateOpts = fun(Prefix) -> + Filter([{level, cuttlefish:conf_get(Prefix ++ ".deflate_opts.level", Conf, undefined)}, + {mem_level, cuttlefish:conf_get(Prefix ++ ".deflate_opts.mem_level", Conf, undefined)}, + {strategy, cuttlefish:conf_get(Prefix ++ ".deflate_opts.strategy", Conf, undefined)}, + {server_context_takeover, cuttlefish:conf_get(Prefix ++ ".deflate_opts.server_context_takeover", Conf, undefined)}, + {client_context_takeover, cuttlefish:conf_get(Prefix ++ ".deflate_opts.client_context_takeover", Conf, undefined)}, + {server_max_windows_bits, cuttlefish:conf_get(Prefix ++ ".deflate_opts.server_max_window_bits", Conf, undefined)}, + {client_max_windows_bits, cuttlefish:conf_get(Prefix ++ ".deflate_opts.client_max_window_bits", Conf, undefined)}]) + end, + TcpOpts = fun(Prefix) -> + Filter([{backlog, cuttlefish:conf_get(Prefix ++ ".backlog", Conf, undefined)}, + {send_timeout, cuttlefish:conf_get(Prefix ++ ".send_timeout", Conf, undefined)}, + {send_timeout_close, cuttlefish:conf_get(Prefix ++ ".send_timeout_close", Conf, undefined)}, + {recbuf, cuttlefish:conf_get(Prefix ++ ".recbuf", Conf, undefined)}, + {sndbuf, cuttlefish:conf_get(Prefix ++ ".sndbuf", Conf, undefined)}, + {buffer, cuttlefish:conf_get(Prefix ++ ".buffer", Conf, undefined)}, + {high_watermark, cuttlefish:conf_get(Prefix ++ ".high_watermark", Conf, undefined)}, + {nodelay, cuttlefish:conf_get(Prefix ++ ".nodelay", Conf, true)}, + {reuseaddr, cuttlefish:conf_get(Prefix ++ ".reuseaddr", Conf, undefined)}]) + end, + SplitFun = fun(undefined) -> undefined; (S) -> string:tokens(S, ",") end, + MapPSKCiphers = fun(PSKCiphers) -> + lists:map( + fun("PSK-AES128-CBC-SHA") -> {psk, aes_128_cbc, sha}; + ("PSK-AES256-CBC-SHA") -> {psk, aes_256_cbc, sha}; + ("PSK-3DES-EDE-CBC-SHA") -> {psk, '3des_ede_cbc', sha}; + ("PSK-RC4-SHA") -> {psk, rc4_128, sha} + end, PSKCiphers) + end, + SslOpts = fun(Prefix) -> + Versions = case SplitFun(cuttlefish:conf_get(Prefix ++ ".tls_versions", Conf, undefined)) of + undefined -> undefined; + L -> [list_to_atom(V) || V <- L] + end, + TLSCiphers = cuttlefish:conf_get(Prefix++".ciphers", Conf, undefined), + PSKCiphers = cuttlefish:conf_get(Prefix++".psk_ciphers", Conf, undefined), + Ciphers = + case {TLSCiphers, PSKCiphers} of + {undefined, undefined} -> + cuttlefish:invalid(Prefix++".ciphers or "++Prefix++".psk_ciphers is absent"); + {TLSCiphers, undefined} -> + SplitFun(TLSCiphers); + {undefined, PSKCiphers} -> + MapPSKCiphers(SplitFun(PSKCiphers)); + {_TLSCiphers, _PSKCiphers} -> + cuttlefish:invalid(Prefix++".ciphers and "++Prefix++".psk_ciphers cannot be configured at the same time") + end, + UserLookupFun = + case PSKCiphers of + undefined -> undefined; + _ -> {fun emqx_psk:lookup/3, <<>>} + end, + Filter([{versions, Versions}, + {ciphers, Ciphers}, + {user_lookup_fun, UserLookupFun}, + {handshake_timeout, cuttlefish:conf_get(Prefix ++ ".handshake_timeout", Conf, undefined)}, + {depth, cuttlefish:conf_get(Prefix ++ ".depth", Conf, undefined)}, + {password, cuttlefish:conf_get(Prefix ++ ".key_password", Conf, undefined)}, + {dhfile, cuttlefish:conf_get(Prefix ++ ".dhfile", Conf, undefined)}, + {keyfile, cuttlefish:conf_get(Prefix ++ ".keyfile", Conf, undefined)}, + {certfile, cuttlefish:conf_get(Prefix ++ ".certfile", Conf, undefined)}, + {cacertfile, cuttlefish:conf_get(Prefix ++ ".cacertfile", Conf, undefined)}, + {verify, cuttlefish:conf_get(Prefix ++ ".verify", Conf, undefined)}, + {fail_if_no_peer_cert, cuttlefish:conf_get(Prefix ++ ".fail_if_no_peer_cert", Conf, undefined)}, + {secure_renegotiate, cuttlefish:conf_get(Prefix ++ ".secure_renegotiate", Conf, undefined)}, + {reuse_sessions, cuttlefish:conf_get(Prefix ++ ".reuse_sessions", Conf, undefined)}, + {honor_cipher_order, cuttlefish:conf_get(Prefix ++ ".honor_cipher_order", Conf, undefined)}]) + end, + + Listen_fix = fun({Ip, Port}) -> case inet:parse_address(Ip) of + {ok, R} -> {R, Port}; + _ -> {Ip, Port} + end; + (Other) -> Other + end, + + TcpListeners = fun(Type, Name) -> + Prefix = string:join(["listener", Type, Name], "."), + ListenOnN = case cuttlefish:conf_get(Prefix, Conf, undefined) of + undefined -> []; + ListenOn -> Listen_fix(ListenOn) + end, + [#{ proto => Atom(Type) + , name => Name + , listen_on => ListenOnN + , opts => [ {deflate_options, DeflateOpts(Prefix)} + , {tcp_options, TcpOpts(Prefix)} + | LisOpts(Prefix) + ] + } + ] + end, + SslListeners = fun(Type, Name) -> + Prefix = string:join(["listener", Type, Name], "."), + case cuttlefish:conf_get(Prefix, Conf, undefined) of + undefined -> + []; + ListenOn -> + [#{ proto => Atom(Type) + , name => Name + , listen_on => Listen_fix(ListenOn) + , opts => [ {deflate_options, DeflateOpts(Prefix)} + , {tcp_options, TcpOpts(Prefix)} + , {ssl_options, SslOpts(Prefix)} + | LisOpts(Prefix) + ] + } + ] + end + end, + + lists:flatten([TcpListeners(Type, Name) || {["listener", Type, Name], ListenOn} + <- cuttlefish_variable:filter_by_prefix("listener.tcp", Conf) + ++ cuttlefish_variable:filter_by_prefix("listener.ws", Conf)] + ++ + [SslListeners(Type, Name) || {["listener", Type, Name], ListenOn} + <- cuttlefish_variable:filter_by_prefix("listener.ssl", Conf) + ++ cuttlefish_variable:filter_by_prefix("listener.wss", Conf)]) +end}. + +%%-------------------------------------------------------------------- +%% Modules +%%-------------------------------------------------------------------- + +{mapping, "modules.loaded_file", "emqx.modules_loaded_file", [ + {datatype, string} +]}. + +{mapping, "module.presence.qos", "emqx.modules", [ + {default, 1}, + {datatype, integer}, + {validators, ["range:0-2"]} +]}. + +{mapping, "module.subscription.$id.topic", "emqx.modules", [ + {datatype, string} +]}. + +{mapping, "module.subscription.$id.qos", "emqx.modules", [ + {default, 1}, + {datatype, integer}, + {validators, ["range:0-2"]} +]}. + +{mapping, "module.subscription.$id.nl", "emqx.modules", [ + {default, 0}, + {datatype, integer}, + {validators, ["range:0-1"]} +]}. + +{mapping, "module.subscription.$id.rap", "emqx.modules", [ + {default, 0}, + {datatype, integer}, + {validators, ["range:0-1"]} +]}. + +{mapping, "module.subscription.$id.rh", "emqx.modules", [ + {default, 0}, + {datatype, integer}, + {validators, ["range:0-2"]} +]}. + +{mapping, "module.rewrite.rule.$id", "emqx.modules", [ + {datatype, string} +]}. + +{mapping, "module.rewrite.pub.rule.$id", "emqx.modules", [ + {datatype, string} +]}. + +{mapping, "module.rewrite.sub.rule.$id", "emqx.modules", [ + {datatype, string} +]}. + +{translation, "emqx.modules", fun(Conf, _, Conf1) -> + Subscriptions = fun() -> + List = cuttlefish_variable:filter_by_prefix("module.subscription", Conf), + TopicList = [{N, Topic}|| {[_,"subscription",N,"topic"], Topic} <- List], + [{iolist_to_binary(T), #{ qos => cuttlefish:conf_get("module.subscription." ++ N ++ ".qos", Conf, 0), + nl => cuttlefish:conf_get("module.subscription." ++ N ++ ".nl", Conf, 0), + rap => cuttlefish:conf_get("module.subscription." ++ N ++ ".rap", Conf, 0), + rh => cuttlefish:conf_get("module.subscription." ++ N ++ ".rh", Conf, 0) + }} || {N, T} <- TopicList] + end, + Rewrites = fun() -> + Rules = cuttlefish_variable:filter_by_prefix("module.rewrite.rule", Conf), + PubRules = cuttlefish_variable:filter_by_prefix("module.rewrite.pub.rule", Conf), + SubRules = cuttlefish_variable:filter_by_prefix("module.rewrite.sub.rule", Conf), + TotalRules = lists:append( + [ {["module", "rewrite", "pub", "rule", I], Rule} || {["module", "rewrite", "rule", I], Rule} <- Rules] ++ PubRules, + [ {["module", "rewrite", "sub", "rule", I], Rule} || {["module", "rewrite", "rule", I], Rule} <- Rules] ++ SubRules + ), + lists:map(fun({[_, "rewrite", PubOrSub, "rule", I], Rule}) -> + [Topic, Re, Dest] = string:tokens(Rule, " "), + {rewrite, list_to_atom(PubOrSub), list_to_binary(Topic), list_to_binary(Re), list_to_binary(Dest)} + end, TotalRules) + end, + lists:append([ + [{emqx_mod_presence, [{qos, cuttlefish:conf_get("module.presence.qos", Conf, 1)}]}], + [{emqx_mod_subscription, Subscriptions()}], + [{emqx_mod_rewrite, Rewrites()}], + [{emqx_mod_topic_metrics, []}], + [{emqx_mod_delayed, []}], + [{emqx_mod_acl_internal, [{acl_file, cuttlefish:conf_get("acl_file", Conf1)}]}] + ]) +end}. + +%%------------------------------------------------------------------- +%% Plugins +%%------------------------------------------------------------------- + +{mapping, "plugins.etc_dir", "emqx.plugins_etc_dir", [ + {datatype, string} +]}. + +{mapping, "plugins.loaded_file", "emqx.plugins_loaded_file", [ + {datatype, string} +]}. + +{mapping, "plugins.expand_plugins_dir", "emqx.expand_plugins_dir", [ + {datatype, string} +]}. + +%%-------------------------------------------------------------------- +%% Broker +%%-------------------------------------------------------------------- + +{mapping, "broker.sys_interval", "emqx.broker_sys_interval", [ + {datatype, {duration, ms}}, + {default, "1m"} +]}. + +{mapping, "broker.sys_heartbeat", "emqx.broker_sys_heartbeat", [ + {datatype, {duration, ms}}, + {default, "30s"} +]}. + +{mapping, "broker.enable_session_registry", "emqx.enable_session_registry", [ + {default, on}, + {datatype, flag} +]}. + +{mapping, "broker.session_locking_strategy", "emqx.session_locking_strategy", [ + {default, quorum}, + {datatype, {enum, [local,leader,quorum,all]}} +]}. + +%% @doc Shared Subscription Dispatch Strategy. +{mapping, "broker.shared_subscription_strategy", "emqx.shared_subscription_strategy", [ + {default, round_robin}, + {datatype, + {enum, + [random, %% randomly pick a subscriber + round_robin, %% round robin alive subscribers one message after another + sticky, %% pick a random subscriber and stick to it + hash, %% hash client ID to a group member + hash_clientid, + hash_topic + ]}} +]}. + +%% @doc Enable or disable shared dispatch acknowledgement for QoS1 and QoS2 messages +{mapping, "broker.shared_dispatch_ack_enabled", "emqx.shared_dispatch_ack_enabled", + [ {default, false}, + {datatype, {enum, [true, false]}} + ]}. + +{mapping, "broker.route_batch_clean", "emqx.route_batch_clean", [ + {default, on}, + {datatype, flag} +]}. + +%% @doc Performance toggle for subscribe/unsubscribe wildcard topic. +%% Change this toggle only when there are many wildcard topics. +%% key: mnesia translational updates with per-key locks. recommended for single node setup. +%% tab: mnesia translational updates with table lock. recommended for multi-nodes setup. +%% global: global lock protected updates. recommended for larger cluster. +%% NOTE: when changing from/to 'global' lock, it requires all nodes in the cluster +%% +{mapping, "broker.perf.route_lock_type", "emqx.route_lock_type", [ + {default, key}, + {datatype, {enum, [key, tab, global]}} +]}. + +%% @doc Enable trie path compaction. +%% Enabling it significantly improves wildcard topic subscribe rate, +%% if wildcard topics have unique prefixes like: 'sensor/{{id}}/+/', +%% where ID is unique per subscriber. +%% +%% Topic match performance (when publishing) may degrade if messages +%% are mostly published to topics with large number of levels. +%% +%% NOTE: This is a cluster-wide configuration. +%% It rquires all nodes to be stopped before changing it. +{mapping, "broker.perf.trie_compaction", "emqx.trie_compaction", [ + {default, true}, + {datatype, {enum, [true, false]}} +]}. + +%%-------------------------------------------------------------------- +%% System Monitor +%%-------------------------------------------------------------------- + +%% @doc Long GC, don't monitor in production mode for: +%% https://github.com/erlang/otp/blob/feb45017da36be78d4c5784d758ede619fa7bfd3/erts/emulator/beam/erl_gc.c#L421 +{mapping, "sysmon.long_gc", "emqx.sysmon", [ + {default, 0}, + {datatype, [integer, {duration, ms}]} +]}. + +%% @doc Long Schedule(ms) +{mapping, "sysmon.long_schedule", "emqx.sysmon", [ + {default, 240}, + {datatype, [integer, {duration, ms}]} +]}. + +%% @doc Large Heap +{mapping, "sysmon.large_heap", "emqx.sysmon", [ + {default, "8MB"}, + {datatype, bytesize} +]}. + +%% @doc Monitor Busy Port +{mapping, "sysmon.busy_port", "emqx.sysmon", [ + {default, false}, + {datatype, {enum, [true, false]}} +]}. + +%% @doc Monitor Busy Dist Port +{mapping, "sysmon.busy_dist_port", "emqx.sysmon", [ + {default, true}, + {datatype, {enum, [true, false]}} +]}. + +{translation, "emqx.sysmon", fun(Conf) -> + Configs = cuttlefish_variable:filter_by_prefix("sysmon", Conf), + [{list_to_atom(Name), Value} || {[_, Name], Value} <- Configs] +end}. + +%%-------------------------------------------------------------------- +%% Operating System Monitor +%%-------------------------------------------------------------------- + +{mapping, "os_mon.cpu_check_interval", "emqx.os_mon", [ + {default, 60}, + {datatype, {duration, s}} +]}. + +{mapping, "os_mon.cpu_high_watermark", "emqx.os_mon", [ + {default, "80%"}, + {datatype, {percent, float}} +]}. + +{mapping, "os_mon.cpu_low_watermark", "emqx.os_mon", [ + {default, "60%"}, + {datatype, {percent, float}} +]}. + +{mapping, "os_mon.mem_check_interval", "emqx.os_mon", [ + {default, 60}, + {datatype, {duration, s}} +]}. + +{mapping, "os_mon.sysmem_high_watermark", "emqx.os_mon", [ + {default, "70%"}, + {datatype, {percent, float}} +]}. + +{mapping, "os_mon.procmem_high_watermark", "emqx.os_mon", [ + {default, "5%"}, + {datatype, {percent, float}} +]}. + +{translation, "emqx.os_mon", fun(Conf) -> + [{cpu_check_interval, cuttlefish:conf_get("os_mon.cpu_check_interval", Conf)}, + {cpu_high_watermark, cuttlefish:conf_get("os_mon.cpu_high_watermark", Conf) * 100}, + {cpu_low_watermark, cuttlefish:conf_get("os_mon.cpu_low_watermark", Conf) * 100}, + {mem_check_interval, cuttlefish:conf_get("os_mon.mem_check_interval", Conf)}, + {sysmem_high_watermark, cuttlefish:conf_get("os_mon.sysmem_high_watermark", Conf) * 100}, + {procmem_high_watermark, cuttlefish:conf_get("os_mon.procmem_high_watermark", Conf) * 100}] +end}. + +%%-------------------------------------------------------------------- +%% VM Monitor +%%-------------------------------------------------------------------- +{mapping, "vm_mon.check_interval", "emqx.vm_mon", [ + {default, 30}, + {datatype, {duration, s}} +]}. + +{mapping, "vm_mon.process_high_watermark", "emqx.vm_mon", [ + {default, "80%"}, + {datatype, {percent, float}} +]}. + +{mapping, "vm_mon.process_low_watermark", "emqx.vm_mon", [ + {default, "60%"}, + {datatype, {percent, float}} +]}. + +{translation, "emqx.vm_mon", fun(Conf) -> + [{check_interval, cuttlefish:conf_get("vm_mon.check_interval", Conf)}, + {process_high_watermark, cuttlefish:conf_get("vm_mon.process_high_watermark", Conf) * 100}, + {process_low_watermark, cuttlefish:conf_get("vm_mon.process_low_watermark", Conf) * 100}] +end}. + +%%-------------------------------------------------------------------- +%% Alarm +%%-------------------------------------------------------------------- +{mapping, "alarm.actions", "emqx.alarm", [ + {default, "log,publish"}, + {datatype, string} +]}. + +{mapping, "alarm.size_limit", "emqx.alarm", [ + {default, 1000}, + {datatype, integer} +]}. + +{mapping, "alarm.validity_period", "emqx.alarm", [ + {default, "24h"}, + {datatype, {duration, s}} +]}. + +{translation, "emqx.alarm", fun(Conf) -> + [{actions, [list_to_atom(Action) || Action <- string:tokens(cuttlefish:conf_get("alarm.actions", Conf), ",")]}, + {size_limit, cuttlefish:conf_get("alarm.size_limit", Conf)}, + {validity_period, cuttlefish:conf_get("alarm.validity_period", Conf)}] +end}. + +%%-------------------------------------------------------------------- +%% Telemetry +%%-------------------------------------------------------------------- +{mapping, "telemetry.enabled", "emqx.telemetry", [ + {default, false}, + {datatype, {enum, [true, false]}} +]}. + +{mapping, "telemetry.url", "emqx.telemetry", [ + {default, "https://telemetry-emqx-io.bigpar.vercel.app/api/telemetry"}, + {datatype, string} +]}. + +{mapping, "telemetry.report_interval", "emqx.telemetry", [ + {default, "7d"}, + {datatype, {duration, s}} +]}. + +{translation, "emqx.telemetry", fun(Conf) -> + [ {enabled, cuttlefish:conf_get("telemetry.enabled", Conf)} + , {url, cuttlefish:conf_get("telemetry.url", Conf)} + , {report_interval, cuttlefish:conf_get("telemetry.report_interval", Conf)} + ] +end}. diff --git a/rebar.config b/rebar.config index 1d871d944..4c086c312 100644 --- a/rebar.config +++ b/rebar.config @@ -42,7 +42,7 @@ , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.10.2"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.1"}}} , {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {tag, "v4.0.1"}}} % TODO: delete when all apps moved to hocon - , {minirest, {git, "https://github.com/emqx/minirest", {tag, "0.3.5"}}} + , {minirest, {git, "https://github.com/emqx/minirest", {tag, "0.3.6"}}} , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.1"}}} , {replayq, {git, "https://github.com/emqx/replayq", {tag, "0.3.2"}}} , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {branch, "2.0.4"}}} diff --git a/rebar.config.erl b/rebar.config.erl index 987f72c67..d0822f3b2 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -127,23 +127,30 @@ prod_compile_opts() -> prod_overrides() -> [{add, [ {erl_opts, [deterministic]}]}]. +relup_deps(Profile) -> + {post_hooks, [{"(linux|darwin|solaris|freebsd|netbsd|openbsd)", compile, "scripts/inject-deps.escript " ++ atom_to_list(Profile)}]}. + profiles() -> Vsn = get_vsn(), [ {'emqx', [ {erl_opts, prod_compile_opts()} , {relx, relx(Vsn, cloud, bin)} , {overrides, prod_overrides()} + , relup_deps('emqx') ]} , {'emqx-pkg', [ {erl_opts, prod_compile_opts()} , {relx, relx(Vsn, cloud, pkg)} , {overrides, prod_overrides()} + , relup_deps('emqx-pkg') ]} , {'emqx-edge', [ {erl_opts, prod_compile_opts()} , {relx, relx(Vsn, edge, bin)} , {overrides, prod_overrides()} + , relup_deps('emqx-edge') ]} , {'emqx-edge-pkg', [ {erl_opts, prod_compile_opts()} , {relx, relx(Vsn, edge, pkg)} , {overrides, prod_overrides()} + , relup_deps('emqx-edge-pkg') ]} , {check, [ {erl_opts, common_compile_opts()} ]} diff --git a/scripts/inject-deps.escript b/scripts/inject-deps.escript new file mode 100755 index 000000000..0b6371342 --- /dev/null +++ b/scripts/inject-deps.escript @@ -0,0 +1,157 @@ +#!/usr/bin/env escript + +%% This script injects implicit relup dependencies for emqx applications. +%% +%% By 'implicit', it means that it is not feasible to define application +%% dependencies in .app.src files. +%% +%% For instance, during upgrade/downgrade, emqx_dashboard usually requires +%% a restart after (but not before) all plugins are upgraded (and maybe +%% restarted), however, the dependencies are not resolvable at build time +%% when relup is generated. +%% +%% This script is to be executed after compile, with the profile given as the +%% first argument. For each dependency, it modifies the .app file to +%% have the `relup_deps` list extended to application attributes. +%% +%% The `relup_deps` application attribute is then picked up by (EMQ's fork of) +%% `relx` when top-sorting apps to generate relup instructions + +-mode(compile). + +usage() -> + "Usage: " ++ escript:script_name() ++ " emqx|emqx-edge". + +-type app() :: atom(). +-type deps_overlay() :: {re, string()} | app(). + +%% deps/0 returns the dependency overlays. +%% {re, Pattern} to match application names using regexp pattern +-spec deps(string()) -> [{app(), [deps_overlay()]}]. +deps("emqx-edge" ++ _) -> + %% special case for edge + base_deps() ++ [{{re, ".+"}, [{exclude, App} || App <- edge_excludes()]}]; +deps(_Profile) -> + base_deps(). + +edge_excludes() -> + [ emqx_lwm2m + , emqx_auth_ldap + , emqx_auth_pgsql + , emqx_auth_redis + , emqx_auth_mongo + , emqx_lua_hook + , emqx_exhook + , emqx_exproto + , emqx_prometheus + , emqx_psk_file + ]. + +base_deps() -> + %% make sure emqx_dashboard depends on all other emqx_xxx apps + %% so the appup instructions for emqx_dashboard is always the last + %% to be executed + [ {emqx_dashboard, [{re, "emqx_.*"}]} + , {emqx_management, [{re, "emqx_.*"}, {exclude, emqx_dashboard}]} + , {{re, "emqx_.*"}, [emqx]} + ]. + +main([Profile | _]) -> + ok = inject(Profile); +main(_Args) -> + io:format(standard_error, "~s", [usage()]), + erlang:halt(1). + +expand_names({Name, Deps}, AppNames) -> + Names = match_pattern(Name, AppNames), + [{N, Deps} || N <- Names]. + +%% merge k-v pairs with v1 ++ v2 +merge([], Acc) -> Acc; +merge([{K, V0} | Rest], Acc) -> + V = case lists:keyfind(K, 1, Acc) of + {K, V1} -> V1 ++ V0; + false -> V0 + end, + NewAcc = lists:keystore(K, 1, Acc, {K, V}), + merge(Rest, NewAcc). + +expand_deps([], _AppNames, Acc) -> Acc; +expand_deps([{exclude, Dep} | Deps], AppNames, Acc) -> + Matches = expand_deps([Dep], AppNames, []), + expand_deps(Deps, AppNames, Acc -- Matches); +expand_deps([Dep | Deps], AppNames, Acc) -> + NewAcc = add_to_list(Acc, match_pattern(Dep, AppNames)), + expand_deps(Deps, AppNames, NewAcc). + +inject(Profile) -> + LibDir = lib_dir(Profile), + AppNames = list_apps(LibDir), + Deps0 = lists:flatmap(fun(Dep) -> expand_names(Dep, AppNames) end, deps(Profile)), + Deps1 = merge(Deps0, []), + Deps2 = lists:map(fun({Name, DepsX}) -> + NewDeps = expand_deps(DepsX, AppNames, []), + {Name, NewDeps} + end, Deps1), + lists:foreach(fun({App, Deps}) -> inject(App, Deps, LibDir) end, Deps2). + +%% list the profile/lib dir to get all apps +list_apps(LibDir) -> + Apps = filelib:wildcard("*", LibDir), + lists:foldl(fun(App, Acc) -> [App || is_app(LibDir, App)] ++ Acc end, [], Apps). + +is_app(_LibDir, "." ++ _) -> false; %% ignore hidden dir +is_app(LibDir, AppName) -> + Path = filename:join([ebin_dir(LibDir, AppName), AppName ++ ".app"]), + filelib:is_regular(Path) orelse error({unknown_app, AppName, Path}). %% wtf + +lib_dir(Profile) -> + filename:join(["_build", Profile, lib]). + +ebin_dir(LibDir, AppName) -> filename:join([LibDir, AppName, "ebin"]). + +inject(App0, DepsToAdd, LibDir) -> + App = str(App0), + AppEbinDir = ebin_dir(LibDir, App), + [AppFile0] = filelib:wildcard("*.app", AppEbinDir), + AppFile = filename:join(AppEbinDir, AppFile0), + {ok, [{application, AppName, Props}]} = file:consult(AppFile), + Deps0 = case lists:keyfind(relup_deps, 1, Props) of + {_, X} -> X; + false -> [] + end, + %% merge extra deps, but do not self-include + Deps = add_to_list(Deps0, DepsToAdd) -- [App0], + case Deps =:= [] of + true -> ok; + _ -> + NewProps = lists:keystore(relup_deps, 1, Props, {relup_deps, Deps}), + AppSpec = {application, AppName, NewProps}, + AppSpecIoData = io_lib:format("~p.", [AppSpec]), + io:format(user, "updated_relup_deps for ~p~n", [App]), + file:write_file(AppFile, AppSpecIoData) + end. + +str(A) when is_atom(A) -> atom_to_list(A). + +match_pattern({re, Re}, AppNames) -> + Match = fun(AppName) -> re:run(AppName, Re) =/= nomatch end, + AppNamesToAdd = lists:filter(Match, AppNames), + AppsToAdd = lists:map(fun(N) -> list_to_atom(N) end, AppNamesToAdd), + case AppsToAdd =:= [] of + true -> error({nomatch, Re}); + false -> AppsToAdd + end; +match_pattern(NameAtom, AppNames) -> + case lists:member(str(NameAtom), AppNames) of + true -> [NameAtom]; + false -> error({notfound, NameAtom}) + end. + +%% Append elements to list without duplication. No reordering. +add_to_list(List, []) -> List; +add_to_list(List, [H | T]) -> + case lists:member(H, List) of + true -> add_to_list(List, T); + false -> add_to_list(List ++ [H], T) + end.