diff --git a/.ci/fvt_tests/http_server/rebar.config b/.ci/fvt_tests/http_server/rebar.config index 29cc392ea..314679264 100644 --- a/.ci/fvt_tests/http_server/rebar.config +++ b/.ci/fvt_tests/http_server/rebar.config @@ -1,7 +1,7 @@ {erl_opts, [debug_info]}. {deps, [ - {minirest, {git, "https://github.com/emqx/minirest.git", {tag, "0.3.5"}}} + {minirest, {git, "https://github.com/emqx/minirest.git", {tag, "0.3.6"}}} ]}. {shell, [ diff --git a/.ci/fvt_tests/relup.lux b/.ci/fvt_tests/relup.lux index 33c35cb3e..91c11d3ae 100644 --- a/.ci/fvt_tests/relup.lux +++ b/.ci/fvt_tests/relup.lux @@ -43,7 +43,7 @@ !sed -i '/emqx_telemetry/d' data/loaded_plugins !./bin/emqx start - ?EMQ X (.*) is started successfully! + ?EMQ X .* is started successfully! ?SH-PROMPT !./bin/emqx_ctl cluster join emqx@127.0.0.1 @@ -99,6 +99,10 @@ """ ?SH-PROMPT + !./bin/emqx_ctl plugins list | grep emqx_management + ?Plugin\(emqx_management.*active=true\) + ?SH-PROMPT + [shell emqx2] !echo "" > log/emqx.log.1 ?SH-PROMPT @@ -120,6 +124,10 @@ """ ?SH-PROMPT + !./bin/emqx_ctl plugins list | grep emqx_management + ?Plugin\(emqx_management.*active=true\) + ?SH-PROMPT + [shell bench] ???publish complete ??SH-PROMPT: diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 2090f722e..b983eaa67 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -83,6 +83,7 @@ jobs: - name: build env: PYTHON: python + DIAGNOSTIC: 1 run: | $env:PATH = "${{ steps.install_erlang.outputs.erlpath }}\bin;$env:PATH" @@ -168,9 +169,11 @@ jobs: - name: build run: | . $HOME/.kerl/${{ matrix.erl_otp }}/activate - make -C source ensure-rebar3 - sudo cp source/rebar3 /usr/local/bin/rebar3 - make -C source ${{ matrix.profile }}-zip + cd source + make ensure-rebar3 + sudo cp rebar3 /usr/local/bin/rebar3 + rm -rf _build/${{ matrix.profile }}/lib + make ${{ matrix.profile }}-zip - name: test run: | cd source diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index 6c9bbf04a..bf85578c5 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -38,6 +38,11 @@ jobs: run: make ${EMQX_NAME}-zip - name: build deb/rpm packages run: make ${EMQX_NAME}-pkg + - uses: actions/upload-artifact@v1 + if: failure() + with: + name: rebar3.crashdump + path: ./rebar3.crashdump - name: pakcages test run: | export CODE_PATH=$GITHUB_WORKSPACE @@ -94,6 +99,11 @@ jobs: make ensure-rebar3 sudo cp rebar3 /usr/local/bin/rebar3 make ${EMQX_NAME}-zip + - uses: actions/upload-artifact@v1 + if: failure() + with: + name: rebar3.crashdump + path: ./rebar3.crashdump - name: test run: | pkg_name=$(basename _packages/${EMQX_NAME}/emqx-*.zip) diff --git a/apps/emqx/etc/emqx.conf b/apps/emqx/etc/emqx.conf index d2b5fd11d..3387a6439 100644 --- a/apps/emqx/etc/emqx.conf +++ b/apps/emqx/etc/emqx.conf @@ -355,8 +355,8 @@ rpc.port_discovery = stateless ## Number of outgoing RPC connections. ## ## Value: Interger [0-256] -## Defaults to NumberOfCPUSchedulers / 2 when set to 0 -#rpc.tcp_client_num = 0 +## Default = 1 +#rpc.tcp_client_num = 1 ## RCP Client connect timeout. ## diff --git a/apps/emqx/include/emqx_mqtt.hrl b/apps/emqx/include/emqx_mqtt.hrl index e6e9bffe5..5dd9a317c 100644 --- a/apps/emqx/include/emqx_mqtt.hrl +++ b/apps/emqx/include/emqx_mqtt.hrl @@ -30,11 +30,13 @@ %% MQTT Protocol Version and Names %%-------------------------------------------------------------------- +-define(MQTT_SN_PROTO_V1, 1). -define(MQTT_PROTO_V3, 3). -define(MQTT_PROTO_V4, 4). -define(MQTT_PROTO_V5, 5). -define(PROTOCOL_NAMES, [ + {?MQTT_SN_PROTO_V1, <<"MQTT-SN">>}, %% XXX:Compatible with emqx-sn plug-in {?MQTT_PROTO_V3, <<"MQIsdp">>}, {?MQTT_PROTO_V4, <<"MQTT">>}, {?MQTT_PROTO_V5, <<"MQTT">>}]). diff --git a/apps/emqx/src/emqx.appup.src b/apps/emqx/src/emqx.appup.src index b51a7f3b7..4f9f00673 100644 --- a/apps/emqx/src/emqx.appup.src +++ b/apps/emqx/src/emqx.appup.src @@ -1,12 +1,31 @@ %% -*- mode: erlang -*- {VSN, - [{"4.3.2", - [{load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, + [ + {"4.3.4", + [{load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, + {load_module,emqx_app,brutal_purge,soft_purge,[]} + ]}, + {"4.3.3", + [{load_module,emqx_packet,brutal_purge,soft_purge,[]}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, + {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_cm,brutal_purge,soft_purge,[]}, + {load_module,emqx_app,brutal_purge,soft_purge,[]} + ]}, + {"4.3.2", + [{load_module,emqx_packet,brutal_purge,soft_purge,[]}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, + {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_connection,brutal_purge,soft_purge,[]}]}, + {load_module,emqx_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_cm,brutal_purge,soft_purge,[]} + ]}, {"4.3.1", - [{load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, + [{load_module,emqx_packet,brutal_purge,soft_purge,[]}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, + {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, @@ -18,7 +37,9 @@ {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}]}, {"4.3.0", - [{load_module,emqx_logger_jsonfmt,brutal_purge,soft_purge,[]}, + [{load_module,emqx_packet,brutal_purge,soft_purge,[]}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, + {load_module,emqx_logger_jsonfmt,brutal_purge,soft_purge,[]}, {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_congestion,brutal_purge,soft_purge,[]}, {load_module,emqx_connection,brutal_purge,soft_purge,[]}, @@ -34,13 +55,32 @@ {apply,{emqx_metrics,upgrade_retained_delayed_counter_type,[]}}, {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}]}, {<<".*">>,[]}], - [{"4.3.2", - [{load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, + [ + {"4.3.4", + [{load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, + {load_module,emqx_app,brutal_purge,soft_purge,[]} + ]}, + {"4.3.3", + [{load_module,emqx_packet,brutal_purge,soft_purge,[]}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, + {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_cm,brutal_purge,soft_purge,[]}, + {load_module,emqx_app,brutal_purge,soft_purge,[]} + ]}, + {"4.3.2", + [{load_module,emqx_packet,brutal_purge,soft_purge,[]}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, + {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}, {load_module,emqx_channel,brutal_purge,soft_purge,[]}, {load_module,emqx_app,brutal_purge,soft_purge,[]}, - {load_module,emqx_connection,brutal_purge,soft_purge,[]}]}, + {load_module,emqx_connection,brutal_purge,soft_purge,[]}, + {load_module,emqx_cm,brutal_purge,soft_purge,[]} + ]}, {"4.3.1", - [{load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, + [{load_module,emqx_packet,brutal_purge,soft_purge,[]}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, + {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_frame,brutal_purge,soft_purge,[]}, {load_module,emqx_cm,brutal_purge,soft_purge,[]}, @@ -52,7 +92,9 @@ {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]}, {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}]}, {"4.3.0", - [{load_module,emqx_logger_jsonfmt,brutal_purge,soft_purge,[]}, + [{load_module,emqx_packet,brutal_purge,soft_purge,[]}, + {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]}, + {load_module,emqx_logger_jsonfmt,brutal_purge,soft_purge,[]}, {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_connection,brutal_purge,soft_purge,[]}, {load_module,emqx_congestion,brutal_purge,soft_purge,[]}, diff --git a/apps/emqx/src/emqx_cm.erl b/apps/emqx/src/emqx_cm.erl index b15a2ff79..6eb375aba 100644 --- a/apps/emqx/src/emqx_cm.erl +++ b/apps/emqx/src/emqx_cm.erl @@ -294,6 +294,9 @@ do_discard_session(ClientId, Pid) -> _ : {noproc, _} -> % emqx_connection: gen_server:call ?tp(debug, "session_already_gone", #{pid => Pid}), ok; + _ : {'EXIT', {noproc, _}} -> % rpc_call/3 + ?tp(debug, "session_already_gone", #{pid => Pid}), + ok; _ : {{shutdown, _}, _} -> ?tp(debug, "session_already_shutdown", #{pid => Pid}), ok; diff --git a/apps/emqx/src/emqx_shared_sub.erl b/apps/emqx/src/emqx_shared_sub.erl index 4707f63db..97aa778f3 100644 --- a/apps/emqx/src/emqx_shared_sub.erl +++ b/apps/emqx/src/emqx_shared_sub.erl @@ -336,9 +336,13 @@ handle_info({mnesia_table_event, {write, NewRecord, _}}, State = #state{pmon = P #emqx_shared_subscription{subpid = SubPid} = NewRecord, {noreply, update_stats(State#state{pmon = emqx_pmon:monitor(SubPid, PMon)})}; -handle_info({mnesia_table_event, {delete_object, OldRecord, _}}, State = #state{pmon = PMon}) -> - #emqx_shared_subscription{subpid = SubPid} = OldRecord, - {noreply, update_stats(State#state{pmon = emqx_pmon:demonitor(SubPid, PMon)})}; +%% The subscriber may have subscribed multiple topics, so we need to keep monitoring the PID until +%% it `unsubscribed` the last topic. +%% The trick is we don't demonitor the subscriber here, and (after a long time) it will eventually +%% be disconnected. +% handle_info({mnesia_table_event, {delete_object, OldRecord, _}}, State = #state{pmon = PMon}) -> +% #emqx_shared_subscription{subpid = SubPid} = OldRecord, +% {noreply, update_stats(State#state{pmon = emqx_pmon:demonitor(SubPid, PMon)})}; handle_info({mnesia_table_event, _Event}, State) -> {noreply, State}; @@ -348,8 +352,7 @@ handle_info({'DOWN', _MRef, process, SubPid, _Reason}, State = #state{pmon = PMo cleanup_down(SubPid), {noreply, update_stats(State#state{pmon = emqx_pmon:erase(SubPid, PMon)})}; -handle_info(Info, State) -> - ?LOG(error, "Unexpected info: ~p", [Info]), +handle_info(_Info, State) -> {noreply, State}. terminate(_Reason, _State) -> diff --git a/apps/emqx/src/emqx_ws_connection.erl b/apps/emqx/src/emqx_ws_connection.erl index 7bc68c271..d686e1611 100644 --- a/apps/emqx/src/emqx_ws_connection.erl +++ b/apps/emqx/src/emqx_ws_connection.erl @@ -403,7 +403,10 @@ websocket_close(Reason, State) -> terminate(Reason, _Req, #state{channel = Channel}) -> ?LOG(debug, "Terminated due to ~p", [Reason]), - emqx_channel:terminate(Reason, Channel). + emqx_channel:terminate(Reason, Channel); + +terminate(_Reason, _Req, _UnExpectedState) -> + ok. %%-------------------------------------------------------------------- %% Handle call diff --git a/apps/emqx_dashboard/src/emqx_dashboard.appup.src b/apps/emqx_dashboard/src/emqx_dashboard.appup.src new file mode 100644 index 000000000..678bd3b22 --- /dev/null +++ b/apps/emqx_dashboard/src/emqx_dashboard.appup.src @@ -0,0 +1,16 @@ +%% -*- mode: erlang -*- +{VSN, + [ {"4.3.0", + %% load all plugins + %% NOTE: this depends on the fact that emqx_dashboard is always + %% the last application gets upgraded + [ {apply, {emqx_plugins, load, []}} + ]}, + {<<".*">>, []} + ], + [ {"4.3.0", + [ {apply, {emqx_plugins, load, []}} + ]}, + {<<".*">>, []} + ] +}. diff --git a/apps/emqx_exhook/src/emqx_exhook.app.src b/apps/emqx_exhook/src/emqx_exhook.app.src index 452d2a742..e703cfe5e 100644 --- a/apps/emqx_exhook/src/emqx_exhook.app.src +++ b/apps/emqx_exhook/src/emqx_exhook.app.src @@ -1,6 +1,6 @@ {application, emqx_exhook, [{description, "EMQ X Extension for Hook"}, - {vsn, "4.3.1"}, + {vsn, "4.3.2"}, {modules, []}, {registered, []}, {mod, {emqx_exhook_app, []}}, diff --git a/apps/emqx_exhook/src/emqx_exhook.appup.src b/apps/emqx_exhook/src/emqx_exhook.appup.src index 2811c1554..26e84d88f 100644 --- a/apps/emqx_exhook/src/emqx_exhook.appup.src +++ b/apps/emqx_exhook/src/emqx_exhook.appup.src @@ -1,14 +1,22 @@ %% -*-: erlang -*- {VSN, [ + {"4.3.1", [ + {load_module, emqx_exhook_server, brutal_purge, soft_purge, []} + ]}, {"4.3.0", [ - {load_module, emqx_exhook_pb, brutal_purge, soft_purge, []} + {load_module, emqx_exhook_pb, brutal_purge, soft_purge, []}, + {load_module, emqx_exhook_server, brutal_purge, soft_purge, []} ]}, {<<".*">>, []} ], [ + {"4.3.1", [ + {load_module, emqx_exhook_server, brutal_purge, soft_purge, []} + ]}, {"4.3.0", [ - {load_module, emqx_exhook_pb, brutal_purge, soft_purge, []} + {load_module, emqx_exhook_pb, brutal_purge, soft_purge, []}, + {load_module, emqx_exhook_server, brutal_purge, soft_purge, []} ]}, {<<".*">>, []} ] diff --git a/apps/emqx_exhook/src/emqx_exhook_server.erl b/apps/emqx_exhook/src/emqx_exhook_server.erl index 848a3f59d..f4965e4ca 100644 --- a/apps/emqx_exhook/src/emqx_exhook_server.erl +++ b/apps/emqx_exhook/src/emqx_exhook_server.erl @@ -122,7 +122,7 @@ channel_opts(Opts) -> Scheme = proplists:get_value(scheme, Opts), Host = proplists:get_value(host, Opts), Port = proplists:get_value(port, Opts), - SvrAddr = lists:flatten(io_lib:format("~s://~s:~w", [Scheme, Host, Port])), + SvrAddr = format_http_uri(Scheme, Host, Port), ClientOpts = case Scheme of https -> SslOpts = lists:keydelete(ssl, 1, proplists:get_value(ssl_options, Opts, [])), @@ -133,6 +133,13 @@ channel_opts(Opts) -> end, {SvrAddr, ClientOpts}. +format_http_uri(Scheme, Host0, Port) -> + Host = case is_tuple(Host0) of + true -> inet:ntoa(Host0); + _ -> Host0 + end, + lists:flatten(io_lib:format("~s://~s:~w", [Scheme, Host, Port])). + -spec unload(server()) -> ok. unload(#server{name = Name, hookspec = HookSpecs}) -> _ = do_deinit(Name), diff --git a/apps/emqx_lua_hook/.gitignore b/apps/emqx_lua_hook/.gitignore deleted file mode 100644 index af616ea1a..000000000 --- a/apps/emqx_lua_hook/.gitignore +++ /dev/null @@ -1,19 +0,0 @@ -deps/ -ebin/ -_rel/ -.erlang.mk/ -*.d -data/ -*.iml -.idea/ -logs/ -*.beam -.DS_Store -erlang.mk -_build/ -rebar.lock -rebar3.crashdump -bbmustache/ -*.conf.rendered -.rebar3 -*.swp diff --git a/apps/emqx_lua_hook/README.md b/apps/emqx_lua_hook/README.md deleted file mode 100644 index a3f65a094..000000000 --- a/apps/emqx_lua_hook/README.md +++ /dev/null @@ -1,338 +0,0 @@ - -# emqx-lua-hook - -This plugin makes it possible to write hooks in lua scripts. - -Lua virtual machine is implemented by [luerl](https://github.com/rvirding/luerl) which supports Lua 5.2. Following features may not work properly: -* label and goto -* tail-call optimisation in return -* only limited standard libraries -* proper handling of `__metatable` - -For the supported functions, please refer to luerl's [project page](https://github.com/rvirding/luerl). - -Lua scripts are stored in 'data/scripts' directory, and will be loaded automatically. If a script is changed during runtime, it should be reloaded to take effect. - -Each lua script could export several functions binding with emqx hooks, triggered by message publish, topic subscribe, client connect, etc. Different lua scripts may export same type function, binding with a same event. But their order being triggered is not guaranteed. - -To start this plugin, run following command: - -```shell -bin/emqx_ctl plugins load emqx_lua_hook -``` - - -## NOTE - -* Since lua VM is run on erlang VM, its performance is poor. Please do NOT write long or complicated lua scripts which may degrade entire system. -* It's hard to debug lua script in emqx environment. Recommended to unit test your lua script in your host first. If everything is OK, deploy it to emqx 'data/scripts' directory. -* Global variable will lost its value for each call. Do NOT use global variable in lua scripts. - - -# Example - -Suppose your emqx is installed in /emqx, and the lua script directory should be /emqx/data/scripts. - -Make a new file called "test.lua" and put following code into this file: - -```lua -function on_message_publish(clientid, username, topic, payload, qos, retain) - return topic, "hello", qos, retain -end - -function register_hook() - return "on_message_publish" -end -``` - -Execute following command to start emq-lua-hook and load scripts in 'data/scripts' directory. - -``` -/emqx/bin/emqx_ctl plugins load emqx_lua_hook -``` - -Now let's take a look at what will happend. - -- Start a mqtt client, such as mqtt.fx. -- Subscribe a topic="a/b". -- Send a message, topic="a/b", payload="123" -- Subscriber will get a message with topic="a/b" and payload="hello". test.lua modifies the payload. - -If there are "test1.lua", "test2.lua" and "test3.lua" in /emqx/data/scripts, all these files will be loaded once emq-lua-hook get started. - -If test2.lua has been changed, restart emq-lua-hook to reload all scripts, or execute following command to reload test2.lua only: - -``` -/emqx/bin/emqx_ctl luahook reload test2.lua -``` - - -# Hook API - -You can find all example codes in the `examples.lua` file. - -## on_client_connected - -```lua -function on_client_connected(clientId, userName, returncode) - return 0 -end -``` -This API is called after a mqtt client has establish a connection with broker. - -### Input -* clientid : a string, mqtt client id. -* username : a string mqtt username -* returncode : a string, has following values - - success : Connection accepted - - Others is failed reason - -### Output -Needless - -## on_client_disconnected - -```lua -function on_client_disconnected(clientId, username, error) - return -end -``` -This API is called after a mqtt client has disconnected. - -### Input -* clientId : a string, mqtt client id. -* username : a string mqtt username -* error : a string, denote the disconnection reason. - -### Output -Needless - -## on_client_subscribe - -```lua -function on_client_subscribe(clientId, username, topic) - -- do your job here - if some_condition then - return new_topic - else - return false - end -end -``` -This API is called before mqtt engine process client's subscribe command. It is possible to change topic or cancel it. - -### Input -* clientid : a string, mqtt client id. -* username : a string mqtt username -* topic : a string, mqtt message's topic - -### Output -* new_topic : a string, change mqtt message's topic -* false : cancel subscription - - -## on_client_unsubscribe - -```lua - function on_client_unsubscribe(clientId, username, topic) - -- do your job here - if some_condition then - return new_topic - else - return false - end -end -``` -This API is called before mqtt engine process client's unsubscribe command. It is possible to change topic or cancel it. - -### Input -* clientid : a string, mqtt client id. -* username : a string mqtt username -* topic : a string, mqtt message's topic - -### Output -* new_topic : a string, change mqtt message's topic -* false : cancel unsubscription - - -## on_session_subscribed - -```lua -function on_session_subscribed(ClientId, Username, Topic) - return -end -``` -This API is called after a subscription has been done. - -### Input -* clientid : a string, mqtt client id. -* username : a string mqtt username -* topic : a string, mqtt's topic filter. - -### Output -Needless - - -## on_session_unsubscribed - -```lua -function on_session_unsubscribed(clientid, username, topic) - return -end -``` -This API is called after a unsubscription has been done. - -### Input -* clientid : a string, mqtt client id. -* username : a string mqtt username -* topic : a string, mqtt's topic filter. - -### Output -Needless - -## on_message_delivered - -```lua -function on_message_delivered(clientid, username, topic, payload, qos, retain) - -- do your job here - return topic, payload, qos, retain -end -``` -This API is called after a message has been pushed to mqtt clients. - -### Input -* clientId : a string, mqtt client id. -* username : a string mqtt username -* topic : a string, mqtt message's topic -* payload : a string, mqtt message's payload -* qos : a number, mqtt message's QOS (0, 1, 2) -* retain : a boolean, mqtt message's retain flag - -### Output -Needless - -## on_message_acked - -```lua -function on_message_acked(clientId, username, topic, payload, qos, retain) - return -end -``` -This API is called after a message has been acknowledged. - -### Input -* clientId : a string, mqtt client id. -* username : a string mqtt username -* topic : a string, mqtt message's topic -* payload : a string, mqtt message's payload -* qos : a number, mqtt message's QOS (0, 1, 2) -* retain : a boolean, mqtt message's retain flag - -### Output -Needless - -## on_message_publish - -```lua -function on_message_publish(clientid, username, topic, payload, qos, retain) - -- do your job here - if some_condition then - return new_topic, new_payload, new_qos, new_retain - else - return false - end -end -``` -This API is called before publishing message into mqtt engine. It's possible to change message or cancel publish in this API. - -### Input -* clientid : a string, mqtt client id of publisher. -* username : a string, mqtt username of publisher -* topic : a string, mqtt message's topic -* payload : a string, mqtt message's payload -* qos : a number, mqtt message's QOS (0, 1, 2) -* retain : a boolean, mqtt message's retain flag - -### Output -* new_topic : a string, change mqtt message's topic -* new_payload : a string, change mqtt message's payload -* new_qos : a number, change mqtt message's QOS -* new_retain : a boolean, change mqtt message's retain flag -* false : cancel publishing this mqtt message - -## register_hook - -```lua -function register_hook() - return "hook_name" -end - --- Or register multiple callbacks - -function register_hook() - return "hook_name1", "hook_name2", ... , "hook_nameX" -end -``` - -This API exports hook(s) implemented in its lua script. - -### Output -* hook_name must be a string, which is equal to the hook API(s) implemented. Possible values: - - "on_client_connected" - - "on_client_disconnected" - - "on_client_subscribe" - - "on_client_unsubscribe" - - "on_session_subscribed" - - "on_session_unsubscribed" - - "on_message_delivered" - - "on_message_acked" - - "on_message_publish" - -# management command - -## load - -```shell -emqx_ctl luahook load script_name -``` -This command will load lua file "script_name" in 'data/scripts' directory, into emqx hook. - -## unload - -```shell -emqx_ctl luahook unload script_name -``` -This command will unload lua file "script_name" out of emqx hook. - -## reload - -```shell -emqx_ctl luahook reload script_name -``` -This command will reload lua file "script_name" in 'data/scripts'. It is useful if a lua script has been modified and apply it immediately. - -## enable - -```shell -emqx_ctl luahook enable script_name -``` -This command will rename lua file "script_name.x" to "script_name", and load it immediately. - -## disable - -```shell -emqx_ctl luahook disable script_name -``` -This command will unload this script, and rename lua file "script_name" to "script_name.x", which will not be loaded during next boot. - - -License -------- - -Apache License Version 2.0 - -Author ------- - -EMQ X Team. - diff --git a/apps/emqx_lua_hook/etc/emqx_lua_hook.conf b/apps/emqx_lua_hook/etc/emqx_lua_hook.conf deleted file mode 100644 index f0256afae..000000000 --- a/apps/emqx_lua_hook/etc/emqx_lua_hook.conf +++ /dev/null @@ -1,4 +0,0 @@ -##-------------------------------------------------------------------- -## EMQ X Lua Hook -##-------------------------------------------------------------------- - diff --git a/apps/emqx_lua_hook/examples.lua b/apps/emqx_lua_hook/examples.lua deleted file mode 100644 index bc36eb771..000000000 --- a/apps/emqx_lua_hook/examples.lua +++ /dev/null @@ -1,71 +0,0 @@ --- --- Given all funcation names needed register to system --- -function register_hook() - return "on_client_connected", - "on_client_disconnected", - "on_client_subscribe", - "on_client_unsubscribe", - "on_session_subscribed", - "on_session_unsubscribed", - "on_message_delivered", - "on_message_acked", - "on_message_publish" -end - ----------------------------------------------------------------------- --- Callback Functions - -function on_client_connected(clientid, username, returncode) - print("Lua: on_client_connected - " .. clientid) - -- do your job here - return -end - -function on_client_disconnected(clientid, username, reason) - print("Lua: on_client_disconnected - " .. clientid) - -- do your job here - return -end - -function on_client_subscribe(clientid, username, topic) - print("Lua: on_client_subscribe - " .. clientid) - -- do your job here - return topic -end - -function on_client_unsubscribe(clientid, username, topic) - print("Lua: on_client_unsubscribe - " .. clientid) - -- do your job here - return topic -end - -function on_session_subscribed(clientid, username, topic) - print("Lua: on_session_subscribed - " .. clientid) - -- do your job here - return -end - -function on_session_unsubscribed(clientid, username, topic) - print("Lua: on_session_unsubscribed - " .. clientid) - -- do your job here - return -end - -function on_message_delivered(clientid, username, topic, payload, qos, retain) - print("Lua: on_message_delivered - " .. clientid) - -- do your job here - return topic, payload, qos, retain -end - -function on_message_acked(clientid, username, topic, payload, qos, retain) - print("Lua: on_message_acked- " .. clientid) - -- do your job here - return -end - -function on_message_publish(clientid, username, topic, payload, qos, retain) - print("Lua: on_message_publish - " .. clientid) - -- do your job here - return topic, payload, qos, retain -end diff --git a/apps/emqx_lua_hook/include/emqx_lua_hook.hrl b/apps/emqx_lua_hook/include/emqx_lua_hook.hrl deleted file mode 100644 index fb7010c2b..000000000 --- a/apps/emqx_lua_hook/include/emqx_lua_hook.hrl +++ /dev/null @@ -1,18 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --define(LOG(Level, Format, Args), emqx_logger:Level("Lua Hook: " ++ Format, Args)). - diff --git a/apps/emqx_lua_hook/priv/emqx_lua_hook.schema b/apps/emqx_lua_hook/priv/emqx_lua_hook.schema deleted file mode 100644 index 8b1378917..000000000 --- a/apps/emqx_lua_hook/priv/emqx_lua_hook.schema +++ /dev/null @@ -1 +0,0 @@ - diff --git a/apps/emqx_lua_hook/rebar.config b/apps/emqx_lua_hook/rebar.config deleted file mode 100644 index 97a06a77c..000000000 --- a/apps/emqx_lua_hook/rebar.config +++ /dev/null @@ -1,21 +0,0 @@ -{deps, - [{luerl, {git, "https://github.com/emqx/luerl", {tag, "v0.3.1"}}} - ]}. - -{edoc_opts, [{preprocess, true}]}. -{erl_opts, [warn_unused_vars, - warn_shadow_vars, - warn_unused_import, - warn_obsolete_guard, - debug_info, - compressed, - {parse_transform} - ]}. -{overrides, [{add, [{erl_opts, [compressed]}]}]}. - -{xref_checks, [undefined_function_calls, undefined_functions, - locals_not_used, deprecated_function_calls, - warnings_as_errors, deprecated_functions]}. -{cover_enabled, true}. -{cover_opts, [verbose]}. -{cover_export_enabled, true}. diff --git a/apps/emqx_lua_hook/src/emqx_lua_hook.app.src b/apps/emqx_lua_hook/src/emqx_lua_hook.app.src deleted file mode 100644 index 627c8e29d..000000000 --- a/apps/emqx_lua_hook/src/emqx_lua_hook.app.src +++ /dev/null @@ -1,14 +0,0 @@ -{application, emqx_lua_hook, - [{description, "EMQ X Lua Hooks"}, - {vsn, "4.3.0"}, % strict semver, bump manually! - {modules, []}, - {registered, []}, - {applications, [kernel,stdlib]}, - {mod, {emqx_lua_hook_app,[]}}, - {env,[]}, - {licenses, ["Apache-2.0"]}, - {maintainers, ["EMQ X Team "]}, - {links, [{"Homepage", "https://emqx.io/"}, - {"Github", "https://github.com/emqx/emqx-lua-hook"} - ]} - ]}. diff --git a/apps/emqx_lua_hook/src/emqx_lua_hook.erl b/apps/emqx_lua_hook/src/emqx_lua_hook.erl deleted file mode 100644 index 6cb3ec0da..000000000 --- a/apps/emqx_lua_hook/src/emqx_lua_hook.erl +++ /dev/null @@ -1,199 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_lua_hook). - --behaviour(gen_server). - --include("emqx_lua_hook.hrl"). --include_lib("luerl/include/luerl.hrl"). - --export([ start_link/0 - , stop/0 - ]). - --export([ load_scripts/0 - , unload_scripts/0 - , load_script/1 - , unload_script/1 - ]). - --export([ init/1 - , handle_call/3 - , handle_cast/2 - , handle_info/2 - , terminate/2 - , code_change/3 - ]). - --export([lua_dir/0]). - --define(SERVER, ?MODULE). - --record(state, {loaded_scripts = []}). - -start_link() -> - gen_server:start_link({local, ?SERVER}, ?MODULE, {}, []). - -stop() -> - gen_server:call(?SERVER, stop). - -load_scripts() -> - gen_server:call(?SERVER, load_scripts). - -unload_scripts() -> - gen_server:call(?SERVER, unload_scrips). - -load_script(ScriptName) -> - gen_server:call(?SERVER, {load_script, ScriptName}). - -unload_script(ScriptName) -> - gen_server:call(?SERVER, {unload_script, ScriptName}). - -lua_dir() -> - filename:join([emqx:get_env(data_dir, "data"), "scripts"]). - -%%----------------------------------------------------------------------------- -%% gen_server callbacks -%%----------------------------------------------------------------------------- - -init({}) -> - {ok, #state{}}. - -handle_call(stop, _From, State) -> - {stop, normal, ok, State}; - -handle_call(load_scripts, _From, State) -> - {reply, ok, State#state{loaded_scripts = do_loadall()}, hibernate}; - -handle_call(unload_scrips, _From, State=#state{loaded_scripts = Scripts}) -> - do_unloadall(Scripts), - {reply, ok, State#state{loaded_scripts = []}, hibernate}; - -handle_call({load_script, ScriptName}, _From, State=#state{loaded_scripts = Scripts}) -> - {Ret, NewScripts} = case do_load(ScriptName) of - error -> {error, Scripts}; - {ScriptName, LuaState} -> - case lists:member({ScriptName, LuaState}, Scripts) of - true -> {ok, Scripts}; - false -> {ok, lists:append([{ScriptName, LuaState}], Scripts)} - end - end, - {reply, Ret, State#state{loaded_scripts = NewScripts}, hibernate}; - -handle_call({unload_script, ScriptName}, _From, State=#state{loaded_scripts = Scripts}) -> - case proplists:get_all_values(ScriptName, Scripts) of - [] -> - {reply, ok, State, hibernate}; - LuaStates -> - lists:foreach(fun(LuaState) -> - % Unload first! If this gen_server has been crashed, loaded_scripts will be empty - do_unload({ScriptName, LuaState}) - end, LuaStates), - NewScripts = proplists:delete(ScriptName, Scripts), - {reply, ok, State#state{loaded_scripts = NewScripts}, hibernate} - end; - -handle_call(Request, From, State) -> - ?LOG(error, "Unknown Request=~p from ~p", [Request, From]), - {reply, ignored, State, hibernate}. - -handle_cast(Msg, State) -> - ?LOG(error, "unexpected cast: ~p", [Msg]), - {noreply, State, hibernate}. - -handle_info(Info, State) -> - ?LOG(error, "unexpected info: ~p", [Info]), - {noreply, State}. - -terminate(_Reason, #state{loaded_scripts = Scripts}) -> - do_unloadall(Scripts), - ok. - -code_change(_OldVsn, State, _Extra) -> - {ok, State}. - -%% ------------------------------------------------------------------ -%% Internal Function Definitions -%% ------------------------------------------------------------------ - -do_loadall() -> - FileList = filelib:wildcard(filename:join([lua_dir(), "*.lua"])), - List = [do_load(X) || X <- FileList], - [X || X <- List, is_tuple(X)]. - -do_load(FileName) -> - case catch luerl:dofile(FileName) of - {'EXIT', St00} -> - ?LOG(error, "Failed to load lua script ~p due to error ~p", [FileName, St00]), - error; - {_Ret, St0=#luerl{}} -> - case catch luerl:call_function([register_hook], [], St0) of - {'EXIT', St1} -> - ?LOG(error, "Failed to execute register_hook function in lua script ~p, which has syntax error, St1=~p", [FileName, St1]), - error; - {Ret1, St1} -> - ?LOG(debug, "Register lua script ~p", [FileName]), - _ = do_register_hooks(Ret1, FileName, St1), - {FileName, St1}; - Other -> - ?LOG(error, "Failed to load lua script ~p, register_hook() raise exception ~p", [FileName, Other]), - error - end; - Exception -> - ?LOG(error, "Failed to load lua script ~p with error ~p", [FileName, Exception]), - error - end. - -do_register(<<"on_message_publish">>, ScriptName, St) -> - emqx_lua_script:register_on_message_publish(ScriptName, St); -do_register(<<"on_message_delivered">>, ScriptName, St) -> - emqx_lua_script:register_on_message_delivered(ScriptName, St); -do_register(<<"on_message_acked">>, ScriptName, St) -> - emqx_lua_script:register_on_message_acked(ScriptName, St); -do_register(<<"on_client_connected">>, ScriptName, St) -> - emqx_lua_script:register_on_client_connected(ScriptName, St); -do_register(<<"on_client_subscribe">>, ScriptName, St) -> - emqx_lua_script:register_on_client_subscribe(ScriptName, St); -do_register(<<"on_client_unsubscribe">>, ScriptName, St) -> - emqx_lua_script:register_on_client_unsubscribe(ScriptName, St); -do_register(<<"on_client_disconnected">>, ScriptName, St) -> - emqx_lua_script:register_on_client_disconnected(ScriptName, St); -do_register(<<"on_session_subscribed">>, ScriptName, St) -> - emqx_lua_script:register_on_session_subscribed(ScriptName, St); -do_register(<<"on_client_authenticate">>, ScriptName, St) -> - emqx_lua_script:register_on_client_authenticate(ScriptName, St); -do_register(<<"on_client_check_acl">>, ScriptName, St) -> - emqx_lua_script:register_on_client_check_acl(ScriptName, St); -do_register(Hook, ScriptName, _St) -> - ?LOG(error, "Discard unknown hook ~p ScriptName=~p", [Hook, ScriptName]). - -do_register_hooks([], _ScriptName, _St) -> - ok; -do_register_hooks([H|T], ScriptName, St) -> - _ = do_register(H, ScriptName, St), - do_register_hooks(T, ScriptName, St); -do_register_hooks(Hook = <<$o, $n, _Rest/binary>>, ScriptName, St) -> - do_register(Hook, ScriptName, St); -do_register_hooks(Hook, ScriptName, _St) -> - ?LOG(error, "Discard unknown hook type ~p from ~p", [Hook, ScriptName]). - -do_unloadall(Scripts) -> - lists:foreach(fun do_unload/1, Scripts). - -do_unload(Script) -> - emqx_lua_script:unregister_hooks(Script), - ok. diff --git a/apps/emqx_lua_hook/src/emqx_lua_hook_app.erl b/apps/emqx_lua_hook/src/emqx_lua_hook_app.erl deleted file mode 100644 index 6b0ec3574..000000000 --- a/apps/emqx_lua_hook/src/emqx_lua_hook_app.erl +++ /dev/null @@ -1,40 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_lua_hook_app). - --behaviour(application). - --emqx_plugin(?MODULE). - --export([ start/2 - , stop/1 - , prep_stop/1 - ]). - -start(_Type, _Args) -> - {ok, Sup} = emqx_lua_hook_sup:start_link(), - emqx_lua_hook:load_scripts(), - emqx_lua_hook_cli:load(), - {ok, Sup}. - -prep_stop(State) -> - emqx_lua_hook:unload_scripts(), - emqx_lua_hook_cli:unload(), - State. - -stop(_State) -> - ok. diff --git a/apps/emqx_lua_hook/src/emqx_lua_hook_cli.erl b/apps/emqx_lua_hook/src/emqx_lua_hook_cli.erl deleted file mode 100644 index e95733239..000000000 --- a/apps/emqx_lua_hook/src/emqx_lua_hook_cli.erl +++ /dev/null @@ -1,88 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_lua_hook_cli). - --export([ load/0 - , cmd/1 - , unload/0 - ]). - --include("emqx_lua_hook.hrl"). --include_lib("luerl/include/luerl.hrl"). - --define(PRINT(Format, Args), io:format(Format, Args)). --define(PRINT_CMD(Cmd, Descr), io:format("~-48s# ~s~n", [Cmd, Descr])). --define(USAGE(CmdList), [?PRINT_CMD(Cmd, Descr) || {Cmd, Descr} <- CmdList]). - -load() -> - emqx_ctl:register_command(luahook, {?MODULE, cmd}, []). - -unload() -> - emqx_ctl:unregister_command(luahook). - -cmd(["load", Script]) -> - case emqx_lua_hook:load_script(fullname(Script)) of - ok -> emqx_ctl:print("Load ~p successfully~n", [Script]); - error -> emqx_ctl:print("Load ~p error~n", [Script]) - end; - -cmd(["reload", Script]) -> - FullName = fullname(Script), - emqx_lua_hook:unload_script(FullName), - case emqx_lua_hook:load_script(FullName) of - ok -> emqx_ctl:print("Reload ~p successfully~n", [Script]); - error -> emqx_ctl:print("Reload ~p error~n", [Script]) - end; - -cmd(["unload", Script]) -> - emqx_lua_hook:unload_script(fullname(Script)), - emqx_ctl:print("Unload ~p successfully~n", [Script]); - -cmd(["enable", Script]) -> - FullName = fullname(Script), - case file:rename(fullnamedisable(Script), FullName) of - ok -> case emqx_lua_hook:load_script(FullName) of - ok -> - emqx_ctl:print("Enable ~p successfully~n", [Script]); - error -> - emqx_ctl:print("Fail to enable ~p~n", [Script]) - end; - {error, Reason} -> - emqx_ctl:print("Fail to enable ~p due to ~p~n", [Script, Reason]) - end; - -cmd(["disable", Script]) -> - FullName = fullname(Script), - emqx_lua_hook:unload_script(FullName), - case file:rename(FullName, fullnamedisable(Script)) of - ok -> - emqx_ctl:print("Disable ~p successfully~n", [Script]); - {error, Reason} -> - emqx_ctl:print("Fail to disable ~p due to ~p~n", [Script, Reason]) - end; - -cmd(_) -> - emqx_ctl:usage([{"luahook load