diff --git a/.gitignore b/.gitignore index 613802978..0982fe30f 100644 --- a/.gitignore +++ b/.gitignore @@ -7,8 +7,6 @@ deps erl_crash.dump ebin !ebin/.placeholder -rel/emqttd -rel/emqttd* .concrete/DEV_MODE .rebar test/ebin/*.beam @@ -26,3 +24,6 @@ eunit.coverdata test/ct.cover.spec logs ct.coverdata +.idea/ +emqttd.iml +_rel/ diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index 45d0a819c..000000000 --- a/.gitmodules +++ /dev/null @@ -1,43 +0,0 @@ -[submodule "plugins/emqttd_dashboard"] - path = plugins/emqttd_dashboard - url = https://github.com/emqtt/emqttd_dashboard.git - branch = stable -[submodule "plugins/emqttd_plugin_template"] - path = plugins/emqttd_plugin_template - url = https://github.com/emqtt/emqttd_plugin_template.git - branch = stable -[submodule "plugins/emqttd_plugin_pgsql"] - path = plugins/emqttd_plugin_pgsql - url = https://github.com/emqtt/emqttd_plugin_pgsql.git - branch = stable -[submodule "plugins/emqttd_plugin_mysql"] - path = plugins/emqttd_plugin_mysql - url = https://github.com/emqtt/emqttd_plugin_mysql.git - branch = stable -[submodule "plugins/emqttd_sockjs"] - path = plugins/emqttd_sockjs - url = https://github.com/emqtt/emqttd_sockjs.git - branch = stable -[submodule "plugins/emqttd_stomp"] - path = plugins/emqttd_stomp - url = https://github.com/emqtt/emqttd_stomp.git - branch = stable -[submodule "plugins/emqttd_recon"] - path = plugins/emqttd_recon - url = https://github.com/emqtt/emqttd_recon.git - branch = stable -[submodule "plugins/emqttd_plugin_redis"] - path = plugins/emqttd_plugin_redis - url = https://github.com/emqtt/emqttd_plugin_redis.git - branch = stable -[submodule "plugins/emqttd_reloader"] - path = plugins/emqttd_reloader - url = https://github.com/emqtt/emqttd_reloader.git - branch = stable -[submodule "plugins/emqttd_plugin_mongo"] - path = plugins/emqttd_plugin_mongo - url = https://github.com/emqtt/emqttd_plugin_mongo.git -[submodule "plugins/emqttd_auth_http"] - path = plugins/emqttd_auth_http - url = https://github.com/emqtt/emqttd_auth_http.git - branch = stable diff --git a/Makefile b/Makefile index 659126c0d..49642f180 100644 --- a/Makefile +++ b/Makefile @@ -1,66 +1,31 @@ -.PHONY: rel deps test plugins +PROJECT = emqttd +PROJECT_DESCRIPTION = Erlang MQTT Broker +PROJECT_VERSION = 2.0 -APP = emqttd -BASE_DIR = $(shell pwd) -REBAR = $(BASE_DIR)/rebar -DIST = $(BASE_DIR)/rel/$(APP) +DEPS = gproc lager gen_logger gen_conf esockd mochiweb -all: submods compile +dep_gproc = git https://github.com/uwiger/gproc.git +dep_lager = git https://github.com/basho/lager.git +dep_gen_conf = git https://github.com/emqtt/gen_conf.git +dep_gen_logger = git https://github.com/emqtt/gen_logger.git +dep_esockd = git https://github.com/emqtt/esockd.git udp +dep_mochiweb = git https://github.com/emqtt/mochiweb.git -submods: - @git submodule update --init +ERLC_OPTS += +'{parse_transform, lager_transform}' -compile: deps - @$(REBAR) compile +TEST_ERLC_OPTS += +debug_info +TEST_ERLC_OPTS += +'{parse_transform, lager_transform}' -deps: - @$(REBAR) get-deps +EUNIT_OPTS = verbose +# EUNIT_ERL_OPTS = -update-deps: - @$(REBAR) update-deps +CT_SUITES = emqttd emqttd_access emqttd_lib emqttd_mod emqttd_net \ + emqttd_mqueue emqttd_protocol emqttd_topic emqttd_trie +CT_OPTS = -cover test/ct.cover.spec -erl_args -name emqttd_ct@127.0.0.1 -xref: - @$(REBAR) xref skip_deps=true +COVER = true -clean: - @$(REBAR) clean - -test: - ERL_FLAGS="-config rel/files/emqttd.test.config" $(REBAR) -v skip_deps=true ct - #$(REBAR) skip_deps=true eunit - -edoc: - @$(REBAR) doc - -rel: compile - @cd rel && $(REBAR) generate -f - -plugins: - @for plugin in ./plugins/* ; do \ - if [ -d $${plugin} ]; then \ - mkdir -p $(DIST)/$${plugin}/ ; \ - cp -R $${plugin}/ebin $(DIST)/$${plugin}/ ; \ - [ -d "$${plugin}/priv" ] && cp -R $${plugin}/priv $(DIST)/$${plugin}/ ; \ - [ -d "$${plugin}/etc" ] && cp -R $${plugin}/etc $(DIST)/$${plugin}/ ; \ - echo "$${plugin} copied" ; \ - fi \ - done - -dist: rel plugins - -PLT = $(BASE_DIR)/.emqttd_dialyzer.plt -APPS = erts kernel stdlib sasl crypto ssl os_mon syntax_tools \ - public_key mnesia inets compiler - -check_plt: compile - dialyzer --check_plt --plt $(PLT) --apps $(APPS) \ - deps/*/ebin ./ebin plugins/*/ebin - -build_plt: compile - dialyzer --build_plt --output_plt $(PLT) --apps $(APPS) \ - deps/*/ebin ./ebin plugins/*/ebin - -dialyzer: compile - dialyzer -Wno_return --plt $(PLT) deps/*/ebin ./ebin plugins/*/ebin +include erlang.mk +app:: rebar.config diff --git a/README.md b/README.md index c4f4b1eff..fd4ff4457 100644 --- a/README.md +++ b/README.md @@ -114,11 +114,11 @@ unzip emqttd-ubuntu64-0.16.0-beta-20160216.zip && cd emqttd Installing from source: ``` -git clone https://github.com/emqtt/emqttd.git +git clone https://github.com/emqtt/emqttd-relx.git -cd emqttd && make && make dist +cd emqttd-relx && make -cd rel/emqttd && ./bin/emqttd console +cd _rel/emqttd && ./bin/emqttd console ``` ## Documents diff --git a/docs/source/_static/images/publish.png b/docs/source/_static/images/publish.png new file mode 100644 index 000000000..0884fc224 Binary files /dev/null and b/docs/source/_static/images/publish.png differ diff --git a/docs/source/changes.rst b/docs/source/changes.rst index e81633c82..33c400101 100644 --- a/docs/source/changes.rst +++ b/docs/source/changes.rst @@ -5,6 +5,124 @@ Changes ======= +.. _release_2.0: + +------------------------------------- +Version 2.0-beta1 (West of West Lake) +------------------------------------- + +*Release Date: 2016-08-30* + +*Release Name: West of West Lake* + +EMQ - Shortened Project Name +---------------------------- + +Adopt a shortened project name: EMQ(Erlang/Enterprise/Elastic MQTT Broker)，E means Erlang/OTP, Enterprise and Elastic. + +Improve the Release Management +------------------------------ + +In order to iterate the project fast, we will adopt a new release management strategy since 2.0. There will be two or three 'Preview Release' named beta1, beta2 or beta3, and then one or two 'Release Candidate' named rc1, rc2 before a Major version is production ready. + +Seperate Rel from Application +----------------------------- + +We split the emqttd 1.x project into two projects since 2.0-beta1 release to resolve the plugins' dependency issue. + +A new project named `emqttd-relx`_ is created and responsible for buiding the emqttd application and the plugins:: + + git clone https://github.com/emqtt/emqttd-relx.git + + cd emqttd-relx && make + + cd _rel/emqttd && ./bin/emqttd console + +erlang.mk and relx +------------------ + +The rebar which is used in 1.x release is replaced by `erlang.mk`_ and `relx`_ tools since 2.0-beta1 release. + +You can check the 'Makefile' and 'relx.config' in the release project of the borker: `emqttd-relx`_ . + +Improve Git Branch Management +----------------------------- + ++------------+-------------------------------------------+ +| stable | 1.x Stable Branch | ++------------+-------------------------------------------+ +| master | 2.x Master Branch | ++------------+-------------------------------------------+ +| emq10 | 1.x Developement Branch | ++------------+-------------------------------------------+ +| emq20 | 2.x Development Branch | ++------------+-------------------------------------------+ +| emq30 | 3.x Development Branch | ++------------+-------------------------------------------+ +| issue#{id} | BugFix Branch | ++------------+-------------------------------------------+ + +New Config Syntax +----------------- + +Since 2.0-beta1 release the configuration file of the broker and plugins adopt a new syntax like rebar.config and relx.config: + +etc/emqttd.conf for example:: + + %% Max ClientId Length Allowed. + {mqtt_max_clientid_len, 512}. + + %% Max Packet Size Allowed, 64K by default. + {mqtt_max_packet_size, 65536}. + + %% Client Idle Timeout. + {mqtt_client_idle_timeout, 30}. % Second + +MQTT-SN Protocol Plugin +----------------------- + +The MQTT-SN Protocol Plugin `emqttd_sn`_ has been ready in 2.0-beta1 release. The default UDP port of MQTT-SN is 1884. + +Load the plugin:: + + ./bin/emqttd_ctl plugins load emqttd_sn + +Improve the PubSub Design +------------------------- + +.. image:: _static/images/publish.png + +Improve the Plugin Management +----------------------------- + +The plugin of EMQ 2.0 broker is a normal erlang application which depends on and extends 'emqttd'. You can create a standalone plugin application project, and add it to `emqttd-relx`_ Makefile as a DEP. + +All the plugins' config files will be copied to emqttd/etc/plugins/ folder when making emqttd brinary packages in `emqttd-relx`_ project:: + + ▾ emqttd/ + ▾ etc/ + ▸ modules/ + ▾ plugins/ + emqtt_coap.conf + emqttd.conf + emqttd_auth_http.conf + emqttd_auth_mongo.conf + emqttd_auth_mysql.conf + emqttd_auth_pgsql.conf + emqttd_auth_redis.conf + emqttd_coap.conf + emqttd_dashboard.conf + emqttd_plugin_template.conf + emqttd_recon.conf + emqttd_reloader.conf + emqttd_sn.conf + emqttd_stomp.conf + +EMQ 2.0 Documentation +--------------------- + +http://emqtt.io/docs/v2/index.html + .. _release_1.1.3: ------------- diff --git a/docs/source/cluster.rst b/docs/source/cluster.rst index 8ebd992ba..96ef37070 100644 --- a/docs/source/cluster.rst +++ b/docs/source/cluster.rst @@ -245,7 +245,7 @@ The Firewall If there is a firewall between clustered nodes, the cluster requires to open 4369 port used by epmd daemon, and a port segment for nodes' communication. -Configure the port segment in etc/emqttd.config, for example: +Configure the port segment in releases/2.0/sys.config, for example: .. code-block:: erlang diff --git a/docs/source/coap.rst b/docs/source/coap.rst index 32d482e17..41d73e618 100644 --- a/docs/source/coap.rst +++ b/docs/source/coap.rst @@ -1,3 +1,8 @@ -============== + +.. _coap: + +============= CoAP Protocol -============== +============= + + diff --git a/docs/source/commands.rst b/docs/source/commands.rst index 7b4243c96..c4e8645c5 100644 --- a/docs/source/commands.rst +++ b/docs/source/commands.rst @@ -20,7 +20,7 @@ Show running status of the broker:: $ ./bin/emqttd_ctl status Node 'emqttd@127.0.0.1' is started - emqttd 1.1 is running + emqttd 2.0 is running .. _command_broker:: @@ -382,10 +382,6 @@ Query the subscription table of the broker: +--------------------------------------------+--------------------------------------+ | subscriptions show | Show a subscription | +--------------------------------------------+--------------------------------------+ -| subscriptions add | Add a static subscription manually | -+--------------------------------------------+--------------------------------------+ -| subscriptions del | Remove a static subscription manually| -+--------------------------------------------+--------------------------------------+ subscriptions list ------------------ @@ -415,22 +411,6 @@ Show the subscriptions of a MQTT client:: clientid: [{<<"x">>,1},{<<"topic2">>,1},{<<"topic3">>,1}] -subscriptions add ------------------------------------------- - -Add a static subscription manually:: - - $ ./bin/emqttd_ctl subscriptions add clientid new_topic 1 - ok - -subscriptions del ------------------------------------- - -Remove a static subscription manually:: - - $ ./bin/emqttd_ctl subscriptions del clientid new_topic - ok - .. _command_plugins:: ------- diff --git a/docs/source/conf.py b/docs/source/conf.py index f96ed0579..3fafdfd06 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -48,7 +48,7 @@ source_suffix = '.rst' master_doc = 'index' # General information about the project. -project = u'Erlang MQTT Broker' +project = u'EMQ 2.0 - Erlang MQTT Broker' copyright = u'2016, Feng Lee' # The version info for the project you're documenting, acts as replacement for @@ -56,9 +56,9 @@ copyright = u'2016, Feng Lee' # built documents. # # The short X.Y version. -version = '1.0' +version = '2.0' # The full version, including alpha/beta/rc tags. -release = '1.0' +release = '2.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/docs/source/config.rst b/docs/source/config.rst index 7ddab18d9..f109a246b 100644 --- a/docs/source/config.rst +++ b/docs/source/config.rst @@ -5,24 +5,66 @@ Configuration ============= -Configuration files of the broker are under 'etc/' folder, including: +The two main configuration files of the broker are under 'etc/' folder: +-------------------+-----------------------------------+ | File | Description | +-------------------+-----------------------------------+ | etc/vm.args | Erlang VM Arguments | +-------------------+-----------------------------------+ -| etc/emqttd.config | emqttd broker Config | -+-------------------+-----------------------------------+ -| etc/acl.config | ACL Config | -+-------------------+-----------------------------------+ -| etc/clients.config| ClientId Authentication | -+-------------------+-----------------------------------+ -| etc/rewrite.config| Rewrite Rules | -+-------------------+-----------------------------------+ -| etc/ssl/* | SSL certificate and key files | +| etc/emqttd.conf | emqttd broker Config | +-------------------+-----------------------------------+ +---------------------------- +Plugins' Configuration Files +---------------------------- + ++----------------------------------------+-----------------------------------+ +| File | Description | ++----------------------------------------+-----------------------------------+ +| etc/plugins/emqttd_auth_http.conf | HTTP Auth/ACL Plugin Config | ++----------------------------------------+-----------------------------------+ +| etc/plugins/emqttd_auth_mongo.conf | MongoDB Auth/ACL Plugin Config | ++----------------------------------------+-----------------------------------+ +| etc/plugins/emqttd_auth_mysql.conf | MySQL Auth/ACL Plugin Config | ++----------------------------------------+-----------------------------------+ +| etc/plugins/emqttd_auth_pgsql.conf | Postgre Auth/ACL Plugin Config | ++----------------------------------------+-----------------------------------+ +| etc/plugins/emqttd_auth_redis.conf | Redis Auth/ACL Plugin Config | ++----------------------------------------+-----------------------------------+ +| etc/plugins/emqttd_coap.conf | CoAP Protocol Plugin Config | ++----------------------------------------+-----------------------------------+ +| etc/plugins/emqttd_dashboard.conf | Dashboard Plugin Config | ++----------------------------------------+-----------------------------------+ +| etc/plugins/emqttd_plugin_template.conf| Template Plugin Config | ++----------------------------------------+-----------------------------------+ +| etc/plugins/emqttd_recon.conf | Recon Plugin Config | ++----------------------------------------+-----------------------------------+ +| etc/plugins/emqttd_reloader.conf | Reloader Plugin Config | ++----------------------------------------+-----------------------------------+ +| etc/plugins/emqttd_sn.conf | MQTT-SN Protocal Plugin Config | ++----------------------------------------+-----------------------------------+ +| etc/plugins/emqttd_stomp.conf | Stomp Protocl Plugin Config | ++----------------------------------------+-----------------------------------+ + +---------------------------- +Modules' Configuration Files +---------------------------- + +The modules' configuration files are in etc/modules/ folder, and referrenced by etc/emqttd.conf: + ++----------------------------+-----------------------------------+ +| File | Description | ++----------------------------+-----------------------------------+ +| etc/modules/acl.config | Internal ACL Rules | ++----------------------------+-----------------------------------+ +| etc/modules/client.config | Config for ClientId Auth Module | ++----------------------------+-----------------------------------+ +| etc/modules/rewrite.config | Config for Rewrite Module | ++----------------------------+-----------------------------------+ +| etc/ssl/* | SSL Certfile and Keyfile | ++-----------------------------+----------------------------------+ + ----------- etc/vm.args ----------- @@ -102,44 +144,11 @@ The name and cookie of Erlang Node should be configured when clustering:: ## Cookie for distributed erlang -setcookie emqttdsecretcookie ------------------ -etc/emqttd.config ------------------ - -This is the main emqttd broker configuration file. - -File Syntax ------------ - -The file use the standard Erlang config syntax and consists of a list of erlang applications and their environments. - -.. code-block:: erlang - - [{kernel, [ - {start_timer, true}, - {start_pg2, true} - ]}, - {sasl, [ - {sasl_error_logger, {file, "log/emqttd_sasl.log"}} - ]}, - - ... - - {emqttd, [ - ... - ]} - ]. - -The file adopts Erlang Term Syntax: - -1. [ ]: List, seperated by comma -2. { }: Tuple, Usually {Env, Value} -3. % : comment - +------------------ Log Level and File ------------------ -Logger of emqttd broker is implemented by 'lager' application: +Logger of emqttd broker is implemented by 'lager' application, which is configured in releases/2.0/sys.config: .. code-block:: erlang @@ -173,44 +182,47 @@ Configure log handlers: ]} ]} -emqttd Application ------------------- +--------------- +etc/emqttd.conf +--------------- -The MQTT broker is implemented by erlang 'emqttd' application: +This is the main configuration file for emqttd broker. + +File Syntax +----------- + +The file uses the Erlang term syntax which is like rebar.config or relx.config: + +1. [ ]: List, seperated by comma +2. { }: Tuple, Usually {Env, Value} +3. % : comment + +MQTT Protocol Parameters +------------------------ + +Maximum ClientId Length +....................... .. code-block:: erlang - {emqttd, [ - %% Authentication and Authorization - {access, [ - ... - ]}, - %% MQTT Protocol Options - {mqtt, [ - ... - ]}, - %% Broker Options - {broker, [ - ... - ]}, - %% Modules - {modules, [ - ... - ]}, - %% Plugins - {plugins, [ - ... - ]}, + %% Max ClientId Length Allowed. + {mqtt_max_clientid_len, 512}. - %% Listeners - {listeners, [ - ... - ]}, +Maximum Packet Size +................... - %% Erlang System Monitor - {sysmon, [ - ]} - ]} +.. code-block:: erlang + + %% Max Packet Size Allowed, 64K by default. + {mqtt_max_packet_size, 65536}. + +MQTT Client Idle Timeout +........................ + +.. code-block:: erlang + + %% Client Idle Timeout. + {mqtt_client_idle_timeout, 30}. % Second Pluggable Authentication ------------------------ @@ -221,54 +233,44 @@ The broker provides Username, ClientId, LDAP and anonymous authentication module .. code-block:: erlang - %% Authetication. Anonymous Default - {auth, [ - %% Authentication with username, password - %% Add users: ./bin/emqttd_ctl users add Username Password - %% {username, [{"test", "public"}]}, + %%-------------------------------------------------------------------- + %% Authentication + %%-------------------------------------------------------------------- - %% Authentication with clientid - % {clientid, [{password, no}, {file, "etc/clients.config"}]}, + %% Anonymous: Allow all + {auth, anonymous, []}. - %% Authentication with LDAP - % {ldap, [ - % {servers, ["localhost"]}, - % {port, 389}, - % {timeout, 30}, - % {user_dn, "uid=$u,ou=People,dc=example,dc=com"}, - % {ssl, fasle}, - % {sslopts, [ - % {"certfile", "ssl.crt"}, - % {"keyfile", "ssl.key"}]} - % ]}, + %% Authentication with username, password + {auth, username, [{passwd, "etc/modules/passwd.conf"}]}. - %% Allow all - {anonymous, []} - ]}, + %% Authentication with clientId + {auth, clientid, [{config, "etc/modules/client.conf"}, {password, no}]}. The modules enabled at the same time compose an authentication chain:: - ---------------- ---------------- ------------- - Client --> | Username | -ignore-> | ClientID | -ignore-> | Anonymous | - ---------------- ---------------- ------------- + ---------------- ---------------- -------------- + Client --> | Anonymous | -ignore-> | Username | -ignore-> | ClientID | + ---------------- ---------------- -------------- | | | \|/ \|/ \|/ allow | deny allow | deny allow | deny -.. NOTE:: There are also MySQL、PostgreSQL、Redis、MongoDB Authentication Plugins. +.. NOTE:: There are also MySQL, Postgre, Redis, MongoDB and HTTP Authentication Plugins. Username Authentication ....................... .. code-block:: erlang - {username, [{client1, "passwd1"}, {client2, "passwd2"}]}, + %% Authentication with username, password + {auth, username, [{passwd, "etc/modules/passwd.conf"}]}. Two ways to configure users: -1. Configure username and plain password directly:: +1. Configure username and plain password in etc/modules/passwd.conf:: - {username, [{client1, "passwd1"}, {client2, "passwd2"}]}, + {"user1", "passwd1"}. + {"user2", "passwd2"}. 2. Add user by './bin/emqttd_ctl users' command:: @@ -279,388 +281,34 @@ ClientID Authentication .. code-block:: erlang - {clientid, [{password, no}, {file, "etc/clients.config"}]}, + %% Authentication with clientId + {auth, clientid, [{config, "etc/modules/client.conf"}, {password, no}]}. Configure ClientIDs in etc/clients.config:: - testclientid0 - testclientid1 127.0.0.1 - testclientid2 192.168.0.1/24 - -LDAP Authentication -................... - -.. code-block:: erlang - - {ldap, [ - {servers, ["localhost"]}, - {port, 389}, - {timeout, 30}, - {user_dn, "uid=$u,ou=People,dc=example,dc=com"}, - {ssl, fasle}, - {sslopts, [ - {"certfile", "ssl.crt"}, - {"keyfile", "ssl.key"}]} - ]}, - + "testclientid0". + {"testclientid1", "127.0.0.1"}. + {"testclientid2", "192.168.0.1/24"}. Anonymous Authentication ........................ Allow any client to connect to the broker:: - {anonymous, []} + %% Anonymous: Allow all + {auth, anonymous, []}. - -ACL ---- +ACL(Authorization) +------------------ Enable the default ACL module: .. code-block:: erlang - {acl, [ - %% Internal ACL module - {internal, [{file, "etc/acl.config"}, {nomatch, allow}]} - ]} + %% Internal ACL config + {acl, internal, [{config, "etc/modules/acl.conf"}, {nomatch, allow}]}. -MQTT Packet and ClientID ------------------------- - -.. code-block:: erlang - - {packet, [ - - %% Max ClientId Length Allowed - {max_clientid_len, 1024}, - - %% Max Packet Size Allowed, 64K default - {max_packet_size, 65536} - ]}, - -MQTT Client Idle Timeout ------------------------- - -.. code-block:: erlang - - {client, [ - %% Socket is connected, but no 'CONNECT' packet received - {idle_timeout, 10} - ]}, - -MQTT Session ------------- - -.. code-block:: erlang - - {session, [ - %% Max number of QoS 1 and 2 messages that can be “in flight” at one time. - %% 0 means no limit - {max_inflight, 100}, - - %% Retry interval for unacked QoS1/2 messages. - {unack_retry_interval, 20}, - - %% Awaiting PUBREL Timeout - {await_rel_timeout, 20}, - - %% Max Packets that Awaiting PUBREL, 0 means no limit - {max_awaiting_rel, 0}, - - %% Interval of Statistics Collection(seconds) - {collect_interval, 20}, - - %% Expired after 2 day (unit: minute) - {expired_after, 2880} - - ]}, - -Session parameters: - -+----------------------+----------------------------------------------------------+ -| max_inflight | Max number of QoS1/2 messages that can be delivered in | -| | the same time | -+----------------------+----------------------------------------------------------+ -| unack_retry_interval | Retry interval for unacked QoS1/2 messages. | -+----------------------+----------------------------------------------------------+ -| await_rel_timeout | Awaiting PUBREL Timeout | -+----------------------+----------------------------------------------------------+ -| max_awaiting_rel | Max number of Packets that Awaiting PUBREL | -+----------------------+----------------------------------------------------------+ -| collect_interval | Interval of Statistics Collection | -+----------------------+----------------------------------------------------------+ -| expired_after | Expired after (unit: minute) | -+----------------------+----------------------------------------------------------+ - -MQTT Message Queue ------------------- - -The message queue of session stores: - -1. Offline messages for persistent session. - -2. Pending messages for inflight window is full - -Queue parameters: - -.. code-block:: erlang - - {queue, [ - %% simple | priority - {type, simple}, - - %% Topic Priority: 0~255, Default is 0 - %% {priority, [{"topic/1", 10}, {"topic/2", 8}]}, - - %% Max queue length. Enqueued messages when persistent client disconnected, - %% or inflight window is full. - {max_length, infinity}, - - %% Low-water mark of queued messages - {low_watermark, 0.2}, - - %% High-water mark of queued messages - {high_watermark, 0.6}, - - %% Queue Qos0 messages? - {queue_qos0, true} - ]} - -+----------------------+---------------------------------------------------+ -| type | Queue type: simple or priority | -+----------------------+---------------------------------------------------+ -| priority | Topic priority | -+----------------------+---------------------------------------------------+ -| max_length | Max Queue size, infinity means no limit | -+----------------------+---------------------------------------------------+ -| low_watermark | Low watermark | -+----------------------+---------------------------------------------------+ -| high_watermark | High watermark | -+----------------------+---------------------------------------------------+ -| queue_qos0 | If Qos0 message queued? | -+----------------------+---------------------------------------------------+ - -Sys Interval of Broker ------------------------ - -.. code-block:: erlang - - %% System interval of publishing $SYS messages - {sys_interval, 60}, - -Retained messages ------------------ - -.. code-block:: erlang - - {retained, [ - %% Expired after seconds, never expired if 0 - {expired_after, 0}, - - %% Maximum number of retained messages - {max_message_num, 100000}, - - %% Max Payload Size of retained message - {max_playload_size, 65536} - ]}, - -PubSub and Router ------------------ - -.. code-block:: erlang - - {pubsub, [ - %% PubSub Pool - {pool_size, 8}, - - %% Subscription: true | false - {subscription, true}, - - %% Route aging time(seconds) - {route_aging, 5} - ]}, - -Bridge Parameters ------------------ - -.. code-block:: erlang - - {bridge, [ - %% Bridge Queue Size - {max_queue_len, 10000}, - - %% Ping Interval of bridge node - {ping_down_interval, 1} - ]} - - -Enable Modules --------------- - -'presence' module will publish presence message to $SYS topic when a client connected or disconnected:: - - {presence, [{qos, 0}]}, - -'subscription' module forces the client to subscribe some topics when connected to the broker: - -.. code-block:: erlang - - %% Subscribe topics automatically when client connected - {subscription, [ - %% Subscription from stored table - stored, - - %% $u will be replaced with username - {"$Q/username/$u", 1}, - - %% $c will be replaced with clientid - {"$Q/client/$c", 1} - ]} - -'rewrite' module supports to rewrite the topic path: - -.. code-block:: erlang - - %% Rewrite rules - {rewrite, [{file, "etc/rewrite.config"}]} - -Plugins Folder --------------- - -.. code-block:: erlang - - {plugins, [ - %% Plugin App Library Dir - {plugins_dir, "./plugins"}, - - %% File to store loaded plugin names. - {loaded_file, "./data/loaded_plugins"} - ]}, - - -TCP Listeners -------------- - -Configure the TCP listeners for MQTT, MQTT(SSL) and HTTP Protocols. - -The most important parameter is 'max_clients' - max concurrent clients allowed. - -The TCP Ports occupied by emqttd broker by default: - -+-----------+-----------------------------------+ -| 1883 | MQTT Port | -+-----------+-----------------------------------+ -| 8883 | MQTT(SSL) Port | -+-----------+-----------------------------------+ -| 8083 | MQTT(WebSocket), HTTP API Port | -+-----------+-----------------------------------+ - -.. code-block:: erlang - - {listeners, [ - - {mqtt, 1883, [ - %% Size of acceptor pool - {acceptors, 16}, - - %% Maximum number of concurrent clients - {max_clients, 8192}, - - %% Socket Access Control - {access, [{allow, all}]}, - - %% Connection Options - {connopts, [ - %% Rate Limit. Format is 'burst, rate', Unit is KB/Sec - %% {rate_limit, "100,10"} %% 100K burst, 10K rate - ]}, - - %% Socket Options - {sockopts, [ - %Set buffer if hight thoughtput - %{recbuf, 4096}, - %{sndbuf, 4096}, - %{buffer, 4096}, - %{nodelay, true}, - {backlog, 1024} - ]} - ]}, - - {mqtts, 8883, [ - %% Size of acceptor pool - {acceptors, 4}, - - %% Maximum number of concurrent clients - {max_clients, 512}, - - %% Socket Access Control - {access, [{allow, all}]}, - - %% SSL certificate and key files - {ssl, [{certfile, "etc/ssl/ssl.crt"}, - {keyfile, "etc/ssl/ssl.key"}]}, - - %% Socket Options - {sockopts, [ - {backlog, 1024} - %{buffer, 4096}, - ]} - ]}, - %% WebSocket over HTTPS Listener - %% {https, 8083, [ - %% %% Size of acceptor pool - %% {acceptors, 4}, - %% %% Maximum number of concurrent clients - %% {max_clients, 512}, - %% %% Socket Access Control - %% {access, [{allow, all}]}, - %% %% SSL certificate and key files - %% {ssl, [{certfile, "etc/ssl/ssl.crt"}, - %% {keyfile, "etc/ssl/ssl.key"}]}, - %% %% Socket Options - %% {sockopts, [ - %% %{buffer, 4096}, - %% {backlog, 1024} - %% ]} - %%]}, - - %% HTTP and WebSocket Listener - {http, 8083, [ - %% Size of acceptor pool - {acceptors, 4}, - %% Maximum number of concurrent clients - {max_clients, 64}, - %% Socket Access Control - {access, [{allow, all}]}, - %% Socket Options - {sockopts, [ - {backlog, 1024} - %{buffer, 4096}, - ]} - ]} - ]}, - -Listener Parameters: - -+-------------+----------------------------------------------------------------+ -| acceptors | TCP Acceptor Pool | -+-------------+----------------------------------------------------------------+ -| max_clients | Maximum number of concurrent TCP connections allowed | -+-------------+----------------------------------------------------------------+ -| access | Access Control by IP, for example: [{allow, "192.168.1.0/24"}] | -+-------------+----------------------------------------------------------------+ -| connopts | Rate Limit Control, for example: {rate_limit, "100,10"} | -+-------------+----------------------------------------------------------------+ -| sockopts | TCP Socket parameters | -+-------------+----------------------------------------------------------------+ - -.. _config_acl: - --------------- -etc/acl.config --------------- - -The 'etc/acl.config' is the default ACL config for emqttd broker. The rules by default: +Define ACL rules in etc/modules/acl.conf. The rules by default: .. code-block:: erlang @@ -686,35 +334,194 @@ An ACL rule is an Erlang tuple. The Access control module of emqttd broker match \|/ \|/ \|/ allow | deny allow | deny allow | deny -.. _config_rewrite: - ------------------- -etc/clients.config ------------------- - -Enable ClientId Authentication in 'etc/emqttd.config': +Sys Interval of Broker +---------------------- .. code-block:: erlang - {auth, [ - %% Authentication with clientid - {clientid, [{password, no}, {file, "etc/clients.config"}]} - ]}, + %% System interval of publishing $SYS messages + {broker_sys_interval, 60}. -Configure all allowed ClientIDs, IP Addresses in etc/clients.config:: +Retained Message Configuration +------------------------------ - testclientid0 - testclientid1 127.0.0.1 - testclientid2 192.168.0.1/24 +Expiration of Retained Message +............................... ------------------- -etc/rewrite.config ------------------- +.. code:: erlang -The Rewrite Rules for emqttd_mod_rewrite: + %% Expired after seconds, never expired if 0 + {retained_expired_after, 0}. + +Maximum Number of Retained Message +................................... + +.. code:: erlang + + %% Max number of retained messages + {retained_max_message_num, 100000}. + +Maximum Size of Retained Message +................................ + +.. code:: erlang + + %% Max Payload Size of retained message + {retained_max_playload_size, 65536}. + +MQTT Session +------------ .. code-block:: erlang + %% Max number of QoS 1 and 2 messages that can be “inflight” at one time. + %% 0 means no limit + {session_max_inflight, 100}. + + %% Retry interval for redelivering QoS1/2 messages. + {session_unack_retry_interval, 60}. + + %% Awaiting PUBREL Timeout + {session_await_rel_timeout, 20}. + + %% Max Packets that Awaiting PUBREL, 0 means no limit + {session_max_awaiting_rel, 0}. + + %% Statistics Collection Interval(seconds) + {session_collect_interval, 0}. + + %% Expired after 2 day (unit: minute) + {session_expired_after, 2880}. + +Session parameters: + ++------------------------------+----------------------------------------------------------+ +| session_max_inflight | Max number of QoS1/2 messages that can be delivered in | +| | the same time | ++------------------------------+----------------------------------------------------------+ +| session_unack_retry_interval | Retry interval for unacked QoS1/2 messages. | ++------------------------------+----------------------------------------------------------+ +| session_await_rel_timeout | Awaiting PUBREL Timeout | ++------------------------------+----------------------------------------------------------+ +| session_max_awaiting_rel | Max number of Packets that Awaiting PUBREL | ++------------------------------+----------------------------------------------------------+ +| session_collect_interval | Interval of Statistics Collection | ++------------------------------+----------------------------------------------------------+ +| session_expired_after | Expired after (unit: minute) | ++------------------------------+----------------------------------------------------------+ + +MQTT Message Queue +------------------ + +The message queue of session stores: + +1. Offline messages for persistent session. + +2. Pending messages for inflight window is full + +Queue parameters: + +.. code-block:: erlang + + %% Type: simple | priority + {queue_type, simple}. + + %% Topic Priority: 0~255, Default is 0 + %% {queue_priority, [{"topic/1", 10}, {"topic/2", 8}]}. + + %% Max queue length. Enqueued messages when persistent client disconnected, + %% or inflight window is full. + {queue_max_length, infinity}. + + %% Low-water mark of queued messages + {queue_low_watermark, 0.2}. + + %% High-water mark of queued messages + {queue_high_watermark, 0.6}. + + %% Queue Qos0 messages? + {queue_qos0, true}. + ++----------------------+---------------------------------------------------+ +| queue_type | Queue type: simple or priority | ++----------------------+---------------------------------------------------+ +| queue_priority | Topic priority | ++----------------------+---------------------------------------------------+ +| queue_max_length | Max Queue size, infinity means no limit | ++----------------------+---------------------------------------------------+ +| queue_low_watermark | Low watermark | ++----------------------+---------------------------------------------------+ +| queue_high_watermark | High watermark | ++----------------------+---------------------------------------------------+ +| queue_qos0 | If Qos0 message queued? | ++----------------------+---------------------------------------------------+ + +PubSub and Router +----------------- + +PubSub Pool Size +................ + +.. code-block:: erlang + + %% PubSub Pool Size. Default should be scheduler numbers. + {pubsub_pool_size, 8}. + +MQTT Bridge Parameters +---------------------- + +Max MQueue Size of Bridge +......................... + +.. code:: erlang + + %% TODO: Bridge Queue Size + {bridge_max_queue_len, 10000}. + +Ping Interval of Bridge +....................... + +.. code:: erlang + + %% Ping Interval of bridge node + {bridge_ping_down_interval, 1}. % second + +Extended Modules +---------------- + +Presence Module +............... + +'presence' module will publish presence message to $SYS topic when a client connected or disconnected: + +.. code:: erlang + + %% Client presence management module. Publish presence messages when + %% client connected or disconnected. + {module, presence, [{qos, 0}]}. + +Subscription Module +................... + +'subscription' module forces the client to subscribe some topics when connected to the broker: + +.. code:: erlang + + %% Subscribe topics automatically when client connected + {module, subscription, [{"$client/$c", 1}]}. + +Rewrite Module +.............. + +'rewrite' module supports to rewrite the topic path: + +.. code:: erlang + + %% [Rewrite](https://github.com/emqtt/emqttd/wiki/Rewrite) + {module, rewrite, [{config, "etc/modules/rewrite.conf"}]}. + +Configure rewrite rules in etc/modules/rewrite.conf:: + {topic, "x/#", [ {rewrite, "^x/y/(.+)$", "z/y/$1"}, {rewrite, "^x/(.+)$", "y/$1"} @@ -723,3 +530,161 @@ The Rewrite Rules for emqttd_mod_rewrite: {topic, "y/+/z/#", [ {rewrite, "^y/(.+)/z/(.+)$", "y/z/$2"} ]}. + +Plugins Folder +-------------- + +.. code:: erlang + + %% Dir of plugins' config + {plugins_etc_dir, "etc/plugins/"}. + + %% File to store loaded plugin names. + {plugins_loaded_file, "data/loaded_plugins"}. + + +TCP Listeners +------------- + +Configure the TCP listeners for MQTT, MQTT(SSL) and HTTP Protocols. + +The most important parameter is 'max_clients' - max concurrent clients allowed. + +The TCP Ports occupied by emqttd broker by default: + ++-----------+-----------------------------------+ +| 1883 | MQTT Port | ++-----------+-----------------------------------+ +| 8883 | MQTT(SSL) Port | ++-----------+-----------------------------------+ +| 8083 | MQTT(WebSocket), HTTP API Port | ++-----------+-----------------------------------+ + +.. code-block:: erlang + +Listener Parameters: + ++-------------+----------------------------------------------------------------+ +| acceptors | TCP Acceptor Pool | ++-------------+----------------------------------------------------------------+ +| max_clients | Maximum number of concurrent TCP connections allowed | ++-------------+----------------------------------------------------------------+ +| access | Access Control by IP, for example: [{allow, "192.168.1.0/24"}] | ++-------------+----------------------------------------------------------------+ +| connopts | Rate Limit Control, for example: {rate_limit, "100,10"} | ++-------------+----------------------------------------------------------------+ +| sockopts | TCP Socket parameters | ++-------------+----------------------------------------------------------------+ + +1883 - Plain MQTT +................. + +.. code-block:: erlang + + %% Plain MQTT + {listener, mqtt, 1883, [ + %% Size of acceptor pool + {acceptors, 16}, + + %% Maximum number of concurrent clients + {max_clients, 512}, + + %% Mount point prefix + %% {mount_point, "prefix/"}, + + %% Socket Access Control + {access, [{allow, all}]}, + + %% Connection Options + {connopts, [ + %% Rate Limit. Format is 'burst, rate', Unit is KB/Sec + %% {rate_limit, "100,10"} %% 100K burst, 10K rate + ]}, + + %% Socket Options + {sockopts, [ + %Set buffer if hight thoughtput + %{recbuf, 4096}, + %{sndbuf, 4096}, + %{buffer, 4096}, + %{nodelay, true}, + {backlog, 1024} + ]} + ]}. + +8883 - MQTT(SSL) +................ + +.. code-block:: erlang + + %% MQTT/SSL + {listener, mqtts, 8883, [ + %% Size of acceptor pool + {acceptors, 4}, + + %% Maximum number of concurrent clients + {max_clients, 512}, + + %% Mount point prefix + %% {mount_point, "secure/"}, + + %% Socket Access Control + {access, [{allow, all}]}, + + %% SSL certificate and key files + {ssl, [{certfile, "etc/ssl/ssl.crt"}, + {keyfile, "etc/ssl/ssl.key"}]}, + + %% Socket Options + {sockopts, [ + {backlog, 1024} + %{buffer, 4096}, + ]} + ]}. + +8083 - MQTT(WebSocket) +...................... + +.. code-block:: erlang + + %% HTTP and WebSocket Listener + {listener, http, 8083, [ + %% Size of acceptor pool + {acceptors, 4}, + + %% Maximum number of concurrent clients + {max_clients, 64}, + + %% Socket Access Control + {access, [{allow, all}]}, + + %% Socket Options + {sockopts, [ + {backlog, 1024} + %{buffer, 4096}, + ]} + ]}. + +Erlang VM Monitor +----------------- + +.. code:: erlang + + %% Long GC, don't monitor in production mode for: + %% https://github.com/erlang/otp/blob/feb45017da36be78d4c5784d758ede619fa7bfd3/erts/emulator/beam/erl_gc.c#L421 + + {sysmon_long_gc, false}. + + %% Long Schedule(ms) + {sysmon_long_schedule, 240}. + + %% 8M words. 32MB on 32-bit VM, 64MB on 64-bit VM. + %% 8 * 1024 * 1024 + {sysmon_large_heap, 8388608}. + + %% Busy Port + {sysmon_busy_port, false}. + + %% Busy Dist Port + {sysmon_busy_dist_port, true}. + diff --git a/docs/source/design.rst b/docs/source/design.rst index a9d081cc2..adc1f10b8 100644 --- a/docs/source/design.rst +++ b/docs/source/design.rst @@ -15,6 +15,17 @@ The emqttd broker 1.0 is more like a network Switch or Router, not a traditional .. image:: _static/images/concept.png +The EMQ 2.0 seperated the Message Flow Plane and Monitor/Control Plane, the Architecture is something like:: + + Control Plane + -------------------- + | | + FrontEnd -> | Flow Plane | -> BackEnd + | | + Session Router + --------------------- + Monitor Plane + Design Philosophy ----------------- @@ -478,3 +489,32 @@ http://github.com/emqtt/emqttd_plugin_template .. _eSockd: https://github.com/emqtt/esockd .. _Chain-of-responsibility_pattern: https://en.wikipedia.org/wiki/Chain-of-responsibility_pattern .. _emqttd_plugin_template: https://github.com/emqtt/emqttd_plugin_template/blob/master/src/emqttd_plugin_template.erl + +----------------- +Mnesia/ETS Tables +----------------- + ++--------------------+--------+----------------------------------------+ +| Table | Type | Description | ++====================+========+========================================+ +| mqtt_trie | mnesia | Trie Table | ++--------------------+--------+----------------------------------------+ +| mqtt_trie_node | mnesia | Trie Node Table | ++--------------------+--------+----------------------------------------+ +| mqtt_route | mnesia | Global Route Table | ++--------------------+--------+----------------------------------------+ +| mqtt_local_route | mnesia | Local Route Table | ++--------------------+--------+----------------------------------------+ +| mqtt_pubsub | ets | PubSub Tab | ++--------------------+--------+----------------------------------------+ +| mqtt_subscriber | ets | Subscriber Tab | ++--------------------+--------+----------------------------------------+ +| mqtt_subscription | ets | Subscription Tab | ++--------------------+--------+----------------------------------------+ +| mqtt_session | mnesia | Global Session Table | ++--------------------+--------+----------------------------------------+ +| mqtt_local_session | ets | Local Session Table | ++--------------------+--------+----------------------------------------+ +| mqtt_client | ets | Client Table | ++--------------------+--------+----------------------------------------+ + diff --git a/docs/source/getstarted.rst b/docs/source/getstarted.rst index 55b11164f..e5d0fbc6e 100644 --- a/docs/source/getstarted.rst +++ b/docs/source/getstarted.rst @@ -35,6 +35,8 @@ Features * MQTT Over WebSocket(SSL) * HTTP Publish API * STOMP protocol +* MQTT-SN Protocol +* CoAP Protocol * STOMP over SockJS * $SYS/# Topics * ClientID Authentication @@ -63,7 +65,7 @@ Installing on Mac, for example: .. code-block:: bash - unzip emqttd-macosx-1.1-beta-20160601.zip && cd emqttd + unzip emqttd-macosx-2.0-beta1-20160830.zip && cd emqttd # Start emqttd ./bin/emqttd start @@ -119,8 +121,6 @@ Modules +-------------------------+--------------------------------------------+ | emqttd_auth_username | Authentication with Username and Password | +-------------------------+--------------------------------------------+ -| emqttd_auth_ldap | Authentication with LDAP | -+-------------------------+--------------------------------------------+ | emqttd_mod_presence | Publish presence message to $SYS topics | | | when client connected or disconnected | +-------------------------+--------------------------------------------+ @@ -136,22 +136,16 @@ Enable 'emqttd_auth_username' module: .. code-block:: erlang - {access, [ - %% Authetication. Anonymous Default - {auth, [ - %% Authentication with username, password - {username, []}, - - ... + %% Authentication with username, password + {auth, username, [{passwd, "etc/modules/passwd.conf"}]}. Enable 'emqttd_mod_presence' module: .. code-block:: erlang - {modules, [ - %% Client presence management module. - %% Publish messages when client connected or disconnected - {presence, [{qos, 0}]} + %% Client presence management module. Publish presence messages when + %% client connected or disconnected. + {module, presence, [{qos, 0}]}. Plugins ------- @@ -163,17 +157,21 @@ A plugin is an Erlang application to extend the emqttd broker. +----------------------------+-----------------------------------+ | `emqttd_dashboard`_ | Web Dashboard | +----------------------------+-----------------------------------+ +| `emqttd_auth_ldap`_ | LDAP Auth Plugin | ++----------------------------+-----------------------------------+ | `emqttd_auth_http`_ | Authentication/ACL with HTTP API | +----------------------------+-----------------------------------+ -| `emqttd_plugin_mysql`_ | Authentication with MySQL | +| `emqttd_auth_mysql` _ | Authentication with MySQL | +----------------------------+-----------------------------------+ -| `emqttd_plugin_pgsql`_ | Authentication with PostgreSQL | +| `emqttd_auth_pgsql`_ | Authentication with PostgreSQL | +----------------------------+-----------------------------------+ -| `emqttd_plugin_redis`_ | Authentication with Redis | +| `emqttd_auth_redis`_ | Authentication with Redis | +----------------------------+-----------------------------------+ | `emqttd_plugin_mongo`_ | Authentication with MongoDB | +----------------------------+-----------------------------------+ -| `emqttd_stomp`_ | STOMP Protocol Plugin | +| `emqttd_sn`_ | MQTT-SN Protocol Plugin | ++----------------------------+-----------------------------------+ +| `emqttd_stomp`_ | STOMP Protocol Plugin | +----------------------------+-----------------------------------+ | `emqttd_sockjs`_ | SockJS(Stomp) Plugin | +----------------------------+-----------------------------------+ @@ -182,9 +180,9 @@ A plugin is an Erlang application to extend the emqttd broker. A plugin could be enabled by 'bin/emqttd_ctl plugins load' command. -For example, enable 'emqttd_plugin_pgsql' plugin:: +For example, enable 'emqttd_auth_pgsql' plugin:: - ./bin/emqttd_ctl plugins load emqttd_plugin_pgsql + ./bin/emqttd_ctl plugins load emqttd_auth_pgsql ----------------------- One Million Connections @@ -238,26 +236,27 @@ emqttd/etc/vm.args:: emqttd broker ------------- -emqttd/etc/emqttd.config: +emqttd/etc/emqttd.conf: .. code-block:: erlang - {mqtt, 1883, [ - %% Size of acceptor pool - {acceptors, 64}, + {listener, mqtt, 1883, [ + %% Size of acceptor pool + {acceptors, 64}, - %% Maximum number of concurrent clients - {max_clients, 1000000}, + %% Maximum number of concurrent clients + {max_clients, 1000000}, - %% Socket Access Control - {access, [{allow, all}]}, + %% Socket Access Control + {access, [{allow, all}]}, - %% Connection Options - {connopts, [ - %% Rate Limit. Format is 'burst, rate', Unit is KB/Sec - %% {rate_limit, "100,10"} %% 100K burst, 10K rate - ]}, - ... + %% Connection Options + {connopts, [ + %% Rate Limit. Format is 'burst, rate', Unit is KB/Sec + %% {rate_limit, "100,10"} %% 100K burst, 10K rate + ]}, + ... + ]}. Test Client ----------- @@ -291,11 +290,15 @@ GitHub: https://github.com/emqtt .. _emqttd_plugin_template: https://github.com/emqtt/emqttd_plugin_template .. _emqttd_dashboard: https://github.com/emqtt/emqttd_dashboard +.. _emqttd_auth_ldap: https://github.com/emqtt/emqttd_auth_ldap .. _emqttd_auth_http: https://github.com/emqtt/emqttd_auth_http -.. _emqttd_plugin_mysql: https://github.com/emqtt/emqttd_plugin_mysql -.. _emqttd_plugin_pgsql: https://github.com/emqtt/emqttd_plugin_pgsql -.. _emqttd_plugin_redis: https://github.com/emqtt/emqttd_plugin_redis -.. _emqttd_plugin_mongo: https://github.com/emqtt/emqttd_plugin_mongo +.. _emqttd_auth_mysql: https://github.com/emqtt/emqttd_plugin_mysql +.. _emqttd_auth_pgsql: https://github.com/emqtt/emqttd_plugin_pgsql +.. _emqttd_auth_redis: https://github.com/emqtt/emqttd_plugin_redis +.. _emqttd_auth_mongo: https://github.com/emqtt/emqttd_plugin_mongo +.. _emqttd_reloader: https://github.com/emqtt/emqttd_reloader .. _emqttd_stomp: https://github.com/emqtt/emqttd_stomp .. _emqttd_sockjs: https://github.com/emqtt/emqttd_sockjs .. _emqttd_recon: https://github.com/emqtt/emqttd_recon +.. _emqttd_sn: https://github.com/emqtt/emqttd_sn + diff --git a/docs/source/index.rst b/docs/source/index.rst index 4051035b1..2db72ecdd 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -35,8 +35,6 @@ Sensors, Mobiles, Web Browsers and Application Servers could be connected by emq | Author: | Feng Lee | +---------------+-----------------------------------------+ -.. NOTE:: MQTT-SN，CoAP Protocols are planned to 1.x release. - Contents: .. toctree:: diff --git a/docs/source/install.rst b/docs/source/install.rst index 8bb68e4f8..83a51d2bd 100644 --- a/docs/source/install.rst +++ b/docs/source/install.rst @@ -35,7 +35,7 @@ Download binary packages from: http://emqtt.io/downloads The package name consists of platform, version and release time. -For example: emqttd-centos64-1.1-beta-20160601.zip +For example: emqttd-centos64-2.0-beta1-20160830.zip .. _install_on_linux: @@ -47,7 +47,7 @@ Download CentOS Package from: http://emqtt.io/downloads/latest/centos, and then .. code-block:: bash - unzip emqttd-centos64-1.1-beta-20160601.zip + unzip emqttd-centos64-2.0-beta-20160830.zip Start the broker in console mode: @@ -80,7 +80,7 @@ If the broker is started successfully, console will print: mqtt listen on 0.0.0.0:1883 with 16 acceptors. mqtts listen on 0.0.0.0:8883 with 4 acceptors. http listen on 0.0.0.0:8083 with 4 acceptors. - Erlang MQTT Broker 1.1 is running now + Erlang MQTT Broker 2.0 is running now Eshell V6.4 (abort with ^G) (emqttd@127.0.0.1)1> @@ -100,7 +100,7 @@ Check the running status of the broker: $ ./bin/emqttd_ctl status Node 'emqttd@127.0.0.1' is started - emqttd 1.1 is running + emqttd 2.0 is running Or check the status by URL:: @@ -130,7 +130,7 @@ We could install the broker on Mac OS X to develop and debug MQTT applications. Download Mac Package from: http://emqtt.io/downloads/latest/macosx -Configure 'lager' log level in 'etc/emqttd.config', all MQTT messages recevied/sent will be printed on console: +Configure 'lager' log level in 'releases/2.0/sys.config', all MQTT messages recevied/sent will be printed on console: .. code-block:: erlang @@ -198,15 +198,15 @@ When all dependencies are ready, clone the emqttd project from github.com and bu .. code-block:: bash - git clone https://github.com/emqtt/emqttd.git + git clone https://github.com/emqtt/emqttd-relx.git - cd emqttd + cd emqttd-relx && make - make && make dist + cd _rel/emqttd && ./bin/emqttd console The binary package output in folder:: - rel/emqttd + _rel/emqttd .. _tcp_ports: @@ -228,19 +228,20 @@ The TCP ports used can be configured in etc/emqttd.config: .. code-block:: erlang - {listeners, [ - {mqtt, 1883, [ - ... - ]}, + %% Plain MQTT + {listener, mqtt, 1883, [ + ... + ]}. - {mqtts, 8883, [ - ... - ]}, - %% HTTP and WebSocket Listener - {http, 8083, [ - ... - ]} - ]}, + %% MQTT/SSL + {listener, mqtts, 8883, [ + ... + ]}. + + %% HTTP and WebSocket Listener + {listener, http, 8083, [ + ... + ]}. The 18083 port is used by Web Dashboard of the broker. Default login: admin, Password: public @@ -255,7 +256,7 @@ Two main configuration files of the emqttd broker: +-------------------+-----------------------------------+ | etc/vm.args | Erlang VM Arguments | +-------------------+-----------------------------------+ -| etc/emqttd.config | emqttd broker Config | +| etc/emqttd.conf | emqttd broker Config | +-------------------+-----------------------------------+ Two important parameters in etc/vm.args: @@ -277,17 +278,18 @@ The maximum number of allowed MQTT clients: .. code-block:: erlang - {listeners, [ - {mqtt, 1883, [ - %% TCP Acceptor Pool - {acceptors, 16}, + %% Plain MQTT + {listener, mqtt, 1883, [ - %% Maximum number of concurrent MQTT clients - {max_clients, 8192}, + %% Size of acceptor pool + {acceptors, 16}, - ... + %% Maximum number of concurrent clients + {max_clients, 8192}, - ]}, + ... + + ]}. .. _init_d_emqttd: diff --git a/docs/source/mqtt-sn.rst b/docs/source/mqtt-sn.rst new file mode 100644 index 000000000..22bcabf6f --- /dev/null +++ b/docs/source/mqtt-sn.rst @@ -0,0 +1,10 @@ + +.. _mqtt_sn: + +TODO:... + +================ +MQTT-SN Protocol +================ + + diff --git a/docs/source/plugins.rst b/docs/source/plugins.rst index 6c4fa552a..ab4543e06 100644 --- a/docs/source/plugins.rst +++ b/docs/source/plugins.rst @@ -7,24 +7,28 @@ Plugins The emqttd broker could be extended by plugins. Users could develop plugins to customize authentication, ACL and functions of the broker, or integrate the broker with other systems. -The plugins that emqtt project released: +The plugins that emqttd 2.0 released: +---------------------------+---------------------------+ | Plugin | Description | +===========================+===========================+ +| `emqttd_dashboard`_ | Web Dashboard | ++---------------------------+---------------------------+ | `emqttd_plugin_template`_ | Template Plugin | +---------------------------+---------------------------+ -| `emqttd_dashboard`_ | Web Dashboard | +| `emqttd_auth_ldap`_ | LDAP Auth | +---------------------------+---------------------------+ | `emqttd_auth_http`_ | HTTP Auth/ACL Plugin | +---------------------------+---------------------------+ -| `emqttd_plugin_mysql`_ | MySQL Auth/ACL Plugin | +| `emqttd_auth_mysql`_ | MySQL Auth/ACL Plugin | +---------------------------+---------------------------+ -| `emqttd_plugin_pgsql`_ | PostgreSQL Auth/ACL Plugin| +| `emqttd_auth_pgsql`_ | PostgreSQL Auth/ACL Plugin| +---------------------------+---------------------------+ -| `emqttd_plugin_redis`_ | Redis Auth/ACL Plugin | +| `emqttd_auth_redis`_ | Redis Auth/ACL Plugin | +---------------------------+---------------------------+ -| `emqttd_plugin_mongo`_ | MongoDB Auth/ACL Plugin | +| `emqttd_auth_mongo`_ | MongoDB Auth/ACL Plugin | ++---------------------------+---------------------------+ +| `emqttd_sn`_ | MQTT-SN Protocol Plugin | +---------------------------+---------------------------+ | `emqttd_stomp`_ | STOMP Protocol Plugin | +---------------------------+---------------------------+ @@ -39,17 +43,9 @@ The plugins that emqtt project released: emqttd_plugin_template - Template Plugin ---------------------------------------- -A plugin is just a normal Erlang application under the 'emqttd/plugins' folder. Each plugin has e configuration file: 'etc/plugin.config'. +A plugin is just a normal Erlang application which has its own configuration file: 'etc/.config'. -plugins/emqttd_plugin_template is a demo plugin. The folder structure: - -+------------------------+---------------------------+ -| File | Description | -+========================+===========================+ -| etc/plugin.config | Plugin config file | -+------------------------+---------------------------+ -| ebin/ | Erlang program files | -+------------------------+---------------------------+ +emqttd_plugin_template is a demo plugin. Load, unload Plugin ------------------- @@ -78,22 +74,51 @@ The Web Dashboard for emqttd broker. The plugin will be loaded automatically whe .. image:: _static/images/dashboard.png -Configure Dashboard -------------------- +Configure Dashboard Plugin +-------------------------- -emqttd_dashboard/etc/plugin.config: +etc/plugins/emqttd_dashboard.conf: .. code-block:: erlang - [ - {emqttd_dashboard, [ - {listener, - {emqttd_dashboard, 18083, [ - {acceptors, 4}, - {max_clients, 512}]} - } + {listener, + {dashboard, 18083, [ + {acceptors, 4}, + {max_clients, 512} ]} - ]. + }. + +---------------------------------- +emqttd_auth_ldap: LDAP Auth Plugin +---------------------------------- + +LDAP Auth Plugin: https://github.com/emqtt/emqttd_auth_ldap + +.. NOTE:: Supported in 2.0-beta1 release + +Configure LDAP Plugin +--------------------- + +etc/plugins/emqttd_auth_ldap.conf: + +.. code-block:: erlang + + {ldap, [ + {servers, ["localhost"]}, + {port, 389}, + {timeout, 30}, + {user_dn, "uid=$u,ou=People,dc=example,dc=com"}, + {ssl, fasle}, + {sslopts, [ + {certfile, "ssl.crt"}, + {keyfile, "ssl.key"} + ]} + ]}. + +Load LDAP Plugin +---------------- + +./bin/emqttd_ctl plugins load emqttd_auth_ldap --------------------------------------- emqttd_auth_http - HTTP Auth/ACL Plugin @@ -103,61 +128,57 @@ MQTT Authentication/ACL with HTTP API: https://github.com/emqtt/emqttd_auth_http .. NOTE:: Supported in 1.1 release -Configure emqttd_auth_http/etc/plugin.config --------------------------------------------- +Configure HTTP Auth/ACL Plugin +------------------------------ -.. code:: erlang +etc/plugins/emqttd_auth_http.conf: - [ +.. code-block:: erlang - {emqttd_auth_http, [ + %% Variables: %u = username, %c = clientid, %a = ipaddress, %t = topic - %% Variables: %u = username, %c = clientid, %a = ipaddress, %t = topic - - {super_req, [ - {method, post}, - {url, "http://localhost:8080/mqtt/superuser"}, - {params, [ - {username, "%u"}, - {clientid, "%c"} - ]} - ]}, - - {auth_req, [ - {method, post}, - {url, "http://localhost:8080/mqtt/auth"}, - {params, [ - {clientid, "%c"}, - {username, "%u"}, - {password, "%P"} - ]} - ]}, - - %% 'access' parameter: sub = 1, pub = 2 - - {acl_req, [ - {method, post}, - {url, "http://localhost:8080/mqtt/acl"}, - {params, [ - {access, "%A"}, - {username, "%u"}, - {clientid, "%c"}, - {ipaddr, "%a"}, - {topic, "%t"} - ]} - ]} + {super_req, [ + {method, post}, + {url, "http://localhost:8080/mqtt/superuser"}, + {params, [ + {username, "%u"}, + {clientid, "%c"} ]} + ]}. - ]. + {auth_req, [ + {method, post}, + {url, "http://localhost:8080/mqtt/auth"}, + {params, [ + {clientid, "%c"}, + {username, "%u"}, + {password, "%P"} + ]} + ]}. -HTTP API --------- + %% 'access' parameter: sub = 1, pub = 2 + + {acl_req, [ + {method, post}, + {url, "http://localhost:8080/mqtt/acl"}, + {params, [ + {access, "%A"}, + {username, "%u"}, + {clientid, "%c"}, + {ipaddr, "%a"}, + {topic, "%t"} + ]} + ]}. + + +HTTP Auth/ACL API +----------------- Return 200 if ok Return 4xx if unauthorized -Load emqttd_auth_http plugin +Load HTTP Auth/ACL Plugin ---------------------------- .. code:: bash @@ -211,64 +232,57 @@ MQTT ACL Table (6,1,'127.0.0.1',NULL,NULL,2,'#'), (7,1,NULL,'dashboard',NULL,1,'$SYS/#'); -Configure emqttd_plugin_mysql/etc/plugin.config ------------------------------------------------ +Configure MySQL Auth/ACL Plugin +------------------------------- -Configure MySQL host, username, password and database: +etc/plugins/emqttd_plugin_mysql.conf: .. code-block:: erlang - [ + {mysql_pool, [ + %% pool options + {pool_size, 8}, + {auto_reconnect, 1}, - {emqttd_plugin_mysql, [ + %% mysql options + {host, "localhost"}, + {port, 3306}, + {user, ""}, + {password, ""}, + {database, "mqtt"}, + {encoding, utf8}, + {keep_alive, true} + ]}. - {mysql_pool, [ - %% ecpool options - {pool_size, 8}, - {auto_reconnect, 3}, + %% Variables: %u = username, %c = clientid, %a = ipaddress - %% mysql options - {host, "localhost"}, - {port, 3306}, - {user, ""}, - {password, ""}, - {database, "mqtt"}, - {encoding, utf8} - ]}, + %% Superuser Query + {superquery, "select is_superuser from mqtt_user where username = '%u' limit 1"}. - %% Variables: %u = username, %c = clientid, %a = ipaddress + %% Authentication Query: select password only + {authquery, "select password from mqtt_user where username = '%u' limit 1"}. - %% Superuser Query - {superquery, "select is_superuser from mqtt_user where username = '%u' limit 1"}, + %% hash algorithm: plain, md5, sha, sha256, pbkdf2? + {password_hash, sha256}. - %% Authentication Query: select password only - {authquery, "select password from mqtt_user where username = '%u' limit 1"}, + %% select password with salt + %% {authquery, "select password, salt from mqtt_user where username = '%u'"}. - %% hash algorithm: plain, md5, sha, sha256, pbkdf2? - {password_hash, sha256}, + %% sha256 with salt prefix + %% {password_hash, {salt, sha256}}. - %% select password with salt - %% {authquery, "select password, salt from mqtt_user where username = '%u'"}, + %% sha256 with salt suffix + %% {password_hash, {sha256, salt}}. - %% sha256 with salt prefix - %% {password_hash, {salt, sha256}}, + %% '%a' = ipaddress, '%u' = username, '%c' = clientid + %% Comment this query, the acl will be disabled + {aclquery, "select allow, ipaddr, username, clientid, access, topic from mqtt_acl where ipaddr = '%a' or username = '%u' or username = '$all' or clientid = '%c'"}. - %% sha256 with salt suffix - %% {password_hash, {sha256, salt}}, + %% If no ACL rules matched, return... + {acl_nomatch, allow}. - %% '%a' = ipaddress, '%u' = username, '%c' = clientid - %% Comment this query, the acl will be disabled - {aclquery, "select allow, ipaddr, username, clientid, access, topic from mqtt_acl where ipaddr = '%a' or username = '%u' or username = '$all' or clientid = '%c'"}, - - %% If no ACL rules matched, return... - {acl_nomatch, allow} - - ]} - - ]. - -Load emqttd_plugin_mysql plugin -------------------------------- +Load MySQL Auth/ACL plugin +-------------------------- .. code-block:: bash @@ -280,8 +294,8 @@ emqttd_plugin_pgsql - PostgreSQL Auth/ACL Plugin MQTT Authentication, ACL with PostgreSQL Database. -MQTT User Table ---------------- +Postgre MQTT User Table +----------------------- .. code-block:: sql @@ -293,8 +307,8 @@ MQTT User Table salt character varying(40) ); -MQTT ACL Table --------------- +Postgre MQTT ACL Table +---------------------- .. code-block:: sql @@ -317,67 +331,62 @@ MQTT ACL Table (6,1,'127.0.0.1',NULL,NULL,2,'#'), (7,1,NULL,'dashboard',NULL,1,'$SYS/#'); -Configure emqttd_plugin_pgsql/etc/plugin.config +Configure Postgre Auth/ACL Plugin ----------------------------------------------- +Plugin Config: etc/plugins/emqttd_plugin_pgsql.conf. + Configure host, username, password and database of PostgreSQL: .. code-block:: erlang - [ + {pgsql_pool, [ + %% pool options + {pool_size, 8}, + {auto_reconnect, 3}, - {emqttd_plugin_pgsql, [ + %% pgsql options + {host, "localhost"}, + {port, 5432}, + {ssl, false}, + {username, "feng"}, + {password, ""}, + {database, "mqtt"}, + {encoding, utf8} + ]}. - {pgsql_pool, [ - %% ecpool options - {pool_size, 8}, - {auto_reconnect, 3}, + %% Variables: %u = username, %c = clientid, %a = ipaddress - %% pgsql options - {host, "localhost"}, - {port, 5432}, - {ssl, false}, - {username, "feng"}, - {password, ""}, - {database, "mqtt"}, - {encoding, utf8} - ]}, + %% Superuser Query + {superquery, "select is_superuser from mqtt_user where username = '%u' limit 1"}. - %% Variables: %u = username, %c = clientid, %a = ipaddress + %% Authentication Query: select password only + {authquery, "select password from mqtt_user where username = '%u' limit 1"}. - %% Superuser Query - {superquery, "select is_superuser from mqtt_user where username = '%u' limit 1"}, + %% hash algorithm: plain, md5, sha, sha256, pbkdf2? + {password_hash, sha256}. - %% Authentication Query: select password only - {authquery, "select password from mqtt_user where username = '%u' limit 1"}, + %% select password with salt + %% {authquery, "select password, salt from mqtt_user where username = '%u'"}. - %% hash algorithm: plain, md5, sha, sha256, pbkdf2? - {password_hash, sha256}, + %% sha256 with salt prefix + %% {password_hash, {salt, sha256}}. - %% select password with salt - %% {authquery, "select password, salt from mqtt_user where username = '%u'"}, + %% sha256 with salt suffix + %% {password_hash, {sha256, salt}}. - %% sha256 with salt prefix - %% {password_hash, {salt, sha256}}, + %% Comment this query, the acl will be disabled. Notice: don't edit this query! + {aclquery, "select allow, ipaddr, username, clientid, access, topic from mqtt_acl where ipaddr = '%a' or username = '%u' or username = '$all' or clientid = '%c'"}. - %% sha256 with salt suffix - %% {password_hash, {sha256, salt}}, + %% If no rules matched, return... + {acl_nomatch, allow}. - %% Comment this query, the acl will be disabled. Notice: don't edit this query! - {aclquery, "select allow, ipaddr, username, clientid, access, topic from mqtt_acl - where ipaddr = '%a' or username = '%u' or username = '$all' or clientid = '%c'"}, - - %% If no rules matched, return... - {acl_nomatch, allow} - ]} - ]. - -Load emqttd_plugin_pgsql Plugin -------------------------------- +Load Postgre Auth/ACL Plugin +----------------------------- .. code-block:: bash - ./bin/emqttd_ctl plugins load emqttd_plugin_pgsql + ./bin/emqttd_ctl plugins load emqttd_auth_pgsql ------------------------------------------- emqttd_plugin_redis - Redis Auth/ACL Plugin @@ -385,58 +394,56 @@ emqttd_plugin_redis - Redis Auth/ACL Plugin MQTT Authentication, ACL with Redis: https://github.com/emqtt/emqttd_plugin_redis -Configure emqttd_plugin_redis/etc/plugin.config ------------------------------------------------ +Configure Redis Auth/ACL Plugin +------------------------------- + +etc/plugins/emqttd_auth_redis.conf: .. code-block:: erlang - [ - {emqttd_plugin_redis, [ + {redis_pool, [ + %% pool options + {pool_size, 8}, + {auto_reconnect, 2}, - {eredis_pool, [ - %% ecpool options - {pool_size, 8}, - {auto_reconnect, 2}, + %% redis options + {host, "127.0.0.1"}, + {port, 6379}, + {database, 0}, + {password, ""} + ]}. - %% eredis options - {host, "127.0.0.1"}, - {port, 6379}, - {database, 0}, - {password, ""} - ]}, + %% Variables: %u = username, %c = clientid - %% Variables: %u = username, %c = clientid + %% HMGET mqtt_user:%u is_superuser + {supercmd, ["HGET", "mqtt_user:%u", "is_superuser"]}. - %% HMGET mqtt_user:%u is_superuser - {supercmd, ["HGET", "mqtt_user:%u", "is_superuser"]}, + %% HMGET mqtt_user:%u password + {authcmd, ["HGET", "mqtt_user:%u", "password"]}. - %% HMGET mqtt_user:%u password - {authcmd, ["HGET", "mqtt_user:%u", "password"]}, + %% Password hash algorithm: plain, md5, sha, sha256, pbkdf2? + {password_hash, sha256}. - %% Password hash algorithm: plain, md5, sha, sha256, pbkdf2? - {password_hash, sha256}, + %% SMEMBERS mqtt_acl:%u + {aclcmd, ["SMEMBERS", "mqtt_acl:%u"]}. - %% SMEMBERS mqtt_acl:%u - {aclcmd, ["SMEMBERS", "mqtt_acl:%u"]}, + %% If no rules matched, return... + {acl_nomatch, deny}. - %% If no rules matched, return... - {acl_nomatch, deny}, + %% Load Subscriptions form Redis when client connected. + {subcmd, ["HGETALL", "mqtt_subs:%u"]}. - %% Load Subscriptions form Redis when client connected. - {subcmd, ["HGETALL", "mqtt_subs:%u"]} - ]} - ]. -User HASH ---------- +Redis User HASH +--------------- Set a 'user' hash with 'password' field, for example:: HSET mqtt_user: is_superuser 1 HSET mqtt_user: password "passwd" -ACL Rule SET ------------- +Redis ACL Rule SET +------------------ The plugin uses a redis SET to store ACL rules:: @@ -444,8 +451,8 @@ The plugin uses a redis SET to store ACL rules:: SADD mqtt_acl: "subscribe topic2" SADD mqtt_acl: "pubsub topic3" -Subscription HASH ------------------ +Redis Subscription HASH +----------------------- The plugin can store static subscriptions in a redis Hash:: @@ -453,12 +460,12 @@ The plugin can store static subscriptions in a redis Hash:: HSET mqtt_subs: topic2 1 HSET mqtt_subs: topic3 2 -Load emqttd_plugin_redis Plugin -------------------------------- +Load Redis Auth/ACL Plugin +-------------------------- .. code-block:: bash - ./bin/emqttd_ctl plugins load emqttd_plugin_redis + ./bin/emqttd_ctl plugins load emqttd_auth_redis --------------------------------------------- emqttd_plugin_mongo - MongoDB Auth/ACL Plugin @@ -466,55 +473,51 @@ emqttd_plugin_mongo - MongoDB Auth/ACL Plugin MQTT Authentication, ACL with MongoDB: https://github.com/emqtt/emqttd_plugin_mongo -Configure emqttd_plugin_mongo/etc/plugin.config ------------------------------------------------ +Configure MongoDB Auth/ACL Plugin +--------------------------------- + +etc/plugins/emqttd_plugin_mongo.conf: .. code-block:: erlang - [ - {emqttd_plugin_mongo, [ + {mongo_pool, [ + {pool_size, 8}, + {auto_reconnect, 3}, - {mongo_pool, [ - {pool_size, 8}, - {auto_reconnect, 3}, + %% Mongodb Opts + {host, "localhost"}, + {port, 27017}, + %% {login, ""}, + %% {password, ""}, + {database, "mqtt"} + ]}. - %% Mongodb Driver Opts - {host, "localhost"}, - {port, 27017}, - %% {login, ""}, - %% {password, ""}, - {database, "mqtt"} - ]}, + %% Variables: %u = username, %c = clientid - %% Variables: %u = username, %c = clientid + %% Superuser Query + {superquery, pool, [ + {collection, "mqtt_user"}, + {super_field, "is_superuser"}, + {selector, {"username", "%u"}} + ]}. - %% Superuser Query - {superquery, [ - {collection, "mqtt_user"}, - {super_field, "is_superuser"}, - {selector, {"username", "%u"}} - ]}, + %% Authentication Query + {authquery, pool, [ + {collection, "mqtt_user"}, + {password_field, "password"}, + %% Hash Algorithm: plain, md5, sha, sha256, pbkdf2? + {password_hash, sha256}, + {selector, {"username", "%u"}} + ]}. - %% Authentication Query - {authquery, [ - {collection, "mqtt_user"}, - {password_field, "password"}, - %% Hash Algorithm: plain, md5, sha, sha256, pbkdf2? - {password_hash, sha256}, - {selector, {"username", "%u"}} - ]}, + %% ACL Query: "%u" = username, "%c" = clientid + {aclquery, pool, [ + {collection, "mqtt_acl"}, + {selector, {"username", "%u"}} + ]}. - %% ACL Query: "%u" = username, "%c" = clientid - {aclquery, [ - {collection, "mqtt_acl"}, - {selector, {"username", "%u"}} - ]}, - - %% If no ACL rules matched, return... - {acl_nomatch, deny} - - ]} - ]. + %% If no ACL rules matched, return... + {acl_nomatch, deny}. MongoDB Database ---------------- @@ -526,8 +529,8 @@ MongoDB Database db.createCollection("mqtt_acl") db.mqtt_user.ensureIndex({"username":1}) -User Collection ---------------- +MongoDB User Collection +----------------------- .. code-block:: json @@ -543,8 +546,8 @@ For example:: db.mqtt_user.insert({username: "test", password: "password hash", is_superuser: false}) db.mqtt_user:insert({username: "root", is_superuser: true}) -ACL Collection --------------- +MongoDB ACL Collection +---------------------- .. code-block:: json @@ -561,12 +564,34 @@ For example:: db.mqtt_acl.insert({username: "test", publish: ["t/1", "t/2"], subscribe: ["user/%u", "client/%c"]}) db.mqtt_acl.insert({username: "admin", pubsub: ["#"]}) -Load emqttd_plugin_mongo Plugin -------------------------------- +Load MongoDB Auth/ACL Plugin +---------------------------- .. code-block:: bash - ./bin/emqttd_ctl plugins load emqttd_plugin_mongo + ./bin/emqttd_ctl plugins load emqttd_auth_mongo + +--------------------------- +emqttd_sn: MQTT-SN Protocol +-------------------------- + +MQTT-SN Protocol/Gateway Plugin. + +Configure MQTT-SN Plugin +------------------------- + +.. NOTE:: UDP Port for MQTT-SN: 1884 + +etc/plugins/emqttd_sn.conf:: + + {listener, {1884, []}}. + +Load MQTT-SN Plugin +------------------- + +.. code:: + + ./bin/emqttd_ctl plugins load emqttd_sn ----------------------------- emqttd_stomp - STOMP Protocol @@ -574,48 +599,41 @@ emqttd_stomp - STOMP Protocol Support STOMP 1.0/1.1/1.2 clients to connect to emqttd broker and communicate with MQTT Clients. -Configure emqttd_stomp/etc/plugin.config ----------------------------------------- +Configure Stomp Plugin +---------------------- + +etc/plugins/emqttd_stomp.conf: .. NOTE:: Default Port for STOMP Protocol: 61613 .. code-block:: erlang - [ - {emqttd_stomp, [ + {default_user, [ + {login, "guest"}, + {passcode, "guest"} + ]}. - {default_user, [ - {login, "guest"}, - {passcode, "guest"} - ]}, + {allow_anonymous, true}. - {allow_anonymous, true}, + {frame, [ + {max_headers, 10}, + {max_header_length, 1024}, + {max_body_length, 8192} + ]}. - %%TODO: unused... - {frame, [ - {max_headers, 10}, - {max_header_length, 1024}, - {max_body_length, 8192} - ]}, + {listener, emqttd_stomp, 61613, [ + {acceptors, 4}, + {max_clients, 512} + ]}. - {listeners, [ - {emqttd_stomp, 61613, [ - {acceptors, 4}, - {max_clients, 512} - ]} - ]} - ]} - ]. - -Load emqttd_stomp Plugin ------------------------- +Load Stomp Plugin +----------------- .. code-block:: bash ./bin/emqttd_ctl plugins load emqttd_stomp - ----------------------------------- emqttd_sockjs - STOMP/SockJS Plugin ----------------------------------- @@ -629,18 +647,21 @@ Configure emqttd_sockjs .. code-block:: erlang - [ - {emqttd_sockjs, [ + {sockjs, []}. - {sockjs, []}, - - {cowboy_listener, {stomp_sockjs, 61616, 4}}, + {cowboy_listener, {stomp_sockjs, 61616, 4}}. + %% TODO: unused... + {stomp, [ + {frame, [ + {max_headers, 10}, + {max_header_length, 1024}, + {max_body_length, 8192} ]} - ]. + ]}. -Load emqttd_sockjs Plugin -------------------------- +Load SockJS Plugin +------------------ .. NOTE:: emqttd_stomp Plugin required. @@ -661,8 +682,8 @@ emqttd_recon - Recon Plugin The plugin loads `recon`_ library on a running emqttd broker. Recon libray helps debug and optimize an Erlang application. -Load emqttd_recon Plugin ------------------------- +Load Recon Plugin +----------------- .. code-block:: bash @@ -689,8 +710,8 @@ Erlang Module Reloader for Development .. NOTE:: Don't load the plugin in production! -Load emqttd_reloader Plugin ---------------------------- +Load 'Reloader' Plugin +---------------------- .. code-block:: bash @@ -712,15 +733,22 @@ Plugin Development Guide Create a Plugin Project ----------------------- -Clone emqttd source from github.com:: +Clone emqttd_plugin_template source from github.com:: - git clone https://github.com/emqtt/emqttd.git + git clone https://github.com/emqtt/emqttd_plugin_template.git -Create a plugin project under 'plugins' folder:: +Create a plugin project with erlang.mk and depends on 'emqttd' application, the 'Makefile':: - cd plugins && mkdir emqttd_my_plugin + PROJECT = emqttd_plugin_abc + PROJECT_DESCRIPTION = emqttd abc plugin + PROJECT_VERSION = 1.0 - cd emqttd_my_plugin && rebar create-app appid=emqttd_my_plugin + DEPS = emqttd + dep_emqttd = git https://github.com/emqtt/emqttd emq20 + + COVER = true + + include erlang.mk Template Plugin: https://github.com/emqtt/emqttd_plugin_template @@ -735,7 +763,7 @@ emqttd_auth_demo.erl - demo authentication module: -behaviour(emqttd_auth_mod). - -include("../../../include/emqttd.hrl"). + -include_lib("emqttd/include/emqttd.hrl"). -export([init/1, check/3, description/0]). @@ -754,7 +782,7 @@ emqttd_acl_demo.erl - demo ACL module: -module(emqttd_acl_demo). - -include("../../../include/emqttd.hrl"). + -include_lib("emqttd/include/emqttd.hrl"). %% ACL callbacks -export([init/1, check_acl/2, reload_acl/1, description/0]). @@ -830,7 +858,7 @@ emqttd_cli_demo.erl: -module(emqttd_cli_demo). - -include("../../../include/emqttd_cli.hrl"). + -include_lib("emqttd/include/emqttd_cli.hrl"). -export([cmd/1]). @@ -850,13 +878,14 @@ There will be a new CLI after the plugin loaded:: ./bin/emqttd_ctl cmd arg1 arg2 - .. _emqttd_dashboard: https://github.com/emqtt/emqttd_dashboard +.. _emqttd_auth_ldap: https://github.com/emqtt/emqttd_auth_ldap .. _emqttd_auth_http: https://github.com/emqtt/emqttd_auth_http -.. _emqttd_plugin_mysql: https://github.com/emqtt/emqttd_plugin_mysql -.. _emqttd_plugin_pgsql: https://github.com/emqtt/emqttd_plugin_pgsql -.. _emqttd_plugin_redis: https://github.com/emqtt/emqttd_plugin_redis -.. _emqttd_plugin_mongo: https://github.com/emqtt/emqttd_plugin_mongo +.. _emqttd_auth_mysql: https://github.com/emqtt/emqttd_plugin_mysql +.. _emqttd_auth_pgsql: https://github.com/emqtt/emqttd_plugin_pgsql +.. _emqttd_auth_redis: https://github.com/emqtt/emqttd_plugin_redis +.. _emqttd_auth_mongo: https://github.com/emqtt/emqttd_plugin_mongo +.. _emqttd_sn: https://github.com/emqtt/emqttd_sn .. _emqttd_stomp: https://github.com/emqtt/emqttd_stomp .. _emqttd_sockjs: https://github.com/emqtt/emqttd_sockjs .. _emqttd_recon: https://github.com/emqtt/emqttd_recon diff --git a/docs/source/tables.md b/docs/source/tables.md new file mode 100644 index 000000000..9268e7822 --- /dev/null +++ b/docs/source/tables.md @@ -0,0 +1,31 @@ + +.. _tables: + +====== +Tables +====== + ++--------------------+--------+----------------------------------------+ +| Table | Type | Description | ++====================+========+========================================+ +| mqtt_trie | mnesia | Trie Table | ++--------------------+--------+----------------------------------------+ +| mqtt_trie_node | mnesia | Trie Node Table | ++--------------------+--------+----------------------------------------+ +| mqtt_route | mnesia | Global Route Table | ++--------------------+--------+----------------------------------------+ +| mqtt_local_route | mnesia | Local Route Table | ++--------------------+--------+----------------------------------------+ +| mqtt_pubsub | ets | PubSub Tab | ++--------------------+--------+----------------------------------------+ +| mqtt_subscriber | ets | Subscriber Tab | ++--------------------+--------+----------------------------------------+ +| mqtt_subscription | ets | Subscription Tab | ++--------------------+--------+----------------------------------------+ +| mqtt_session | mnesia | Global Session Table | ++--------------------+--------+----------------------------------------+ +| mqtt_local_session | ets | Local Session Table | ++--------------------+--------+----------------------------------------+ +| mqtt_client | ets | Client Table | ++--------------------+--------+----------------------------------------+ + diff --git a/ebin/.placeholder b/ebin/.placeholder deleted file mode 100644 index 5a885e5ce..000000000 --- a/ebin/.placeholder +++ /dev/null @@ -1 +0,0 @@ -emqttd plugin cannot include "emqttd/include/emqttd.hrl" without this directory:( diff --git a/erlang.mk b/erlang.mk new file mode 100644 index 000000000..e348d4493 --- /dev/null +++ b/erlang.mk @@ -0,0 +1,2741 @@ +# Copyright (c) 2013-2015, Loïc Hoguin +# +# Permission to use, copy, modify, and/or distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +.PHONY: all app apps deps search rel docs install-docs check tests clean distclean help erlang-mk + +ERLANG_MK_FILENAME := $(realpath $(lastword $(MAKEFILE_LIST))) + +ERLANG_MK_VERSION = 2.0.0-pre.2-130-gc6fe5ea + +# Core configuration. + +PROJECT ?= $(notdir $(CURDIR)) +PROJECT := $(strip $(PROJECT)) + +PROJECT_VERSION ?= rolling +PROJECT_MOD ?= $(PROJECT)_app + +# Verbosity. + +V ?= 0 + +verbose_0 = @ +verbose_2 = set -x; +verbose = $(verbose_$(V)) + +gen_verbose_0 = @echo " GEN " $@; +gen_verbose_2 = set -x; +gen_verbose = $(gen_verbose_$(V)) + +# Temporary files directory. + +ERLANG_MK_TMP ?= $(CURDIR)/.erlang.mk +export ERLANG_MK_TMP + +# "erl" command. + +ERL = erl +A0 -noinput -boot start_clean + +# Platform detection. + +ifeq ($(PLATFORM),) +UNAME_S := $(shell uname -s) + +ifeq ($(UNAME_S),Linux) +PLATFORM = linux +else ifeq ($(UNAME_S),Darwin) +PLATFORM = darwin +else ifeq ($(UNAME_S),SunOS) +PLATFORM = solaris +else ifeq ($(UNAME_S),GNU) +PLATFORM = gnu +else ifeq ($(UNAME_S),FreeBSD) +PLATFORM = freebsd +else ifeq ($(UNAME_S),NetBSD) +PLATFORM = netbsd +else ifeq ($(UNAME_S),OpenBSD) +PLATFORM = openbsd +else ifeq ($(UNAME_S),DragonFly) +PLATFORM = dragonfly +else ifeq ($(shell uname -o),Msys) +PLATFORM = msys2 +else +$(error Unable to detect platform. Please open a ticket with the output of uname -a.) +endif + +export PLATFORM +endif + +# Core targets. + +all:: deps app rel + +# Noop to avoid a Make warning when there's nothing to do. +rel:: + $(verbose) : + +check:: tests + +clean:: clean-crashdump + +clean-crashdump: +ifneq ($(wildcard erl_crash.dump),) + $(gen_verbose) rm -f erl_crash.dump +endif + +distclean:: clean distclean-tmp + +distclean-tmp: + $(gen_verbose) rm -rf $(ERLANG_MK_TMP) + +help:: + $(verbose) printf "%s\n" \ + "erlang.mk (version $(ERLANG_MK_VERSION)) is distributed under the terms of the ISC License." \ + "Copyright (c) 2013-2015 Loïc Hoguin " \ + "" \ + "Usage: [V=1] $(MAKE) [target]..." \ + "" \ + "Core targets:" \ + " all Run deps, app and rel targets in that order" \ + " app Compile the project" \ + " deps Fetch dependencies (if needed) and compile them" \ + " search q=... Search for a package in the built-in index" \ + " rel Build a release for this project, if applicable" \ + " docs Build the documentation for this project" \ + " install-docs Install the man pages for this project" \ + " check Compile and run all tests and analysis for this project" \ + " tests Run the tests for this project" \ + " clean Delete temporary and output files from most targets" \ + " distclean Delete all temporary and output files" \ + " help Display this help and exit" \ + " erlang-mk Update erlang.mk to the latest version" + +# Core functions. + +empty := +space := $(empty) $(empty) +tab := $(empty) $(empty) +comma := , + +define newline + + +endef + +define comma_list +$(subst $(space),$(comma),$(strip $(1))) +endef + +# Adding erlang.mk to make Erlang scripts who call init:get_plain_arguments() happy. +define erlang +$(ERL) $(2) -pz $(ERLANG_MK_TMP)/rebar/ebin -eval "$(subst $(newline),,$(subst ",\",$(1)))" -- erlang.mk +endef + +ifeq ($(PLATFORM),msys2) +core_native_path = $(subst \,\\\\,$(shell cygpath -w $1)) +else +core_native_path = $1 +endif + +ifeq ($(shell which wget 2>/dev/null | wc -l), 1) +define core_http_get + wget --no-check-certificate -O $(1) $(2)|| rm $(1) +endef +else +define core_http_get.erl + ssl:start(), + inets:start(), + case httpc:request(get, {"$(2)", []}, [{autoredirect, true}], []) of + {ok, {{_, 200, _}, _, Body}} -> + case file:write_file("$(1)", Body) of + ok -> ok; + {error, R1} -> halt(R1) + end; + {error, R2} -> + halt(R2) + end, + halt(0). +endef + +define core_http_get + $(call erlang,$(call core_http_get.erl,$(call core_native_path,$1),$2)) +endef +endif + +core_eq = $(and $(findstring $(1),$(2)),$(findstring $(2),$(1))) + +core_find = $(if $(wildcard $1),$(shell find $(1:%/=%) -type f -name $(subst *,\*,$2))) + +core_lc = $(subst A,a,$(subst B,b,$(subst C,c,$(subst D,d,$(subst E,e,$(subst F,f,$(subst G,g,$(subst H,h,$(subst I,i,$(subst J,j,$(subst K,k,$(subst L,l,$(subst M,m,$(subst N,n,$(subst O,o,$(subst P,p,$(subst Q,q,$(subst R,r,$(subst S,s,$(subst T,t,$(subst U,u,$(subst V,v,$(subst W,w,$(subst X,x,$(subst Y,y,$(subst Z,z,$(1))))))))))))))))))))))))))) + +core_ls = $(filter-out $(1),$(shell echo $(1))) + +# @todo Use a solution that does not require using perl. +core_relpath = $(shell perl -e 'use File::Spec; print File::Spec->abs2rel(@ARGV) . "\n"' $1 $2) + +# Automated update. + +ERLANG_MK_REPO ?= https://github.com/ninenines/erlang.mk +ERLANG_MK_COMMIT ?= +ERLANG_MK_BUILD_CONFIG ?= build.config +ERLANG_MK_BUILD_DIR ?= .erlang.mk.build + +erlang-mk: + git clone $(ERLANG_MK_REPO) $(ERLANG_MK_BUILD_DIR) +ifdef ERLANG_MK_COMMIT + cd $(ERLANG_MK_BUILD_DIR) && git checkout $(ERLANG_MK_COMMIT) +endif + if [ -f $(ERLANG_MK_BUILD_CONFIG) ]; then cp $(ERLANG_MK_BUILD_CONFIG) $(ERLANG_MK_BUILD_DIR)/build.config; fi + $(MAKE) -C $(ERLANG_MK_BUILD_DIR) + cp $(ERLANG_MK_BUILD_DIR)/erlang.mk ./erlang.mk + rm -rf $(ERLANG_MK_BUILD_DIR) + +# Copyright (c) 2015, Loïc Hoguin +# This file is part of erlang.mk and subject to the terms of the ISC License. + +.PHONY: search + +define pkg_print + $(verbose) printf "%s\n" \ + $(if $(call core_eq,$(1),$(pkg_$(1)_name)),,"Pkg name: $(1)") \ + "App name: $(pkg_$(1)_name)" \ + "Description: $(pkg_$(1)_description)" \ + "Home page: $(pkg_$(1)_homepage)" \ + "Fetch with: $(pkg_$(1)_fetch)" \ + "Repository: $(pkg_$(1)_repo)" \ + "Commit: $(pkg_$(1)_commit)" \ + "" + +endef + +search: +ifdef q + $(foreach p,$(PACKAGES), \ + $(if $(findstring $(call core_lc,$(q)),$(call core_lc,$(pkg_$(p)_name) $(pkg_$(p)_description))), \ + $(call pkg_print,$(p)))) +else + $(foreach p,$(PACKAGES),$(call pkg_print,$(p))) +endif + +# Copyright (c) 2013-2015, Loïc Hoguin +# This file is part of erlang.mk and subject to the terms of the ISC License. + +.PHONY: distclean-deps + +# Configuration. + +ifdef OTP_DEPS +$(warning The variable OTP_DEPS is deprecated in favor of LOCAL_DEPS.) +endif + +IGNORE_DEPS ?= +export IGNORE_DEPS + +APPS_DIR ?= $(CURDIR)/apps +export APPS_DIR + +DEPS_DIR ?= $(CURDIR)/deps +export DEPS_DIR + +REBAR_DEPS_DIR = $(DEPS_DIR) +export REBAR_DEPS_DIR + +dep_name = $(if $(dep_$(1)),$(1),$(if $(pkg_$(1)_name),$(pkg_$(1)_name),$(1))) +dep_repo = $(patsubst git://github.com/%,https://github.com/%, \ + $(if $(dep_$(1)),$(word 2,$(dep_$(1))),$(pkg_$(1)_repo))) +dep_commit = $(if $(dep_$(1)_commit),$(dep_$(1)_commit),$(if $(dep_$(1)),$(word 3,$(dep_$(1))),$(pkg_$(1)_commit))) + +ALL_APPS_DIRS = $(if $(wildcard $(APPS_DIR)/),$(filter-out $(APPS_DIR),$(shell find $(APPS_DIR) -maxdepth 1 -type d))) +ALL_DEPS_DIRS = $(addprefix $(DEPS_DIR)/,$(foreach dep,$(filter-out $(IGNORE_DEPS),$(BUILD_DEPS) $(DEPS)),$(call dep_name,$(dep)))) + +ifeq ($(filter $(APPS_DIR) $(DEPS_DIR),$(subst :, ,$(ERL_LIBS))),) +ifeq ($(ERL_LIBS),) + ERL_LIBS = $(APPS_DIR):$(DEPS_DIR) +else + ERL_LIBS := $(ERL_LIBS):$(APPS_DIR):$(DEPS_DIR) +endif +endif +export ERL_LIBS + +export NO_AUTOPATCH + +# Verbosity. + +dep_verbose_0 = @echo " DEP " $(1); +dep_verbose_2 = set -x; +dep_verbose = $(dep_verbose_$(V)) + +# Core targets. + +ifdef IS_APP +apps:: +else +apps:: $(ALL_APPS_DIRS) +ifeq ($(IS_APP)$(IS_DEP),) + $(verbose) rm -f $(ERLANG_MK_TMP)/apps.log +endif + $(verbose) mkdir -p $(ERLANG_MK_TMP) +# Create ebin directory for all apps to make sure Erlang recognizes them +# as proper OTP applications when using -include_lib. This is a temporary +# fix, a proper fix would be to compile apps/* in the right order. + $(verbose) for dep in $(ALL_APPS_DIRS) ; do \ + mkdir -p $$dep/ebin || exit $$?; \ + done + $(verbose) for dep in $(ALL_APPS_DIRS) ; do \ + if grep -qs ^$$dep$$ $(ERLANG_MK_TMP)/apps.log; then \ + :; \ + else \ + echo $$dep >> $(ERLANG_MK_TMP)/apps.log; \ + $(MAKE) -C $$dep IS_APP=1 || exit $$?; \ + fi \ + done +endif + +ifneq ($(SKIP_DEPS),) +deps:: +else +deps:: $(ALL_DEPS_DIRS) apps +ifeq ($(IS_APP)$(IS_DEP),) + $(verbose) rm -f $(ERLANG_MK_TMP)/deps.log +endif + $(verbose) mkdir -p $(ERLANG_MK_TMP) + $(verbose) for dep in $(ALL_DEPS_DIRS) ; do \ + if grep -qs ^$$dep$$ $(ERLANG_MK_TMP)/deps.log; then \ + :; \ + else \ + echo $$dep >> $(ERLANG_MK_TMP)/deps.log; \ + if [ -f $$dep/GNUmakefile ] || [ -f $$dep/makefile ] || [ -f $$dep/Makefile ]; then \ + $(MAKE) -C $$dep IS_DEP=1 || exit $$?; \ + else \ + echo "Error: No Makefile to build dependency $$dep."; \ + exit 2; \ + fi \ + fi \ + done +endif + +# Deps related targets. + +# @todo rename GNUmakefile and makefile into Makefile first, if they exist +# While Makefile file could be GNUmakefile or makefile, +# in practice only Makefile is needed so far. +define dep_autopatch + if [ -f $(DEPS_DIR)/$(1)/erlang.mk ]; then \ + $(call erlang,$(call dep_autopatch_appsrc.erl,$(1))); \ + $(call dep_autopatch_erlang_mk,$(1)); \ + elif [ -f $(DEPS_DIR)/$(1)/Makefile ]; then \ + if [ 0 != `grep -c "include ../\w*\.mk" $(DEPS_DIR)/$(1)/Makefile` ]; then \ + $(call dep_autopatch2,$(1)); \ + elif [ 0 != `grep -ci rebar $(DEPS_DIR)/$(1)/Makefile` ]; then \ + $(call dep_autopatch2,$(1)); \ + elif [ -n "`find $(DEPS_DIR)/$(1)/ -type f -name \*.mk -not -name erlang.mk -exec grep -i rebar '{}' \;`" ]; then \ + $(call dep_autopatch2,$(1)); \ + else \ + $(call erlang,$(call dep_autopatch_app.erl,$(1))); \ + fi \ + else \ + if [ ! -d $(DEPS_DIR)/$(1)/src/ ]; then \ + $(call dep_autopatch_noop,$(1)); \ + else \ + $(call dep_autopatch2,$(1)); \ + fi \ + fi +endef + +define dep_autopatch2 + if [ -f $(DEPS_DIR)/$1/src/$1.app.src.script ]; then \ + $(call erlang,$(call dep_autopatch_appsrc_script.erl,$(1))); \ + fi; \ + $(call erlang,$(call dep_autopatch_appsrc.erl,$(1))); \ + if [ -f $(DEPS_DIR)/$(1)/rebar -o -f $(DEPS_DIR)/$(1)/rebar.config -o -f $(DEPS_DIR)/$(1)/rebar.config.script ]; then \ + $(call dep_autopatch_fetch_rebar); \ + $(call dep_autopatch_rebar,$(1)); \ + else \ + $(call dep_autopatch_gen,$(1)); \ + fi +endef + +define dep_autopatch_noop + printf "noop:\n" > $(DEPS_DIR)/$(1)/Makefile +endef + +# Overwrite erlang.mk with the current file by default. +ifeq ($(NO_AUTOPATCH_ERLANG_MK),) +define dep_autopatch_erlang_mk + echo "include $(call core_relpath,$(dir $(ERLANG_MK_FILENAME)),$(DEPS_DIR)/app)/erlang.mk" \ + > $(DEPS_DIR)/$1/erlang.mk +endef +else +define dep_autopatch_erlang_mk + : +endef +endif + +define dep_autopatch_gen + printf "%s\n" \ + "ERLC_OPTS = +debug_info" \ + "include ../../erlang.mk" > $(DEPS_DIR)/$(1)/Makefile +endef + +define dep_autopatch_fetch_rebar + mkdir -p $(ERLANG_MK_TMP); \ + if [ ! -d $(ERLANG_MK_TMP)/rebar ]; then \ + git clone -q -n -- https://github.com/rebar/rebar $(ERLANG_MK_TMP)/rebar; \ + cd $(ERLANG_MK_TMP)/rebar; \ + git checkout -q 791db716b5a3a7671e0b351f95ddf24b848ee173; \ + $(MAKE); \ + cd -; \ + fi +endef + +define dep_autopatch_rebar + if [ -f $(DEPS_DIR)/$(1)/Makefile ]; then \ + mv $(DEPS_DIR)/$(1)/Makefile $(DEPS_DIR)/$(1)/Makefile.orig.mk; \ + fi; \ + $(call erlang,$(call dep_autopatch_rebar.erl,$(1))); \ + rm -f $(DEPS_DIR)/$(1)/ebin/$(1).app +endef + +define dep_autopatch_rebar.erl + application:load(rebar), + application:set_env(rebar, log_level, debug), + Conf1 = case file:consult("$(call core_native_path,$(DEPS_DIR)/$1/rebar.config)") of + {ok, Conf0} -> Conf0; + _ -> [] + end, + {Conf, OsEnv} = fun() -> + case filelib:is_file("$(call core_native_path,$(DEPS_DIR)/$1/rebar.config.script)") of + false -> {Conf1, []}; + true -> + Bindings0 = erl_eval:new_bindings(), + Bindings1 = erl_eval:add_binding('CONFIG', Conf1, Bindings0), + Bindings = erl_eval:add_binding('SCRIPT', "$(call core_native_path,$(DEPS_DIR)/$1/rebar.config.script)", Bindings1), + Before = os:getenv(), + {ok, Conf2} = file:script("$(call core_native_path,$(DEPS_DIR)/$1/rebar.config.script)", Bindings), + {Conf2, lists:foldl(fun(E, Acc) -> lists:delete(E, Acc) end, os:getenv(), Before)} + end + end(), + Write = fun (Text) -> + file:write_file("$(call core_native_path,$(DEPS_DIR)/$1/Makefile)", Text, [append]) + end, + Escape = fun (Text) -> + re:replace(Text, "\\\\$$", "\$$$$", [global, {return, list}]) + end, + Write("IGNORE_DEPS += edown eper eunit_formatters meck node_package " + "rebar_lock_deps_plugin rebar_vsn_plugin reltool_util\n"), + Write("C_SRC_DIR = /path/do/not/exist\n"), + Write("C_SRC_TYPE = rebar\n"), + Write("DRV_CFLAGS = -fPIC\nexport DRV_CFLAGS\n"), + Write(["ERLANG_ARCH = ", rebar_utils:wordsize(), "\nexport ERLANG_ARCH\n"]), + fun() -> + Write("ERLC_OPTS = +debug_info\nexport ERLC_OPTS\n"), + case lists:keyfind(erl_opts, 1, Conf) of + false -> ok; + {_, ErlOpts} -> + lists:foreach(fun + ({d, D}) -> + Write("ERLC_OPTS += -D" ++ atom_to_list(D) ++ "=1\n"); + ({i, I}) -> + Write(["ERLC_OPTS += -I ", I, "\n"]); + ({platform_define, Regex, D}) -> + case rebar_utils:is_arch(Regex) of + true -> Write("ERLC_OPTS += -D" ++ atom_to_list(D) ++ "=1\n"); + false -> ok + end; + ({parse_transform, PT}) -> + Write("ERLC_OPTS += +'{parse_transform, " ++ atom_to_list(PT) ++ "}'\n"); + (_) -> ok + end, ErlOpts) + end, + Write("\n") + end(), + fun() -> + File = case lists:keyfind(deps, 1, Conf) of + false -> []; + {_, Deps} -> + [begin case case Dep of + {N, S} when is_atom(N), is_list(S) -> {N, {hex, S}}; + {N, S} when is_tuple(S) -> {N, S}; + {N, _, S} -> {N, S}; + {N, _, S, _} -> {N, S}; + _ -> false + end of + false -> ok; + {Name, Source} -> + {Method, Repo, Commit} = case Source of + {hex, V} -> {hex, V, undefined}; + {git, R} -> {git, R, master}; + {M, R, {branch, C}} -> {M, R, C}; + {M, R, {ref, C}} -> {M, R, C}; + {M, R, {tag, C}} -> {M, R, C}; + {M, R, C} -> {M, R, C} + end, + Write(io_lib:format("DEPS += ~s\ndep_~s = ~s ~s ~s~n", [Name, Name, Method, Repo, Commit])) + end end || Dep <- Deps] + end + end(), + fun() -> + case lists:keyfind(erl_first_files, 1, Conf) of + false -> ok; + {_, Files} -> + Names = [[" ", case lists:reverse(F) of + "lre." ++ Elif -> lists:reverse(Elif); + Elif -> lists:reverse(Elif) + end] || "src/" ++ F <- Files], + Write(io_lib:format("COMPILE_FIRST +=~s\n", [Names])) + end + end(), + Write("\n\nrebar_dep: preprocess pre-deps deps pre-app app\n"), + Write("\npreprocess::\n"), + Write("\npre-deps::\n"), + Write("\npre-app::\n"), + PatchHook = fun(Cmd) -> + case Cmd of + "make -C" ++ Cmd1 -> "$$\(MAKE) -C" ++ Escape(Cmd1); + "gmake -C" ++ Cmd1 -> "$$\(MAKE) -C" ++ Escape(Cmd1); + "make " ++ Cmd1 -> "$$\(MAKE) -f Makefile.orig.mk " ++ Escape(Cmd1); + "gmake " ++ Cmd1 -> "$$\(MAKE) -f Makefile.orig.mk " ++ Escape(Cmd1); + _ -> Escape(Cmd) + end + end, + fun() -> + case lists:keyfind(pre_hooks, 1, Conf) of + false -> ok; + {_, Hooks} -> + [case H of + {'get-deps', Cmd} -> + Write("\npre-deps::\n\t" ++ PatchHook(Cmd) ++ "\n"); + {compile, Cmd} -> + Write("\npre-app::\n\tCC=$$\(CC) " ++ PatchHook(Cmd) ++ "\n"); + {Regex, compile, Cmd} -> + case rebar_utils:is_arch(Regex) of + true -> Write("\npre-app::\n\tCC=$$\(CC) " ++ PatchHook(Cmd) ++ "\n"); + false -> ok + end; + _ -> ok + end || H <- Hooks] + end + end(), + ShellToMk = fun(V) -> + re:replace(re:replace(V, "(\\\\$$)(\\\\w*)", "\\\\1(\\\\2)", [global]), + "-Werror\\\\b", "", [{return, list}, global]) + end, + PortSpecs = fun() -> + case lists:keyfind(port_specs, 1, Conf) of + false -> + case filelib:is_dir("$(call core_native_path,$(DEPS_DIR)/$1/c_src)") of + false -> []; + true -> + [{"priv/" ++ proplists:get_value(so_name, Conf, "$(1)_drv.so"), + proplists:get_value(port_sources, Conf, ["c_src/*.c"]), []}] + end; + {_, Specs} -> + lists:flatten([case S of + {Output, Input} -> {ShellToMk(Output), Input, []}; + {Regex, Output, Input} -> + case rebar_utils:is_arch(Regex) of + true -> {ShellToMk(Output), Input, []}; + false -> [] + end; + {Regex, Output, Input, [{env, Env}]} -> + case rebar_utils:is_arch(Regex) of + true -> {ShellToMk(Output), Input, Env}; + false -> [] + end + end || S <- Specs]) + end + end(), + PortSpecWrite = fun (Text) -> + file:write_file("$(call core_native_path,$(DEPS_DIR)/$1/c_src/Makefile.erlang.mk)", Text, [append]) + end, + case PortSpecs of + [] -> ok; + _ -> + Write("\npre-app::\n\t$$\(MAKE) -f c_src/Makefile.erlang.mk\n"), + PortSpecWrite(io_lib:format("ERL_CFLAGS = -finline-functions -Wall -fPIC -I \\"~s/erts-~s/include\\" -I \\"~s\\"\n", + [code:root_dir(), erlang:system_info(version), code:lib_dir(erl_interface, include)])), + PortSpecWrite(io_lib:format("ERL_LDFLAGS = -L \\"~s\\" -lerl_interface -lei\n", + [code:lib_dir(erl_interface, lib)])), + [PortSpecWrite(["\n", E, "\n"]) || E <- OsEnv], + FilterEnv = fun(Env) -> + lists:flatten([case E of + {_, _} -> E; + {Regex, K, V} -> + case rebar_utils:is_arch(Regex) of + true -> {K, V}; + false -> [] + end + end || E <- Env]) + end, + MergeEnv = fun(Env) -> + lists:foldl(fun ({K, V}, Acc) -> + case lists:keyfind(K, 1, Acc) of + false -> [{K, rebar_utils:expand_env_variable(V, K, "")}|Acc]; + {_, V0} -> [{K, rebar_utils:expand_env_variable(V, K, V0)}|Acc] + end + end, [], Env) + end, + PortEnv = case lists:keyfind(port_env, 1, Conf) of + false -> []; + {_, PortEnv0} -> FilterEnv(PortEnv0) + end, + PortSpec = fun ({Output, Input0, Env}) -> + filelib:ensure_dir("$(call core_native_path,$(DEPS_DIR)/$1/)" ++ Output), + Input = [[" ", I] || I <- Input0], + PortSpecWrite([ + [["\n", K, " = ", ShellToMk(V)] || {K, V} <- lists:reverse(MergeEnv(PortEnv))], + case $(PLATFORM) of + darwin -> "\n\nLDFLAGS += -flat_namespace -undefined suppress"; + _ -> "" + end, + "\n\nall:: ", Output, "\n\n", + "%.o: %.c\n\t$$\(CC) -c -o $$\@ $$\< $$\(CFLAGS) $$\(ERL_CFLAGS) $$\(DRV_CFLAGS) $$\(EXE_CFLAGS)\n\n", + "%.o: %.C\n\t$$\(CXX) -c -o $$\@ $$\< $$\(CXXFLAGS) $$\(ERL_CFLAGS) $$\(DRV_CFLAGS) $$\(EXE_CFLAGS)\n\n", + "%.o: %.cc\n\t$$\(CXX) -c -o $$\@ $$\< $$\(CXXFLAGS) $$\(ERL_CFLAGS) $$\(DRV_CFLAGS) $$\(EXE_CFLAGS)\n\n", + "%.o: %.cpp\n\t$$\(CXX) -c -o $$\@ $$\< $$\(CXXFLAGS) $$\(ERL_CFLAGS) $$\(DRV_CFLAGS) $$\(EXE_CFLAGS)\n\n", + [[Output, ": ", K, " = ", ShellToMk(V), "\n"] || {K, V} <- lists:reverse(MergeEnv(FilterEnv(Env)))], + Output, ": $$\(foreach ext,.c .C .cc .cpp,", + "$$\(patsubst %$$\(ext),%.o,$$\(filter %$$\(ext),$$\(wildcard", Input, "))))\n", + "\t$$\(CC) -o $$\@ $$\? $$\(LDFLAGS) $$\(ERL_LDFLAGS) $$\(DRV_LDFLAGS) $$\(EXE_LDFLAGS)", + case {filename:extension(Output), $(PLATFORM)} of + {[], _} -> "\n"; + {_, darwin} -> "\n"; + _ -> " -shared\n" + end]) + end, + [PortSpec(S) || S <- PortSpecs] + end, + Write("\ninclude $(call core_relpath,$(dir $(ERLANG_MK_FILENAME)),$(DEPS_DIR)/app)/erlang.mk"), + RunPlugin = fun(Plugin, Step) -> + case erlang:function_exported(Plugin, Step, 2) of + false -> ok; + true -> + c:cd("$(call core_native_path,$(DEPS_DIR)/$1/)"), + Ret = Plugin:Step({config, "", Conf, dict:new(), dict:new(), dict:new(), + dict:store(base_dir, "", dict:new())}, undefined), + io:format("rebar plugin ~p step ~p ret ~p~n", [Plugin, Step, Ret]) + end + end, + fun() -> + case lists:keyfind(plugins, 1, Conf) of + false -> ok; + {_, Plugins} -> + [begin + case lists:keyfind(deps, 1, Conf) of + false -> ok; + {_, Deps} -> + case lists:keyfind(P, 1, Deps) of + false -> ok; + _ -> + Path = "$(call core_native_path,$(DEPS_DIR)/)" ++ atom_to_list(P), + io:format("~s", [os:cmd("$(MAKE) -C $(call core_native_path,$(DEPS_DIR)/$1) " ++ Path)]), + io:format("~s", [os:cmd("$(MAKE) -C " ++ Path ++ " IS_DEP=1")]), + code:add_patha(Path ++ "/ebin") + end + end + end || P <- Plugins], + [case code:load_file(P) of + {module, P} -> ok; + _ -> + case lists:keyfind(plugin_dir, 1, Conf) of + false -> ok; + {_, PluginsDir} -> + ErlFile = "$(call core_native_path,$(DEPS_DIR)/$1/)" ++ PluginsDir ++ "/" ++ atom_to_list(P) ++ ".erl", + {ok, P, Bin} = compile:file(ErlFile, [binary]), + {module, P} = code:load_binary(P, ErlFile, Bin) + end + end || P <- Plugins], + [RunPlugin(P, preprocess) || P <- Plugins], + [RunPlugin(P, pre_compile) || P <- Plugins], + [RunPlugin(P, compile) || P <- Plugins] + end + end(), + halt() +endef + +define dep_autopatch_app.erl + UpdateModules = fun(App) -> + case filelib:is_regular(App) of + false -> ok; + true -> + {ok, [{application, '$(1)', L0}]} = file:consult(App), + Mods = filelib:fold_files("$(call core_native_path,$(DEPS_DIR)/$1/src)", "\\\\.erl$$", true, + fun (F, Acc) -> [list_to_atom(filename:rootname(filename:basename(F)))|Acc] end, []), + L = lists:keystore(modules, 1, L0, {modules, Mods}), + ok = file:write_file(App, io_lib:format("~p.~n", [{application, '$(1)', L}])) + end + end, + UpdateModules("$(call core_native_path,$(DEPS_DIR)/$1/ebin/$1.app)"), + halt() +endef + +define dep_autopatch_appsrc_script.erl + AppSrc = "$(call core_native_path,$(DEPS_DIR)/$1/src/$1.app.src)", + AppSrcScript = AppSrc ++ ".script", + Bindings = erl_eval:new_bindings(), + {ok, Conf} = file:script(AppSrcScript, Bindings), + ok = file:write_file(AppSrc, io_lib:format("~p.~n", [Conf])), + halt() +endef + +define dep_autopatch_appsrc.erl + AppSrcOut = "$(call core_native_path,$(DEPS_DIR)/$1/src/$1.app.src)", + AppSrcIn = case filelib:is_regular(AppSrcOut) of false -> "$(call core_native_path,$(DEPS_DIR)/$1/ebin/$1.app)"; true -> AppSrcOut end, + case filelib:is_regular(AppSrcIn) of + false -> ok; + true -> + {ok, [{application, $(1), L0}]} = file:consult(AppSrcIn), + L1 = lists:keystore(modules, 1, L0, {modules, []}), + L2 = case lists:keyfind(vsn, 1, L1) of {_, git} -> lists:keyreplace(vsn, 1, L1, {vsn, "git"}); _ -> L1 end, + L3 = case lists:keyfind(registered, 1, L2) of false -> [{registered, []}|L2]; _ -> L2 end, + ok = file:write_file(AppSrcOut, io_lib:format("~p.~n", [{application, $(1), L3}])), + case AppSrcOut of AppSrcIn -> ok; _ -> ok = file:delete(AppSrcIn) end + end, + halt() +endef + +define dep_fetch_git + git clone -q -n -- $(call dep_repo,$(1)) $(DEPS_DIR)/$(call dep_name,$(1)); \ + cd $(DEPS_DIR)/$(call dep_name,$(1)) && git checkout -q $(call dep_commit,$(1)); +endef + +define dep_fetch_git-submodule + git submodule update --init -- $(DEPS_DIR)/$1; +endef + +define dep_fetch_hg + hg clone -q -U $(call dep_repo,$(1)) $(DEPS_DIR)/$(call dep_name,$(1)); \ + cd $(DEPS_DIR)/$(call dep_name,$(1)) && hg update -q $(call dep_commit,$(1)); +endef + +define dep_fetch_svn + svn checkout -q $(call dep_repo,$(1)) $(DEPS_DIR)/$(call dep_name,$(1)); +endef + +define dep_fetch_cp + cp -R $(call dep_repo,$(1)) $(DEPS_DIR)/$(call dep_name,$(1)); +endef + +define dep_fetch_hex.erl + ssl:start(), + inets:start(), + {ok, {{_, 200, _}, _, Body}} = httpc:request(get, + {"https://s3.amazonaws.com/s3.hex.pm/tarballs/$(1)-$(2).tar", []}, + [], [{body_format, binary}]), + {ok, Files} = erl_tar:extract({binary, Body}, [memory]), + {_, Source} = lists:keyfind("contents.tar.gz", 1, Files), + ok = erl_tar:extract({binary, Source}, [{cwd, "$(call core_native_path,$(DEPS_DIR)/$1)"}, compressed]), + halt() +endef + +# Hex only has a package version. No need to look in the Erlang.mk packages. +define dep_fetch_hex + $(call erlang,$(call dep_fetch_hex.erl,$(1),$(strip $(word 2,$(dep_$(1)))))); +endef + +define dep_fetch_fail + echo "Error: Unknown or invalid dependency: $(1)." >&2; \ + exit 78; +endef + +# Kept for compatibility purposes with older Erlang.mk configuration. +define dep_fetch_legacy + $(warning WARNING: '$(1)' dependency configuration uses deprecated format.) \ + git clone -q -n -- $(word 1,$(dep_$(1))) $(DEPS_DIR)/$(1); \ + cd $(DEPS_DIR)/$(1) && git checkout -q $(if $(word 2,$(dep_$(1))),$(word 2,$(dep_$(1))),master); +endef + +define dep_fetch + $(if $(dep_$(1)), \ + $(if $(dep_fetch_$(word 1,$(dep_$(1)))), \ + $(word 1,$(dep_$(1))), \ + $(if $(IS_DEP),legacy,fail)), \ + $(if $(filter $(1),$(PACKAGES)), \ + $(pkg_$(1)_fetch), \ + fail)) +endef + +define dep_target +$(DEPS_DIR)/$(call dep_name,$1): + $(eval DEP_NAME := $(call dep_name,$1)) + $(eval DEP_STR := $(if $(filter-out $1,$(DEP_NAME)),$1,"$1 ($(DEP_NAME))")) + $(verbose) if test -d $(APPS_DIR)/$(DEP_NAME); then \ + echo "Error: Dependency" $(DEP_STR) "conflicts with application found in $(APPS_DIR)/$(DEP_NAME)."; \ + exit 17; \ + fi + $(verbose) mkdir -p $(DEPS_DIR) + $(dep_verbose) $(call dep_fetch_$(strip $(call dep_fetch,$(1))),$(1)) + $(verbose) if [ -f $(DEPS_DIR)/$(1)/configure.ac -o -f $(DEPS_DIR)/$(1)/configure.in ] \ + && [ ! -f $(DEPS_DIR)/$(1)/configure ]; then \ + echo " AUTO " $(1); \ + cd $(DEPS_DIR)/$(1) && autoreconf -Wall -vif -I m4; \ + fi + - $(verbose) if [ -f $(DEPS_DIR)/$(DEP_NAME)/configure ]; then \ + echo " CONF " $(DEP_STR); \ + cd $(DEPS_DIR)/$(DEP_NAME) && ./configure; \ + fi +ifeq ($(filter $(1),$(NO_AUTOPATCH)),) + $(verbose) if [ "$(1)" = "amqp_client" -a "$(RABBITMQ_CLIENT_PATCH)" ]; then \ + if [ ! -d $(DEPS_DIR)/rabbitmq-codegen ]; then \ + echo " PATCH Downloading rabbitmq-codegen"; \ + git clone https://github.com/rabbitmq/rabbitmq-codegen.git $(DEPS_DIR)/rabbitmq-codegen; \ + fi; \ + if [ ! -d $(DEPS_DIR)/rabbitmq-server ]; then \ + echo " PATCH Downloading rabbitmq-server"; \ + git clone https://github.com/rabbitmq/rabbitmq-server.git $(DEPS_DIR)/rabbitmq-server; \ + fi; \ + ln -s $(DEPS_DIR)/amqp_client/deps/rabbit_common-0.0.0 $(DEPS_DIR)/rabbit_common; \ + elif [ "$(1)" = "rabbit" -a "$(RABBITMQ_SERVER_PATCH)" ]; then \ + if [ ! -d $(DEPS_DIR)/rabbitmq-codegen ]; then \ + echo " PATCH Downloading rabbitmq-codegen"; \ + git clone https://github.com/rabbitmq/rabbitmq-codegen.git $(DEPS_DIR)/rabbitmq-codegen; \ + fi \ + else \ + $$(call dep_autopatch,$(DEP_NAME)) \ + fi +endif +endef + +$(foreach dep,$(BUILD_DEPS) $(DEPS),$(eval $(call dep_target,$(dep)))) + +ifndef IS_APP +clean:: clean-apps + +clean-apps: + $(verbose) for dep in $(ALL_APPS_DIRS) ; do \ + $(MAKE) -C $$dep clean IS_APP=1 || exit $$?; \ + done + +distclean:: distclean-apps + +distclean-apps: + $(verbose) for dep in $(ALL_APPS_DIRS) ; do \ + $(MAKE) -C $$dep distclean IS_APP=1 || exit $$?; \ + done +endif + +ifndef SKIP_DEPS +distclean:: distclean-deps + +distclean-deps: + $(gen_verbose) rm -rf $(DEPS_DIR) +endif + +# External plugins. + +DEP_PLUGINS ?= + +define core_dep_plugin +-include $(DEPS_DIR)/$(1) + +$(DEPS_DIR)/$(1): $(DEPS_DIR)/$(2) ; +endef + +$(foreach p,$(DEP_PLUGINS),\ + $(eval $(if $(findstring /,$p),\ + $(call core_dep_plugin,$p,$(firstword $(subst /, ,$p))),\ + $(call core_dep_plugin,$p/plugins.mk,$p)))) + +# Copyright (c) 2013-2015, Loïc Hoguin +# This file is part of erlang.mk and subject to the terms of the ISC License. + +# Configuration. + +DTL_FULL_PATH ?= +DTL_PATH ?= templates/ +DTL_SUFFIX ?= _dtl +DTL_OPTS ?= + +# Verbosity. + +dtl_verbose_0 = @echo " DTL " $(filter %.dtl,$(?F)); +dtl_verbose = $(dtl_verbose_$(V)) + +# Core targets. + +DTL_FILES = $(sort $(call core_find,$(DTL_PATH),*.dtl)) + +ifneq ($(DTL_FILES),) + +ifdef DTL_FULL_PATH +BEAM_FILES += $(addprefix ebin/,$(patsubst %.dtl,%_dtl.beam,$(subst /,_,$(DTL_FILES:$(DTL_PATH)%=%)))) +else +BEAM_FILES += $(addprefix ebin/,$(patsubst %.dtl,%_dtl.beam,$(notdir $(DTL_FILES)))) +endif + +# Rebuild templates when the Makefile changes. +$(DTL_FILES): $(MAKEFILE_LIST) + @touch $@ + +define erlydtl_compile.erl + [begin + Module0 = case "$(strip $(DTL_FULL_PATH))" of + "" -> + filename:basename(F, ".dtl"); + _ -> + "$(DTL_PATH)" ++ F2 = filename:rootname(F, ".dtl"), + re:replace(F2, "/", "_", [{return, list}, global]) + end, + Module = list_to_atom(string:to_lower(Module0) ++ "$(DTL_SUFFIX)"), + case erlydtl:compile(F, Module, [$(DTL_OPTS)] ++ [{out_dir, "ebin/"}, return_errors, {doc_root, "templates"}]) of + ok -> ok; + {ok, _} -> ok + end + end || F <- string:tokens("$(1)", " ")], + halt(). +endef + +ebin/$(PROJECT).app:: $(DTL_FILES) | ebin/ + $(if $(strip $?),\ + $(dtl_verbose) $(call erlang,$(call erlydtl_compile.erl,$?),-pa ebin/ $(DEPS_DIR)/erlydtl/ebin/)) + +endif + +# Copyright (c) 2015, Loïc Hoguin +# This file is part of erlang.mk and subject to the terms of the ISC License. + +# Verbosity. + +proto_verbose_0 = @echo " PROTO " $(filter %.proto,$(?F)); +proto_verbose = $(proto_verbose_$(V)) + +# Core targets. + +define compile_proto + $(verbose) mkdir -p ebin/ include/ + $(proto_verbose) $(call erlang,$(call compile_proto.erl,$(1))) + $(proto_verbose) erlc +debug_info -o ebin/ ebin/*.erl + $(verbose) rm ebin/*.erl +endef + +define compile_proto.erl + [begin + Dir = filename:dirname(filename:dirname(F)), + protobuffs_compile:generate_source(F, + [{output_include_dir, Dir ++ "/include"}, + {output_src_dir, Dir ++ "/ebin"}]) + end || F <- string:tokens("$(1)", " ")], + halt(). +endef + +ifneq ($(wildcard src/),) +ebin/$(PROJECT).app:: $(sort $(call core_find,src/,*.proto)) + $(if $(strip $?),$(call compile_proto,$?)) +endif + +# Copyright (c) 2013-2015, Loïc Hoguin +# This file is part of erlang.mk and subject to the terms of the ISC License. + +.PHONY: clean-app + +# Configuration. + +ERLC_OPTS ?= -Werror +debug_info +warn_export_vars +warn_shadow_vars \ + +warn_obsolete_guard # +bin_opt_info +warn_export_all +warn_missing_spec +COMPILE_FIRST ?= +COMPILE_FIRST_PATHS = $(addprefix src/,$(addsuffix .erl,$(COMPILE_FIRST))) +ERLC_EXCLUDE ?= +ERLC_EXCLUDE_PATHS = $(addprefix src/,$(addsuffix .erl,$(ERLC_EXCLUDE))) + +ERLC_MIB_OPTS ?= +COMPILE_MIB_FIRST ?= +COMPILE_MIB_FIRST_PATHS = $(addprefix mibs/,$(addsuffix .mib,$(COMPILE_MIB_FIRST))) + +# Verbosity. + +app_verbose_0 = @echo " APP " $(PROJECT); +app_verbose_2 = set -x; +app_verbose = $(app_verbose_$(V)) + +appsrc_verbose_0 = @echo " APP " $(PROJECT).app.src; +appsrc_verbose_2 = set -x; +appsrc_verbose = $(appsrc_verbose_$(V)) + +makedep_verbose_0 = @echo " DEPEND" $(PROJECT).d; +makedep_verbose_2 = set -x; +makedep_verbose = $(makedep_verbose_$(V)) + +erlc_verbose_0 = @echo " ERLC " $(filter-out $(patsubst %,%.erl,$(ERLC_EXCLUDE)),\ + $(filter %.erl %.core,$(?F))); +erlc_verbose_2 = set -x; +erlc_verbose = $(erlc_verbose_$(V)) + +xyrl_verbose_0 = @echo " XYRL " $(filter %.xrl %.yrl,$(?F)); +xyrl_verbose_2 = set -x; +xyrl_verbose = $(xyrl_verbose_$(V)) + +asn1_verbose_0 = @echo " ASN1 " $(filter %.asn1,$(?F)); +asn1_verbose_2 = set -x; +asn1_verbose = $(asn1_verbose_$(V)) + +mib_verbose_0 = @echo " MIB " $(filter %.bin %.mib,$(?F)); +mib_verbose_2 = set -x; +mib_verbose = $(mib_verbose_$(V)) + +ifneq ($(wildcard src/),) + +# Targets. + +ifeq ($(wildcard ebin/test),) +app:: deps $(PROJECT).d + $(verbose) $(MAKE) --no-print-directory app-build +else +app:: clean deps $(PROJECT).d + $(verbose) $(MAKE) --no-print-directory app-build +endif + +ifeq ($(wildcard src/$(PROJECT_MOD).erl),) +define app_file +{application, $(PROJECT), [ + {description, "$(PROJECT_DESCRIPTION)"}, + {vsn, "$(PROJECT_VERSION)"},$(if $(IS_DEP), + {id$(comma)$(space)"$(1)"}$(comma)) + {modules, [$(call comma_list,$(2))]}, + {registered, []}, + {applications, [$(call comma_list,kernel stdlib $(OTP_DEPS) $(LOCAL_DEPS) $(DEPS))]} +]}. +endef +else +define app_file +{application, $(PROJECT), [ + {description, "$(PROJECT_DESCRIPTION)"}, + {vsn, "$(PROJECT_VERSION)"},$(if $(IS_DEP), + {id$(comma)$(space)"$(1)"}$(comma)) + {modules, [$(call comma_list,$(2))]}, + {registered, [$(call comma_list,$(PROJECT)_sup $(PROJECT_REGISTERED))]}, + {applications, [$(call comma_list,kernel stdlib $(OTP_DEPS) $(LOCAL_DEPS) $(DEPS))]}, + {mod, {$(PROJECT_MOD), []}} +]}. +endef +endif + +app-build: ebin/$(PROJECT).app + $(verbose) : + +# Source files. + +ERL_FILES = $(sort $(call core_find,src/,*.erl)) +CORE_FILES = $(sort $(call core_find,src/,*.core)) + +# ASN.1 files. + +ifneq ($(wildcard asn1/),) +ASN1_FILES = $(sort $(call core_find,asn1/,*.asn1)) +ERL_FILES += $(addprefix src/,$(patsubst %.asn1,%.erl,$(notdir $(ASN1_FILES)))) + +define compile_asn1 + $(verbose) mkdir -p include/ + $(asn1_verbose) erlc -v -I include/ -o asn1/ +noobj $(1) + $(verbose) mv asn1/*.erl src/ + $(verbose) mv asn1/*.hrl include/ + $(verbose) mv asn1/*.asn1db include/ +endef + +$(PROJECT).d:: $(ASN1_FILES) + $(if $(strip $?),$(call compile_asn1,$?)) +endif + +# SNMP MIB files. + +ifneq ($(wildcard mibs/),) +MIB_FILES = $(sort $(call core_find,mibs/,*.mib)) + +$(PROJECT).d:: $(COMPILE_MIB_FIRST_PATHS) $(MIB_FILES) + $(verbose) mkdir -p include/ priv/mibs/ + $(mib_verbose) erlc -v $(ERLC_MIB_OPTS) -o priv/mibs/ -I priv/mibs/ $? + $(mib_verbose) erlc -o include/ -- $(addprefix priv/mibs/,$(patsubst %.mib,%.bin,$(notdir $?))) +endif + +# Leex and Yecc files. + +XRL_FILES = $(sort $(call core_find,src/,*.xrl)) +XRL_ERL_FILES = $(addprefix src/,$(patsubst %.xrl,%.erl,$(notdir $(XRL_FILES)))) +ERL_FILES += $(XRL_ERL_FILES) + +YRL_FILES = $(sort $(call core_find,src/,*.yrl)) +YRL_ERL_FILES = $(addprefix src/,$(patsubst %.yrl,%.erl,$(notdir $(YRL_FILES)))) +ERL_FILES += $(YRL_ERL_FILES) + +$(PROJECT).d:: $(XRL_FILES) $(YRL_FILES) + $(if $(strip $?),$(xyrl_verbose) erlc -v -o src/ $?) + +# Erlang and Core Erlang files. + +define makedep.erl + E = ets:new(makedep, [bag]), + G = digraph:new([acyclic]), + ErlFiles = lists:usort(string:tokens("$(ERL_FILES)", " ")), + Modules = [{list_to_atom(filename:basename(F, ".erl")), F} || F <- ErlFiles], + Add = fun (Mod, Dep) -> + case lists:keyfind(Dep, 1, Modules) of + false -> ok; + {_, DepFile} -> + {_, ModFile} = lists:keyfind(Mod, 1, Modules), + ets:insert(E, {ModFile, DepFile}), + digraph:add_vertex(G, Mod), + digraph:add_vertex(G, Dep), + digraph:add_edge(G, Mod, Dep) + end + end, + AddHd = fun (F, Mod, DepFile) -> + case file:open(DepFile, [read]) of + {error, enoent} -> ok; + {ok, Fd} -> + F(F, Fd, Mod), + {_, ModFile} = lists:keyfind(Mod, 1, Modules), + ets:insert(E, {ModFile, DepFile}) + end + end, + Attr = fun + (F, Mod, behavior, Dep) -> Add(Mod, Dep); + (F, Mod, behaviour, Dep) -> Add(Mod, Dep); + (F, Mod, compile, {parse_transform, Dep}) -> Add(Mod, Dep); + (F, Mod, compile, Opts) when is_list(Opts) -> + case proplists:get_value(parse_transform, Opts) of + undefined -> ok; + Dep -> Add(Mod, Dep) + end; + (F, Mod, include, Hrl) -> + case filelib:is_file("include/" ++ Hrl) of + true -> AddHd(F, Mod, "include/" ++ Hrl); + false -> + case filelib:is_file("src/" ++ Hrl) of + true -> AddHd(F, Mod, "src/" ++ Hrl); + false -> false + end + end; + (F, Mod, include_lib, "$1/include/" ++ Hrl) -> AddHd(F, Mod, "include/" ++ Hrl); + (F, Mod, include_lib, Hrl) -> AddHd(F, Mod, "include/" ++ Hrl); + (F, Mod, import, {Imp, _}) -> + case filelib:is_file("src/" ++ atom_to_list(Imp) ++ ".erl") of + false -> ok; + true -> Add(Mod, Imp) + end; + (_, _, _, _) -> ok + end, + MakeDepend = fun(F, Fd, Mod) -> + case io:parse_erl_form(Fd, undefined) of + {ok, {attribute, _, Key, Value}, _} -> + Attr(F, Mod, Key, Value), + F(F, Fd, Mod); + {eof, _} -> + file:close(Fd); + _ -> + F(F, Fd, Mod) + end + end, + [begin + Mod = list_to_atom(filename:basename(F, ".erl")), + {ok, Fd} = file:open(F, [read]), + MakeDepend(MakeDepend, Fd, Mod) + end || F <- ErlFiles], + Depend = sofs:to_external(sofs:relation_to_family(sofs:relation(ets:tab2list(E)))), + CompileFirst = [X || X <- lists:reverse(digraph_utils:topsort(G)), [] =/= digraph:in_neighbours(G, X)], + ok = file:write_file("$(1)", [ + [[F, "::", [[" ", D] || D <- Deps], "; @touch \$$@\n"] || {F, Deps} <- Depend], + "\nCOMPILE_FIRST +=", [[" ", atom_to_list(CF)] || CF <- CompileFirst], "\n" + ]), + halt() +endef + +ifeq ($(if $(NO_MAKEDEP),$(wildcard $(PROJECT).d),),) +$(PROJECT).d:: $(ERL_FILES) $(call core_find,include/,*.hrl) + $(makedep_verbose) $(call erlang,$(call makedep.erl,$@)) +endif + +# Rebuild everything when the Makefile changes. +$(ERL_FILES) $(CORE_FILES) $(ASN1_FILES) $(MIB_FILES) $(XRL_FILES) $(YRL_FILES):: $(MAKEFILE_LIST) + @touch $@ + +-include $(PROJECT).d + +ebin/$(PROJECT).app:: ebin/ + +ebin/: + $(verbose) mkdir -p ebin/ + +define compile_erl + $(erlc_verbose) erlc -v $(if $(IS_DEP),$(filter-out -Werror,$(ERLC_OPTS)),$(ERLC_OPTS)) -o ebin/ \ + -pa ebin/ -I include/ $(filter-out $(ERLC_EXCLUDE_PATHS),$(COMPILE_FIRST_PATHS) $(1)) +endef + +ebin/$(PROJECT).app:: $(ERL_FILES) $(CORE_FILES) $(wildcard src/$(PROJECT).app.src) + $(eval FILES_TO_COMPILE := $(filter-out src/$(PROJECT).app.src,$?)) + $(if $(strip $(FILES_TO_COMPILE)),$(call compile_erl,$(FILES_TO_COMPILE))) + $(eval GITDESCRIBE := $(shell git describe --dirty --abbrev=7 --tags --always --first-parent 2>/dev/null || true)) + $(eval MODULES := $(patsubst %,'%',$(sort $(notdir $(basename \ + $(filter-out $(ERLC_EXCLUDE_PATHS),$(ERL_FILES) $(CORE_FILES) $(BEAM_FILES))))))) +ifeq ($(wildcard src/$(PROJECT).app.src),) + $(app_verbose) printf "$(subst $(newline),\n,$(subst ",\",$(call app_file,$(GITDESCRIBE),$(MODULES))))" \ + > ebin/$(PROJECT).app +else + $(verbose) if [ -z "$$(grep -e '^[^%]*{\s*modules\s*,' src/$(PROJECT).app.src)" ]; then \ + echo "Empty modules entry not found in $(PROJECT).app.src. Please consult the erlang.mk README for instructions." >&2; \ + exit 1; \ + fi + $(appsrc_verbose) cat src/$(PROJECT).app.src \ + | sed "s/{[[:space:]]*modules[[:space:]]*,[[:space:]]*\[\]}/{modules, \[$(call comma_list,$(MODULES))\]}/" \ + | sed "s/{id,[[:space:]]*\"git\"}/{id, \"$(subst /,\/,$(GITDESCRIBE))\"}/" \ + > ebin/$(PROJECT).app +endif + +clean:: clean-app + +clean-app: + $(gen_verbose) rm -rf $(PROJECT).d ebin/ priv/mibs/ $(XRL_ERL_FILES) $(YRL_ERL_FILES) \ + $(addprefix include/,$(patsubst %.mib,%.hrl,$(notdir $(MIB_FILES)))) \ + $(addprefix include/,$(patsubst %.asn1,%.hrl,$(notdir $(ASN1_FILES)))) \ + $(addprefix include/,$(patsubst %.asn1,%.asn1db,$(notdir $(ASN1_FILES)))) \ + $(addprefix src/,$(patsubst %.asn1,%.erl,$(notdir $(ASN1_FILES)))) + +endif + +# Copyright (c) 2015, Viktor Söderqvist +# This file is part of erlang.mk and subject to the terms of the ISC License. + +.PHONY: docs-deps + +# Configuration. + +ALL_DOC_DEPS_DIRS = $(addprefix $(DEPS_DIR)/,$(DOC_DEPS)) + +# Targets. + +$(foreach dep,$(DOC_DEPS),$(eval $(call dep_target,$(dep)))) + +ifneq ($(SKIP_DEPS),) +doc-deps: +else +doc-deps: $(ALL_DOC_DEPS_DIRS) + $(verbose) for dep in $(ALL_DOC_DEPS_DIRS) ; do $(MAKE) -C $$dep; done +endif + +# Copyright (c) 2015, Loïc Hoguin +# This file is part of erlang.mk and subject to the terms of the ISC License. + +.PHONY: rel-deps + +# Configuration. + +ALL_REL_DEPS_DIRS = $(addprefix $(DEPS_DIR)/,$(REL_DEPS)) + +# Targets. + +$(foreach dep,$(REL_DEPS),$(eval $(call dep_target,$(dep)))) + +ifneq ($(SKIP_DEPS),) +rel-deps: +else +rel-deps: $(ALL_REL_DEPS_DIRS) + $(verbose) for dep in $(ALL_REL_DEPS_DIRS) ; do $(MAKE) -C $$dep; done +endif + +# Copyright (c) 2015, Loïc Hoguin +# This file is part of erlang.mk and subject to the terms of the ISC License. + +.PHONY: test-deps test-dir test-build clean-test-dir + +# Configuration. + +TEST_DIR ?= $(CURDIR)/test + +ALL_TEST_DEPS_DIRS = $(addprefix $(DEPS_DIR)/,$(TEST_DEPS)) + +TEST_ERLC_OPTS ?= +debug_info +warn_export_vars +warn_shadow_vars +warn_obsolete_guard +TEST_ERLC_OPTS += -DTEST=1 + +# Targets. + +$(foreach dep,$(TEST_DEPS),$(eval $(call dep_target,$(dep)))) + +ifneq ($(SKIP_DEPS),) +test-deps: +else +test-deps: $(ALL_TEST_DEPS_DIRS) + $(verbose) for dep in $(ALL_TEST_DEPS_DIRS) ; do $(MAKE) -C $$dep IS_DEP=1; done +endif + +ifneq ($(wildcard $(TEST_DIR)),) +test-dir: + $(gen_verbose) erlc -v $(TEST_ERLC_OPTS) -I include/ -o $(TEST_DIR) \ + $(call core_find,$(TEST_DIR)/,*.erl) -pa ebin/ +endif + +ifeq ($(wildcard src),) +test-build:: ERLC_OPTS=$(TEST_ERLC_OPTS) +test-build:: clean deps test-deps + $(verbose) $(MAKE) --no-print-directory test-dir ERLC_OPTS="$(TEST_ERLC_OPTS)" +else +ifeq ($(wildcard ebin/test),) +test-build:: ERLC_OPTS=$(TEST_ERLC_OPTS) +test-build:: clean deps test-deps $(PROJECT).d + $(verbose) $(MAKE) --no-print-directory app-build test-dir ERLC_OPTS="$(TEST_ERLC_OPTS)" + $(gen_verbose) touch ebin/test +else +test-build:: ERLC_OPTS=$(TEST_ERLC_OPTS) +test-build:: deps test-deps $(PROJECT).d + $(verbose) $(MAKE) --no-print-directory app-build test-dir ERLC_OPTS="$(TEST_ERLC_OPTS)" +endif + +clean:: clean-test-dir + +clean-test-dir: +ifneq ($(wildcard $(TEST_DIR)/*.beam),) + $(gen_verbose) rm -f $(TEST_DIR)/*.beam +endif +endif + +# Copyright (c) 2015, Loïc Hoguin +# This file is part of erlang.mk and subject to the terms of the ISC License. + +.PHONY: rebar.config + +# We strip out -Werror because we don't want to fail due to +# warnings when used as a dependency. + +compat_prepare_erlc_opts = $(shell echo "$1" | sed 's/, */,/g') + +define compat_convert_erlc_opts +$(if $(filter-out -Werror,$1),\ + $(if $(findstring +,$1),\ + $(shell echo $1 | cut -b 2-))) +endef + +define compat_erlc_opts_to_list +[$(call comma_list,$(foreach o,$(call compat_prepare_erlc_opts,$1),$(call compat_convert_erlc_opts,$o)))] +endef + +define compat_rebar_config +{deps, [ +$(call comma_list,$(foreach d,$(DEPS),\ + $(if $(filter hex,$(call dep_fetch,$d)),\ + {$(call dep_name,$d)$(comma)"$(call dep_repo,$d)"},\ + {$(call dep_name,$d)$(comma)".*"$(comma){git,"$(call dep_repo,$d)"$(comma)"$(call dep_commit,$d)"}}))) +]}. +{erl_opts, $(call compat_erlc_opts_to_list,$(ERLC_OPTS))}. +endef + +$(eval _compat_rebar_config = $$(compat_rebar_config)) +$(eval export _compat_rebar_config) + +rebar.config: + $(gen_verbose) echo "$${_compat_rebar_config}" > rebar.config + +# Copyright (c) 2015, Loïc Hoguin +# This file is part of erlang.mk and subject to the terms of the ISC License. + +.PHONY: asciidoc asciidoc-guide asciidoc-manual install-asciidoc distclean-asciidoc + +MAN_INSTALL_PATH ?= /usr/local/share/man +MAN_SECTIONS ?= 3 7 + +docs:: asciidoc + +asciidoc: asciidoc-guide asciidoc-manual + +ifeq ($(wildcard doc/src/guide/book.asciidoc),) +asciidoc-guide: +else +asciidoc-guide: distclean-asciidoc doc-deps + a2x -v -f pdf doc/src/guide/book.asciidoc && mv doc/src/guide/book.pdf doc/guide.pdf + a2x -v -f chunked doc/src/guide/book.asciidoc && mv doc/src/guide/book.chunked/ doc/html/ +endif + +ifeq ($(wildcard doc/src/manual/*.asciidoc),) +asciidoc-manual: +else +asciidoc-manual: distclean-asciidoc doc-deps + for f in doc/src/manual/*.asciidoc ; do \ + a2x -v -f manpage $$f ; \ + done + for s in $(MAN_SECTIONS); do \ + mkdir -p doc/man$$s/ ; \ + mv doc/src/manual/*.$$s doc/man$$s/ ; \ + gzip doc/man$$s/*.$$s ; \ + done + +install-docs:: install-asciidoc + +install-asciidoc: asciidoc-manual + for s in $(MAN_SECTIONS); do \ + mkdir -p $(MAN_INSTALL_PATH)/man$$s/ ; \ + install -g `id -u` -o `id -g` -m 0644 doc/man$$s/*.gz $(MAN_INSTALL_PATH)/man$$s/ ; \ + done +endif + +distclean:: distclean-asciidoc + +distclean-asciidoc: + $(gen_verbose) rm -rf doc/html/ doc/guide.pdf doc/man3/ doc/man7/ + +# Copyright (c) 2014-2015, Loïc Hoguin +# This file is part of erlang.mk and subject to the terms of the ISC License. + +.PHONY: bootstrap bootstrap-lib bootstrap-rel new list-templates + +# Core targets. + +help:: + $(verbose) printf "%s\n" "" \ + "Bootstrap targets:" \ + " bootstrap Generate a skeleton of an OTP application" \ + " bootstrap-lib Generate a skeleton of an OTP library" \ + " bootstrap-rel Generate the files needed to build a release" \ + " new-app in=NAME Create a new local OTP application NAME" \ + " new-lib in=NAME Create a new local OTP library NAME" \ + " new t=TPL n=NAME Generate a module NAME based on the template TPL" \ + " new t=T n=N in=APP Generate a module NAME based on the template TPL in APP" \ + " list-templates List available templates" + +# Bootstrap templates. + +define bs_appsrc +{application, $p, [ + {description, ""}, + {vsn, "0.1.0"}, + {id, "git"}, + {modules, []}, + {registered, []}, + {applications, [ + kernel, + stdlib + ]}, + {mod, {$p_app, []}}, + {env, []} +]}. +endef + +define bs_appsrc_lib +{application, $p, [ + {description, ""}, + {vsn, "0.1.0"}, + {id, "git"}, + {modules, []}, + {registered, []}, + {applications, [ + kernel, + stdlib + ]} +]}. +endef + +# To prevent autocompletion issues with ZSH, we add "include erlang.mk" +# separately during the actual bootstrap. +ifdef SP +define bs_Makefile +PROJECT = $p +PROJECT_DESCRIPTION = New project +PROJECT_VERSION = 0.0.1 + +# Whitespace to be used when creating files from templates. +SP = $(SP) + +endef +else +define bs_Makefile +PROJECT = $p +PROJECT_DESCRIPTION = New project +PROJECT_VERSION = 0.0.1 + +endef +endif + +define bs_apps_Makefile +PROJECT = $p +PROJECT_DESCRIPTION = New project +PROJECT_VERSION = 0.0.1 + +include $(call core_relpath,$(dir $(ERLANG_MK_FILENAME)),$(APPS_DIR)/app)/erlang.mk +endef + +define bs_app +-module($p_app). +-behaviour(application). + +-export([start/2]). +-export([stop/1]). + +start(_Type, _Args) -> + $p_sup:start_link(). + +stop(_State) -> + ok. +endef + +define bs_relx_config +{release, {$p_release, "1"}, [$p]}. +{extended_start_script, true}. +{sys_config, "rel/sys.config"}. +{vm_args, "rel/vm.args"}. +endef + +define bs_sys_config +[ +]. +endef + +define bs_vm_args +-name $p@127.0.0.1 +-setcookie $p +-heart +endef + +# Normal templates. + +define tpl_supervisor +-module($(n)). +-behaviour(supervisor). + +-export([start_link/0]). +-export([init/1]). + +start_link() -> + supervisor:start_link({local, ?MODULE}, ?MODULE, []). + +init([]) -> + Procs = [], + {ok, {{one_for_one, 1, 5}, Procs}}. +endef + +define tpl_gen_server +-module($(n)). +-behaviour(gen_server). + +%% API. +-export([start_link/0]). + +%% gen_server. +-export([init/1]). +-export([handle_call/3]). +-export([handle_cast/2]). +-export([handle_info/2]). +-export([terminate/2]). +-export([code_change/3]). + +-record(state, { +}). + +%% API. + +-spec start_link() -> {ok, pid()}. +start_link() -> + gen_server:start_link(?MODULE, [], []). + +%% gen_server. + +init([]) -> + {ok, #state{}}. + +handle_call(_Request, _From, State) -> + {reply, ignored, State}. + +handle_cast(_Msg, State) -> + {noreply, State}. + +handle_info(_Info, State) -> + {noreply, State}. + +terminate(_Reason, _State) -> + ok. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. +endef + +define tpl_module +-module($(n)). +-export([]). +endef + +define tpl_cowboy_http +-module($(n)). +-behaviour(cowboy_http_handler). + +-export([init/3]). +-export([handle/2]). +-export([terminate/3]). + +-record(state, { +}). + +init(_, Req, _Opts) -> + {ok, Req, #state{}}. + +handle(Req, State=#state{}) -> + {ok, Req2} = cowboy_req:reply(200, Req), + {ok, Req2, State}. + +terminate(_Reason, _Req, _State) -> + ok. +endef + +define tpl_gen_fsm +-module($(n)). +-behaviour(gen_fsm). + +%% API. +-export([start_link/0]). + +%% gen_fsm. +-export([init/1]). +-export([state_name/2]). +-export([handle_event/3]). +-export([state_name/3]). +-export([handle_sync_event/4]). +-export([handle_info/3]). +-export([terminate/3]). +-export([code_change/4]). + +-record(state, { +}). + +%% API. + +-spec start_link() -> {ok, pid()}. +start_link() -> + gen_fsm:start_link(?MODULE, [], []). + +%% gen_fsm. + +init([]) -> + {ok, state_name, #state{}}. + +state_name(_Event, StateData) -> + {next_state, state_name, StateData}. + +handle_event(_Event, StateName, StateData) -> + {next_state, StateName, StateData}. + +state_name(_Event, _From, StateData) -> + {reply, ignored, state_name, StateData}. + +handle_sync_event(_Event, _From, StateName, StateData) -> + {reply, ignored, StateName, StateData}. + +handle_info(_Info, StateName, StateData) -> + {next_state, StateName, StateData}. + +terminate(_Reason, _StateName, _StateData) -> + ok. + +code_change(_OldVsn, StateName, StateData, _Extra) -> + {ok, StateName, StateData}. +endef + +define tpl_cowboy_loop +-module($(n)). +-behaviour(cowboy_loop_handler). + +-export([init/3]). +-export([info/3]). +-export([terminate/3]). + +-record(state, { +}). + +init(_, Req, _Opts) -> + {loop, Req, #state{}, 5000, hibernate}. + +info(_Info, Req, State) -> + {loop, Req, State, hibernate}. + +terminate(_Reason, _Req, _State) -> + ok. +endef + +define tpl_cowboy_rest +-module($(n)). + +-export([init/3]). +-export([content_types_provided/2]). +-export([get_html/2]). + +init(_, _Req, _Opts) -> + {upgrade, protocol, cowboy_rest}. + +content_types_provided(Req, State) -> + {[{{<<"text">>, <<"html">>, '*'}, get_html}], Req, State}. + +get_html(Req, State) -> + {<<"This is REST!">>, Req, State}. +endef + +define tpl_cowboy_ws +-module($(n)). +-behaviour(cowboy_websocket_handler). + +-export([init/3]). +-export([websocket_init/3]). +-export([websocket_handle/3]). +-export([websocket_info/3]). +-export([websocket_terminate/3]). + +-record(state, { +}). + +init(_, _, _) -> + {upgrade, protocol, cowboy_websocket}. + +websocket_init(_, Req, _Opts) -> + Req2 = cowboy_req:compact(Req), + {ok, Req2, #state{}}. + +websocket_handle({text, Data}, Req, State) -> + {reply, {text, Data}, Req, State}; +websocket_handle({binary, Data}, Req, State) -> + {reply, {binary, Data}, Req, State}; +websocket_handle(_Frame, Req, State) -> + {ok, Req, State}. + +websocket_info(_Info, Req, State) -> + {ok, Req, State}. + +websocket_terminate(_Reason, _Req, _State) -> + ok. +endef + +define tpl_ranch_protocol +-module($(n)). +-behaviour(ranch_protocol). + +-export([start_link/4]). +-export([init/4]). + +-type opts() :: []. +-export_type([opts/0]). + +-record(state, { + socket :: inet:socket(), + transport :: module() +}). + +start_link(Ref, Socket, Transport, Opts) -> + Pid = spawn_link(?MODULE, init, [Ref, Socket, Transport, Opts]), + {ok, Pid}. + +-spec init(ranch:ref(), inet:socket(), module(), opts()) -> ok. +init(Ref, Socket, Transport, _Opts) -> + ok = ranch:accept_ack(Ref), + loop(#state{socket=Socket, transport=Transport}). + +loop(State) -> + loop(State). +endef + +# Plugin-specific targets. + +define render_template + $(verbose) printf -- '$(subst $(newline),\n,$(subst %,%%,$(subst ','\'',$(subst $(tab),$(WS),$(call $(1))))))\n' > $(2) +endef + +ifndef WS +ifdef SP +WS = $(subst a,,a $(wordlist 1,$(SP),a a a a a a a a a a a a a a a a a a a a)) +else +WS = $(tab) +endif +endif + +bootstrap: +ifneq ($(wildcard src/),) + $(error Error: src/ directory already exists) +endif + $(eval p := $(PROJECT)) + $(eval n := $(PROJECT)_sup) + $(call render_template,bs_Makefile,Makefile) + $(verbose) echo "include erlang.mk" >> Makefile + $(verbose) mkdir src/ +ifdef LEGACY + $(call render_template,bs_appsrc,src/$(PROJECT).app.src) +endif + $(call render_template,bs_app,src/$(PROJECT)_app.erl) + $(call render_template,tpl_supervisor,src/$(PROJECT)_sup.erl) + +bootstrap-lib: +ifneq ($(wildcard src/),) + $(error Error: src/ directory already exists) +endif + $(eval p := $(PROJECT)) + $(call render_template,bs_Makefile,Makefile) + $(verbose) echo "include erlang.mk" >> Makefile + $(verbose) mkdir src/ +ifdef LEGACY + $(call render_template,bs_appsrc_lib,src/$(PROJECT).app.src) +endif + +bootstrap-rel: +ifneq ($(wildcard relx.config),) + $(error Error: relx.config already exists) +endif +ifneq ($(wildcard rel/),) + $(error Error: rel/ directory already exists) +endif + $(eval p := $(PROJECT)) + $(call render_template,bs_relx_config,relx.config) + $(verbose) mkdir rel/ + $(call render_template,bs_sys_config,rel/sys.config) + $(call render_template,bs_vm_args,rel/vm.args) + +new-app: +ifndef in + $(error Usage: $(MAKE) new-app in=APP) +endif +ifneq ($(wildcard $(APPS_DIR)/$in),) + $(error Error: Application $in already exists) +endif + $(eval p := $(in)) + $(eval n := $(in)_sup) + $(verbose) mkdir -p $(APPS_DIR)/$p/src/ + $(call render_template,bs_apps_Makefile,$(APPS_DIR)/$p/Makefile) +ifdef LEGACY + $(call render_template,bs_appsrc,$(APPS_DIR)/$p/src/$p.app.src) +endif + $(call render_template,bs_app,$(APPS_DIR)/$p/src/$p_app.erl) + $(call render_template,tpl_supervisor,$(APPS_DIR)/$p/src/$p_sup.erl) + +new-lib: +ifndef in + $(error Usage: $(MAKE) new-lib in=APP) +endif +ifneq ($(wildcard $(APPS_DIR)/$in),) + $(error Error: Application $in already exists) +endif + $(eval p := $(in)) + $(verbose) mkdir -p $(APPS_DIR)/$p/src/ + $(call render_template,bs_apps_Makefile,$(APPS_DIR)/$p/Makefile) +ifdef LEGACY + $(call render_template,bs_appsrc_lib,$(APPS_DIR)/$p/src/$p.app.src) +endif + +new: +ifeq ($(wildcard src/)$(in),) + $(error Error: src/ directory does not exist) +endif +ifndef t + $(error Usage: $(MAKE) new t=TEMPLATE n=NAME [in=APP]) +endif +ifndef tpl_$(t) + $(error Unknown template) +endif +ifndef n + $(error Usage: $(MAKE) new t=TEMPLATE n=NAME [in=APP]) +endif +ifdef in + $(verbose) $(MAKE) -C $(APPS_DIR)/$(in)/ new t=$t n=$n in= +else + $(call render_template,tpl_$(t),src/$(n).erl) +endif + +list-templates: + $(verbose) echo Available templates: $(sort $(patsubst tpl_%,%,$(filter tpl_%,$(.VARIABLES)))) + +# Copyright (c) 2014-2015, Loïc Hoguin +# This file is part of erlang.mk and subject to the terms of the ISC License. + +.PHONY: clean-c_src distclean-c_src-env + +# Configuration. + +C_SRC_DIR ?= $(CURDIR)/c_src +C_SRC_ENV ?= $(C_SRC_DIR)/env.mk +C_SRC_OUTPUT ?= $(CURDIR)/priv/$(PROJECT) +C_SRC_TYPE ?= shared + +# System type and C compiler/flags. + +ifeq ($(PLATFORM),msys2) + C_SRC_OUTPUT_EXECUTABLE_EXTENSION ?= .exe + C_SRC_OUTPUT_SHARED_EXTENSION ?= .dll +else + C_SRC_OUTPUT_EXECUTABLE_EXTENSION ?= + C_SRC_OUTPUT_SHARED_EXTENSION ?= .so +endif + +ifeq ($(C_SRC_TYPE),shared) + C_SRC_OUTPUT_FILE = $(C_SRC_OUTPUT)$(C_SRC_OUTPUT_SHARED_EXTENSION) +else + C_SRC_OUTPUT_FILE = $(C_SRC_OUTPUT)$(C_SRC_OUTPUT_EXECUTABLE_EXTENSION) +endif + +ifeq ($(PLATFORM),msys2) +# We hardcode the compiler used on MSYS2. The default CC=cc does +# not produce working code. The "gcc" MSYS2 package also doesn't. + CC = /mingw64/bin/gcc + export CC + CFLAGS ?= -O3 -std=c99 -finline-functions -Wall -Wmissing-prototypes + CXXFLAGS ?= -O3 -finline-functions -Wall +else ifeq ($(PLATFORM),darwin) + CC ?= cc + CFLAGS ?= -O3 -std=c99 -arch x86_64 -finline-functions -Wall -Wmissing-prototypes + CXXFLAGS ?= -O3 -arch x86_64 -finline-functions -Wall + LDFLAGS ?= -arch x86_64 -flat_namespace -undefined suppress +else ifeq ($(PLATFORM),freebsd) + CC ?= cc + CFLAGS ?= -O3 -std=c99 -finline-functions -Wall -Wmissing-prototypes + CXXFLAGS ?= -O3 -finline-functions -Wall +else ifeq ($(PLATFORM),linux) + CC ?= gcc + CFLAGS ?= -O3 -std=c99 -finline-functions -Wall -Wmissing-prototypes + CXXFLAGS ?= -O3 -finline-functions -Wall +endif + +ifneq ($(PLATFORM),msys2) + CFLAGS += -fPIC + CXXFLAGS += -fPIC +endif + +CFLAGS += -I"$(ERTS_INCLUDE_DIR)" -I"$(ERL_INTERFACE_INCLUDE_DIR)" +CXXFLAGS += -I"$(ERTS_INCLUDE_DIR)" -I"$(ERL_INTERFACE_INCLUDE_DIR)" + +LDLIBS += -L"$(ERL_INTERFACE_LIB_DIR)" -lerl_interface -lei + +# Verbosity. + +c_verbose_0 = @echo " C " $(?F); +c_verbose = $(c_verbose_$(V)) + +cpp_verbose_0 = @echo " CPP " $(?F); +cpp_verbose = $(cpp_verbose_$(V)) + +link_verbose_0 = @echo " LD " $(@F); +link_verbose = $(link_verbose_$(V)) + +# Targets. + +ifeq ($(wildcard $(C_SRC_DIR)),) +else ifneq ($(wildcard $(C_SRC_DIR)/Makefile),) +app:: app-c_src + +test-build:: app-c_src + +app-c_src: + $(MAKE) -C $(C_SRC_DIR) + +clean:: + $(MAKE) -C $(C_SRC_DIR) clean + +else + +ifeq ($(SOURCES),) +SOURCES := $(sort $(foreach pat,*.c *.C *.cc *.cpp,$(call core_find,$(C_SRC_DIR)/,$(pat)))) +endif +OBJECTS = $(addsuffix .o, $(basename $(SOURCES))) + +COMPILE_C = $(c_verbose) $(CC) $(CFLAGS) $(CPPFLAGS) -c +COMPILE_CPP = $(cpp_verbose) $(CXX) $(CXXFLAGS) $(CPPFLAGS) -c + +app:: $(C_SRC_ENV) $(C_SRC_OUTPUT_FILE) + +test-build:: $(C_SRC_ENV) $(C_SRC_OUTPUT_FILE) + +$(C_SRC_OUTPUT_FILE): $(OBJECTS) + $(verbose) mkdir -p priv/ + $(link_verbose) $(CC) $(OBJECTS) \ + $(LDFLAGS) $(if $(filter $(C_SRC_TYPE),shared),-shared) $(LDLIBS) \ + -o $(C_SRC_OUTPUT_FILE) + +%.o: %.c + $(COMPILE_C) $(OUTPUT_OPTION) $< + +%.o: %.cc + $(COMPILE_CPP) $(OUTPUT_OPTION) $< + +%.o: %.C + $(COMPILE_CPP) $(OUTPUT_OPTION) $< + +%.o: %.cpp + $(COMPILE_CPP) $(OUTPUT_OPTION) $< + +clean:: clean-c_src + +clean-c_src: + $(gen_verbose) rm -f $(C_SRC_OUTPUT_FILE) $(OBJECTS) + +endif + +ifneq ($(wildcard $(C_SRC_DIR)),) +$(C_SRC_ENV): + $(verbose) $(ERL) -eval "file:write_file(\"$(call core_native_path,$(C_SRC_ENV))\", \ + io_lib:format( \ + \"ERTS_INCLUDE_DIR ?= ~s/erts-~s/include/~n\" \ + \"ERL_INTERFACE_INCLUDE_DIR ?= ~s~n\" \ + \"ERL_INTERFACE_LIB_DIR ?= ~s~n\", \ + [code:root_dir(), erlang:system_info(version), \ + code:lib_dir(erl_interface, include), \ + code:lib_dir(erl_interface, lib)])), \ + halt()." + +distclean:: distclean-c_src-env + +distclean-c_src-env: + $(gen_verbose) rm -f $(C_SRC_ENV) + +-include $(C_SRC_ENV) +endif + +# Templates. + +define bs_c_nif +#include "erl_nif.h" + +static int loads = 0; + +static int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info) +{ + /* Initialize private data. */ + *priv_data = NULL; + + loads++; + + return 0; +} + +static int upgrade(ErlNifEnv* env, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info) +{ + /* Convert the private data to the new version. */ + *priv_data = *old_priv_data; + + loads++; + + return 0; +} + +static void unload(ErlNifEnv* env, void* priv_data) +{ + if (loads == 1) { + /* Destroy the private data. */ + } + + loads--; +} + +static ERL_NIF_TERM hello(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + if (enif_is_atom(env, argv[0])) { + return enif_make_tuple2(env, + enif_make_atom(env, "hello"), + argv[0]); + } + + return enif_make_tuple2(env, + enif_make_atom(env, "error"), + enif_make_atom(env, "badarg")); +} + +static ErlNifFunc nif_funcs[] = { + {"hello", 1, hello} +}; + +ERL_NIF_INIT($n, nif_funcs, load, NULL, upgrade, unload) +endef + +define bs_erl_nif +-module($n). + +-export([hello/1]). + +-on_load(on_load/0). +on_load() -> + PrivDir = case code:priv_dir(?MODULE) of + {error, _} -> + AppPath = filename:dirname(filename:dirname(code:which(?MODULE))), + filename:join(AppPath, "priv"); + Path -> + Path + end, + erlang:load_nif(filename:join(PrivDir, atom_to_list(?MODULE)), 0). + +hello(_) -> + erlang:nif_error({not_loaded, ?MODULE}). +endef + +new-nif: +ifneq ($(wildcard $(C_SRC_DIR)/$n.c),) + $(error Error: $(C_SRC_DIR)/$n.c already exists) +endif +ifneq ($(wildcard src/$n.erl),) + $(error Error: src/$n.erl already exists) +endif +ifdef in + $(verbose) $(MAKE) -C $(APPS_DIR)/$(in)/ new-nif n=$n in= +else + $(verbose) mkdir -p $(C_SRC_DIR) src/ + $(call render_template,bs_c_nif,$(C_SRC_DIR)/$n.c) + $(call render_template,bs_erl_nif,src/$n.erl) +endif + +# Copyright (c) 2015, Loïc Hoguin +# This file is part of erlang.mk and subject to the terms of the ISC License. + +.PHONY: ci ci-setup distclean-kerl + +KERL ?= $(CURDIR)/kerl +export KERL + +KERL_URL ?= https://raw.githubusercontent.com/yrashk/kerl/master/kerl + +OTP_GIT ?= https://github.com/erlang/otp + +CI_INSTALL_DIR ?= $(HOME)/erlang +CI_OTP ?= + +ifeq ($(strip $(CI_OTP)),) +ci:: +else +ci:: $(addprefix ci-,$(CI_OTP)) + +ci-prepare: $(addprefix $(CI_INSTALL_DIR)/,$(CI_OTP)) + +ci-setup:: + +ci_verbose_0 = @echo " CI " $(1); +ci_verbose = $(ci_verbose_$(V)) + +define ci_target +ci-$(1): $(CI_INSTALL_DIR)/$(1) + $(ci_verbose) \ + PATH="$(CI_INSTALL_DIR)/$(1)/bin:$(PATH)" \ + CI_OTP_RELEASE="$(1)" \ + CT_OPTS="-label $(1)" \ + $(MAKE) clean ci-setup tests +endef + +$(foreach otp,$(CI_OTP),$(eval $(call ci_target,$(otp)))) + +define ci_otp_target +ifeq ($(wildcard $(CI_INSTALL_DIR)/$(1)),) +$(CI_INSTALL_DIR)/$(1): $(KERL) + $(KERL) build git $(OTP_GIT) $(1) $(1) + $(KERL) install $(1) $(CI_INSTALL_DIR)/$(1) +endif +endef + +$(foreach otp,$(CI_OTP),$(eval $(call ci_otp_target,$(otp)))) + +$(KERL): + $(gen_verbose) $(call core_http_get,$(KERL),$(KERL_URL)) + $(verbose) chmod +x $(KERL) + +help:: + $(verbose) printf "%s\n" "" \ + "Continuous Integration targets:" \ + " ci Run '$(MAKE) tests' on all configured Erlang versions." \ + "" \ + "The CI_OTP variable must be defined with the Erlang versions" \ + "that must be tested. For example: CI_OTP = OTP-17.3.4 OTP-17.5.3" + +distclean:: distclean-kerl + +distclean-kerl: + $(gen_verbose) rm -rf $(KERL) +endif + +# Copyright (c) 2013-2015, Loïc Hoguin +# This file is part of erlang.mk and subject to the terms of the ISC License. + +.PHONY: ct apps-ct distclean-ct + +# Configuration. + +CT_OPTS ?= +ifneq ($(wildcard $(TEST_DIR)),) + CT_SUITES ?= $(sort $(subst _SUITE.erl,,$(notdir $(call core_find,$(TEST_DIR)/,*_SUITE.erl)))) +else + CT_SUITES ?= +endif + +# Core targets. + +tests:: ct + +distclean:: distclean-ct + +help:: + $(verbose) printf "%s\n" "" \ + "Common_test targets:" \ + " ct Run all the common_test suites for this project" \ + "" \ + "All your common_test suites have their associated targets." \ + "A suite named http_SUITE can be ran using the ct-http target." + +# Plugin-specific targets. + +CT_RUN = ct_run \ + -no_auto_compile \ + -noinput \ + -pa $(CURDIR)/ebin $(DEPS_DIR)/*/ebin $(APPS_DIR)/*/ebin $(TEST_DIR) \ + -dir $(TEST_DIR) \ + -logdir $(CURDIR)/logs + +ifeq ($(CT_SUITES),) +ct: $(if $(IS_APP),,apps-ct) +else +ct: test-build $(if $(IS_APP),,apps-ct) + $(verbose) mkdir -p $(CURDIR)/logs/ + $(gen_verbose) $(CT_RUN) -sname ct_$(PROJECT) -suite $(addsuffix _SUITE,$(CT_SUITES)) $(CT_OPTS) +endif + +ifneq ($(ALL_APPS_DIRS),) +define ct_app_target +apps-ct-$1: + $(MAKE) -C $1 ct IS_APP=1 +endef + +$(foreach app,$(ALL_APPS_DIRS),$(eval $(call ct_app_target,$(app)))) + +apps-ct: test-build $(addprefix apps-ct-,$(ALL_APPS_DIRS)) +endif + +ifndef t +CT_EXTRA = +else +ifeq (,$(findstring :,$t)) +CT_EXTRA = -group $t +else +t_words = $(subst :, ,$t) +CT_EXTRA = -group $(firstword $(t_words)) -case $(lastword $(t_words)) +endif +endif + +define ct_suite_target +ct-$(1): test-build + $(verbose) mkdir -p $(CURDIR)/logs/ + $(gen_verbose) $(CT_RUN) -sname ct_$(PROJECT) -suite $(addsuffix _SUITE,$(1)) $(CT_EXTRA) $(CT_OPTS) +endef + +$(foreach test,$(CT_SUITES),$(eval $(call ct_suite_target,$(test)))) + +distclean-ct: + $(gen_verbose) rm -rf $(CURDIR)/logs/ + +# Copyright (c) 2013-2015, Loïc Hoguin +# This file is part of erlang.mk and subject to the terms of the ISC License. + +.PHONY: plt distclean-plt dialyze + +# Configuration. + +DIALYZER_PLT ?= $(CURDIR)/.$(PROJECT).plt +export DIALYZER_PLT + +PLT_APPS ?= +DIALYZER_DIRS ?= --src -r $(wildcard src) $(ALL_APPS_DIRS) +DIALYZER_OPTS ?= -Werror_handling -Wrace_conditions -Wunmatched_returns # -Wunderspecs + +# Core targets. + +check:: dialyze + +distclean:: distclean-plt + +help:: + $(verbose) printf "%s\n" "" \ + "Dialyzer targets:" \ + " plt Build a PLT file for this project" \ + " dialyze Analyze the project using Dialyzer" + +# Plugin-specific targets. + +define filter_opts.erl + Opts = binary:split(<<"$1">>, <<"-">>, [global]), + Filtered = lists:reverse(lists:foldl(fun + (O = <<"pa ", _/bits>>, Acc) -> [O|Acc]; + (O = <<"D ", _/bits>>, Acc) -> [O|Acc]; + (O = <<"I ", _/bits>>, Acc) -> [O|Acc]; + (_, Acc) -> Acc + end, [], Opts)), + io:format("~s~n", [[["-", O] || O <- Filtered]]), + halt(). +endef + +$(DIALYZER_PLT): deps app + $(verbose) dialyzer --build_plt --apps erts kernel stdlib $(PLT_APPS) $(OTP_DEPS) $(LOCAL_DEPS) $(DEPS) + +plt: $(DIALYZER_PLT) + +distclean-plt: + $(gen_verbose) rm -f $(DIALYZER_PLT) + +ifneq ($(wildcard $(DIALYZER_PLT)),) +dialyze: +else +dialyze: $(DIALYZER_PLT) +endif + $(verbose) dialyzer --no_native `$(call erlang,$(call filter_opts.erl,$(ERLC_OPTS)))` $(DIALYZER_DIRS) $(DIALYZER_OPTS) + +# Copyright (c) 2013-2015, Loïc Hoguin +# This file is part of erlang.mk and subject to the terms of the ISC License. + +.PHONY: distclean-edoc edoc + +# Configuration. + +EDOC_OPTS ?= + +# Core targets. + +ifneq ($(wildcard doc/overview.edoc),) +docs:: edoc +endif + +distclean:: distclean-edoc + +# Plugin-specific targets. + +edoc: distclean-edoc doc-deps + $(gen_verbose) $(ERL) -eval 'edoc:application($(PROJECT), ".", [$(EDOC_OPTS)]), halt().' + +distclean-edoc: + $(gen_verbose) rm -f doc/*.css doc/*.html doc/*.png doc/edoc-info + +# Copyright (c) 2014 Dave Cottlehuber +# This file is part of erlang.mk and subject to the terms of the ISC License. + +.PHONY: distclean-escript escript + +# Configuration. + +ESCRIPT_NAME ?= $(PROJECT) +ESCRIPT_FILE ?= $(ESCRIPT_NAME) + +ESCRIPT_COMMENT ?= This is an -*- erlang -*- file + +ESCRIPT_BEAMS ?= "ebin/*", "deps/*/ebin/*" +ESCRIPT_SYS_CONFIG ?= "rel/sys.config" +ESCRIPT_EMU_ARGS ?= -pa . \ + -sasl errlog_type error \ + -escript main $(ESCRIPT_NAME) +ESCRIPT_SHEBANG ?= /usr/bin/env escript +ESCRIPT_STATIC ?= "deps/*/priv/**", "priv/**" + +# Core targets. + +distclean:: distclean-escript + +help:: + $(verbose) printf "%s\n" "" \ + "Escript targets:" \ + " escript Build an executable escript archive" \ + +# Plugin-specific targets. + +# Based on https://github.com/synrc/mad/blob/master/src/mad_bundle.erl +# Copyright (c) 2013 Maxim Sokhatsky, Synrc Research Center +# Modified MIT License, https://github.com/synrc/mad/blob/master/LICENSE : +# Software may only be used for the great good and the true happiness of all +# sentient beings. + +define ESCRIPT_RAW +'Read = fun(F) -> {ok, B} = file:read_file(filename:absname(F)), B end,'\ +'Files = fun(L) -> A = lists:concat([filelib:wildcard(X)||X<- L ]),'\ +' [F || F <- A, not filelib:is_dir(F) ] end,'\ +'Squash = fun(L) -> [{filename:basename(F), Read(F) } || F <- L ] end,'\ +'Zip = fun(A, L) -> {ok,{_,Z}} = zip:create(A, L, [{compress,all},memory]), Z end,'\ +'Ez = fun(Escript) ->'\ +' Static = Files([$(ESCRIPT_STATIC)]),'\ +' Beams = Squash(Files([$(ESCRIPT_BEAMS), $(ESCRIPT_SYS_CONFIG)])),'\ +' Archive = Beams ++ [{ "static.gz", Zip("static.gz", Static)}],'\ +' escript:create(Escript, [ $(ESCRIPT_OPTIONS)'\ +' {archive, Archive, [memory]},'\ +' {shebang, "$(ESCRIPT_SHEBANG)"},'\ +' {comment, "$(ESCRIPT_COMMENT)"},'\ +' {emu_args, " $(ESCRIPT_EMU_ARGS)"}'\ +' ]),'\ +' file:change_mode(Escript, 8#755)'\ +'end,'\ +'Ez("$(ESCRIPT_FILE)"),'\ +'halt().' +endef + +ESCRIPT_COMMAND = $(subst ' ',,$(ESCRIPT_RAW)) + +escript:: distclean-escript deps app + $(gen_verbose) $(ERL) -eval $(ESCRIPT_COMMAND) + +distclean-escript: + $(gen_verbose) rm -f $(ESCRIPT_NAME) + +# Copyright (c) 2014, Enrique Fernandez +# Copyright (c) 2015, Loïc Hoguin +# This file is contributed to erlang.mk and subject to the terms of the ISC License. + +.PHONY: eunit apps-eunit + +# Configuration + +EUNIT_OPTS ?= +EUNIT_ERL_OPTS ?= + +# Core targets. + +tests:: eunit + +help:: + $(verbose) printf "%s\n" "" \ + "EUnit targets:" \ + " eunit Run all the EUnit tests for this project" + +# Plugin-specific targets. + +define eunit.erl + case "$(COVER)" of + "" -> ok; + _ -> + case cover:compile_beam_directory("ebin") of + {error, _} -> halt(1); + _ -> ok + end + end, + case eunit:test($1, [$(EUNIT_OPTS)]) of + ok -> ok; + error -> halt(2) + end, + case "$(COVER)" of + "" -> ok; + _ -> + cover:export("eunit.coverdata") + end, + halt() +endef + +EUNIT_ERL_OPTS += -pa $(TEST_DIR) $(DEPS_DIR)/*/ebin $(APPS_DIR)/*/ebin $(CURDIR)/ebin + +ifdef t +ifeq (,$(findstring :,$(t))) +eunit: test-build + $(gen_verbose) $(call erlang,$(call eunit.erl,['$(t)']),$(EUNIT_ERL_OPTS)) +else +eunit: test-build + $(gen_verbose) $(call erlang,$(call eunit.erl,fun $(t)/0),$(EUNIT_ERL_OPTS)) +endif +else +EUNIT_EBIN_MODS = $(notdir $(basename $(ERL_FILES) $(BEAM_FILES))) +EUNIT_TEST_MODS = $(notdir $(basename $(call core_find,$(TEST_DIR)/,*.erl))) + +EUNIT_MODS = $(foreach mod,$(EUNIT_EBIN_MODS) $(filter-out \ + $(patsubst %,%_tests,$(EUNIT_EBIN_MODS)),$(EUNIT_TEST_MODS)),'$(mod)') + +eunit: test-build $(if $(IS_APP),,apps-eunit) + $(gen_verbose) $(call erlang,$(call eunit.erl,[$(call comma_list,$(EUNIT_MODS))]),$(EUNIT_ERL_OPTS)) + +ifneq ($(ALL_APPS_DIRS),) +apps-eunit: + $(verbose) for app in $(ALL_APPS_DIRS); do $(MAKE) -C $$app eunit IS_APP=1; done +endif +endif + +# Copyright (c) 2013-2015, Loïc Hoguin +# This file is part of erlang.mk and subject to the terms of the ISC License. + +.PHONY: relx-rel distclean-relx-rel distclean-relx run + +# Configuration. + +RELX ?= $(CURDIR)/relx +RELX_CONFIG ?= $(CURDIR)/relx.config + +RELX_URL ?= https://github.com/erlware/relx/releases/download/v3.19.0/relx +RELX_OPTS ?= +RELX_OUTPUT_DIR ?= _rel + +ifeq ($(firstword $(RELX_OPTS)),-o) + RELX_OUTPUT_DIR = $(word 2,$(RELX_OPTS)) +else + RELX_OPTS += -o $(RELX_OUTPUT_DIR) +endif + +# Core targets. + +ifeq ($(IS_DEP),) +ifneq ($(wildcard $(RELX_CONFIG)),) +rel:: relx-rel +endif +endif + +distclean:: distclean-relx-rel distclean-relx + +# Plugin-specific targets. + +$(RELX): + $(gen_verbose) $(call core_http_get,$(RELX),$(RELX_URL)) + $(verbose) chmod +x $(RELX) + +relx-rel: $(RELX) rel-deps app + $(verbose) $(RELX) -c $(RELX_CONFIG) $(RELX_OPTS) + +distclean-relx-rel: + $(gen_verbose) rm -rf $(RELX_OUTPUT_DIR) + +distclean-relx: + $(gen_verbose) rm -rf $(RELX) + +# Run target. + +ifeq ($(wildcard $(RELX_CONFIG)),) +run: +else + +define get_relx_release.erl + {ok, Config} = file:consult("$(RELX_CONFIG)"), + {release, {Name, _}, _} = lists:keyfind(release, 1, Config), + io:format("~s", [Name]), + halt(0). +endef + +RELX_RELEASE = `$(call erlang,$(get_relx_release.erl))` + +run: all + $(verbose) $(RELX_OUTPUT_DIR)/$(RELX_RELEASE)/bin/$(RELX_RELEASE) console + +help:: + $(verbose) printf "%s\n" "" \ + "Relx targets:" \ + " run Compile the project, build the release and run it" + +endif + +# Copyright (c) 2014, M Robert Martin +# Copyright (c) 2015, Loïc Hoguin +# This file is contributed to erlang.mk and subject to the terms of the ISC License. + +.PHONY: shell + +# Configuration. + +SHELL_ERL ?= erl +SHELL_PATHS ?= $(CURDIR)/ebin $(APPS_DIR)/*/ebin $(DEPS_DIR)/*/ebin +SHELL_OPTS ?= + +ALL_SHELL_DEPS_DIRS = $(addprefix $(DEPS_DIR)/,$(SHELL_DEPS)) + +# Core targets + +help:: + $(verbose) printf "%s\n" "" \ + "Shell targets:" \ + " shell Run an erlang shell with SHELL_OPTS or reasonable default" + +# Plugin-specific targets. + +$(foreach dep,$(SHELL_DEPS),$(eval $(call dep_target,$(dep)))) + +build-shell-deps: $(ALL_SHELL_DEPS_DIRS) + $(verbose) for dep in $(ALL_SHELL_DEPS_DIRS) ; do $(MAKE) -C $$dep ; done + +shell: build-shell-deps + $(gen_verbose) $(SHELL_ERL) -pa $(SHELL_PATHS) $(SHELL_OPTS) + +# Copyright (c) 2015, Loïc Hoguin +# This file is part of erlang.mk and subject to the terms of the ISC License. + +ifeq ($(filter triq,$(DEPS) $(TEST_DEPS)),triq) +.PHONY: triq + +# Targets. + +tests:: triq + +define triq_check.erl + code:add_pathsa(["$(CURDIR)/ebin", "$(DEPS_DIR)/*/ebin"]), + try + case $(1) of + all -> [true] =:= lists:usort([triq:check(M) || M <- [$(call comma_list,$(3))]]); + module -> triq:check($(2)); + function -> triq:check($(2)) + end + of + true -> halt(0); + _ -> halt(1) + catch error:undef -> + io:format("Undefined property or module~n"), + halt(0) + end. +endef + +ifdef t +ifeq (,$(findstring :,$(t))) +triq: test-build + $(verbose) $(call erlang,$(call triq_check.erl,module,$(t))) +else +triq: test-build + $(verbose) echo Testing $(t)/0 + $(verbose) $(call erlang,$(call triq_check.erl,function,$(t)())) +endif +else +triq: test-build + $(eval MODULES := $(patsubst %,'%',$(sort $(notdir $(basename $(wildcard ebin/*.beam)))))) + $(gen_verbose) $(call erlang,$(call triq_check.erl,all,undefined,$(MODULES))) +endif +endif + +# Copyright (c) 2015, Erlang Solutions Ltd. +# This file is part of erlang.mk and subject to the terms of the ISC License. + +.PHONY: xref distclean-xref + +# Configuration. + +ifeq ($(XREF_CONFIG),) + XREF_ARGS := +else + XREF_ARGS := -c $(XREF_CONFIG) +endif + +XREFR ?= $(CURDIR)/xrefr +export XREFR + +XREFR_URL ?= https://github.com/inaka/xref_runner/releases/download/0.2.2/xrefr + +# Core targets. + +help:: + $(verbose) printf "%s\n" "" \ + "Xref targets:" \ + " xref Run Xrefr using $XREF_CONFIG as config file if defined" + +distclean:: distclean-xref + +# Plugin-specific targets. + +$(XREFR): + $(gen_verbose) $(call core_http_get,$(XREFR),$(XREFR_URL)) + $(verbose) chmod +x $(XREFR) + +xref: deps app $(XREFR) + $(gen_verbose) $(XREFR) $(XREFR_ARGS) + +distclean-xref: + $(gen_verbose) rm -rf $(XREFR) + +# Copyright 2015, Viktor Söderqvist +# This file is part of erlang.mk and subject to the terms of the ISC License. + +COVER_REPORT_DIR = cover + +# Hook in coverage to ct + +ifdef COVER +ifdef CT_RUN +# All modules in 'ebin' +COVER_MODS = $(notdir $(basename $(call core_ls,ebin/*.beam))) + +test-build:: $(TEST_DIR)/ct.cover.spec + +$(TEST_DIR)/ct.cover.spec: + $(verbose) echo Cover mods: $(COVER_MODS) + $(gen_verbose) printf "%s\n" \ + '{incl_mods,[$(subst $(space),$(comma),$(COVER_MODS))]}.' \ + '{export,"$(CURDIR)/ct.coverdata"}.' > $@ + +CT_RUN += -cover $(TEST_DIR)/ct.cover.spec +endif +endif + +# Core targets + +ifdef COVER +ifneq ($(COVER_REPORT_DIR),) +tests:: + $(verbose) $(MAKE) --no-print-directory cover-report +endif +endif + +clean:: coverdata-clean + +ifneq ($(COVER_REPORT_DIR),) +distclean:: cover-report-clean +endif + +help:: + $(verbose) printf "%s\n" "" \ + "Cover targets:" \ + " cover-report Generate a HTML coverage report from previously collected" \ + " cover data." \ + " all.coverdata Merge {eunit,ct}.coverdata into one coverdata file." \ + "" \ + "If COVER=1 is set, coverage data is generated by the targets eunit and ct. The" \ + "target tests additionally generates a HTML coverage report from the combined" \ + "coverdata files from each of these testing tools. HTML reports can be disabled" \ + "by setting COVER_REPORT_DIR to empty." + +# Plugin specific targets + +COVERDATA = $(filter-out all.coverdata,$(wildcard *.coverdata)) + +.PHONY: coverdata-clean +coverdata-clean: + $(gen_verbose) rm -f *.coverdata ct.cover.spec + +# Merge all coverdata files into one. +all.coverdata: $(COVERDATA) + $(gen_verbose) $(ERL) -eval ' \ + $(foreach f,$(COVERDATA),cover:import("$(f)") == ok orelse halt(1),) \ + cover:export("$@"), halt(0).' + +# These are only defined if COVER_REPORT_DIR is non-empty. Set COVER_REPORT_DIR to +# empty if you want the coverdata files but not the HTML report. +ifneq ($(COVER_REPORT_DIR),) + +.PHONY: cover-report-clean cover-report + +cover-report-clean: + $(gen_verbose) rm -rf $(COVER_REPORT_DIR) + +ifeq ($(COVERDATA),) +cover-report: +else + +# Modules which include eunit.hrl always contain one line without coverage +# because eunit defines test/0 which is never called. We compensate for this. +EUNIT_HRL_MODS = $(subst $(space),$(comma),$(shell \ + grep -e '^\s*-include.*include/eunit\.hrl"' src/*.erl \ + | sed "s/^src\/\(.*\)\.erl:.*/'\1'/" | uniq)) + +define cover_report.erl + $(foreach f,$(COVERDATA),cover:import("$(f)") == ok orelse halt(1),) + Ms = cover:imported_modules(), + [cover:analyse_to_file(M, "$(COVER_REPORT_DIR)/" ++ atom_to_list(M) + ++ ".COVER.html", [html]) || M <- Ms], + Report = [begin {ok, R} = cover:analyse(M, module), R end || M <- Ms], + EunitHrlMods = [$(EUNIT_HRL_MODS)], + Report1 = [{M, {Y, case lists:member(M, EunitHrlMods) of + true -> N - 1; false -> N end}} || {M, {Y, N}} <- Report], + TotalY = lists:sum([Y || {_, {Y, _}} <- Report1]), + TotalN = lists:sum([N || {_, {_, N}} <- Report1]), + Perc = fun(Y, N) -> case Y + N of 0 -> 100; S -> round(100 * Y / S) end end, + TotalPerc = Perc(TotalY, TotalN), + {ok, F} = file:open("$(COVER_REPORT_DIR)/index.html", [write]), + io:format(F, "~n" + "~n" + "Coverage report~n" + "~n", []), + io:format(F, "

Coverage

~n

Total: ~p%

~n", [TotalPerc]), + io:format(F, "~n", []), + [io:format(F, "" + "~n", + [M, M, Perc(Y, N)]) || {M, {Y, N}} <- Report1], + How = "$(subst $(space),$(comma)$(space),$(basename $(COVERDATA)))", + Date = "$(shell date -u "+%Y-%m-%dT%H:%M:%SZ")", + io:format(F, "
ModuleCoverage
~p~p%
~n" + "

Generated using ~s and erlang.mk on ~s.

~n" + "", [How, Date]), + halt(). +endef + +cover-report: + $(gen_verbose) mkdir -p $(COVER_REPORT_DIR) + $(gen_verbose) $(call erlang,$(cover_report.erl)) + +endif +endif # ifneq ($(COVER_REPORT_DIR),) diff --git a/include/emqttd.hrl b/include/emqttd.hrl index b68b583f0..0059d11a3 100644 --- a/include/emqttd.hrl +++ b/include/emqttd.hrl @@ -14,8 +14,6 @@ %% limitations under the License. %%-------------------------------------------------------------------- -%% MQTT Broker Header - %%-------------------------------------------------------------------- %% Banner %%-------------------------------------------------------------------- @@ -26,59 +24,43 @@ -define(PROTOCOL_VERSION, "MQTT/3.1.1"). --define(ERTS_MINIMUM, "6.0"). +-define(ERTS_MINIMUM, "7.0"). -%% System Topics. --define(SYSTOP, <<"$SYS">>). +%%-------------------------------------------------------------------- +%% Sys/Queue/Share Topics' Prefix +%%-------------------------------------------------------------------- -%% Queue Topics. --define(QTop, <<"$Q">>). +-define(SYSTOP, <<"$SYS/">>). %% System Topic + +-define(QUEUE, <<"$queue/">>). %% Queue Topic + +-define(SHARE, <<"$share/">>). %% Shared Topic %%-------------------------------------------------------------------- %% PubSub %%-------------------------------------------------------------------- --type pubsub() :: publish | subscribe. +-type(pubsub() :: publish | subscribe). --define(IS_PUBSUB(PS), (PS =:= publish orelse PS =:= subscribe)). +-define(PUBSUB(PS), (PS =:= publish orelse PS =:= subscribe)). %%-------------------------------------------------------------------- %% MQTT Topic %%-------------------------------------------------------------------- + -record(mqtt_topic, { - topic :: binary(), - flags :: [retained | static] + topic :: binary(), + flags = [] :: [retained | static] }). --type mqtt_topic() :: #mqtt_topic{}. - -%%-------------------------------------------------------------------- -%% MQTT Subscription -%%-------------------------------------------------------------------- --record(mqtt_subscription, { - subid :: binary() | atom(), - topic :: binary(), - qos = 0 :: 0 | 1 | 2 -}). - --type mqtt_subscription() :: #mqtt_subscription{}. - -%%-------------------------------------------------------------------- -%% MQTT Route -%%-------------------------------------------------------------------- --record(mqtt_route, { - topic :: binary(), - node :: node() -}). - --type mqtt_route() :: #mqtt_route{}. +-type(mqtt_topic() :: #mqtt_topic{}). %%-------------------------------------------------------------------- %% MQTT Client %%-------------------------------------------------------------------- --type ws_header_key() :: atom() | binary() | string(). --type ws_header_val() :: atom() | binary() | string() | integer(). +-type(ws_header_key() :: atom() | binary() | string()). +-type(ws_header_val() :: atom() | binary() | string() | integer()). -record(mqtt_client, { client_id :: binary() | undefined, @@ -93,41 +75,63 @@ connected_at :: erlang:timestamp() }). --type mqtt_client() :: #mqtt_client{}. +-type(mqtt_client() :: #mqtt_client{}). %%-------------------------------------------------------------------- %% MQTT Session %%-------------------------------------------------------------------- + -record(mqtt_session, { - client_id :: binary(), - sess_pid :: pid(), - persistent :: boolean() + client_id :: binary(), + sess_pid :: pid(), + persistent :: boolean() }). --type mqtt_session() :: #mqtt_session{}. +-type(mqtt_session() :: #mqtt_session{}). %%-------------------------------------------------------------------- %% MQTT Message %%-------------------------------------------------------------------- --type mqtt_msgid() :: binary() | undefined. --type mqtt_pktid() :: 1..16#ffff | undefined. +-type(mqtt_msgid() :: binary() | undefined). +-type(mqtt_pktid() :: 1..16#ffff | undefined). -record(mqtt_message, { - msgid :: mqtt_msgid(), %% Global unique message ID - pktid :: mqtt_pktid(), %% PacketId - topic :: binary(), %% Topic that the message is published to - from :: binary() | atom(), %% ClientId of the publisher - sender :: binary() | undefined, %% Username of the publisher - qos = 0 :: 0 | 1 | 2, %% Message QoS - flags = [] :: [retain | dup | sys], %% Message Flags - retain = false :: boolean(), %% Retain flag - dup = false :: boolean(), %% Dup flag - sys = false :: boolean(), %% $SYS flag - payload :: binary(), %% Payload - timestamp :: erlang:timestamp() %% os:timestamp + id :: mqtt_msgid(), %% Global unique message ID + pktid :: mqtt_pktid(), %% PacketId + from :: {binary(), undefined | binary()}, %% ClientId and Username + topic :: binary(), %% Topic that the message is published to + qos = 0 :: 0 | 1 | 2, %% Message QoS + flags = [] :: [retain | dup | sys], %% Message Flags + retain = false :: boolean(), %% Retain flag + dup = false :: boolean(), %% Dup flag + sys = false :: boolean(), %% $SYS flag + headers = [] :: list(), + payload :: binary(), %% Payload + timestamp :: pos_integer() %% os:timestamp to seconds }). --type mqtt_message() :: #mqtt_message{}. +-type(mqtt_message() :: #mqtt_message{}). + +%%-------------------------------------------------------------------- +%% MQTT Delivery +%%-------------------------------------------------------------------- +-record(mqtt_delivery, { + sender :: pid(), %% Pid of the sender/publisher + message :: mqtt_message(), %% Message + flows :: list() +}). + +-type(mqtt_delivery() :: #mqtt_delivery{}). + +%%-------------------------------------------------------------------- +%% MQTT Route +%%-------------------------------------------------------------------- +-record(mqtt_route, { + topic :: binary(), + node :: node() +}). + +-type(mqtt_route() :: #mqtt_route{}). %%-------------------------------------------------------------------- %% MQTT Alarm @@ -140,7 +144,7 @@ timestamp :: erlang:timestamp() %% Timestamp }). --type mqtt_alarm() :: #mqtt_alarm{}. +-type(mqtt_alarm() :: #mqtt_alarm{}). %%-------------------------------------------------------------------- %% MQTT Plugin @@ -149,11 +153,10 @@ name, version, descr, - config, active = false }). --type mqtt_plugin() :: #mqtt_plugin{}. +-type(mqtt_plugin() :: #mqtt_plugin{}). %%-------------------------------------------------------------------- %% MQTT CLI Command @@ -168,5 +171,5 @@ descr }). --type mqtt_cli() :: #mqtt_cli{}. +-type(mqtt_cli() :: #mqtt_cli{}). diff --git a/include/emqttd_protocol.hrl b/include/emqttd_protocol.hrl index 61b82f02d..8a5e5d0ca 100644 --- a/include/emqttd_protocol.hrl +++ b/include/emqttd_protocol.hrl @@ -26,7 +26,7 @@ {?MQTT_PROTO_V31, <<"MQIsdp">>}, {?MQTT_PROTO_V311, <<"MQTT">>}]). --type mqtt_vsn() :: ?MQTT_PROTO_V31 | ?MQTT_PROTO_V311. +-type(mqtt_vsn() :: ?MQTT_PROTO_V31 | ?MQTT_PROTO_V311). %%-------------------------------------------------------------------- %% MQTT QoS @@ -41,11 +41,11 @@ -define(IS_QOS(I), (I >= ?QOS0 andalso I =< ?QOS2)). --type mqtt_qos() :: ?QOS0 | ?QOS1 | ?QOS2. +-type(mqtt_qos() :: ?QOS0 | ?QOS1 | ?QOS2). --type mqtt_qos_name() :: qos0 | at_most_once | +-type(mqtt_qos_name() :: qos0 | at_most_once | qos1 | at_least_once | - qos2 | exactly_once. + qos2 | exactly_once). -define(QOS_I(Name), begin @@ -102,7 +102,7 @@ 'PINGRESP', 'DISCONNECT']). --type mqtt_packet_type() :: ?RESERVED..?DISCONNECT. +-type(mqtt_packet_type() :: ?RESERVED..?DISCONNECT). %%-------------------------------------------------------------------- %% MQTT Connect Return Codes @@ -114,7 +114,7 @@ -define(CONNACK_CREDENTIALS, 4). %% Username or password is malformed -define(CONNACK_AUTH, 5). %% Client is not authorized to connect --type mqtt_connack() :: ?CONNACK_ACCEPT..?CONNACK_AUTH. +-type(mqtt_connack() :: ?CONNACK_ACCEPT..?CONNACK_AUTH). %%-------------------------------------------------------------------- %% MQTT Parser and Serializer @@ -135,8 +135,9 @@ %%-------------------------------------------------------------------- %% MQTT Packets %%-------------------------------------------------------------------- --type mqtt_client_id() :: binary(). --type mqtt_packet_id() :: 1..16#ffff | undefined. +-type(mqtt_client_id() :: binary()). +-type(mqtt_username() :: binary() | undefined). +-type(mqtt_packet_id() :: 1..16#ffff | undefined). -record(mqtt_packet_connect, { client_id = <<>> :: mqtt_client_id(), diff --git a/include/emqttd_trie.hrl b/include/emqttd_trie.hrl index d077da5fb..e701d90cd 100644 --- a/include/emqttd_trie.hrl +++ b/include/emqttd_trie.hrl @@ -14,7 +14,7 @@ %% limitations under the License. %%-------------------------------------------------------------------- --type trie_node_id() :: binary() | atom(). +-type(trie_node_id() :: binary() | atom()). -record(trie_node, { node_id :: trie_node_id(), @@ -24,12 +24,12 @@ }). -record(trie_edge, { - node_id :: trie_node_id(), - word :: binary() | atom() + node_id :: trie_node_id(), + word :: binary() | atom() }). -record(trie, { - edge :: #trie_edge{}, - node_id :: trie_node_id() + edge :: #trie_edge{}, + node_id :: trie_node_id() }). diff --git a/plugins/emqttd_auth_http b/plugins/emqttd_auth_http deleted file mode 160000 index d5a40dbea..000000000 --- a/plugins/emqttd_auth_http +++ /dev/null @@ -1 +0,0 @@ -Subproject commit d5a40dbea347b3aa4a7631dec40cc09ee5ec4024 diff --git a/plugins/emqttd_dashboard b/plugins/emqttd_dashboard deleted file mode 160000 index 95a3d0964..000000000 --- a/plugins/emqttd_dashboard +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 95a3d0964b2d6d4541fdabe6c07869a14f4c990d diff --git a/plugins/emqttd_plugin_mongo b/plugins/emqttd_plugin_mongo deleted file mode 160000 index 7ed2f9c51..000000000 --- a/plugins/emqttd_plugin_mongo +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 7ed2f9c51f0f3a2ea9b0003705b0dd30c3f3ba9e diff --git a/plugins/emqttd_plugin_mysql b/plugins/emqttd_plugin_mysql deleted file mode 160000 index e91b17f28..000000000 --- a/plugins/emqttd_plugin_mysql +++ /dev/null @@ -1 +0,0 @@ -Subproject commit e91b17f2880756ae0c411373594e5e226478c4fb diff --git a/plugins/emqttd_plugin_pgsql b/plugins/emqttd_plugin_pgsql deleted file mode 160000 index f4c571e74..000000000 --- a/plugins/emqttd_plugin_pgsql +++ /dev/null @@ -1 +0,0 @@ -Subproject commit f4c571e74fbc02c1f02368ba69e5b08f4468e4e8 diff --git a/plugins/emqttd_plugin_redis b/plugins/emqttd_plugin_redis deleted file mode 160000 index 10bda5043..000000000 --- a/plugins/emqttd_plugin_redis +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 10bda5043e437f8bd5c4997d4929006fbb6931e1 diff --git a/plugins/emqttd_plugin_template b/plugins/emqttd_plugin_template deleted file mode 160000 index c05985ae5..000000000 --- a/plugins/emqttd_plugin_template +++ /dev/null @@ -1 +0,0 @@ -Subproject commit c05985ae5828ea8a1cc364cc9a0ad764ea009a5d diff --git a/plugins/emqttd_recon b/plugins/emqttd_recon deleted file mode 160000 index d4879c24f..000000000 --- a/plugins/emqttd_recon +++ /dev/null @@ -1 +0,0 @@ -Subproject commit d4879c24f055e4215b5a984cfed72ecd5945df2d diff --git a/plugins/emqttd_reloader b/plugins/emqttd_reloader deleted file mode 160000 index 0e1da2339..000000000 --- a/plugins/emqttd_reloader +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 0e1da23391951ad7f7d83e07d26b78bb7d0c3191 diff --git a/plugins/emqttd_stomp b/plugins/emqttd_stomp deleted file mode 160000 index ccf192844..000000000 --- a/plugins/emqttd_stomp +++ /dev/null @@ -1 +0,0 @@ -Subproject commit ccf19284402fa305ef392577aa2ce6edc18890ae diff --git a/rebar b/rebar deleted file mode 100755 index c2b7e2022..000000000 Binary files a/rebar and /dev/null differ diff --git a/rebar.config b/rebar.config index dc548c0b1..5306e0ae0 100644 --- a/rebar.config +++ b/rebar.config @@ -1,48 +1,4 @@ -%% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*- -%% ex: ts=4 sw=4 ft=erlang et - -{require_min_otp_vsn, "R17"}. - -%% fail_on_warning, -{erl_opts, [debug_info, {parse_transform, lager_transform}]}. - -{erl_opts, [warn_export_all, - warn_unused_import, - {i, "include"}, - {src_dirs, ["src"]}]}. - -{validate_app_modules, true}. - -{erl_first_files, ["src/gen_server2.erl", - "src/emqttd_auth_mod.erl", - "src/emqttd_acl_mod.erl"]}. - -{eunit_opts, []}. %%verbose - -{ct_dir, "test"}. - -{ct_log_dir, "logs"}. - -{ct_extra_params, "-name ct_emqttd@127.0.0.1 -config rel/files/emqttd.test.config"}. - -{ct_use_short_names, false}. - -{xref_checks, [undefined_function_calls]}. - -{cover_enabled, true}. - -%% plugins cannot find emqttd.hrl without ".." lib dirs:( -%% but this setting will make deps apps collision -%% comment in 0.13.0 release -%% {lib_dirs, ["../"]}. - -{sub_dirs, ["rel", "plugins/*/"]}. - {deps, [ - {gproc, ".*", {git, "git://github.com/uwiger/gproc.git", {branch, "master"}}}, - {lager, ".*", {git, "git://github.com/basho/lager.git", {branch, "master"}}}, - {esockd, ".*", {git, "git://github.com/emqtt/esockd.git", {tag, "4.0"}}}, - {mochiweb, "4.*", {git, "git://github.com/emqtt/mochiweb.git", {tag, "4.1.1"}}} +{gproc,".*",{git,"https://github.com/uwiger/gproc.git",""}},{lager,".*",{git,"https://github.com/basho/lager.git",""}},{gen_logger,".*",{git,"https://github.com/emqtt/gen_logger.git",""}},{gen_conf,".*",{git,"https://github.com/emqtt/gen_conf.git",""}},{esockd,".*",{git,"https://github.com/emqtt/esockd.git","udp"}},{mochiweb,".*",{git,"https://github.com/emqtt/mochiweb.git",""}} ]}. - -{recursive_cmds, [ct, eunit, clean]}. +{erl_opts, [{parse_transform,lager_transform}]}. diff --git a/rel/files/acl.config b/rel/files/acl.config deleted file mode 100644 index 9b1d512a6..000000000 --- a/rel/files/acl.config +++ /dev/null @@ -1,28 +0,0 @@ -%%%----------------------------------------------------------------------------- -%%% -%%% [ACL](https://github.com/emqtt/emqttd/wiki/ACL) -%%% -%%% -type who() :: all | binary() | -%%% {ipaddr, esockd_access:cidr()} | -%%% {client, binary()} | -%%% {user, binary()}. -%%% -%%% -type access() :: subscribe | publish | pubsub. -%%% -%%% -type topic() :: binary(). -%%% -%%% -type rule() :: {allow, all} | -%%% {allow, who(), access(), list(topic())} | -%%% {deny, all} | -%%% {deny, who(), access(), list(topic())}. -%%% -%%%----------------------------------------------------------------------------- - -{allow, {user, "dashboard"}, subscribe, ["$SYS/#"]}. - -{allow, {ipaddr, "127.0.0.1"}, pubsub, ["$SYS/#", "#"]}. - -{deny, all, subscribe, ["$SYS/#", {eq, "#"}]}. - -{allow, all}. - diff --git a/rel/files/clients.config b/rel/files/clients.config deleted file mode 100644 index 2c880c365..000000000 --- a/rel/files/clients.config +++ /dev/null @@ -1,3 +0,0 @@ -testclientid0 -testclientid1 127.0.0.1 -testclientid2 192.168.0.1/24 diff --git a/rel/files/emqttd b/rel/files/emqttd deleted file mode 100755 index b78e68e25..000000000 --- a/rel/files/emqttd +++ /dev/null @@ -1,323 +0,0 @@ -#!/bin/sh -# -*- tab-width:4;indent-tabs-mode:nil -*- -# ex: ts=4 sw=4 et - -# /bin/sh on Solaris is not a POSIX compatible shell, but /usr/bin/ksh is. -if [ `uname -s` = 'SunOS' -a "${POSIX_SHELL}" != "true" ]; then - POSIX_SHELL="true" - export POSIX_SHELL - # To support 'whoami' add /usr/ucb to path - PATH=/usr/ucb:$PATH - export PATH - exec /usr/bin/ksh $0 "$@" -fi -unset POSIX_SHELL # clear it so if we invoke other scripts, they run as ksh as well - -RUNNER_SCRIPT_DIR={{runner_script_dir}} -RUNNER_SCRIPT=${0##*/} - -RUNNER_BASE_DIR={{runner_base_dir}} -RUNNER_ETC_DIR={{runner_etc_dir}} -RUNNER_LIB_DIR={{platform_lib_dir}} -RUNNER_LOG_DIR={{runner_log_dir}} -RUNNER_DATA_DIR=$RUNNER_BASE_DIR/data -RUNNER_PLUGINS_DIR=$RUNNER_BASE_DIR/plugins - -# Note the trailing slash on $PIPE_DIR/ -PIPE_DIR={{pipe_dir}} -RUNNER_USER={{runner_user}} -PLATFORM_DATA_DIR={{platform_data_dir}} -SSL_DIST_CONFIG=$PLATFORM_DATA_DIR/ssl_distribution.args_file -RIAK_VERSION="git" - -WHOAMI=$(whoami) - -# Make sure this script is running as the appropriate user -if ([ "$RUNNER_USER" ] && [ "x$WHOAMI" != "x$RUNNER_USER" ]); then - type sudo > /dev/null 2>&1 - if [ $? -ne 0 ]; then - echo "sudo doesn't appear to be installed and your EUID isn't $RUNNER_USER" 1>&2 - exit 1 - fi - echo "Attempting to restart script through sudo -H -u $RUNNER_USER" >&2 - exec sudo -H -u $RUNNER_USER -i $RUNNER_SCRIPT_DIR/$RUNNER_SCRIPT $@ -fi - -# Warn the user if ulimit -n is less than 1024 -ULIMIT_F=`ulimit -n` -if [ "$ULIMIT_F" -lt 1024 ]; then - echo "!!!!" - echo "!!!! WARNING: ulimit -n is ${ULIMIT_F}; 1024 is the recommended minimum." - echo "!!!!" -fi - -# Make sure CWD is set to runner base dir -cd $RUNNER_BASE_DIR - -# Make sure log directory exists -mkdir -p $RUNNER_LOG_DIR - -# Make sure the data directory exists -mkdir -p $PLATFORM_DATA_DIR - -# Warn the user if they don't have write permissions on the log dir -if [ ! -w $RUNNER_LOG_DIR ]; then - echo "!!!!" - echo "!!!! WARNING: $RUNNER_LOG_DIR not writable; logs and crash dumps unavailable." - echo "!!!!" -fi - -# Extract the target node name from node.args -NAME_ARG=`egrep '^\-s?name' $RUNNER_ETC_DIR/vm.args` -if [ -z "$NAME_ARG" ]; then - echo "vm.args needs to have either -name or -sname parameter." - exit 1 -fi -NODE_NAME=${NAME_ARG##* } - -# Extract the target cookie -COOKIE_ARG=`grep '^\-setcookie' $RUNNER_ETC_DIR/vm.args` -if [ -z "$COOKIE_ARG" ]; then - echo "vm.args needs to have a -setcookie parameter." - exit 1 -fi - -# Identify the script name -SCRIPT=`basename $0` - -# Parse out release and erts info -START_ERL=`cat $RUNNER_BASE_DIR/releases/start_erl.data` -ERTS_VSN=${START_ERL% *} -APP_VSN=${START_ERL#* } - -# Add ERTS bin dir to our path -ERTS_PATH=$RUNNER_BASE_DIR/erts-$ERTS_VSN/bin - -# Setup command to control the node -NODETOOL="$ERTS_PATH/escript $ERTS_PATH/nodetool $NAME_ARG $COOKIE_ARG" -NODETOOL_LITE="$ERTS_PATH/escript $ERTS_PATH/nodetool" - -# Common functions - -# Ping node without allowing nodetool to take stdin -ping_node() { - $NODETOOL ping < /dev/null -} - -# Set the PID global variable, return 1 on error -get_pid() { - PID=`$NODETOOL getpid < /dev/null` - ES=$? - if [ "$ES" -ne 0 ]; then - echo "Node is not running!" - return 1 - fi - - # don't allow empty or init pid's - if [ -z $PID ] || [ "$PID" -le 1 ]; then - return 1 - fi - - return 0 -} - - -# Scrape out SSL distribution config info from vm.args into $SSL_DIST_CONFIG -rm -f $SSL_DIST_CONFIG -sed -n '/Begin SSL distribution items/,/End SSL distribution items/p' \ - $RUNNER_ETC_DIR/vm.args > $SSL_DIST_CONFIG - -# Check the first argument for instructions -case "$1" in - start) - # Make sure there is not already a node running - RES=`ping_node` - if [ "$RES" = "pong" ]; then - echo "Node is already running!" - exit 1 - fi - # Sanity check the emqttd.config file - RES=`$NODETOOL_LITE chkconfig $RUNNER_ETC_DIR/emqttd.config` - if [ $? != 0 ]; then - echo "Error reading $RUNNER_ETC_DIR/emqttd.config" - echo $RES - exit 1 - fi - HEART_COMMAND="$RUNNER_SCRIPT_DIR/$SCRIPT start" - export HEART_COMMAND - mkdir -p $PIPE_DIR - $ERTS_PATH/run_erl -daemon $PIPE_DIR $RUNNER_LOG_DIR \ - "exec $RUNNER_SCRIPT_DIR/$SCRIPT console" 2>&1 - - # Wait for the node to come up. We can't just ping it because - # distributed erlang comes up for a second before emqttd crashes - # (eg. in the case of an unwriteable disk). Once the node comes - # up we check for the node watcher process. If that's running - # then we assume things are good enough. This will at least let - # the user know when emqttd is crashing right after startup. - WAIT=${WAIT_FOR_ERLANG:-15} - while [ $WAIT -gt 0 ]; do - WAIT=`expr $WAIT - 1` - sleep 1 - RES=`ping_node` - if [ "$?" -ne 0 ]; then - continue - fi - echo "emqttd is started successfully!" - exit 0 - done - echo "emqttd failed to start within ${WAIT_FOR_ERLANG:-15} seconds," - echo "see the output of 'emqttd console' for more information." - echo "If you want to wait longer, set the environment variable" - echo "WAIT_FOR_ERLANG to the number of seconds to wait." - exit 1 - ;; - - stop) - UNAME_S=`uname -s` - case $UNAME_S in - Darwin) - # Make sure we explicitly set this because iTerm.app doesn't for - # some reason. - COMMAND_MODE=unix2003 - esac - - # Get the PID from nodetool - get_pid - GPR=$? - if [ "$GPR" -ne 0 ] || [ -z $PID ]; then - exit $GPR - fi - - # Tell nodetool to initiate a stop - $NODETOOL stop - ES=$? - if [ "$ES" -ne 0 ]; then - exit $ES - fi - - # Wait for the node to completely stop... - while `kill -s 0 $PID 2>/dev/null`; - do - sleep 1 - done - ;; - - restart) - ## Restart the VM without exiting the process - $NODETOOL restart - ES=$? - if [ "$ES" -ne 0 ]; then - exit $ES - fi - ;; - - reboot) - ## Restart the VM completely (uses heart to restart it) - $NODETOOL reboot - ES=$? - if [ "$ES" -ne 0 ]; then - exit $ES - fi - ;; - - ping) - ## See if the VM is alive - ping_node - ES=$? - if [ "$ES" -ne 0 ]; then - exit $ES - fi - ;; - - attach) - if [ "$2" = "-f" ]; then - echo "Forcing connection..." - else - # Make sure a node is running - RES=`ping_node` - ES=$? - if [ "$ES" -ne 0 ]; then - echo "Node is not running!" - exit $ES - fi - fi - - shift - exec $ERTS_PATH/to_erl $PIPE_DIR - ;; - - console) - RES=`ping_node` - if [ "$RES" = "pong" ]; then - echo "Node is already running - use '$SCRIPT attach' instead" - exit 1 - fi - # Sanity check the emqttd.config file - RES=`$NODETOOL_LITE chkconfig $RUNNER_ETC_DIR/emqttd.config` - if [ $? != 0 ]; then - echo "Error reading $RUNNER_ETC_DIR/emqttd.config" - echo $RES - exit 1 - fi - # Setup beam-required vars - ROOTDIR=$RUNNER_BASE_DIR - ERL_LIBS=$ROOTDIR/plugins - BINDIR=$ROOTDIR/erts-$ERTS_VSN/bin - EMU=beam - PROGNAME=`echo $0 | sed 's/.*\///'` - # Setup Mnesia Dir - MNESIA_DIR="$RUNNER_DATA_DIR/mnesia/$NODE_NAME" - CMD="$BINDIR/erlexec -boot $RUNNER_BASE_DIR/releases/$APP_VSN/$SCRIPT \ - -embedded -config $RUNNER_ETC_DIR/emqttd.config \ - -pa $RUNNER_LIB_DIR/basho-patches \ - -mnesia dir "\"${MNESIA_DIR}\"" \ - -args_file $RUNNER_ETC_DIR/vm.args -- ${1+"$@"}" - export EMU - export ROOTDIR - export ERL_LIBS - export BINDIR - export PROGNAME - - # Dump environment info for logging purposes - echo "Exec: $CMD" - echo "Root: $ROOTDIR" - - # Log the startup - logger -t "$SCRIPT[$$]" "Starting up" - - # Start the VM - exec $CMD - ;; - chkconfig) - RES=`$NODETOOL_LITE chkconfig $RUNNER_ETC_DIR/emqttd.config` - if [ $? != 0 ]; then - echo "Error reading $RUNNER_ETC_DIR/emqttd.config" - echo $RES - exit 1 - fi - echo "config is OK" - ;; - escript) - shift - $ERTS_PATH/escript "$@" - ;; - version) - echo $RIAK_VERSION - ;; - getpid) - # Get the PID from nodetool - get_pid - ES=$? - if [ "$ES" -ne 0 ] || [ -z $PID ]; then - exit $ES - fi - echo $PID - ;; - *) - echo "Usage: $SCRIPT {start|stop|restart|reboot|ping|console|attach|chkconfig|escript|version|getpid}" - exit 1 - ;; -esac - -exit 0 diff --git a/rel/files/emqttd.cmd b/rel/files/emqttd.cmd deleted file mode 100644 index effa49536..000000000 --- a/rel/files/emqttd.cmd +++ /dev/null @@ -1,108 +0,0 @@ -@echo off -@setlocal -@setlocal enabledelayedexpansion - -@set node_name=emqttd - -@rem Get the absolute path to the parent directory, -@rem which is assumed to be the node root. -@for /F "delims=" %%I in ("%~dp0..") do @set node_root=%%~fI - -@set releases_dir=%node_root%\releases -@set runner_etc_dir=%node_root%\etc - -@rem Parse ERTS version and release version from start_erl.data -@for /F "usebackq tokens=1,2" %%I in ("%releases_dir%\start_erl.data") do @( - @call :set_trim erts_version %%I - @call :set_trim release_version %%J -) - -@set vm_args=%runner_etc_dir%\vm.args -@set sys_config=%runner_etc_dir%\emqttd.config -@set node_boot_script=%releases_dir%\%release_version%\%node_name% -@set clean_boot_script=%releases_dir%\%release_version%\start_clean - -@rem extract erlang cookie from vm.args -@for /f "usebackq tokens=1-2" %%I in (`findstr /b \-setcookie "%vm_args%"`) do @set erlang_cookie=%%J - -@set erts_bin=%node_root%\erts-%erts_version%\bin - -@set service_name=%node_name%_%release_version% - -@set erlsrv="%erts_bin%\erlsrv.exe" -@set epmd="%erts_bin%\epmd.exe" -@set escript="%erts_bin%\escript.exe" -@set werl="%erts_bin%\werl.exe" - -@if "%1"=="usage" @goto usage -@if "%1"=="install" @goto install -@if "%1"=="uninstall" @goto uninstall -@if "%1"=="start" @goto start -@if "%1"=="stop" @goto stop -@if "%1"=="restart" @call :stop && @goto start -@if "%1"=="console" @goto console -@if "%1"=="query" @goto query -@if "%1"=="attach" @goto attach -@if "%1"=="upgrade" @goto upgrade -@echo Unknown command: "%1" - -:usage -@echo Usage: %~n0 [install^|uninstall^|start^|stop^|restart^|console^|query^|attach^|upgrade] -@goto :EOF - -:install -@set description=Erlang node %node_name% in %node_root% -@set start_erl=%node_root%\bin\start_erl.cmd -@set args= ++ %node_name% ++ %node_root% -@%erlsrv% add %service_name% -c "%description%" -sname %node_name% -w "%node_root%" -m "%start_erl%" -args "%args%" -stopaction "init:stop()." -@goto :EOF - -:uninstall -@%erlsrv% remove %service_name% -@%epmd% -kill -@goto :EOF - -:start -@%erlsrv% start %service_name% -@goto :EOF - -:stop -@%erlsrv% stop %service_name% -@goto :EOF - -:console -set dest_path=%~dp0 -cd /d !dest_path!..\plugins -set current_path=%cd% -set plugins= -for /d %%P in (*) do ( -set "plugins=!plugins!"!current_path!\%%P\ebin" " -) -cd /d %node_root% - -@start "%node_name% console" %werl% -boot "%node_boot_script%" -config "%sys_config%" -args_file "%vm_args%" -sname %node_name% -pa %plugins% -@goto :EOF - -:query -@%erlsrv% list %service_name% -@exit %ERRORLEVEL% -@goto :EOF - -:attach -@for /f "usebackq" %%I in (`hostname`) do @set hostname=%%I -start "%node_name% attach" %werl% -boot "%clean_boot_script%" -remsh %node_name%@%hostname% -sname console -setcookie %erlang_cookie% -@goto :EOF - -:upgrade -@if "%2"=="" ( - @echo Missing upgrade package argument - @echo Usage: %~n0 upgrade {package base name} - @echo NOTE {package base name} MUST NOT include the .tar.gz suffix - @goto :EOF -) -@%escript% %node_root%\bin\install_upgrade.escript %node_name% %erlang_cookie% %2 -@goto :EOF - -:set_trim -@set %1=%2 -@goto :EOF diff --git a/rel/files/emqttd.config.development b/rel/files/emqttd.config.development deleted file mode 100644 index 8cec54f04..000000000 --- a/rel/files/emqttd.config.development +++ /dev/null @@ -1,303 +0,0 @@ -% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*- -%% ex: ft=erlang ts=4 sw=4 et -[{kernel, [ - {start_timer, true}, - {start_pg2, true} - ]}, - {sasl, [ - {sasl_error_logger, {file, "log/emqttd_sasl.log"}} - ]}, - {ssl, [ - %{versions, ['tlsv1.2', 'tlsv1.1']} - ]}, - {lager, [ - {colored, true}, - {async_threshold, 1000}, - {error_logger_redirect, false}, - {crash_log, "log/emqttd_crash.log"}, - {handlers, [ - {lager_console_backend, info}, - %%NOTICE: Level >= error - %%{lager_emqtt_backend, error}, - {lager_file_backend, [ - {formatter_config, [time, " ", pid, " [",severity,"] ", message, "\n"]}, - {file, "log/emqttd_info.log"}, - {level, info}, - {size, 104857600}, - {date, "$D0"}, - {count, 30} - ]}, - {lager_file_backend, [ - {formatter_config, [time, " ", pid, " [",severity,"] ", message, "\n"]}, - {file, "log/emqttd_error.log"}, - {level, error}, - {size, 104857600}, - {date, "$D0"}, - {count, 30} - ]} - ]} - ]}, - {esockd, [ - {logger, {lager, info}} - ]}, - {emqttd, [ - %% Authentication and Authorization - {access, [ - %% Authetication. Anonymous Default - {auth, [ - %% Authentication with username, password - %{username, []}, - - %% Authentication with clientid - %{clientid, [{password, no}, {file, "etc/clients.config"}]}, - - %% Authentication with LDAP - % {ldap, [ - % {servers, ["localhost"]}, - % {port, 389}, - % {timeout, 30}, - % {user_dn, "uid=$u,ou=People,dc=example,dc=com"}, - % {ssl, fasle}, - % {sslopts, [ - % {certfile, "ssl.crt"}, - % {keyfile, "ssl.key"}]} - % ]}, - - %% Allow all - {anonymous, []} - ]}, - %% ACL config - {acl, [ - %% Internal ACL module - {internal, [{file, "etc/acl.config"}, {nomatch, allow}]} - ]} - ]}, - %% MQTT Protocol Options - {mqtt, [ - %% Packet - {packet, [ - %% Max ClientId Length Allowed - {max_clientid_len, 1024}, - %% Max Packet Size Allowed, 64K default - {max_packet_size, 65536} - ]}, - %% Client - {client, [ - %% Socket is connected, but no 'CONNECT' packet received - {idle_timeout, 10} %% seconds - ]}, - %% Session - {session, [ - %% Max number of QoS 1 and 2 messages that can be “in flight” at one time. - %% 0 means no limit - {max_inflight, 100}, - - %% Retry interval for redelivering QoS1/2 messages. - {unack_retry_interval, 20}, - - %% Awaiting PUBREL Timeout - {await_rel_timeout, 20}, - - %% Max Packets that Awaiting PUBREL, 0 means no limit - {max_awaiting_rel, 0}, - - %% Statistics Collection Interval(seconds) - {collect_interval, 20}, - - %% Expired after 2 day (unit: minute) - {expired_after, 2880} - - ]}, - %% Queue - {queue, [ - %% simple | priority - {type, simple}, - - %% Topic Priority: 0~255, Default is 0 - %% {priority, [{"topic/1", 10}, {"topic/2", 8}]}, - - %% Max queue length. Enqueued messages when persistent client disconnected, - %% or inflight window is full. - {max_length, infinity}, - - %% Low-water mark of queued messages - {low_watermark, 0.2}, - - %% High-water mark of queued messages - {high_watermark, 0.6}, - - %% Queue Qos0 messages? - {queue_qos0, true} - ]} - ]}, - %% Broker Options - {broker, [ - %% System interval of publishing broker $SYS messages - {sys_interval, 60}, - - %% Retained messages - {retained, [ - %% Expired after seconds, never expired if 0 - {expired_after, 0}, - - %% Max number of retained messages - {max_message_num, 100000}, - - %% Max Payload Size of retained message - {max_playload_size, 65536} - ]}, - - %% PubSub and Router - {pubsub, [ - %% Default should be scheduler numbers - {pool_size, 8}, - - %% Store Subscription: true | false - {subscription, true}, - - %% Route aging time(seconds) - {route_aging, 5} - ]}, - - %% Bridge - {bridge, [ - %%TODO: bridge queue size - {max_queue_len, 10000}, - - %% Ping Interval of bridge node - {ping_down_interval, 1} %seconds - ]} - ]}, - %% Modules - {modules, [ - %% Client presence management module. - %% Publish messages when client connected or disconnected - {presence, [{qos, 0}]}, - - %% Subscribe topics automatically when client connected - {subscription, [ - - %% $c will be replaced by clientid - %% {"$queue/clients/$c", 1}, - - %% Static subscriptions from backend - backend - ]} - - %% Rewrite rules - %% {rewrite, [{file, "etc/rewrite.config"}]} - ]}, - %% Plugins - {plugins, [ - %% Plugin App Library Dir - {plugins_dir, "./plugins"}, - - %% File to store loaded plugin names. - {loaded_file, "./data/loaded_plugins"} - ]}, - - %% Listeners - {listeners, [ - {mqtt, 1883, [ - %% Size of acceptor pool - {acceptors, 16}, - - %% Maximum number of concurrent clients - {max_clients, 512}, - - %% Socket Access Control - {access, [{allow, all}]}, - - %% Connection Options - {connopts, [ - %% Rate Limit. Format is 'burst, rate', Unit is KB/Sec - %% {rate_limit, "100,10"} %% 100K burst, 10K rate - ]}, - - %% Socket Options - {sockopts, [ - %Set buffer if hight thoughtput - %{recbuf, 4096}, - %{sndbuf, 4096}, - %{buffer, 4096}, - %{nodelay, true}, - {backlog, 512} - ]} - ]}, - - {mqtts, 8883, [ - %% Size of acceptor pool - {acceptors, 4}, - - %% Maximum number of concurrent clients - {max_clients, 512}, - - %% Socket Access Control - {access, [{allow, all}]}, - - %% SSL certificate and key files - {ssl, [{certfile, "etc/ssl/ssl.crt"}, - {keyfile, "etc/ssl/ssl.key"}]}, - - %% Socket Options - {sockopts, [ - {backlog, 1024} - %{buffer, 4096}, - ]} - ]}, - %% WebSocket over HTTPS Listener - %% {https, 8083, [ - %% %% Size of acceptor pool - %% {acceptors, 4}, - %% %% Maximum number of concurrent clients - %% {max_clients, 512}, - %% %% Socket Access Control - %% {access, [{allow, all}]}, - %% %% SSL certificate and key files - %% {ssl, [{certfile, "etc/ssl/ssl.crt"}, - %% {keyfile, "etc/ssl/ssl.key"}]}, - %% %% Socket Options - %% {sockopts, [ - %% %{buffer, 4096}, - %% {backlog, 1024} - %% ]} - %%]}, - - %% HTTP and WebSocket Listener - {http, 8083, [ - %% Size of acceptor pool - {acceptors, 4}, - %% Maximum number of concurrent clients - {max_clients, 64}, - %% Socket Access Control - {access, [{allow, all}]}, - %% Socket Options - {sockopts, [ - {backlog, 1024} - %{buffer, 4096}, - ]} - ]} - ]}, - - %% Erlang System Monitor - {sysmon, [ - %% Long GC - {long_gc, 100}, - - %% Long Schedule(ms) - {long_schedule, 100}, - - %% 8M words. 32MB on 32-bit VM, 64MB on 64-bit VM. - %% 8 * 1024 * 1024 - {large_heap, 8388608}, - - %% Busy Port - {busy_port, true}, - - %% Busy Dist Port - {busy_dist_port, true} - - ]} - ]} -]. - diff --git a/rel/files/emqttd.config.production b/rel/files/emqttd.config.production deleted file mode 100644 index d41c9f0e9..000000000 --- a/rel/files/emqttd.config.production +++ /dev/null @@ -1,296 +0,0 @@ -% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*- -%% ex: ft=erlang ts=4 sw=4 et -[{kernel, [ - {start_timer, true}, - {start_pg2, true} - ]}, - {sasl, [ - {sasl_error_logger, {file, "log/emqttd_sasl.log"}} - ]}, - {ssl, [ - %{versions, ['tlsv1.2', 'tlsv1.1']} - ]}, - {lager, [ - {colored, true}, - {async_threshold, 5000}, - {error_logger_redirect, false}, - {crash_log, "log/emqttd_crash.log"}, - {handlers, [ - {lager_console_backend, error}, - %%NOTICE: Level >= error - %%{lager_emqtt_backend, error}, - {lager_file_backend, [ - {formatter_config, [time, " ", pid, " [",severity,"] ", message, "\n"]}, - {file, "log/emqttd_error.log"}, - {level, error}, - {size, 104857600}, - {date, "$D0"}, - {count, 30} - ]} - ]} - ]}, - {esockd, [ - {logger, {lager, error}} - ]}, - {emqttd, [ - %% Authentication and Authorization - {access, [ - %% Authetication. Anonymous Default - {auth, [ - %% Authentication with username, password - %{username, []}, - - %% Authentication with clientid - %{clientid, [{password, no}, {file, "etc/clients.config"}]}, - - %% Authentication with LDAP - % {ldap, [ - % {servers, ["localhost"]}, - % {port, 389}, - % {timeout, 30}, - % {user_dn, "uid=$u,ou=People,dc=example,dc=com"}, - % {ssl, fasle}, - % {sslopts, [ - % {certfile, "ssl.crt"}, - % {keyfile, "ssl.key"}]} - % ]}, - - %% Allow all - {anonymous, []} - ]}, - %% ACL config - {acl, [ - %% Internal ACL module - {internal, [{file, "etc/acl.config"}, {nomatch, allow}]} - ]} - ]}, - %% MQTT Protocol Options - {mqtt, [ - %% Packet - {packet, [ - %% Max ClientId Length Allowed - {max_clientid_len, 512}, - %% Max Packet Size Allowed, 64K default - {max_packet_size, 65536} - ]}, - %% Client - {client, [ - %% Socket is connected, but no 'CONNECT' packet received - {idle_timeout, 30} %% seconds - ]}, - %% Session - {session, [ - %% Max number of QoS 1 and 2 messages that can be “in flight” at one time. - %% 0 means no limit - {max_inflight, 100}, - - %% Retry interval for redelivering QoS1/2 messages. - {unack_retry_interval, 60}, - - %% Awaiting PUBREL Timeout - {await_rel_timeout, 20}, - - %% Max Packets that Awaiting PUBREL, 0 means no limit - {max_awaiting_rel, 0}, - - %% Statistics Collection Interval(seconds) - {collect_interval, 0}, - - %% Expired after 2 day (unit: minute) - {expired_after, 2880} - - ]}, - %% Queue - {queue, [ - %% simple | priority - {type, simple}, - - %% Topic Priority: 0~255, Default is 0 - %% {priority, [{"topic/1", 10}, {"topic/2", 8}]}, - - %% Max queue length. Enqueued messages when persistent client disconnected, - %% or inflight window is full. - {max_length, infinity}, - - %% Low-water mark of queued messages - {low_watermark, 0.2}, - - %% High-water mark of queued messages - {high_watermark, 0.6}, - - %% Queue Qos0 messages? - {queue_qos0, true} - ]} - ]}, - %% Broker Options - {broker, [ - %% System interval of publishing broker $SYS messages - {sys_interval, 60}, - - %% Retained messages - {retained, [ - %% Expired after seconds, never expired if 0 - {expired_after, 0}, - - %% Max number of retained messages - {max_message_num, 100000}, - - %% Max Payload Size of retained message - {max_playload_size, 65536} - ]}, - - %% PubSub and Router - {pubsub, [ - %% Default should be scheduler numbers - {pool_size, 8}, - - %% Store Subscription: true | false - {subscription, true}, - - %% Route aging time(seconds) - {route_aging, 5} - ]}, - - %% Bridge - {bridge, [ - %%TODO: bridge queue size - {max_queue_len, 10000}, - - %% Ping Interval of bridge node - {ping_down_interval, 1} %seconds - ]} - ]}, - %% Modules - {modules, [ - %% Client presence management module. - %% Publish messages when client connected or disconnected - {presence, [{qos, 0}]}, - - %% Subscribe topics automatically when client connected - {subscription, [ - - %% $c will be replaced by clientid - %% {"$queue/clients/$c", 1}, - - %% Static subscriptions from backend - backend - ]} - - %% Rewrite rules - %% {rewrite, [{file, "etc/rewrite.config"}]} - ]}, - %% Plugins - {plugins, [ - %% Plugin App Library Dir - {plugins_dir, "./plugins"}, - - %% File to store loaded plugin names. - {loaded_file, "./data/loaded_plugins"} - ]}, - - %% Listeners - {listeners, [ - {mqtt, 1883, [ - %% Size of acceptor pool - {acceptors, 16}, - - %% Maximum number of concurrent clients - {max_clients, 512}, - - %% Socket Access Control - {access, [{allow, all}]}, - - %% Connection Options - {connopts, [ - %% Rate Limit. Format is 'burst, rate', Unit is KB/Sec - %% {rate_limit, "100,10"} %% 100K burst, 10K rate - ]}, - - %% Socket Options - {sockopts, [ - %Set buffer if hight thoughtput - %{recbuf, 4096}, - %{sndbuf, 4096}, - %{buffer, 4096}, - %{nodelay, true}, - {backlog, 1024} - ]} - ]}, - - {mqtts, 8883, [ - %% Size of acceptor pool - {acceptors, 4}, - - %% Maximum number of concurrent clients - {max_clients, 512}, - - %% Socket Access Control - {access, [{allow, all}]}, - - %% SSL certificate and key files - {ssl, [{certfile, "etc/ssl/ssl.crt"}, - {keyfile, "etc/ssl/ssl.key"}]}, - - %% Socket Options - {sockopts, [ - {backlog, 1024} - %{buffer, 4096}, - ]} - ]}, - %% WebSocket over HTTPS Listener - %% {https, 8083, [ - %% %% Size of acceptor pool - %% {acceptors, 4}, - %% %% Maximum number of concurrent clients - %% {max_clients, 512}, - %% %% Socket Access Control - %% {access, [{allow, all}]}, - %% %% SSL certificate and key files - %% {ssl, [{certfile, "etc/ssl/ssl.crt"}, - %% {keyfile, "etc/ssl/ssl.key"}]}, - %% %% Socket Options - %% {sockopts, [ - %% %{buffer, 4096}, - %% {backlog, 1024} - %% ]} - %%]}, - - %% HTTP and WebSocket Listener - {http, 8083, [ - %% Size of acceptor pool - {acceptors, 4}, - %% Maximum number of concurrent clients - {max_clients, 64}, - %% Socket Access Control - {access, [{allow, all}]}, - %% Socket Options - {sockopts, [ - {backlog, 1024} - %{buffer, 4096}, - ]} - ]} - ]}, - - %% Erlang System Monitor - {sysmon, [ - %% Long GC, don't monitor in production mode for: - %% https://github.com/erlang/otp/blob/feb45017da36be78d4c5784d758ede619fa7bfd3/erts/emulator/beam/erl_gc.c#L421 - {long_gc, false}, - - %% Long Schedule(ms) - {long_schedule, 240}, - - %% 8M words. 32MB on 32-bit VM, 64MB on 64-bit VM. - %% 8 * 1024 * 1024 - {large_heap, 8388608}, - - %% Busy Port - {busy_port, false}, - - %% Busy Dist Port - {busy_dist_port, true} - - ]} - ]} -]. - diff --git a/rel/files/emqttd.test.config b/rel/files/emqttd.test.config deleted file mode 100644 index 48ad73252..000000000 --- a/rel/files/emqttd.test.config +++ /dev/null @@ -1,300 +0,0 @@ -% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*- -%% ex: ft=erlang ts=4 sw=4 et -[{kernel, [ - {start_timer, true}, - {start_pg2, true} - ]}, - {sasl, [ - {sasl_error_logger, {file, "emqttd_sasl.log"}} - ]}, - {ssl, [ - %{versions, ['tlsv1.2', 'tlsv1.1']} - ]}, - {lager, [ - {colored, true}, - {async_threshold, 1000}, - {error_logger_redirect, false}, - {crash_log, "log/emqttd_crash.log"}, - {handlers, [ - {lager_console_backend, info}, - %%NOTICE: Level >= error - %%{lager_emqtt_backend, error}, - {lager_file_backend, [ - {formatter_config, [time, " ", pid, " [",severity,"] ", message, "\n"]}, - {file, "log/emqttd_info.log"}, - {level, info}, - {size, 104857600}, - {date, "$D0"}, - {count, 30} - ]}, - {lager_file_backend, [ - {formatter_config, [time, " ", pid, " [",severity,"] ", message, "\n"]}, - {file, "log/emqttd_error.log"}, - {level, error}, - {size, 104857600}, - {date, "$D0"}, - {count, 30} - ]} - ]} - ]}, - {esockd, [ - {logger, {lager, info}} - ]}, - {emqttd, [ - %% Authentication and Authorization - {access, [ - %% Authetication. Anonymous Default - {auth, [ - %% Authentication with username, password - %% {username, [{test, "password"}, {"test1", "password1"}]}, - - %% Authentication with clientid - %{clientid, [{password, no}, {file, "etc/clients.config"}]}, - - %% Authentication with LDAP - % {ldap, [ - % {servers, ["localhost"]}, - % {port, 389}, - % {timeout, 30}, - % {user_dn, "uid=$u,ou=People,dc=example,dc=com"}, - % {ssl, fasle}, - % {sslopts, [ - % {"certfile", "ssl.crt"}, - % {"keyfile", "ssl.key"}]} - % ]}, - - %% Allow all - {anonymous, []} - ]}, - %% ACL config - {acl, [ - %% Internal ACL module - %% {internal, [{file, "testdata/test_acl.config"}, {nomatch, allow}]} - ]} - ]}, - %% MQTT Protocol Options - {mqtt, [ - %% Packet - {packet, [ - %% Max ClientId Length Allowed - {max_clientid_len, 1024}, - %% Max Packet Size Allowed, 64K default - {max_packet_size, 65536} - ]}, - %% Client - {client, [ - %% Socket is connected, but no 'CONNECT' packet received - {idle_timeout, 10} %% seconds - ]}, - %% Session - {session, [ - %% Max number of QoS 1 and 2 messages that can be “in flight” at one time. - %% 0 means no limit - {max_inflight, 100}, - - %% Retry interval for redelivering QoS1/2 messages. - {unack_retry_interval, 20}, - - %% Awaiting PUBREL Timeout - {await_rel_timeout, 20}, - - %% Max Packets that Awaiting PUBREL, 0 means no limit - {max_awaiting_rel, 0}, - - %% Statistics Collection Interval(seconds) - {collect_interval, 20}, - - %% Expired after 2 day (unit: minute) - {expired_after, 2880} - - ]}, - %% Queue - {queue, [ - %% simple | priority - {type, simple}, - - %% Topic Priority: 0~255, Default is 0 - %% {priority, [{"topic/1", 10}, {"topic/2", 8}]}, - - %% Max queue length. Enqueued messages when persistent client disconnected, - %% or inflight window is full. - {max_length, infinity}, - - %% Low-water mark of queued messages - {low_watermark, 0.2}, - - %% High-water mark of queued messages - {high_watermark, 0.6}, - - %% Queue Qos0 messages? - {queue_qos0, true} - ]} - ]}, - %% Broker Options - {broker, [ - %% System interval of publishing broker $SYS messages - {sys_interval, 60}, - - %% Retained messages - {retained, [ - %% Expired after seconds, never expired if 0 - {expired_after, 0}, - - %% Max number of retained messages - {max_message_num, 100000}, - - %% Max Payload Size of retained message - {max_playload_size, 65536} - ]}, - - %% PubSub and Router - {pubsub, [ - %% Default should be scheduler numbers - {pool_size, 8}, - - %% Route aging time(seconds) - {route_aging, 5} - ]}, - - %% Bridge - {bridge, [ - %%TODO: bridge queue size - {max_queue_len, 10000}, - - %% Ping Interval of bridge node - {ping_down_interval, 1} %seconds - ]} - ]}, - %% Modules - {modules, [ - %% Client presence management module. - %% Publish messages when client connected or disconnected - {presence, [{qos, 0}]}, - - %% Subscribe topics automatically when client connected - {subscription, [ - - %% $c will be replaced by clientid - %% {"$queue/clients/$c", 1}, - - %% Static subscriptions from backend - backend - ]} - - %% Rewrite rules - %% {rewrite, [{file, "etc/rewrite.config"}]} - ]}, - %% Plugins - {plugins, [ - %% Plugin App Library Dir - {plugins_dir, "./plugins"}, - - %% File to store loaded plugin names. - {loaded_file, "./data/loaded_plugins"} - ]}, - - %% Listeners - {listeners, [ - {mqtt, 1883, [ - %% Size of acceptor pool - {acceptors, 16}, - - %% Maximum number of concurrent clients - {max_clients, 512}, - - %% Socket Access Control - {access, [{allow, all}]}, - - %% Connection Options - {connopts, [ - %% Rate Limit. Format is 'burst, rate', Unit is KB/Sec - %% {rate_limit, "100,10"} %% 100K burst, 10K rate - ]}, - - %% Socket Options - {sockopts, [ - %Set buffer if hight thoughtput - %{recbuf, 4096}, - %{sndbuf, 4096}, - %{buffer, 4096}, - %{nodelay, true}, - {backlog, 512} - ]} - ]}, - - {mqtts, 8883, [ - %% Size of acceptor pool - {acceptors, 4}, - - %% Maximum number of concurrent clients - {max_clients, 512}, - - %% Socket Access Control - {access, [{allow, all}]}, - - %% SSL certificate and key files - {ssl, [{certfile, "etc/ssl/ssl.crt"}, - {keyfile, "etc/ssl/ssl.key"}]}, - - %% Socket Options - {sockopts, [ - {backlog, 1024} - %{buffer, 4096}, - ]} - ]}, - %% WebSocket over HTTPS Listener - %% {https, 8083, [ - %% %% Size of acceptor pool - %% {acceptors, 4}, - %% %% Maximum number of concurrent clients - %% {max_clients, 512}, - %% %% Socket Access Control - %% {access, [{allow, all}]}, - %% %% SSL certificate and key files - %% {ssl, [{certfile, "etc/ssl/ssl.crt"}, - %% {keyfile, "etc/ssl/ssl.key"}]}, - %% %% Socket Options - %% {sockopts, [ - %% %{buffer, 4096}, - %% {backlog, 1024} - %% ]} - %%]}, - - %% HTTP and WebSocket Listener - {http, 8083, [ - %% Size of acceptor pool - {acceptors, 4}, - %% Maximum number of concurrent clients - {max_clients, 64}, - %% Socket Access Control - {access, [{allow, all}]}, - %% Socket Options - {sockopts, [ - {backlog, 1024} - %{buffer, 4096}, - ]} - ]} - ]}, - - %% Erlang System Monitor - {sysmon, [ - %% Long GC - {long_gc, 100}, - - %% Long Schedule(ms) - {long_schedule, 100}, - - %% 8M words. 32MB on 32-bit VM, 64MB on 64-bit VM. - %% 8 * 1024 * 1024 - {large_heap, 8388608}, - - %% Busy Port - {busy_port, true}, - - %% Busy Dist Port - {busy_dist_port, true} - - ]} - ]} -]. - diff --git a/rel/files/emqttd_ctl b/rel/files/emqttd_ctl deleted file mode 100755 index 4292b893e..000000000 --- a/rel/files/emqttd_ctl +++ /dev/null @@ -1,91 +0,0 @@ -#!/bin/sh -# -*- tab-width:4;indent-tabs-mode:nil -*- -# ex: ts=4 sw=4 et - -# /bin/sh on Solaris is not a POSIX compatible shell, but /usr/bin/ksh is. -if [ `uname -s` = 'SunOS' -a "${POSIX_SHELL}" != "true" ]; then - POSIX_SHELL="true" - export POSIX_SHELL - # To support 'whoami' add /usr/ucb to path - PATH=/usr/ucb:$PATH - export PATH - exec /usr/bin/ksh $0 "$@" -fi -unset POSIX_SHELL # clear it so if we invoke other scripts, they run as ksh as well - -RUNNER_SCRIPT_DIR={{runner_script_dir}} -RUNNER_SCRIPT=${0##*/} - -RUNNER_BASE_DIR={{runner_base_dir}} -RUNNER_ETC_DIR={{runner_etc_dir}} -RUNNER_LIB_DIR={{platform_lib_dir}} -RUNNER_LOG_DIR={{runner_log_dir}} -RUNNER_USER={{runner_user}} - -WHOAMI=$(whoami) - -# Make sure this script is running as the appropriate user -if ([ "$RUNNER_USER" ] && [ "x$WHOAMI" != "x$RUNNER_USER" ]); then - type sudo > /dev/null 2>&1 - if [ $? -ne 0 ]; then - echo "sudo doesn't appear to be installed and your EUID isn't $RUNNER_USER" 1>&2 - exit 1 - fi - echo "Attempting to restart script through sudo -H -u $RUNNER_USER" >&2 - exec sudo -H -u $RUNNER_USER -i $RUNNER_SCRIPT_DIR/$RUNNER_SCRIPT $@ -fi - -# Make sure CWD is set to runner base dir -cd $RUNNER_BASE_DIR - -# Extract the target node name from node.args -NAME_ARG=`egrep "^ *-s?name" $RUNNER_ETC_DIR/vm.args` -if [ -z "$NAME_ARG" ]; then - echo "vm.args needs to have either -name or -sname parameter." - exit 1 -fi - -# Learn how to specify node name for connection from remote nodes -echo "$NAME_ARG" | grep '^-sname' > /dev/null 2>&1 -if [ "X$?" = "X0" ]; then - NAME_PARAM="-sname" - NAME_HOST="" -else - NAME_PARAM="-name" - echo "$NAME_ARG" | grep '@.*' > /dev/null 2>&1 - if [ "X$?" = "X0" ]; then - NAME_HOST=`echo "${NAME_ARG}" | sed -e 's/.*\(@.*\)$/\1/'` - else - NAME_HOST="" - fi -fi - -# Extract the target cookie -COOKIE_ARG=`grep '\-setcookie' $RUNNER_ETC_DIR/vm.args` -if [ -z "$COOKIE_ARG" ]; then - echo "vm.args needs to have a -setcookie parameter." - exit 1 -fi - -# Identify the script name -SCRIPT=`basename $0` - -# Parse out release and erts info -START_ERL=`cat $RUNNER_BASE_DIR/releases/start_erl.data` -ERTS_VSN=${START_ERL% *} -APP_VSN=${START_ERL#* } - -# Add ERTS bin dir to our path -ERTS_PATH=$RUNNER_BASE_DIR/erts-$ERTS_VSN/bin - -# Setup command to control the node -NODETOOL="$ERTS_PATH/escript $ERTS_PATH/nodetool $NAME_ARG $COOKIE_ARG" - -RES=`$NODETOOL ping` -if [ "$RES" != "pong" ]; then - echo "Node is not running!" - exit 1 -fi - -$NODETOOL rpc emqttd_ctl run $@ - diff --git a/rel/files/emqttd_top b/rel/files/emqttd_top deleted file mode 100755 index 24533c436..000000000 --- a/rel/files/emqttd_top +++ /dev/null @@ -1,117 +0,0 @@ -#!/bin/sh -# -*- tab-width:4;indent-tabs-mode:nil -*- -# ex: ts=4 sw=4 et - -# /bin/sh on Solaris is not a POSIX compatible shell, but /usr/bin/ksh is. -if [ `uname -s` = 'SunOS' -a "${POSIX_SHELL}" != "true" ]; then - POSIX_SHELL="true" - export POSIX_SHELL - # To support 'whoami' add /usr/ucb to path - PATH=/usr/ucb:$PATH - export PATH - exec /usr/bin/ksh $0 "$@" -fi -unset POSIX_SHELL # clear it so if we invoke other scripts, they run as ksh as well - -RUNNER_SCRIPT_DIR={{runner_script_dir}} -RUNNER_SCRIPT=${0##*/} - -RUNNER_BASE_DIR={{runner_base_dir}} -RUNNER_ETC_DIR={{runner_etc_dir}} -RUNNER_LIB_DIR={{platform_lib_dir}} -RUNNER_USER={{runner_user}} - -WHOAMI=$(whoami) - -# Make sure this script is running as the appropriate user -if ([ "$RUNNER_USER" ] && [ "x$WHOAMI" != "x$RUNNER_USER" ]); then - type sudo > /dev/null 2>&1 - if [ $? -ne 0 ]; then - echo "sudo doesn't appear to be installed and your EUID isn't $RUNNER_USER" 1>&2 - exit 1 - fi - echo "Attempting to restart script through sudo -H -u $RUNNER_USER" >&2 - exec sudo -H -u $RUNNER_USER -i $RUNNER_SCRIPT_DIR/$RUNNER_SCRIPT $@ -fi - -# Make sure CWD is set to runner base dir -cd $RUNNER_BASE_DIR - -# Extract the target node name from node.args -NAME_ARG=`egrep "^ *-s?name" $RUNNER_ETC_DIR/vm.args` -if [ -z "$NAME_ARG" ]; then - echo "vm.args needs to have either -name or -sname parameter." - exit 1 -fi - -# Learn how to specify node name for connection from remote nodes -echo "$NAME_ARG" | grep '^-sname' > /dev/null 2>&1 -if [ "X$?" = "X0" ]; then - NAME_PARAM="-sname" - NAME_HOST="" -else - NAME_PARAM="-name" - echo "$NAME_ARG" | grep '@.*' > /dev/null 2>&1 - if [ "X$?" = "X0" ]; then - NAME_HOST=`echo "${NAME_ARG}" | sed -e 's/.*\(@.*\)$/\1/'` - else - NAME_HOST="" - fi -fi - -# Extract the target cookie -COOKIE_ARG=`grep '\-setcookie' $RUNNER_ETC_DIR/vm.args` -if [ -z "$COOKIE_ARG" ]; then - echo "vm.args needs to have a -setcookie parameter." - exit 1 -fi - -# Identify the script name -SCRIPT=`basename $0` - -# Parse out release and erts info -START_ERL=`cat $RUNNER_BASE_DIR/releases/start_erl.data` -ERTS_VSN=${START_ERL% *} -APP_VSN=${START_ERL#* } - -# Add ERTS bin dir to our path -ERTS_PATH=$RUNNER_BASE_DIR/erts-$ERTS_VSN/bin - -NODE_NAME=${NAME_ARG#* } - -# Setup command to control the node -NODETOOL="$ERTS_PATH/escript $ERTS_PATH/nodetool $NAME_ARG $COOKIE_ARG" - -RES=`$NODETOOL ping` -if [ "$RES" != "pong" ]; then - echo "Node is not running!" - exit 1 -fi - -case "$1" in - runtime) - SORTBY="runtime" - ;; - reductions) - SORTBY="reductions" - ;; - memory) - SORTBY="memory" - ;; - msg_q) - SORTBY="msg_q" - ;; - *) - echo "Usage: $SCRIPT {runtime | reductions | memory | msg_q}" - exit 1 - ;; -esac - -MYPID=$$ -ETOP_ARGS="-sort $SORTBY -interval 10 -lines 50 -tracing off" -$ERTS_PATH/erl -noshell -noinput \ - -pa $RUNNER_LIB_DIR/basho-patches \ - -hidden $NAME_PARAM emqttd_top$MYPID$NAME_HOST $COOKIE_ARG \ - -s etop -s erlang halt -output text \ - -node $NODE_NAME $ETOP_ARGS - diff --git a/rel/files/erl b/rel/files/erl deleted file mode 100755 index 6f65e3fc9..000000000 --- a/rel/files/erl +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/sh - -## This script replaces the default "erl" in erts-VSN/bin. This is necessary -## as escript depends on erl and in turn, erl depends on having access to a -## bootscript (start.boot). Note that this script is ONLY invoked as a side-effect -## of running escript -- the embedded node bypasses erl and uses erlexec directly -## (as it should). -## -## Note that this script makes the assumption that there is a start_clean.boot -## file available in $ROOTDIR/release/VSN. - -# Determine the abspath of where this script is executing from. -ERTS_BIN_DIR=$(cd ${0%/*} && pwd) - -# Now determine the root directory -- this script runs from erts-VSN/bin, -# so we simply need to strip off two dirs from the end of the ERTS_BIN_DIR -# path. -ROOTDIR=${ERTS_BIN_DIR%/*/*} - -# Parse out release and erts info -START_ERL=`cat $ROOTDIR/releases/start_erl.data` -ERTS_VSN=${START_ERL% *} -APP_VSN=${START_ERL#* } - -BINDIR=$ROOTDIR/erts-$ERTS_VSN/bin -EMU=beam -PROGNAME=`echo $0 | sed 's/.*\\///'` -CMD="$BINDIR/erlexec" -export EMU -export ROOTDIR -export BINDIR -export PROGNAME - -exec $CMD -boot $ROOTDIR/releases/$APP_VSN/start_clean ${1+"$@"} diff --git a/rel/files/install_upgrade.escript b/rel/files/install_upgrade.escript deleted file mode 100644 index 56cea1963..000000000 --- a/rel/files/install_upgrade.escript +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env escript -%%! -noshell -noinput -%% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*- -%% ex: ft=erlang ts=4 sw=4 et - --define(TIMEOUT, 60000). --define(INFO(Fmt,Args), io:format(Fmt,Args)). - -main([NodeName, Cookie, ReleasePackage]) -> - TargetNode = start_distribution(NodeName, Cookie), - {ok, Vsn} = rpc:call(TargetNode, release_handler, unpack_release, - [ReleasePackage], ?TIMEOUT), - ?INFO("Unpacked Release ~p~n", [Vsn]), - {ok, OtherVsn, Desc} = rpc:call(TargetNode, release_handler, - check_install_release, [Vsn], ?TIMEOUT), - {ok, OtherVsn, Desc} = rpc:call(TargetNode, release_handler, - install_release, [Vsn], ?TIMEOUT), - ?INFO("Installed Release ~p~n", [Vsn]), - ok = rpc:call(TargetNode, release_handler, make_permanent, [Vsn], ?TIMEOUT), - ?INFO("Made Release ~p Permanent~n", [Vsn]); -main(_) -> - init:stop(1). - -start_distribution(NodeName, Cookie) -> - MyNode = make_script_node(NodeName), - {ok, _Pid} = net_kernel:start([MyNode, shortnames]), - erlang:set_cookie(node(), list_to_atom(Cookie)), - TargetNode = make_target_node(NodeName), - case {net_kernel:hidden_connect_node(TargetNode), - net_adm:ping(TargetNode)} of - {true, pong} -> - ok; - {_, pang} -> - io:format("Node ~p not responding to pings.\n", [TargetNode]), - init:stop(1) - end, - TargetNode. - -make_target_node(Node) -> - [_, Host] = string:tokens(atom_to_list(node()), "@"), - list_to_atom(lists:concat([Node, "@", Host])). - -make_script_node(Node) -> - list_to_atom(lists:concat([Node, "_upgrader_", os:getpid()])). diff --git a/rel/files/loaded_plugins b/rel/files/loaded_plugins deleted file mode 100644 index 68ba6a41d..000000000 --- a/rel/files/loaded_plugins +++ /dev/null @@ -1 +0,0 @@ -emqttd_dashboard. diff --git a/rel/files/nodetool b/rel/files/nodetool deleted file mode 100755 index 09c2b86ef..000000000 --- a/rel/files/nodetool +++ /dev/null @@ -1,214 +0,0 @@ -#!/usr/bin/env escript - -%% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*- -%% ex: ft=erlang ts=4 sw=4 et -%% ------------------------------------------------------------------- -%% -%% nodetool: Helper Script for interacting with live nodes -%% -%% ------------------------------------------------------------------- --mode(compile). - -main(Args) -> - ok = start_epmd(), - %% Extract the args - {RestArgs, TargetNode} = process_args(Args, [], undefined), - - %% process_args() has side-effects (e.g. when processing "-name"), - %% so take care of app-starting business first. - [application:start(App) || App <- [crypto, public_key, ssl]], - - %% any commands that don't need a running node - case RestArgs of - ["chkconfig", File] -> - case file:consult(File) of - {ok, Terms} -> - case validate(Terms) of - ok -> - io:format("ok\n"), - halt(0); - {error, Problems} -> - lists:foreach(fun print_issue/1, Problems), - %% halt(1) if any problems were errors - halt(case [x || {error, _} <- Problems] of - [] -> 0; - _ -> 1 - end) - end; - {error, {Line, Mod, Term}} -> - io:format(standard_error, ["Error on line ", file:format_error({Line, Mod, Term}), "\n"], []), - halt(1); - {error, R} -> - io:format(standard_error, ["Error reading config file: ", file:format_error(R), "\n"], []), - halt(1) - end; - _ -> - ok - end, - - %% See if the node is currently running -- if it's not, we'll bail - case {net_kernel:hidden_connect_node(TargetNode), net_adm:ping(TargetNode)} of - {true, pong} -> - ok; - {false,pong} -> - io:format("Failed to connect to node ~p .\n", [TargetNode]), - halt(1); - {_, pang} -> - io:format("Node ~p not responding to pings.\n", [TargetNode]), - halt(1) - end, - - case RestArgs of - ["getpid"] -> - io:format("~p\n", [list_to_integer(rpc:call(TargetNode, os, getpid, []))]); - ["ping"] -> - %% If we got this far, the node already responsed to a ping, so just dump - %% a "pong" - io:format("pong\n"); - ["stop"] -> - io:format("~p\n", [rpc:call(TargetNode, init, stop, [], 60000)]); - ["restart"] -> - io:format("~p\n", [rpc:call(TargetNode, init, restart, [], 60000)]); - ["reboot"] -> - io:format("~p\n", [rpc:call(TargetNode, init, reboot, [], 60000)]); - ["rpc", Module, Function | RpcArgs] -> - case rpc:call(TargetNode, list_to_atom(Module), list_to_atom(Function), - [RpcArgs], 60000) of - ok -> - ok; - {badrpc, Reason} -> - io:format("RPC to ~p failed: ~p\n", [TargetNode, Reason]), - halt(1); - _ -> - halt(1) - end; - ["rpc_infinity", Module, Function | RpcArgs] -> - case rpc:call(TargetNode, list_to_atom(Module), list_to_atom(Function), [RpcArgs], infinity) of - ok -> - ok; - {badrpc, Reason} -> - io:format("RPC to ~p failed: ~p\n", [TargetNode, Reason]), - halt(1); - _ -> - halt(1) - end; - ["rpcterms", Module, Function, ArgsAsString] -> - case rpc:call(TargetNode, list_to_atom(Module), list_to_atom(Function), - consult(ArgsAsString), 60000) of - {badrpc, Reason} -> - io:format("RPC to ~p failed: ~p\n", [TargetNode, Reason]), - halt(1); - Other -> - io:format("~p\n", [Other]) - end; - Other -> - io:format("Other: ~p\n", [Other]), - io:format("Usage: nodetool {chkconfig|getpid|ping|stop|restart|reboot|rpc|rpc_infinity|rpcterms}\n") - end, - net_kernel:stop(). - - -process_args([], Acc, TargetNode) -> - {lists:reverse(Acc), TargetNode}; -process_args(["-setcookie", Cookie | Rest], Acc, TargetNode) -> - erlang:set_cookie(node(), list_to_atom(Cookie)), - process_args(Rest, Acc, TargetNode); -process_args(["-name", TargetName | Rest], Acc, _) -> - ThisNode = append_node_suffix(TargetName, "_maint_"), - {ok, _} = net_kernel:start([ThisNode, longnames]), - process_args(Rest, Acc, nodename(TargetName)); -process_args(["-sname", TargetName | Rest], Acc, _) -> - ThisNode = append_node_suffix(TargetName, "_maint_"), - {ok, _} = net_kernel:start([ThisNode, shortnames]), - process_args(Rest, Acc, nodename(TargetName)); -process_args([Arg | Rest], Acc, Opts) -> - process_args(Rest, [Arg | Acc], Opts). - - -start_epmd() -> - [] = os:cmd(epmd_path() ++ " -daemon"), - ok. - -epmd_path() -> - ErtsBinDir = filename:dirname(escript:script_name()), - Name = "epmd", - case os:find_executable(Name, ErtsBinDir) of - false -> - case os:find_executable(Name) of - false -> - io:format("Could not find epmd.~n"), - halt(1); - GlobalEpmd -> - GlobalEpmd - end; - Epmd -> - Epmd - end. - - -nodename(Name) -> - case string:tokens(Name, "@") of - [_Node, _Host] -> - list_to_atom(Name); - [Node] -> - [_, Host] = string:tokens(atom_to_list(node()), "@"), - list_to_atom(lists:concat([Node, "@", Host])) - end. - -append_node_suffix(Name, Suffix) -> - case string:tokens(Name, "@") of - [Node, Host] -> - list_to_atom(lists:concat([Node, Suffix, os:getpid(), "@", Host])); - [Node] -> - list_to_atom(lists:concat([Node, Suffix, os:getpid()])) - end. - - -%% -%% Given a string or binary, parse it into a list of terms, ala file:consult/0 -%% -consult(Str) when is_list(Str) -> - consult([], Str, []); -consult(Bin) when is_binary(Bin)-> - consult([], binary_to_list(Bin), []). - -consult(Cont, Str, Acc) -> - case erl_scan:tokens(Cont, Str, 0) of - {done, Result, Remaining} -> - case Result of - {ok, Tokens, _} -> - {ok, Term} = erl_parse:parse_term(Tokens), - consult([], Remaining, [Term | Acc]); - {eof, _Other} -> - lists:reverse(Acc); - {error, Info, _} -> - {error, Info} - end; - {more, Cont1} -> - consult(Cont1, eof, Acc) - end. - - -%% -%% Validation functions for checking the emqttd.config -%% -validate([Terms]) -> - Results = [ValidateFun(Terms) || ValidateFun <- get_validation_funs()], - Failures = [Res || Res <- Results, Res /= true], - case Failures of - [] -> - ok; - _ -> - {error, Failures} - end. - -%% Some initial and basic checks for the app.config file -get_validation_funs() -> - [ ]. - -print_issue({warning, Warning}) -> - io:format(standard_error, "Warning in emqttd.config: ~s~n", [Warning]); -print_issue({error, Error}) -> - io:format(standard_error, "Error in emqttd.config: ~s~n", [Error]). - - diff --git a/rel/files/rewrite.config b/rel/files/rewrite.config deleted file mode 100644 index 494a85f74..000000000 --- a/rel/files/rewrite.config +++ /dev/null @@ -1,14 +0,0 @@ -%%%----------------------------------------------------------------------------- -%% -%% [Rewrite](https://github.com/emqtt/emqttd/wiki/Rewrite) -%% -%%%----------------------------------------------------------------------------- - -%{topic, "x/#", [ -% {rewrite, "^x/y/(.+)$", "z/y/$1"}, -% {rewrite, "^x/(.+)$", "y/$1"} -%]}. - -%{topic, "y/+/z/#", [ -% {rewrite, "^y/(.+)/z/(.+)$", "y/z/$2"} -%]}. diff --git a/rel/files/ssl/ssl.crt b/rel/files/ssl/ssl.crt deleted file mode 100644 index 001844674..000000000 --- a/rel/files/ssl/ssl.crt +++ /dev/null @@ -1,17 +0,0 @@ ------BEGIN CERTIFICATE----- -MIICuTCCAiICCQC8+3PPaqATfDANBgkqhkiG9w0BAQUFADCBoDELMAkGA1UEBhMC -Q0gxETAPBgNVBAgTCFpoZUppYW5nMREwDwYDVQQHEwhIYW5nWmhvdTEUMBIGA1UE -ChMLWGlhb0xpIFRlY2gxHzAdBgNVBAsTFkluZm9ybWF0aW9uIFRlY2hub2xvZ3kx -EzARBgNVBAMTCnQuZW1xdHQuaW8xHzAdBgkqhkiG9w0BCQEWEGZlbmcgYXQgZW1x -dHQuaW8wHhcNMTUwMjI1MTc0NjQwWhcNMTYwMjI1MTc0NjQwWjCBoDELMAkGA1UE -BhMCQ0gxETAPBgNVBAgTCFpoZUppYW5nMREwDwYDVQQHEwhIYW5nWmhvdTEUMBIG -A1UEChMLWGlhb0xpIFRlY2gxHzAdBgNVBAsTFkluZm9ybWF0aW9uIFRlY2hub2xv -Z3kxEzARBgNVBAMTCnQuZW1xdHQuaW8xHzAdBgkqhkiG9w0BCQEWEGZlbmcgYXQg -ZW1xdHQuaW8wgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBALAtN2OHsvltOk+9 -AtlwMtKuaWW2WpV/S0lRRG9x9k8pyd5PJeeYAr2jVsoWnZInb1CoEOHFcwxZLjv3 -gEvz+X+//W02YyI9hnvCJUpT/+6P0gJEbmTmqL078M6vbtwtiF1YC7mdo0nGAZuK -qedpIoEZbVJavf4S0vXWTsb3s5unAgMBAAEwDQYJKoZIhvcNAQEFBQADgYEAgUR3 -z4uDUsAl+xUorPMBIOS/ncHHVk1XucVv9Wi4chzzZ+4/Y77/fFqP6oxhQ59C9Q8i -iT5wjaE4R1eCge18lPSw3yb1tsTe5B3WkRTzziPq/Q/AsC+DifkkE1YW67leuJV/ -vz74sEi0dudmOVoe6peYxjEH8xXoIUqhnwXt/4Q= ------END CERTIFICATE----- diff --git a/rel/files/ssl/ssl.key b/rel/files/ssl/ssl.key deleted file mode 100644 index 5d5786fac..000000000 --- a/rel/files/ssl/ssl.key +++ /dev/null @@ -1,15 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -MIICXAIBAAKBgQCwLTdjh7L5bTpPvQLZcDLSrmlltlqVf0tJUURvcfZPKcneTyXn -mAK9o1bKFp2SJ29QqBDhxXMMWS4794BL8/l/v/1tNmMiPYZ7wiVKU//uj9ICRG5k -5qi9O/DOr27cLYhdWAu5naNJxgGbiqnnaSKBGW1SWr3+EtL11k7G97ObpwIDAQAB -AoGBAKU1cbiLG0GdtU3rME3ZUj+RQNMZ4u5IVcBmTie4FcN8q4ombKQ2P3O4RX3z -IUZaZp+bS2F8uHt+8cVYPl57Zp5fwbIlv6jWgGpvXLsX8JBQl2OTw38B+hVwJvAM -h0mBzprUOs3KGZyF5cyA4osrZ4QvCZhwId9fAjwLGBF9i1yBAkEA4jWAF1sWQiwF -vY476m+0ihpRwGKjldKHWFZmvoB/AnNV/rXO+HRl3MB5wmO+Dqg3gJZrjGBgDeaV -g9hoQjK6ZwJBAMdg57iKLd8uUb7c4pR8fDdDbeeI5X7WDf2k9emT3BMPJPQ3EiSf -CStn1hRfp31U9CXEnw94rKHhrdMFrYjdzMECQCcWD3f5qTLt4GAMf5XWj199hLq1 -UIbGxdQhuccY9Nk7jJRiXczYb/Fg4KkSCvkFX/G8DAFJdc9xFEyfzAQEN+kCQH3a -nMrvZn9gBLffRKOIZPyZctHZp0xGIHTA4X39GMlrIN+Lt8coIKimlgssSlSiAK+q -iuFAQnC5PXlcNyuTHsECQAMNMY6jXikgSUZfVXitAFX3g9+IbjT9eJ92f60QneW8 -mxWQoqP3fqCSbTEysb7NojEEwppSZtaNgnBb5R4E+mU= ------END RSA PRIVATE KEY----- diff --git a/rel/files/start_erl.cmd b/rel/files/start_erl.cmd deleted file mode 100644 index 8285c0534..000000000 --- a/rel/files/start_erl.cmd +++ /dev/null @@ -1,50 +0,0 @@ -@setlocal -@echo off -@setlocal enabledelayedexpansion - -@rem Parse arguments. erlsrv.exe prepends erl arguments prior to first ++. -@rem Other args are position dependent. -@set args="%*" -@for /F "delims=++ tokens=1,2,3" %%I in (%args%) do @( - @set erl_args=%%I - @call :set_trim node_name %%J - @rem Trim spaces from the left of %%K (node_root), which may have spaces inside - @for /f "tokens=* delims= " %%a in ("%%K") do @set node_root=%%a -) - -@set releases_dir=%node_root%\releases - -@rem parse ERTS version and release version from start_erl.dat -@for /F "usebackq tokens=1,2" %%I in ("%releases_dir%\start_erl.data") do @( - @call :set_trim erts_version %%I - @call :set_trim release_version %%J -) - -@set erl_exe="%node_root%\erts-%erts_version%\bin\erl.exe" -@set boot_file="%releases_dir%\%release_version%\%node_name%" - -@if exist "%releases_dir%\%release_version%\sys.config" ( - @set app_config="%releases_dir%\%release_version%\sys.config" -) else ( - @set app_config="%node_root%\etc\emqttd.config" -) - -@if exist "%releases_dir%\%release_version%\vm.args" ( - @set vm_args="%releases_dir%\%release_version%\vm.args" -) else ( - @set vm_args="%node_root%\etc\vm.args" -) - -set dest_path=%~dp0 -cd /d !dest_path!..\plugins -set current_path=%cd% -set plugins= -for /d %%P in (*) do ( -set "plugins=!plugins!"!current_path!\%%P\ebin" " -) -cd /d %node_root% -@%erl_exe% %erl_args% -boot %boot_file% -config %app_config% -args_file %vm_args% -pa %plugins% - -:set_trim -@set %1=%2 -@goto :EOF diff --git a/rel/files/vm.args b/rel/files/vm.args deleted file mode 100644 index a2ad9302c..000000000 --- a/rel/files/vm.args +++ /dev/null @@ -1,61 +0,0 @@ -##------------------------------------------------------------------------- -## Name of the emqttd node: Name@Host -## -## NOTICE: The Host should be IP address or the fully qualified host name. -## The short hostname cannot work! -##------------------------------------------------------------------------- - --name emqttd@127.0.0.1 -# or -#-name emqttd@localhost. - -## Cookie for distributed erlang --setcookie emqttdsecretcookie - -##------------------------------------------------------------------------- -## Flags -##------------------------------------------------------------------------- - -## Heartbeat management; auto-restarts VM if it dies or becomes unresponsive -## (Disabled by default..use with caution!) -##-heart --smp true - -## Enable kernel poll and a few async threads -+K true - -## 12 threads/core. -+A 48 - -## max process numbers -+P 8192 - -## Sets the maximum number of simultaneously existing ports for this system -+Q 8192 - -## max atom number -## +t - -## Set the distribution buffer busy limit (dist_buf_busy_limit) in kilobytes. -## Valid range is 1-2097151. Default is 1024. -## +zdbbl 8192 - -## Set scheduler bind type. -## +sbt db - -##------------------------------------------------------------------------- -## Env -##------------------------------------------------------------------------- - -## Increase number of concurrent ports/sockets, deprecated in R17 --env ERL_MAX_PORTS 8192 - --env ERTS_MAX_PORTS 8192 - -## Mnesia and SSL will create temporary ets tables. --env ERL_MAX_ETS_TABLES 1024 - -## Tweak GC to run more often --env ERL_FULLSWEEP_AFTER 1000 - --env ERL_CRASH_DUMP log/emqttd_crash.dump diff --git a/rel/reltool.config b/rel/reltool.config deleted file mode 100644 index c79fa74cd..000000000 --- a/rel/reltool.config +++ /dev/null @@ -1,98 +0,0 @@ -%% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*- -%% ex: ft=erlang ts=4 sw=4 et -{sys, [ - {lib_dirs, ["../deps"]}, - {erts, [{mod_cond, derived}, {app_file, strip}]}, - {app_file, strip}, - {rel, "emqttd", git, - [ - kernel, - stdlib, - sasl, - asn1, - syntax_tools, - ssl, - crypto, - eldap, - xmerl, - os_mon, - inets, - goldrush, - compiler, - runtime_tools, - {observer, load}, - lager, - gen_logger, - gproc, - esockd, - mochiweb, - emqttd - ]}, - {rel, "start_clean", "", - [ - kernel, - stdlib - ]}, - {boot_rel, "emqttd"}, - {profile, embedded}, - {incl_cond, exclude}, - %{mod_cond, derived}, - {excl_archive_filters, [".*"]}, %% Do not archive built libs - {excl_sys_filters, ["^bin/(?!start_clean.boot)", - "^erts.*/bin/(dialyzer|typer)", - "^erts.*/(doc|info|include|lib|man|src)"]}, - {excl_app_filters, ["\.gitignore"]}, - {app, kernel, [{incl_cond, include}]}, - {app, stdlib, [{incl_cond, include}]}, - {app, sasl, [{incl_cond, include}]}, - {app, asn1, [{incl_cond, include}]}, - {app, crypto, [{incl_cond, include}]}, - {app, ssl, [{incl_cond, include}]}, - {app, xmerl, [{incl_cond, include}]}, - {app, os_mon, [{incl_cond, include}]}, - {app, syntax_tools, [{incl_cond, include}]}, - {app, public_key, [{incl_cond, include}]}, - {app, mnesia, [{incl_cond, include}]}, - {app, eldap, [{incl_cond, include}]}, - {app, inets, [{incl_cond, include}]}, - {app, compiler, [{incl_cond, include}]}, - {app, runtime_tools, [{incl_cond, include}]}, - {app, observer, [{incl_cond, include}]}, - {app, goldrush, [{incl_cond, include}]}, - {app, gen_logger, [{incl_cond, include}]}, - {app, lager, [{incl_cond, include}]}, - {app, gproc, [{incl_cond, include}]}, - {app, esockd, [{mod_cond, app}, {incl_cond, include}]}, - {app, mochiweb, [{mod_cond, app}, {incl_cond, include}]}, - {app, emqttd, [{mod_cond, app}, {incl_cond, include}, {lib_dir, ".."}]} - ]}. - -{target_dir, "emqttd"}. - -{overlay_vars, "vars.config"}. - -{overlay, [ - {mkdir, "log/"}, - {mkdir, "etc/"}, - {mkdir, "etc/ssl/"}, - {mkdir, "data/"}, - {mkdir, "data/mnesia"}, - {mkdir, "plugins/"}, - {copy, "files/erl", "\{\{erts_vsn\}\}/bin/erl"}, - {template, "files/nodetool", "\{\{erts_vsn\}\}/bin/nodetool"}, - {template, "files/emqttd", "bin/emqttd"}, - {template, "files/emqttd_ctl", "bin/emqttd_ctl"}, - {template, "files/emqttd_top", "bin/emqttd_top"}, - {template, "files/emqttd.cmd", "bin/emqttd.cmd"}, - {copy, "files/start_erl.cmd", "bin/start_erl.cmd"}, - {copy, "files/install_upgrade.escript", "bin/install_upgrade.escript"}, - {copy, "files/ssl/ssl.crt", "etc/ssl/ssl.crt"}, - {copy, "files/ssl/ssl.key", "etc/ssl/ssl.key"}, - {template, "files/emqttd.config.production", "etc/emqttd.config"}, - {template, "files/emqttd.config.development", "etc/emqttd.config.development"}, - {template, "files/acl.config", "etc/acl.config"}, - {template, "files/rewrite.config", "etc/rewrite.config"}, - {template, "files/clients.config", "etc/clients.config"}, - {template, "files/vm.args", "etc/vm.args"}, - {copy, "files/loaded_plugins", "data/loaded_plugins"} - ]}. diff --git a/rel/reltool.config.script b/rel/reltool.config.script deleted file mode 100644 index fa6b571ac..000000000 --- a/rel/reltool.config.script +++ /dev/null @@ -1,18 +0,0 @@ -%% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*- -%% ex: ft=erlang ts=4 sw=4 et - -Sys = proplists:get_value(sys, CONFIG), -IncludeApps = [App || {app, App, _} <- Sys], - -[DepsDir] = proplists:get_value(lib_dirs, Sys), -DepApps = lists:map(fun(AppFile) -> - {ok, [{application, Name, Attrs}]} - = file:consult(filename:join(DepsDir, AppFile)), - Name - end, filelib:wildcard("*/ebin/*.app", DepsDir)), -AppendApps = DepApps -- IncludeApps, -Cond = [{mod_cond, app}, {incl_cond, include}], - -NewSys = lists:append(Sys, [{app, App, Cond} || App <- AppendApps]), - -lists:keyreplace(sys, 1, CONFIG, {sys, NewSys}). diff --git a/rel/vars.config b/rel/vars.config deleted file mode 100644 index 982940a28..000000000 --- a/rel/vars.config +++ /dev/null @@ -1,28 +0,0 @@ -%% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*- -%% ex: ft=erlang ts=4 sw=4 et - -%% Platform-specific installation paths -{platform_bin_dir, "./bin"}. -{platform_data_dir, "./var/data"}. -{platform_etc_dir, "./etc"}. -{platform_lib_dir, "./lib"}. -{platform_log_dir, "./log"}. - -%% -%% etc/emqttd.config -%% - - -%% -%% etc/vm.args -%% - -%% -%% bin/emqttd -%% -{runner_script_dir, "$(cd ${0%/*} && pwd)"}. -{runner_base_dir, "${RUNNER_SCRIPT_DIR%/*}"}. -{runner_etc_dir, "$RUNNER_BASE_DIR/etc"}. -{runner_log_dir, "$RUNNER_BASE_DIR/log"}. -{pipe_dir, "/tmp/$RUNNER_SCRIPT/"}. -{runner_user, ""}. diff --git a/src/emqttd.app.src b/src/emqttd.app.src index 6290ae6b4..1a8dc1d15 100644 --- a/src/emqttd.app.src +++ b/src/emqttd.app.src @@ -1,11 +1,12 @@ {application, emqttd, [ {description, "Erlang MQTT Broker"}, - {vsn, "1.1.3"}, + {vsn, "2.0"}, {id, "emqttd"}, {modules, []}, {registered, []}, - {applications, [kernel, stdlib, gproc, esockd, mochiweb]}, + {applications, [kernel, stdlib, gproc, esockd, mochiweb, + gen_logger, gen_conf]}, {mod, {emqttd_app, []}}, {env, []} ]}. diff --git a/src/emqttd.erl b/src/emqttd.erl index 9bc455fbd..252d77d78 100644 --- a/src/emqttd.erl +++ b/src/emqttd.erl @@ -14,38 +14,66 @@ %% limitations under the License. %%-------------------------------------------------------------------- +%% Facade Module for The EMQTT Broker + -module(emqttd). -include("emqttd.hrl"). -include("emqttd_protocol.hrl"). --export([start/0, env/1, env/2, is_running/1]). +-export([start/0, conf/1, conf/2, env/1, env/2, is_running/1]). %% PubSub API --export([create/2, lookup/2, publish/1, subscribe/1, subscribe/3, - unsubscribe/1, unsubscribe/3]). +-export([subscribe/1, subscribe/2, subscribe/3, publish/1, + unsubscribe/1, unsubscribe/2]). + +%% PubSub Management API +-export([setqos/3, topics/0, subscriptions/1, subscribers/1, + is_subscribed/2, subscriber_down/1]). %% Hooks API -export([hook/4, hook/3, unhook/2, run_hooks/3]). +%% Adapter +-export([adapter/1]). + +%% Debug API +-export([dump/0]). + +-type(subscriber() :: pid() | binary()). + +-type(suboption() :: local | {qos, non_neg_integer()} | {share, {'$queue' | binary()}}). + +-type(pubsub_error() :: {error, {already_subscribed, binary()} + | {subscription_not_found, binary()}}). + +-export_type([subscriber/0, suboption/0, pubsub_error/0]). + -define(APP, ?MODULE). %%-------------------------------------------------------------------- -%% Bootstrap, environment, is_running... +%% Bootstrap, environment, configuration, is_running... %%-------------------------------------------------------------------- %% @doc Start emqttd application. -spec(start() -> ok | {error, any()}). start() -> application:start(?APP). -%% @doc Group environment --spec(env(Group :: atom()) -> list()). -env(Group) -> application:get_env(?APP, Group, []). +%% @doc Get Config +-spec(conf(Key :: atom()) -> any()). +conf(Key) -> gen_conf:value(?APP, Key). + +-spec(conf(Key :: atom(), Default :: any()) -> any()). +conf(Key, Default) -> gen_conf:value(?APP, Key, Default). + +%% @doc Environment +-spec(env(Key:: atom()) -> any()). +env(Key) -> application:get_env(?APP, Key). %% @doc Get environment --spec(env(Group :: atom(), Name :: atom()) -> undefined | any()). -env(Group, Name) -> proplists:get_value(Name, env(Group)). +-spec(env(Key:: atom(), Default:: any()) -> undefined | any()). +env(Key, Default) -> application:get_env(?APP, Key, Default). %% @doc Is running? -spec(is_running(node()) -> boolean()). @@ -57,52 +85,60 @@ is_running(Node) -> end. %%-------------------------------------------------------------------- -%% PubSub APIs that wrap emqttd_server, emqttd_pubsub +%% PubSub APIs %%-------------------------------------------------------------------- -%% @doc Lookup Topic or Subscription --spec(lookup(topic, binary()) -> [mqtt_topic()]; - (subscription, binary()) -> [mqtt_subscription()]). -lookup(topic, Topic) when is_binary(Topic) -> - emqttd_pubsub:lookup_topic(Topic); +%% @doc Subscribe +-spec(subscribe(iodata()) -> ok | {error, any()}). +subscribe(Topic) -> + subscribe(Topic, self()). -lookup(subscription, ClientId) when is_binary(ClientId) -> - emqttd_server:lookup_subscription(ClientId). +-spec(subscribe(iodata(), subscriber()) -> ok | {error, any()}). +subscribe(Topic, Subscriber) -> + subscribe(Topic, Subscriber, []). -%% @doc Create a Topic or Subscription --spec(create(topic | subscription, binary()) -> ok | {error, any()}). -create(topic, Topic) when is_binary(Topic) -> - emqttd_pubsub:create_topic(Topic); - -create(subscription, {ClientId, Topic, Qos}) -> - Subscription = #mqtt_subscription{subid = ClientId, topic = Topic, qos = ?QOS_I(Qos)}, - emqttd_backend:add_subscription(Subscription). +-spec(subscribe(iodata(), subscriber(), [suboption()]) -> ok | pubsub_error()). +subscribe(Topic, Subscriber, Options) -> + with_pubsub(fun(PS) -> PS:subscribe(iolist_to_binary(Topic), Subscriber, Options) end). %% @doc Publish MQTT Message --spec(publish(mqtt_message()) -> ok). -publish(Msg) when is_record(Msg, mqtt_message) -> - emqttd_server:publish(Msg), ok. - -%% @doc Subscribe --spec(subscribe(binary()) -> ok; - ({binary(), binary(), mqtt_qos()}) -> ok). -subscribe(Topic) when is_binary(Topic) -> - emqttd_server:subscribe(Topic); -subscribe({ClientId, Topic, Qos}) -> - subscribe(ClientId, Topic, Qos). - --spec(subscribe(binary(), binary(), mqtt_qos()) -> {ok, mqtt_qos()}). -subscribe(ClientId, Topic, Qos) -> - emqttd_server:subscribe(ClientId, Topic, Qos). +-spec(publish(mqtt_message()) -> {ok, mqtt_delivery()} | ignore). +publish(Msg) -> + with_pubsub(fun(PS) -> PS:publish(Msg) end). %% @doc Unsubscribe --spec(unsubscribe(binary()) -> ok). -unsubscribe(Topic) when is_binary(Topic) -> - emqttd_server:unsubscribe(Topic). +-spec(unsubscribe(iodata()) -> ok | pubsub_error()). +unsubscribe(Topic) -> + unsubscribe(Topic, self()). --spec(unsubscribe(binary(), binary(), mqtt_qos()) -> ok). -unsubscribe(ClientId, Topic, Qos) -> - emqttd_server:unsubscribe(ClientId, Topic, Qos). +-spec(unsubscribe(iodata(), subscriber()) -> ok | pubsub_error()). +unsubscribe(Topic, Subscriber) -> + with_pubsub(fun(PS) -> PS:unsubscribe(iolist_to_binary(Topic), Subscriber) end). + +-spec(setqos(binary(), subscriber(), mqtt_qos()) -> ok). +setqos(Topic, Subscriber, Qos) -> + with_pubsub(fun(PS) -> PS:setqos(iolist_to_binary(Topic), Subscriber, Qos) end). + +-spec(topics() -> [binary()]). +topics() -> emqttd_router:topics(). + +-spec(subscribers(iodata()) -> list(subscriber())). +subscribers(Topic) -> + with_pubsub(fun(PS) -> PS:subscribers(iolist_to_binary(Topic)) end). + +-spec(subscriptions(subscriber()) -> [{binary(), suboption()}]). +subscriptions(Subscriber) -> + with_pubsub(fun(PS) -> PS:subscriptions(Subscriber) end). + +-spec(is_subscribed(iodata(), subscriber()) -> boolean()). +is_subscribed(Topic, Subscriber) -> + with_pubsub(fun(PS) -> PS:is_subscribed(iolist_to_binary(Topic), Subscriber) end). + +-spec(subscriber_down(subscriber()) -> ok). +subscriber_down(Subscriber) -> + with_pubsub(fun(PS) -> PS:subscriber_down(Subscriber) end). + +with_pubsub(Fun) -> Fun(env(pubsub_server, emqttd_server)). %%-------------------------------------------------------------------- %% Hooks API @@ -124,3 +160,17 @@ unhook(Hook, Function) -> run_hooks(Hook, Args, Acc) -> emqttd_hook:run(Hook, Args, Acc). +%%-------------------------------------------------------------------- +%% Adapter +%%-------------------------------------------------------------------- + +adapter(server) -> env(pubsub_server, emqttd_server); +adapter(pubsub) -> env(pubsub_adapter, emqttd_pubsub); +adapter(bridge) -> env(bridge_adapter, emqttd_bridge). + +%%-------------------------------------------------------------------- +%% Debug +%%-------------------------------------------------------------------- + +dump() -> with_pubsub(fun(PS) -> lists:append([PS:dump(), emqttd_router:dump()]) end). + diff --git a/src/emqttd_access_control.erl b/src/emqttd_access_control.erl index 47d6417d4..c2c8fb9b3 100644 --- a/src/emqttd_access_control.erl +++ b/src/emqttd_access_control.erl @@ -23,7 +23,7 @@ -define(SERVER, ?MODULE). %% API Function Exports --export([start_link/0, start_link/1, +-export([start_link/0, auth/2, % authentication check_acl/3, % acl check reload_acl/0, % reload acl @@ -48,11 +48,8 @@ %% @doc Start access control server. -spec(start_link() -> {ok, pid()} | ignore | {error, any()}). -start_link() -> start_link(emqttd:env(access)). - --spec(start_link(Opts :: list()) -> {ok, pid()} | ignore | {error, any()}). -start_link(Opts) -> - gen_server:start_link({local, ?SERVER}, ?MODULE, [Opts], []). +start_link() -> + gen_server:start_link({local, ?SERVER}, ?MODULE, [], []). %% @doc Authenticate MQTT Client. -spec(auth(Client :: mqtt_client(), Password :: password()) -> ok | {error, any()}). @@ -73,7 +70,7 @@ auth(Client, Password, [{Mod, State, _Seq} | Mods]) -> Client :: mqtt_client(), PubSub :: pubsub(), Topic :: binary()). -check_acl(Client, PubSub, Topic) when ?IS_PUBSUB(PubSub) -> +check_acl(Client, PubSub, Topic) when ?PUBSUB(PubSub) -> case lookup_mods(acl) of [] -> allow; AclMods -> check_acl(Client, PubSub, Topic, AclMods) @@ -125,17 +122,14 @@ stop() -> gen_server:call(?MODULE, stop). %% gen_server callbacks %%-------------------------------------------------------------------- -init([Opts]) -> +init([]) -> ets:new(?ACCESS_CONTROL_TAB, [set, named_table, protected, {read_concurrency, true}]), - ets:insert(?ACCESS_CONTROL_TAB, {auth_modules, init_mods(auth, proplists:get_value(auth, Opts))}), - ets:insert(?ACCESS_CONTROL_TAB, {acl_modules, init_mods(acl, proplists:get_value(acl, Opts))}), + ets:insert(?ACCESS_CONTROL_TAB, {auth_modules, init_mods(gen_conf:list(emqttd, auth))}), + ets:insert(?ACCESS_CONTROL_TAB, {acl_modules, init_mods(gen_conf:list(emqttd, acl))}), {ok, #state{}}. -init_mods(auth, AuthMods) -> - [init_mod(authmod(Name), Opts) || {Name, Opts} <- AuthMods]; - -init_mods(acl, AclMods) -> - [init_mod(aclmod(Name), Opts) || {Name, Opts} <- AclMods]. +init_mods(Mods) -> + [init_mod(mod_name(Type, Name), Opts) || {Type, Name, Opts} <- Mods]. init_mod(Mod, Opts) -> {ok, State} = Mod:init(Opts), {Mod, State, 0}. @@ -191,15 +185,14 @@ code_change(_OldVsn, State, _Extra) -> %% Internal functions %%-------------------------------------------------------------------- -authmod(Name) when is_atom(Name) -> - mod(emqttd_auth_, Name). - -aclmod(Name) when is_atom(Name) -> - mod(emqttd_acl_, Name). +mod_name(auth, Name) -> mod(emqttd_auth_, Name); +mod_name(acl, Name) -> mod(emqttd_acl_, Name). + mod(Prefix, Name) -> list_to_atom(lists:concat([Prefix, Name])). if_existed(false, Fun) -> Fun(); + if_existed(_Mod, _Fun) -> {error, already_existed}. diff --git a/src/emqttd_acl_anonymous.erl b/src/emqttd_acl_anonymous.erl new file mode 100644 index 000000000..ef80457fd --- /dev/null +++ b/src/emqttd_acl_anonymous.erl @@ -0,0 +1,35 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2012-2016 Feng Lee . +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqttd_acl_anonymous). + +-behaviour(emqttd_acl_mod). + +%% ACL callbacks +-export([init/1, check_acl/2, reload_acl/1, description/0]). + +init(Opts) -> + {ok, Opts}. + +check_acl(_Who, _State) -> + allow. + +reload_acl(_State) -> + ok. + +description() -> + "Anonymous ACL". + diff --git a/src/emqttd_acl_internal.erl b/src/emqttd_acl_internal.erl index eba5bd905..282fb77a4 100644 --- a/src/emqttd_acl_internal.erl +++ b/src/emqttd_acl_internal.erl @@ -27,7 +27,7 @@ -define(ACL_RULE_TAB, mqtt_acl_rule). --record(state, {acl_file, nomatch = allow}). +-record(state, {config, nomatch = allow}). %%-------------------------------------------------------------------- %% API @@ -46,16 +46,20 @@ all_rules() -> %%-------------------------------------------------------------------- %% @doc Init internal ACL --spec(init(AclOpts :: list()) -> {ok, State :: any()}). -init(AclOpts) -> +-spec(init(Opts :: list()) -> {ok, State :: any()}). +init(Opts) -> ets:new(?ACL_RULE_TAB, [set, public, named_table, {read_concurrency, true}]), - AclFile = proplists:get_value(file, AclOpts), - Default = proplists:get_value(nomatch, AclOpts, allow), - State = #state{acl_file = AclFile, nomatch = Default}, - true = load_rules_from_file(State), - {ok, State}. + case proplists:get_value(config, Opts) of + undefined -> + {ok, #state{}}; + File -> + Default = proplists:get_value(nomatch, Opts, allow), + State = #state{config = File, nomatch = Default}, + true = load_rules_from_file(State), + {ok, State} + end. -load_rules_from_file(#state{acl_file = AclFile}) -> +load_rules_from_file(#state{config = AclFile}) -> {ok, Terms} = file:consult(AclFile), Rules = [emqttd_access_rule:compile(Term) || Term <- Terms], lists:foreach(fun(PubSub) -> @@ -83,6 +87,8 @@ filter(_PubSub, {_AllowDeny, _Who, _, _Topics}) -> PubSub :: pubsub(), Topic :: binary(), State :: #state{}). +check_acl(_Who, #state{config = undefined}) -> + allow; check_acl({Client, PubSub, Topic}, #state{nomatch = Default}) -> case match(Client, Topic, lookup(PubSub)) of {matched, allow} -> allow; @@ -107,6 +113,8 @@ match(Client, Topic, [Rule|Rules]) -> %% @doc Reload ACL -spec(reload_acl(State :: #state{}) -> ok | {error, Reason :: any()}). +reload_acl(#state{config = undefined}) -> + ok; reload_acl(State) -> case catch load_rules_from_file(State) of {'EXIT', Error} -> {error, Error}; @@ -115,5 +123,6 @@ reload_acl(State) -> %% @doc ACL Module Description -spec(description() -> string()). -description() -> "Internal ACL with etc/acl.config". +description() -> + "Internal ACL with etc/acl.conf". diff --git a/src/emqttd_app.erl b/src/emqttd_app.erl index 5c56e824f..2db44c351 100644 --- a/src/emqttd_app.erl +++ b/src/emqttd_app.erl @@ -1,4 +1,4 @@ -%%-------------------------------------------------------------------- +%-------------------------------------------------------------------- %% Copyright (c) 2012-2016 Feng Lee . %% %% Licensed under the Apache License, Version 2.0 (the "License"); @@ -26,12 +26,8 @@ -export([start_listener/1, stop_listener/1, is_mod_enabled/1]). %% MQTT SockOpts --define(MQTT_SOCKOPTS, [ - binary, - {packet, raw}, - {reuseaddr, true}, - {backlog, 512}, - {nodelay, true}]). +-define(MQTT_SOCKOPTS, [binary, {packet, raw}, {reuseaddr, true}, + {backlog, 512}, {nodelay, true}]). -type listener() :: {atom(), esockd:listen_on(), [esockd:option()]}. @@ -46,6 +42,7 @@ Reason :: term()). start(_StartType, _StartArgs) -> print_banner(), + gen_conf:init(emqttd), emqttd_mnesia:start(), {ok, Sup} = emqttd_sup:start_link(), start_servers(Sup), @@ -80,6 +77,7 @@ print_vsn() -> start_servers(Sup) -> Servers = [{"emqttd ctl", emqttd_ctl}, {"emqttd hook", emqttd_hook}, + {"emqttd router", emqttd_router}, {"emqttd pubsub", {supervisor, emqttd_pubsub_sup}}, {"emqttd stats", emqttd_stats}, {"emqttd metrics", emqttd_metrics}, @@ -93,7 +91,7 @@ start_servers(Sup) -> {"emqttd broker", emqttd_broker}, {"emqttd alarm", emqttd_alarm}, {"emqttd mod supervisor", emqttd_mod_sup}, - {"emqttd bridge supervisor", {supervisor, emqttd_bridge_sup}}, + {"emqttd bridge supervisor", {supervisor, emqttd_bridge_sup_sup}}, {"emqttd access control", emqttd_access_control}, {"emqttd system monitor", {supervisor, emqttd_sysmon_sup}}], [start_server(Sup, Server) || Server <- Servers]. @@ -101,17 +99,17 @@ start_servers(Sup) -> start_server(_Sup, {Name, F}) when is_function(F) -> ?PRINT("~s is starting...", [Name]), F(), - ?PRINT_MSG("[done]~n"); + ?PRINT_MSG("[ok]~n"); start_server(Sup, {Name, Server}) -> ?PRINT("~s is starting...", [Name]), start_child(Sup, Server), - ?PRINT_MSG("[done]~n"); + ?PRINT_MSG("[ok]~n"); start_server(Sup, {Name, Server, Opts}) -> ?PRINT("~s is starting...", [ Name]), start_child(Sup, Server, Opts), - ?PRINT_MSG("[done]~n"). + ?PRINT_MSG("[ok]~n"). start_child(Sup, {supervisor, Module}) -> supervisor:start_child(Sup, supervisor_spec(Module)); @@ -149,9 +147,9 @@ worker_spec(M, F, A) -> %% @doc load all modules load_all_mods() -> - lists:foreach(fun load_mod/1, emqttd:env(modules)). + lists:foreach(fun load_mod/1, gen_conf:list(emqttd, module)). -load_mod({Name, Opts}) -> +load_mod({module, Name, Opts}) -> Mod = list_to_atom("emqttd_mod_" ++ atom_to_list(Name)), case catch Mod:load(Opts) of ok -> lager:info("Load module ~s successfully", [Name]); @@ -161,7 +159,7 @@ load_mod({Name, Opts}) -> %% @doc Is module enabled? -spec(is_mod_enabled(Name :: atom()) -> boolean()). -is_mod_enabled(Name) -> emqttd:env(modules, Name) =/= undefined. +is_mod_enabled(Name) -> lists:keyfind(Name, 2, gen_conf:list(emqttd, module)). %%-------------------------------------------------------------------- %% Start Listeners @@ -169,25 +167,27 @@ is_mod_enabled(Name) -> emqttd:env(modules, Name) =/= undefined. %% @doc Start Listeners of the broker. -spec(start_listeners() -> any()). -start_listeners() -> lists:foreach(fun start_listener/1, emqttd:env(listeners)). +start_listeners() -> lists:foreach(fun start_listener/1, gen_conf:list(emqttd, listener)). %% Start mqtt listener -spec(start_listener(listener()) -> any()). -start_listener({mqtt, ListenOn, Opts}) -> start_listener(mqtt, ListenOn, Opts); +start_listener({listener, mqtt, ListenOn, Opts}) -> + start_listener(mqtt, ListenOn, Opts); %% Start mqtt(SSL) listener -start_listener({mqtts, ListenOn, Opts}) -> start_listener(mqtts, ListenOn, Opts); +start_listener({listener, mqtts, ListenOn, Opts}) -> + start_listener(mqtts, ListenOn, Opts); %% Start http listener -start_listener({http, ListenOn, Opts}) -> +start_listener({listener, http, ListenOn, Opts}) -> mochiweb:start_http(http, ListenOn, Opts, {emqttd_http, handle_request, []}); %% Start https listener -start_listener({https, ListenOn, Opts}) -> +start_listener({listener, https, ListenOn, Opts}) -> mochiweb:start_http(https, ListenOn, Opts, {emqttd_http, handle_request, []}). start_listener(Protocol, ListenOn, Opts) -> - MFArgs = {emqttd_client, start_link, [emqttd:env(mqtt)]}, + MFArgs = {emqttd_client, start_link, [emqttd_conf:mqtt()]}, {ok, _} = esockd:open(Protocol, ListenOn, merge_sockopts(Opts), MFArgs). merge_sockopts(Options) -> @@ -200,8 +200,22 @@ merge_sockopts(Options) -> %%-------------------------------------------------------------------- %% @doc Stop Listeners -stop_listeners() -> lists:foreach(fun stop_listener/1, emqttd:env(listeners)). +stop_listeners() -> lists:foreach(fun stop_listener/1, gen_conf:list(listener)). %% @private -stop_listener({Protocol, ListenOn, _Opts}) -> esockd:close(Protocol, ListenOn). +stop_listener({listener, Protocol, ListenOn, _Opts}) -> esockd:close(Protocol, ListenOn). +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). +merge_sockopts_test_() -> + Opts = [{acceptors, 16}, {max_clients, 512}], + ?_assert(merge_sockopts(Opts) == [{sockopts, ?MQTT_SOCKOPTS} | Opts]). + +load_all_mods_test_() -> + ?_assert(load_all_mods() == ok). + +is_mod_enabled_test_() -> + ?_assert(is_mod_enabled(presence) == {module, presence, [{qos, 0}]}), + ?_assert(is_mod_enabled(test) == false). + +-endif. diff --git a/src/emqttd_auth_clientid.erl b/src/emqttd_auth_clientid.erl index 35b71035b..15a751ea8 100644 --- a/src/emqttd_auth_clientid.erl +++ b/src/emqttd_auth_clientid.erl @@ -69,7 +69,8 @@ init(Opts) -> {ram_copies, [node()]}, {attributes, record_info(fields, ?AUTH_CLIENTID_TAB)}]), mnesia:add_table_copy(?AUTH_CLIENTID_TAB, node(), ram_copies), - load(proplists:get_value(file, Opts)), + Clients = load_client_from(proplists:get_value(config, Opts)), + mnesia:transaction(fun() -> [mnesia:write(C) || C<- Clients] end), {ok, Opts}. check(#mqtt_client{client_id = undefined}, _Password, _Opts) -> @@ -93,32 +94,19 @@ description() -> "ClientId authentication module". %% Internal functions %%-------------------------------------------------------------------- -load(undefined) -> +load_client_from(undefined) -> ok; -load(File) -> - {ok, Fd} = file:open(File, [read]), - load(Fd, file:read_line(Fd), []). +load_client_from(File) -> + {ok, Clients} = file:consult(File), + [client(Client) || Client <- Clients]. -load(Fd, {ok, Line}, Clients) when is_list(Line) -> - Clients1 = - case string:tokens(Line, " ") of - [ClientIdS] -> - ClientId = list_to_binary(string:strip(ClientIdS, right, $\n)), - [#mqtt_auth_clientid{client_id = ClientId} | Clients]; - [ClientId, IpAddr0] -> - IpAddr = string:strip(IpAddr0, right, $\n), - [#mqtt_auth_clientid{client_id = list_to_binary(ClientId), - ipaddr = esockd_cidr:parse(IpAddr, true)} | Clients]; - BadLine -> - lager:error("BadLine in clients.config: ~s", [BadLine]), - Clients - end, - load(Fd, file:read_line(Fd), Clients1); +client(ClientId) when is_list(ClientId) -> + #mqtt_auth_clientid{client_id = list_to_binary(ClientId)}; -load(Fd, eof, Clients) -> - mnesia:transaction(fun() -> [mnesia:write(C) || C<- Clients] end), - file:close(Fd). +client({ClientId, IpAddr}) when is_list(ClientId) -> + #mqtt_auth_clientid{client_id = iolist_to_binary(ClientId), + ipaddr = esockd_cidr:parse(IpAddr, true)}. check_clientid_only(ClientId, IpAddr) -> case mnesia:dirty_read(?AUTH_CLIENTID_TAB, ClientId) of diff --git a/src/emqttd_auth_ldap.erl b/src/emqttd_auth_ldap.erl deleted file mode 100644 index 11e1f27f3..000000000 --- a/src/emqttd_auth_ldap.erl +++ /dev/null @@ -1,77 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2012-2016 Feng Lee . -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - -%% @doc LDAP Authentication Module --module(emqttd_auth_ldap). - --include("emqttd.hrl"). - --import(proplists, [get_value/2, get_value/3]). - --behaviour(emqttd_auth_mod). - --export([init/1, check/3, description/0]). - --record(state, {servers, user_dn, options}). - -init(Opts) -> - Servers = get_value(servers, Opts, ["localhost"]), - Port = get_value(port, Opts, 389), - Timeout = get_value(timeout, Opts, 30), - UserDn = get_value(user_dn, Opts), - LdapOpts = - case get_value(ssl, Opts, false) of - true -> - SslOpts = get_value(sslopts, Opts), - [{port, Port}, {timeout, Timeout}, {sslopts, SslOpts}]; - false -> - [{port, Port}, {timeout, Timeout}] - end, - {ok, #state{servers = Servers, user_dn = UserDn, options = LdapOpts}}. - -check(#mqtt_client{username = undefined}, _Password, _State) -> - {error, username_undefined}; -check(_Client, undefined, _State) -> - {error, password_undefined}; -check(_Client, <<>>, _State) -> - {error, password_undefined}; -check(#mqtt_client{username = Username}, Password, - #state{servers = Servers, user_dn = UserDn, options = Options}) -> - case eldap:open(Servers, Options) of - {ok, LDAP} -> - UserDn1 = fill(binary_to_list(Username), UserDn), - ldap_bind(LDAP, UserDn1, binary_to_list(Password)); - {error, Reason} -> - {error, Reason} - end. - -ldap_bind(LDAP, UserDn, Password) -> - case catch eldap:simple_bind(LDAP, UserDn, Password) of - ok -> - ok; - {error, invalidCredentials} -> - {error, invalid_credentials}; - {error, Error} -> - {error, Error}; - {'EXIT', Reason} -> - {error, Reason} - end. - -fill(Username, UserDn) -> - re:replace(UserDn, "\\$u", Username, [global, {return, list}]). - -description() -> "LDAP Authentication Module". - diff --git a/src/emqttd_auth_username.erl b/src/emqttd_auth_username.erl index 3c59b7c20..6a0b1d17f 100644 --- a/src/emqttd_auth_username.erl +++ b/src/emqttd_auth_username.erl @@ -68,7 +68,7 @@ if_enabled(Fun) -> end. hint() -> - ?PRINT_MSG("Please enable '{username, []}' authentication in etc/emqttd.config first.~n"). + ?PRINT_MSG("Please enable '{auth, username, []}' in etc/emqttd.conf first.~n"). %%-------------------------------------------------------------------- %% API @@ -81,7 +81,13 @@ is_enabled() -> -spec(add_user(binary(), binary()) -> ok | {error, any()}). add_user(Username, Password) -> User = #?AUTH_USERNAME_TAB{username = Username, password = hash(Password)}, - ret(mnesia:transaction(fun mnesia:write/1, [User])). + ret(mnesia:transaction(fun insert_user/1, [User])). + +insert_user(User = #?AUTH_USERNAME_TAB{username = Username}) -> + case mnesia:read(?AUTH_USERNAME_TAB, Username) of + [] -> mnesia:write(User); + [_|_] -> mnesia:abort(existed) + end. add_default_user(Username, Password) when is_atom(Username) -> add_default_user(atom_to_list(Username), Password); @@ -110,16 +116,20 @@ all_users() -> mnesia:dirty_all_keys(?AUTH_USERNAME_TAB). %% emqttd_auth_mod callbacks %%-------------------------------------------------------------------- -init(DefautUsers) -> +init(Opts) -> mnesia:create_table(?AUTH_USERNAME_TAB, [ {disc_copies, [node()]}, {attributes, record_info(fields, ?AUTH_USERNAME_TAB)}]), mnesia:add_table_copy(?AUTH_USERNAME_TAB, node(), disc_copies), - lists:foreach(fun({Username, Password}) -> - add_default_user(Username, Password) - end, DefautUsers), + case proplists:get_value(passwd, Opts) of + undefined -> ok; + File -> {ok, DefaultUsers} = file:consult(File), + lists:foreach(fun({Username, Password}) -> + add_default_user(Username, Password) + end, DefaultUsers) + end, emqttd_ctl:register_cmd(users, {?MODULE, cli}, []), - {ok, []}. + {ok, Opts}. check(#mqtt_client{username = undefined}, _Password, _Opts) -> {error, username_undefined}; @@ -127,7 +137,7 @@ check(_User, undefined, _Opts) -> {error, password_undefined}; check(#mqtt_client{username = Username}, Password, _Opts) -> case mnesia:dirty_read(?AUTH_USERNAME_TAB, Username) of - [] -> + [] -> {error, username_not_found}; [#?AUTH_USERNAME_TAB{password = <>}] -> case Hash =:= md5_hash(Salt, Password) of diff --git a/src/emqttd_backend.erl b/src/emqttd_backend.erl deleted file mode 100644 index ead4c59d5..000000000 --- a/src/emqttd_backend.erl +++ /dev/null @@ -1,153 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2012-2016 Feng Lee . -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqttd_backend). - --include("emqttd.hrl"). - --include_lib("stdlib/include/ms_transform.hrl"). - -%% Mnesia Callbacks --export([mnesia/1]). - --boot_mnesia({mnesia, [boot]}). --copy_mnesia({mnesia, [copy]}). - -%% Retained Message API --export([retain_message/1, read_messages/1, match_messages/1, delete_message/1, - expire_messages/1, retained_count/0]). - -%% Static Subscription API --export([add_subscription/1, lookup_subscriptions/1, del_subscriptions/1, - del_subscription/2]). - --record(retained_message, {topic, msg}). - -%%-------------------------------------------------------------------- -%% Mnesia callbacks -%%-------------------------------------------------------------------- - -mnesia(boot) -> - ok = emqttd_mnesia:create_table(retained_message, [ - {type, ordered_set}, - {disc_copies, [node()]}, - {record_name, retained_message}, - {attributes, record_info(fields, retained_message)}, - {storage_properties, [{ets, [compressed]}, - {dets, [{auto_save, 1000}]}]}]), - ok = emqttd_mnesia:create_table(backend_subscription, [ - {type, bag}, - {disc_copies, [node()]}, - {record_name, mqtt_subscription}, - {attributes, record_info(fields, mqtt_subscription)}, - {storage_properties, [{ets, [compressed]}, - {dets, [{auto_save, 5000}]}]}]); - -mnesia(copy) -> - ok = emqttd_mnesia:copy_table(retained_message), - ok = emqttd_mnesia:copy_table(backend_subscription). - -%%-------------------------------------------------------------------- -%% Retained Message -%%-------------------------------------------------------------------- - --spec(retain_message(mqtt_message()) -> ok). -retain_message(Msg = #mqtt_message{topic = Topic}) -> - mnesia:dirty_write(#retained_message{topic = Topic, msg = Msg}). - --spec(read_messages(binary()) -> [mqtt_message()]). -read_messages(Topic) -> - [Msg || #retained_message{msg = Msg} <- mnesia:dirty_read(retained_message, Topic)]. - --spec(match_messages(binary()) -> [mqtt_message()]). -match_messages(Filter) -> - %% TODO: optimize later... - Fun = fun(#retained_message{topic = Name, msg = Msg}, Acc) -> - case emqttd_topic:match(Name, Filter) of - true -> [Msg|Acc]; - false -> Acc - end - end, - mnesia:async_dirty(fun mnesia:foldl/3, [Fun, [], retained_message]). - --spec(delete_message(binary()) -> ok). -delete_message(Topic) -> - mnesia:dirty_delete(retained_message, Topic). - --spec(expire_messages(pos_integer()) -> any()). -expire_messages(Time) when is_integer(Time) -> - mnesia:transaction( - fun() -> - Match = ets:fun2ms( - fun(#retained_message{topic = Topic, msg = #mqtt_message{timestamp = {MegaSecs, Secs, _}}}) - when Time > (MegaSecs * 1000000 + Secs) -> Topic - end), - Topics = mnesia:select(retained_message, Match, write), - lists:foreach(fun(<<"$SYS/", _/binary>>) -> ok; %% ignore $SYS/# messages - (Topic) -> mnesia:delete({retained_message, Topic}) - end, Topics) - end). - --spec(retained_count() -> non_neg_integer()). -retained_count() -> - mnesia:table_info(retained_message, size). - -%%-------------------------------------------------------------------- -%% Static Subscriptions -%%-------------------------------------------------------------------- - -%% @doc Add a static subscription manually. --spec(add_subscription(mqtt_subscription()) -> ok | {error, already_existed}). -add_subscription(Subscription = #mqtt_subscription{subid = SubId, topic = Topic}) -> - Pattern = match_pattern(SubId, Topic), - return(mnesia:transaction(fun() -> - case mnesia:match_object(backend_subscription, Pattern, write) of - [] -> - mnesia:write(backend_subscription, Subscription, write); - [Subscription] -> - mnesia:abort(already_existed); - [Subscription1] -> %% QoS is different - mnesia:delete_object(backend_subscription, Subscription1, write), - mnesia:write(backend_subscription, Subscription, write) - end - end)). - -%% @doc Lookup static subscriptions. --spec(lookup_subscriptions(binary()) -> list(mqtt_subscription())). -lookup_subscriptions(ClientId) when is_binary(ClientId) -> - mnesia:dirty_read(backend_subscription, ClientId). - -%% @doc Delete static subscriptions by ClientId manually. --spec(del_subscriptions(binary()) -> ok). -del_subscriptions(ClientId) when is_binary(ClientId) -> - return(mnesia:transaction(fun mnesia:delete/1, [{backend_subscription, ClientId}])). - -%% @doc Delete a static subscription manually. --spec(del_subscription(binary(), binary()) -> ok). -del_subscription(ClientId, Topic) when is_binary(ClientId) andalso is_binary(Topic) -> - return(mnesia:transaction(fun del_subscription_/1, [match_pattern(ClientId, Topic)])). - -del_subscription_(Pattern) -> - lists:foreach(fun(Subscription) -> - mnesia:delete_object(backend_subscription, Subscription, write) - end, mnesia:match_object(backend_subscription, Pattern, write)). - -match_pattern(SubId, Topic) -> - #mqtt_subscription{subid = SubId, topic = Topic, qos = '_'}. - -return({atomic, ok}) -> ok; -return({aborted, Reason}) -> {error, Reason}. - diff --git a/src/emqttd_bridge_sup.erl b/src/emqttd_bridge_sup.erl index 2d7da927c..e808ee72a 100644 --- a/src/emqttd_bridge_sup.erl +++ b/src/emqttd_bridge_sup.erl @@ -18,56 +18,25 @@ -behavior(supervisor). --export([start_link/0, bridges/0, start_bridge/2, start_bridge/3, stop_bridge/2]). +-export([start_link/3]). -export([init/1]). --define(BRIDGE_ID(Node, Topic), {bridge, Node, Topic}). - %%-------------------------------------------------------------------- %% API %%-------------------------------------------------------------------- %% @doc Start bridge supervisor -start_link() -> - supervisor:start_link({local, ?MODULE}, ?MODULE, []). - -%% @doc List all bridges --spec(bridges() -> [{tuple(), pid()}]). -bridges() -> - [{{Node, Topic}, Pid} || {?BRIDGE_ID(Node, Topic), Pid, worker, _} - <- supervisor:which_children(?MODULE)]. - -%% @doc Start a bridge --spec(start_bridge(atom(), binary()) -> {ok, pid()} | {error, any()}). -start_bridge(Node, Topic) when is_atom(Node) andalso is_binary(Topic) -> - start_bridge(Node, Topic, []). - --spec(start_bridge(atom(), binary(), [emqttd_bridge:option()]) -> {ok, pid()} | {error, any()}). -start_bridge(Node, _Topic, _Options) when Node =:= node() -> - {error, bridge_to_self}; -start_bridge(Node, Topic, Options) when is_atom(Node) andalso is_binary(Topic) -> - Options1 = emqttd_opts:merge(emqttd_broker:env(bridge), Options), - supervisor:start_child(?MODULE, bridge_spec(Node, Topic, Options1)). - -%% @doc Stop a bridge --spec(stop_bridge(atom(), binary()) -> {ok, pid()} | ok). -stop_bridge(Node, Topic) when is_atom(Node) andalso is_binary(Topic) -> - ChildId = ?BRIDGE_ID(Node, Topic), - case supervisor:terminate_child(?MODULE, ChildId) of - ok -> supervisor:delete_child(?MODULE, ChildId); - Error -> Error - end. +-spec(start_link(atom(), binary(), [emqttd_bridge:option()]) -> {ok, pid()} | {error, any()}). +start_link(Node, Topic, Options) -> + supervisor:start_link(?MODULE, [Node, Topic, Options]). %%-------------------------------------------------------------------- %% Supervisor callbacks %%-------------------------------------------------------------------- -init([]) -> - {ok, {{one_for_one, 10, 100}, []}}. - -bridge_spec(Node, Topic, Options) -> - ChildId = ?BRIDGE_ID(Node, Topic), - {ChildId, {emqttd_bridge, start_link, [Node, Topic, Options]}, - transient, 10000, worker, [emqttd_bridge]}. +init([Node, Topic, Options]) -> + {ok, {{one_for_all, 10, 100}, + [{bridge, {emqttd_bridge, start_link, [Node, Topic, Options]}, + transient, 10000, worker, [emqttd_bridge]}]}}. diff --git a/src/emqttd_bridge_sup_sup.erl b/src/emqttd_bridge_sup_sup.erl new file mode 100644 index 000000000..475149afe --- /dev/null +++ b/src/emqttd_bridge_sup_sup.erl @@ -0,0 +1,76 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2012-2016 Feng Lee . +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqttd_bridge_sup_sup). + +-behavior(supervisor). + +-export([start_link/0, bridges/0, start_bridge/2, start_bridge/3, stop_bridge/2]). + +-export([init/1]). + +-define(CHILD_ID(Node, Topic), {bridge_sup, Node, Topic}). + +start_link() -> + supervisor:start_link({local, ?MODULE}, ?MODULE, []). + +%%-------------------------------------------------------------------- +%% API +%%-------------------------------------------------------------------- + +%% @doc List all bridges +-spec(bridges() -> [{node(), binary(), pid()}]). +bridges() -> + [{Node, Topic, Pid} || {?CHILD_ID(Node, Topic), Pid, supervisor, _} + <- supervisor:which_children(?MODULE)]. + +%% @doc Start a bridge +-spec(start_bridge(atom(), binary()) -> {ok, pid()} | {error, any()}). +start_bridge(Node, Topic) when is_atom(Node) andalso is_binary(Topic) -> + start_bridge(Node, Topic, []). + +-spec(start_bridge(atom(), binary(), [emqttd_bridge:option()]) -> {ok, pid()} | {error, any()}). +start_bridge(Node, _Topic, _Options) when Node =:= node() -> + {error, bridge_to_self}; +start_bridge(Node, Topic, Options) when is_atom(Node) andalso is_binary(Topic) -> + Options1 = emqttd_opts:merge(emqttd_conf:bridge(), Options), + supervisor:start_child(?MODULE, bridge_spec(Node, Topic, Options1)). + +%% @doc Stop a bridge +-spec(stop_bridge(atom(), binary()) -> {ok, pid()} | ok). +stop_bridge(Node, Topic) when is_atom(Node) andalso is_binary(Topic) -> + ChildId = ?CHILD_ID(Node, Topic), + case supervisor:terminate_child(?MODULE, ChildId) of + ok -> supervisor:delete_child(?MODULE, ChildId); + Error -> Error + end. + +%%-------------------------------------------------------------------- +%% Supervisor callbacks +%%-------------------------------------------------------------------- + +init([]) -> + {ok, {{one_for_one, 10, 100}, []}}. + +bridge_spec(Node, Topic, Options) -> + SupMod = sup_mod(emqttd:adapter(bridge)), + {?CHILD_ID(Node, Topic), + {SupMod, start_link, [Node, Topic, Options]}, + permanent, infinity, supervisor, [SupMod]}. + +sup_mod(Adaper) -> + list_to_atom(atom_to_list(Adaper) ++ "_sup"). + diff --git a/src/emqttd_broker.erl b/src/emqttd_broker.erl index d95800954..2d53a1328 100644 --- a/src/emqttd_broker.erl +++ b/src/emqttd_broker.erl @@ -29,7 +29,7 @@ -export([subscribe/1, notify/2]). %% Broker API --export([env/1, version/0, uptime/0, datetime/0, sysdescr/0]). +-export([version/0, uptime/0, datetime/0, sysdescr/0]). %% Tick API -export([start_tick/1, stop_tick/1]). @@ -71,10 +71,6 @@ subscribe(EventType) -> notify(EventType, Event) -> gproc:send({p, l, {broker, EventType}}, {notify, EventType, self(), Event}). -%% @doc Get broker env -env(Name) -> - proplists:get_value(Name, emqttd:env(broker)). - %% @doc Get broker version -spec(version() -> string()). version() -> @@ -99,7 +95,7 @@ datetime() -> %% @doc Start a tick timer start_tick(Msg) -> - start_tick(timer:seconds(env(sys_interval)), Msg). + start_tick(timer:seconds(emqttd:conf(broker_sys_interval, 60)), Msg). start_tick(0, _Msg) -> undefined; @@ -119,9 +115,6 @@ stop_tick(TRef) -> init([]) -> emqttd_time:seed(), ets:new(?BROKER_TAB, [set, public, named_table]), - % Create $SYS Topics - emqttd:create(topic, <<"$SYS/brokers">>), - [ok = create_topic(Topic) || Topic <- ?SYSTOP_BROKERS], % Tick {ok, #state{started_at = os:timestamp(), heartbeat = start_tick(1000, heartbeat), @@ -164,9 +157,6 @@ code_change(_OldVsn, State, _Extra) -> %% Internal functions %%-------------------------------------------------------------------- -create_topic(Topic) -> - emqttd:create(topic, emqttd_topic:systop(Topic)). - retain(brokers) -> Payload = list_to_binary(string:join([atom_to_list(N) || N <- emqttd_mnesia:running_nodes()], ",")), diff --git a/src/emqttd_cli.erl b/src/emqttd_cli.erl index a64f086ed..f346be653 100644 --- a/src/emqttd_cli.erl +++ b/src/emqttd_cli.erl @@ -170,24 +170,20 @@ if_client(ClientId, Fun) -> %%-------------------------------------------------------------------- %% @doc Sessions Command sessions(["list"]) -> - [sessions(["list", Type]) || Type <- ["persistent", "transient"]]; + dump(mqtt_local_session); +%% performance issue? sessions(["list", "persistent"]) -> - dump(mqtt_persistent_session); + lists:foreach(fun print/1, ets:match_object(mqtt_local_session, {'_', false, '_', '_'})); +%% performance issue? sessions(["list", "transient"]) -> - dump(mqtt_transient_session); + lists:foreach(fun print/1, ets:match_object(mqtt_local_session, {'_', true, '_', '_'})); sessions(["show", ClientId]) -> - MP = {{bin(ClientId), '_'}, '_'}, - case {ets:match_object(mqtt_transient_session, MP), - ets:match_object(mqtt_persistent_session, MP)} of - {[], []} -> - ?PRINT_MSG("Not Found.~n"); - {[SessInfo], _} -> - print(SessInfo); - {_, [SessInfo]} -> - print(SessInfo) + case ets:lookup(mqtt_local_session, bin(ClientId)) of + [] -> ?PRINT_MSG("Not Found.~n"); + [SessInfo] -> print(SessInfo) end; sessions(_) -> @@ -199,10 +195,10 @@ sessions(_) -> %%-------------------------------------------------------------------- %% @doc Routes Command routes(["list"]) -> - if_could_print(route, fun print/1); + if_could_print(mqtt_route, fun print/1); routes(["show", Topic]) -> - print(mnesia:dirty_read(route, bin(Topic))); + print(mnesia:dirty_read(mqtt_route, bin(Topic))); routes(_) -> ?USAGE([{"routes list", "List all routes"}, @@ -211,54 +207,58 @@ routes(_) -> %%-------------------------------------------------------------------- %% @doc Topics Command topics(["list"]) -> - if_could_print(topic, fun print/1); + lists:foreach(fun(Topic) -> ?PRINT("~s~n", [Topic]) end, emqttd:topics()); topics(["show", Topic]) -> - print(mnesia:dirty_read(topic, bin(Topic))); + print(mnesia:dirty_read(mqtt_route, bin(Topic))); topics(_) -> ?USAGE([{"topics list", "List all topics"}, {"topics show ", "Show a topic"}]). subscriptions(["list"]) -> - if_could_print(subscription, fun print/1); - -subscriptions(["list", "static"]) -> - if_could_print(backend_subscription, fun print/1); + lists:foreach(fun(Subscription) -> + print(subscription, Subscription) + end, []); %%emqttd:subscriptions()); subscriptions(["show", ClientId]) -> - case mnesia:dirty_read(subscription, bin(ClientId)) of + case ets:lookup(mqtt_subscription, bin(ClientId)) of [] -> ?PRINT_MSG("Not Found.~n"); - Records -> print(Records) + Records -> [print(subscription, Subscription) || Subscription <- Records] end; -subscriptions(["add", ClientId, Topic, QoS]) -> - Add = fun(IntQos) -> - Subscription = #mqtt_subscription{subid = bin(ClientId), - topic = bin(Topic), - qos = IntQos}, - case emqttd_backend:add_subscription(Subscription) of - ok -> - ?PRINT_MSG("ok~n"); - {error, already_existed} -> - ?PRINT_MSG("Error: already existed~n"); - {error, Reason} -> - ?PRINT("Error: ~p~n", [Reason]) - end - end, - if_valid_qos(QoS, Add); +%% +%% subscriptions(["add", ClientId, Topic, QoS]) -> +%% Add = fun(IntQos) -> +%% Subscription = #mqtt_subscription{subid = bin(ClientId), +%% topic = bin(Topic), +%% qos = IntQos}, +%% case emqttd_backend:add_subscription(Subscription) of +%% ok -> +%% ?PRINT_MSG("ok~n"); +%% {error, already_existed} -> +%% ?PRINT_MSG("Error: already existed~n"); +%% {error, Reason} -> +%% ?PRINT("Error: ~p~n", [Reason]) +%% end +%% end, +%% if_valid_qos(QoS, Add); +%% -subscriptions(["del", ClientId]) -> - Ok = emqttd_backend:del_subscriptions(bin(ClientId)), - ?PRINT("~p~n", [Ok]); +%% +%% subscriptions(["del", ClientId]) -> +%% Ok = emqttd_backend:del_subscriptions(bin(ClientId)), +%% ?PRINT("~p~n", [Ok]); +%% -subscriptions(["del", ClientId, Topic]) -> - Ok = emqttd_backend:del_subscription(bin(ClientId), bin(Topic)), - ?PRINT("~p~n", [Ok]); +%% +%% subscriptions(["del", ClientId, Topic]) -> +%% Ok = emqttd_backend:del_subscription(bin(ClientId), bin(Topic)), +%% ?PRINT("~p~n", [Ok]); +%% subscriptions(_) -> ?USAGE([{"subscriptions list", "List all subscriptions"}, - {"subscriptions list static", "List all static subscriptions"}, {"subscriptions show ", "Show subscriptions of a client"}, {"subscriptions add ", "Add a static subscription manually"}, {"subscriptions del ", "Delete static subscriptions manually"}, @@ -310,7 +310,7 @@ plugins(_) -> bridges(["list"]) -> foreach(fun({{Node, Topic}, _Pid}) -> ?PRINT("bridge: ~s--~s-->~s~n", [node(), Topic, Node]) - end, emqttd_bridge_sup:bridges()); + end, emqttd_bridge_sup_sup:bridges()); bridges(["options"]) -> ?PRINT_MSG("Options:~n"), @@ -322,20 +322,20 @@ bridges(["options"]) -> ?PRINT_MSG(" qos=2,prefix=abc/,suffix=/yxz,queue=1000~n"); bridges(["start", SNode, Topic]) -> - case emqttd_bridge_sup:start_bridge(list_to_atom(SNode), list_to_binary(Topic)) of + case emqttd_bridge_sup_sup:start_bridge(list_to_atom(SNode), list_to_binary(Topic)) of {ok, _} -> ?PRINT_MSG("bridge is started.~n"); {error, Error} -> ?PRINT("error: ~p~n", [Error]) end; bridges(["start", SNode, Topic, OptStr]) -> Opts = parse_opts(bridge, OptStr), - case emqttd_bridge_sup:start_bridge(list_to_atom(SNode), list_to_binary(Topic), Opts) of + case emqttd_bridge_sup_sup:start_bridge(list_to_atom(SNode), list_to_binary(Topic), Opts) of {ok, _} -> ?PRINT_MSG("bridge is started.~n"); {error, Error} -> ?PRINT("error: ~p~n", [Error]) end; bridges(["stop", SNode, Topic]) -> - case emqttd_bridge_sup:stop_bridge(list_to_atom(SNode), list_to_binary(Topic)) of + case emqttd_bridge_sup_sup:stop_bridge(list_to_atom(SNode), list_to_binary(Topic)) of ok -> ?PRINT_MSG("bridge is stopped.~n"); {error, Error} -> ?PRINT("error: ~p~n", [Error]) end; @@ -491,13 +491,13 @@ print(Routes = [#mqtt_route{topic = Topic} | _]) -> Nodes = [atom_to_list(Node) || #mqtt_route{node = Node} <- Routes], ?PRINT("~s -> ~s~n", [Topic, string:join(Nodes, ",")]); -print(Subscriptions = [#mqtt_subscription{subid = ClientId} | _]) -> - TopicTable = [io_lib:format("~s:~w", [Topic, Qos]) - || #mqtt_subscription{topic = Topic, qos = Qos} <- Subscriptions], - ?PRINT("~s -> ~s~n", [ClientId, string:join(TopicTable, ",")]); +%% print(Subscriptions = [#mqtt_subscription{subid = ClientId} | _]) -> +%% TopicTable = [io_lib:format("~s:~w", [Topic, Qos]) +%% || #mqtt_subscription{topic = Topic, qos = Qos} <- Subscriptions], +%% ?PRINT("~s -> ~s~n", [ClientId, string:join(TopicTable, ",")]); -print(Topics = [#mqtt_topic{}|_]) -> - foreach(fun print/1, Topics); +%% print(Topics = [#mqtt_topic{}|_]) -> +%% foreach(fun print/1, Topics); print(#mqtt_plugin{name = Name, version = Ver, descr = Descr, active = Active}) -> ?PRINT("Plugin(~s, version=~s, description=~s, active=~s)~n", @@ -509,15 +509,14 @@ print(#mqtt_client{client_id = ClientId, clean_sess = CleanSess, username = User [ClientId, CleanSess, Username, emqttd_net:format(Peername), emqttd_time:now_to_secs(ConnectedAt)]); -print(#mqtt_topic{topic = Topic, flags = Flags}) -> - ?PRINT("~s: ~s~n", [Topic, string:join([atom_to_list(F) || F <- Flags], ",")]); +%% print(#mqtt_topic{topic = Topic, flags = Flags}) -> +%% ?PRINT("~s: ~s~n", [Topic, string:join([atom_to_list(F) || F <- Flags], ",")]); print(#mqtt_route{topic = Topic, node = Node}) -> ?PRINT("~s -> ~s~n", [Topic, Node]); -print({{ClientId, _ClientPid}, SessInfo}) -> - InfoKeys = [clean_sess, - max_inflight, +print({ClientId, _ClientPid, CleanSess, SessInfo}) -> + InfoKeys = [max_inflight, inflight_queue, message_queue, message_dropped, @@ -529,7 +528,12 @@ print({{ClientId, _ClientPid}, SessInfo}) -> "message_queue=~w, message_dropped=~w, " "awaiting_rel=~w, awaiting_ack=~w, awaiting_comp=~w, " "created_at=~w)~n", - [ClientId | [format(Key, get_value(Key, SessInfo)) || Key <- InfoKeys]]). + [ClientId, CleanSess | [format(Key, get_value(Key, SessInfo)) || Key <- InfoKeys]]). + +print(subscription, {Sub, Topic, Opts}) when is_pid(Sub) -> + ?PRINT("~p -> ~s: ~p~n", [Sub, Topic, Opts]); +print(subscription, {Sub, Topic, Opts}) -> + ?PRINT("~s -> ~s: ~p~n", [Sub, Topic, Opts]). format(created_at, Val) -> emqttd_time:now_to_secs(Val); diff --git a/src/emqttd_client.erl b/src/emqttd_client.erl index 9d29f4c68..7b7dedcf9 100644 --- a/src/emqttd_client.erl +++ b/src/emqttd_client.erl @@ -39,7 +39,7 @@ %% Client State -record(client_state, {connection, connname, peername, peerhost, peerport, await_recv, conn_state, rate_limit, parser_fun, - proto_state, packet_opts, keepalive}). + proto_state, packet_opts, keepalive, mountpoint}). -define(INFO_KEYS, [peername, peerhost, peerport, await_recv, conn_state]). @@ -87,16 +87,20 @@ init([OriginConn, MqttEnv]) -> end, ConnName = esockd_net:format(PeerName), Self = self(), - SendFun = fun(Data) -> + + %% Send Packet... + SendFun = fun(Packet) -> + Data = emqttd_serializer:serialize(Packet), + ?LOG(debug, "SEND ~p", [Data], #client_state{connname = ConnName}), + emqttd_metrics:inc('bytes/sent', size(Data)), try Connection:async_send(Data) of true -> ok catch error:Error -> Self ! {shutdown, Error} end end, - PktOpts = proplists:get_value(packet, MqttEnv), - ParserFun = emqttd_parser:new(PktOpts), - ProtoState = emqttd_protocol:init(PeerName, SendFun, PktOpts), + ParserFun = emqttd_parser:new(MqttEnv), + ProtoState = emqttd_protocol:init(PeerName, SendFun, MqttEnv), RateLimit = proplists:get_value(rate_limit, Connection:opts()), State = run_socket(#client_state{connection = Connection, connname = ConnName, @@ -108,9 +112,8 @@ init([OriginConn, MqttEnv]) -> rate_limit = RateLimit, parser_fun = ParserFun, proto_state = ProtoState, - packet_opts = PktOpts}), - ClientOpts = proplists:get_value(client, MqttEnv), - IdleTimout = proplists:get_value(idle_timeout, ClientOpts, 10), + packet_opts = MqttEnv}), + IdleTimout = proplists:get_value(client_idle_timeout, MqttEnv, 30), gen_server:enter_loop(?MODULE, [], State, timer:seconds(IdleTimout)). handle_call(session, _From, State = #client_state{proto_state = ProtoState}) -> diff --git a/src/emqttd_cluster.erl b/src/emqttd_cluster.erl index 05c2ecf70..834de2d71 100644 --- a/src/emqttd_cluster.erl +++ b/src/emqttd_cluster.erl @@ -82,6 +82,5 @@ remove(Node) -> end. %% @doc Cluster status -status() -> - emqttd_mnesia:cluster_status(). +status() -> emqttd_mnesia:cluster_status(). diff --git a/src/emqttd_cm.erl b/src/emqttd_cm.erl index f78db2ad0..da98dedd6 100644 --- a/src/emqttd_cm.erl +++ b/src/emqttd_cm.erl @@ -26,7 +26,7 @@ %% API Exports -export([start_link/3]). --export([lookup/1, lookup_proc/1, register/1, unregister/1]). +-export([lookup/1, lookup_proc/1, reg/1, unreg/1]). %% gen_server Function Exports -export([init/1, handle_call/3, handle_cast/2, handle_info/2, @@ -44,23 +44,17 @@ %%-------------------------------------------------------------------- %% @doc Start Client Manager --spec(start_link(Pool, Id, StatsFun) -> {ok, pid()} | ignore | {error, any()} when - Pool :: atom(), - Id :: pos_integer(), - StatsFun :: fun()). +-spec(start_link(atom(), pos_integer(), fun()) -> {ok, pid()} | ignore | {error, any()}). start_link(Pool, Id, StatsFun) -> gen_server2:start_link(?MODULE, [Pool, Id, StatsFun], []). %% @doc Lookup Client by ClientId --spec(lookup(ClientId :: binary()) -> mqtt_client() | undefined). +-spec(lookup(binary()) -> mqtt_client() | undefined). lookup(ClientId) when is_binary(ClientId) -> - case ets:lookup(mqtt_client, ClientId) of - [Client] -> Client; - [] -> undefined - end. + case ets:lookup(mqtt_client, ClientId) of [Client] -> Client; [] -> undefined end. %% @doc Lookup client pid by clientId --spec(lookup_proc(ClientId :: binary()) -> pid() | undefined). +-spec(lookup_proc(binary()) -> pid() | undefined). lookup_proc(ClientId) when is_binary(ClientId) -> try ets:lookup_element(mqtt_client, ClientId, #mqtt_client.client_pid) catch @@ -68,14 +62,14 @@ lookup_proc(ClientId) when is_binary(ClientId) -> end. %% @doc Register ClientId with Pid. --spec(register(Client :: mqtt_client()) -> ok). -register(Client = #mqtt_client{client_id = ClientId}) -> - gen_server2:call(pick(ClientId), {register, Client}, 120000). +-spec(reg(mqtt_client()) -> ok). +reg(Client = #mqtt_client{client_id = ClientId}) -> + gen_server2:call(pick(ClientId), {reg, Client}, 120000). %% @doc Unregister clientId with pid. --spec(unregister(ClientId :: binary()) -> ok). -unregister(ClientId) when is_binary(ClientId) -> - gen_server2:cast(pick(ClientId), {unregister, ClientId, self()}). +-spec(unreg(binary()) -> ok). +unreg(ClientId) when is_binary(ClientId) -> + gen_server2:cast(pick(ClientId), {unreg, ClientId, self()}). pick(ClientId) -> gproc_pool:pick_worker(?POOL, ClientId). @@ -88,22 +82,16 @@ init([Pool, Id, StatsFun]) -> {ok, #state{pool = Pool, id = Id, statsfun = StatsFun, monitors = dict:new()}}. prioritise_call(Req, _From, _Len, _State) -> - case Req of - {register, _Client} -> 2; - _ -> 1 - end. + case Req of {reg, _Client} -> 2; _ -> 1 end. prioritise_cast(Msg, _Len, _State) -> - case Msg of - {unregister, _ClientId, _Pid} -> 9; - _ -> 1 - end. + case Msg of {unreg, _ClientId, _Pid} -> 9; _ -> 1 end. prioritise_info(_Msg, _Len, _State) -> 3. -handle_call({register, Client = #mqtt_client{client_id = ClientId, - client_pid = Pid}}, _From, State) -> +handle_call({reg, Client = #mqtt_client{client_id = ClientId, + client_pid = Pid}}, _From, State) -> case lookup_proc(ClientId) of Pid -> {reply, ok, State}; @@ -115,7 +103,7 @@ handle_call({register, Client = #mqtt_client{client_id = ClientId, handle_call(Req, _From, State) -> ?UNEXPECTED_REQ(Req, State). -handle_cast({unregister, ClientId, Pid}, State) -> +handle_cast({unreg, ClientId, Pid}, State) -> case lookup_proc(ClientId) of Pid -> ets:delete(mqtt_client, ClientId), diff --git a/src/emqttd_conf.erl b/src/emqttd_conf.erl new file mode 100644 index 000000000..c4a78fd9b --- /dev/null +++ b/src/emqttd_conf.erl @@ -0,0 +1,99 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2012-2016 Feng Lee . +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqttd_conf). + +-export([mqtt/0, retained/0, session/0, queue/0, bridge/0, pubsub/0]). + +mqtt() -> + [ + %% Max ClientId Length Allowed. + {max_clientid_len, emqttd:conf(mqtt_max_clientid_len, 512)}, + %% Max Packet Size Allowed, 64K by default. + {max_packet_size, emqttd:conf(mqtt_max_packet_size, 65536)}, + %% Client Idle Timeout. + {client_idle_timeout, emqttd:conf(mqtt_client_idle_timeout, 30)} + ]. + +retained() -> + [ + %% Expired after seconds, never expired if 0 + {expired_after, emqttd:conf(retained_expired_after, 0)}, + %% Max number of retained messages + {max_message_num, emqttd:conf(retained_max_message_num, 100000)}, + %% Max Payload Size of retained message + {max_playload_size, emqttd:conf(retained_max_playload_size, 65536)} + ]. + +session() -> + [ + %% Max number of QoS 1 and 2 messages that can be “inflight” at one time. + %% 0 means no limit + {max_inflight, emqttd:conf(session_max_inflight, 100)}, + + %% Retry interval for redelivering QoS1/2 messages. + {unack_retry_interval, emqttd:conf(session_unack_retry_interval, 60)}, + + %% Awaiting PUBREL Timeout + {await_rel_timeout, emqttd:conf(session_await_rel_timeout, 20)}, + + %% Max Packets that Awaiting PUBREL, 0 means no limit + {max_awaiting_rel, emqttd:conf(session_max_awaiting_rel, 0)}, + + %% Statistics Collection Interval(seconds) + {collect_interval, emqttd:conf(session_collect_interval, 0)}, + + %% Expired after 2 day (unit: minute) + {expired_after, emqttd:conf(session_expired_after, 2880)} + ]. + +queue() -> + [ + %% Type: simple | priority + {type, emqttd:conf(queue_type, simple)}, + + %% Topic Priority: 0~255, Default is 0 + {priority, emqttd:conf(queue_priority, [])}, + + %% Max queue length. Enqueued messages when persistent client disconnected, + %% or inflight window is full. + {max_length, emqttd:conf(queue_max_length, infinity)}, + + %% Low-water mark of queued messages + {low_watermark, emqttd:conf(queue_low_watermark, 0.2)}, + + %% High-water mark of queued messages + {high_watermark, emqttd:conf(queue_high_watermark, 0.6)}, + + %% Queue Qos0 messages? + {queue_qos0, emqttd:conf(queue_qos0, true)} + ]. + +bridge() -> + [ + %% TODO: Bridge Queue Size + {max_queue_len, emqttd:conf(bridge_max_queue_len, 10000)}, + + %% Ping Interval of bridge node + {ping_down_interval, emqttd:conf(bridge_ping_down_interval, 1)} + ]. + +pubsub() -> + [ + %% PubSub and Router. Default should be scheduler numbers. + {pool_size, emqttd:conf(pubsub_pool_size, 8)} + ]. + diff --git a/src/emqttd_ctl.erl b/src/emqttd_ctl.erl index 2f406bef0..1cd5d3055 100644 --- a/src/emqttd_ctl.erl +++ b/src/emqttd_ctl.erl @@ -133,3 +133,20 @@ noreply(State) -> next_seq(State = #state{seq = Seq}) -> State#state{seq = Seq + 1}. +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). +register_cmd_test_() -> + {setup, + fun() -> + {ok, InitState} = emqttd_ctl:init([]), + InitState + end, + fun(State) -> + ok = emqttd_ctl:terminate(shutdown, State) + end, + fun(State = #state{seq = Seq}) -> + emqttd_ctl:handle_cast({register_cmd, test0, {?MODULE, test0}, []}, State), + [?_assertMatch([{{0,test0},{?MODULE, test0}, []}], ets:lookup(?CMD_TAB, {Seq,test0}))] + end + }. +-endif. diff --git a/src/emqttd_guid.erl b/src/emqttd_guid.erl index d9593d3a0..03a4e6904 100644 --- a/src/emqttd_guid.erl +++ b/src/emqttd_guid.erl @@ -29,7 +29,7 @@ %% @end -module(emqttd_guid). --export([gen/0, new/0, timestamp/1]). +-export([gen/0, new/0, timestamp/1, to_hexstr/1, from_hexstr/1, to_base62/1, from_base62/1]). -define(MAX_SEQ, 16#FFFF). @@ -120,3 +120,15 @@ npid() -> PidByte3:8, PidByte4:8>>, NPid. +to_hexstr(<>) -> + list_to_binary(integer_to_list(I, 16)). + +from_hexstr(S) -> + I = list_to_integer(binary_to_list(S), 16), <>. + +to_base62(<>) -> + emqttd_base62:encode(I). + +from_base62(S) -> + I = emqttd_base62:decode(S), <>. + diff --git a/src/emqttd_message.erl b/src/emqttd_message.erl index 7a3632d0e..7dfc4fdc6 100644 --- a/src/emqttd_message.erl +++ b/src/emqttd_message.erl @@ -28,29 +28,21 @@ -export([format/1]). -%% @doc Make a message --spec(make(From, Topic, Payload) -> mqtt_message() when - From :: atom() | binary(), - Topic :: binary(), - Payload :: binary()). -make(From, Topic, Payload) -> - #mqtt_message{topic = Topic, - from = From, - payload = Payload, - timestamp = os:timestamp()}. +-type(msg_from() :: atom() | {binary(), undefined | binary()}). --spec(make(From, Qos, Topic, Payload) -> mqtt_message() when - From :: atom() | binary(), - Qos :: mqtt_qos() | mqtt_qos_name(), - Topic :: binary(), - Payload :: binary()). +%% @doc Make a message +-spec(make(msg_from(), binary(), binary()) -> mqtt_message()). +make(From, Topic, Payload) -> + make(From, ?QOS_0, Topic, Payload). + +-spec(make(msg_from(), mqtt_qos(), binary(), binary()) -> mqtt_message()). make(From, Qos, Topic, Payload) -> - #mqtt_message{msgid = msgid(?QOS_I(Qos)), - topic = Topic, + #mqtt_message{id = msgid(), from = From, qos = ?QOS_I(Qos), + topic = Topic, payload = Payload, - timestamp = os:timestamp()}. + timestamp = emqttd_time:now_to_secs()}. %% @doc Message from Packet -spec(from_packet(mqtt_packet()) -> mqtt_message()). @@ -61,14 +53,14 @@ from_packet(#mqtt_packet{header = #mqtt_packet_header{type = ?PUBLISH, variable = #mqtt_packet_publish{topic_name = Topic, packet_id = PacketId}, payload = Payload}) -> - #mqtt_message{msgid = msgid(Qos), - pktid = PacketId, - qos = Qos, - retain = Retain, - dup = Dup, - topic = Topic, - payload = Payload, - timestamp = os:timestamp()}; + #mqtt_message{id = msgid(), + pktid = PacketId, + qos = Qos, + retain = Retain, + dup = Dup, + topic = Topic, + payload = Payload, + timestamp = emqttd_time:now_to_secs()}; from_packet(#mqtt_packet_connect{will_flag = false}) -> undefined; @@ -79,15 +71,14 @@ from_packet(#mqtt_packet_connect{client_id = ClientId, will_qos = Qos, will_topic = Topic, will_msg = Msg}) -> - #mqtt_message{msgid = msgid(Qos), + #mqtt_message{id = msgid(), topic = Topic, - from = ClientId, - sender = Username, + from = {ClientId, Username}, retain = Retain, qos = Qos, dup = false, payload = Msg, - timestamp = os:timestamp()}. + timestamp = emqttd_time:now_to_secs()}. from_packet(ClientId, Packet) -> Msg = from_packet(Packet), @@ -95,12 +86,9 @@ from_packet(ClientId, Packet) -> from_packet(Username, ClientId, Packet) -> Msg = from_packet(Packet), - Msg#mqtt_message{from = ClientId, sender = Username}. + Msg#mqtt_message{from = {ClientId, Username}}. -msgid(?QOS_0) -> - undefined; -msgid(Qos) when Qos =:= ?QOS_1 orelse Qos =:= ?QOS_2 -> - emqttd_guid:gen(). +msgid() -> emqttd_guid:gen(). %% @doc Message to packet -spec(to_packet(mqtt_message()) -> mqtt_packet()). @@ -150,10 +138,16 @@ unset_flag(retain, Msg = #mqtt_message{retain = true}) -> unset_flag(Flag, Msg) when Flag =:= dup orelse Flag =:= retain -> Msg. %% @doc Format MQTT Message -format(#mqtt_message{msgid = MsgId, pktid = PktId, from = From, sender = Sender, +format(#mqtt_message{id = MsgId, pktid = PktId, from = {ClientId, Username}, qos = Qos, retain = Retain, dup = Dup, topic =Topic}) -> - io_lib:format("Message(Q~p, R~p, D~p, MsgId=~p, PktId=~p, From=~s, Sender=~s, Topic=~s)", - [i(Qos), i(Retain), i(Dup), MsgId, PktId, From, Sender, Topic]). + io_lib:format("Message(Q~p, R~p, D~p, MsgId=~p, PktId=~p, From=~s/~s, Topic=~s)", + [i(Qos), i(Retain), i(Dup), MsgId, PktId, Username, ClientId, Topic]); + +%% TODO:... +format(#mqtt_message{id = MsgId, pktid = PktId, from = From, + qos = Qos, retain = Retain, dup = Dup, topic =Topic}) -> + io_lib:format("Message(Q~p, R~p, D~p, MsgId=~p, PktId=~p, From=~s, Topic=~s)", + [i(Qos), i(Retain), i(Dup), MsgId, PktId, From, Topic]). i(true) -> 1; i(false) -> 0; diff --git a/src/emqttd_metrics.erl b/src/emqttd_metrics.erl index 79bfa95db..651e5037a 100644 --- a/src/emqttd_metrics.erl +++ b/src/emqttd_metrics.erl @@ -243,7 +243,7 @@ init([]) -> % Init metrics [create_metric(Metric) || Metric <- Metrics], % $SYS Topics for metrics - [ok = emqttd:create(topic, metric_topic(Topic)) || {_, Topic} <- Metrics], + % [ok = emqttd:create(topic, metric_topic(Topic)) || {_, Topic} <- Metrics], % Tick to publish metrics {ok, #state{tick_tref = emqttd_broker:start_tick(tick)}, hibernate}. diff --git a/src/emqttd_mod_presence.erl b/src/emqttd_mod_presence.erl index 2db444e8d..7815e88be 100644 --- a/src/emqttd_mod_presence.erl +++ b/src/emqttd_mod_presence.erl @@ -45,12 +45,13 @@ on_client_connected(ConnAck, Client = #mqtt_client{client_id = ClientId, emqttd:publish(emqttd_message:set_flag(sys, Msg)), {ok, Client}. -on_client_disconnected(Reason, ClientId, Opts) -> +on_client_disconnected(Reason, #mqtt_client{client_id = ClientId}, Opts) -> Json = mochijson2:encode([{clientid, ClientId}, {reason, reason(Reason)}, {ts, emqttd_time:now_to_secs()}]), Msg = message(qos(Opts), topic(disconnected, ClientId), Json), - emqttd:publish(emqttd_message:set_flag(sys, Msg)). + emqttd:publish(emqttd_message:set_flag(sys, Msg)), + ok. unload(_Opts) -> emqttd:unhook('client.connected', fun ?MODULE:on_client_connected/3), diff --git a/src/emqttd_mod_rewrite.erl b/src/emqttd_mod_rewrite.erl index 9109d2155..edd6ac41a 100644 --- a/src/emqttd_mod_rewrite.erl +++ b/src/emqttd_mod_rewrite.erl @@ -30,20 +30,23 @@ %%-------------------------------------------------------------------- load(Opts) -> - File = proplists:get_value(file, Opts), - {ok, Terms} = file:consult(File), - Sections = compile(Terms), - emqttd:hook('client.subscribe', fun ?MODULE:rewrite_subscribe/3, [Sections]), - emqttd:hook('client.unsubscribe', fun ?MODULE:rewrite_unsubscribe/3, [Sections]), - emqttd:hook('message.publish', fun ?MODULE:rewrite_publish/2, [Sections]). + case proplists:get_value(config, Opts) of + undefined -> + ok; + File -> + {ok, Terms} = file:consult(File), Sections = compile(Terms), + emqttd:hook('client.subscribe', fun ?MODULE:rewrite_subscribe/3, [Sections]), + emqttd:hook('client.unsubscribe', fun ?MODULE:rewrite_unsubscribe/3, [Sections]), + emqttd:hook('message.publish', fun ?MODULE:rewrite_publish/2, [Sections]) + end. -rewrite_subscribe(_ClientId, TopicTable, Sections) -> - lager:info("Rewrite subscribe: ~p", [TopicTable]), - {ok, [{match_topic(Topic, Sections), Qos} || {Topic, Qos} <- TopicTable]}. +rewrite_subscribe({_ClientId, _Username}, {Topic, Opts}, Sections) -> + lager:info("Rewrite subscribe: ~p", [{Topic, Opts}]), + {ok, {match_topic(Topic, Sections), Opts}}. -rewrite_unsubscribe(_ClientId, Topics, Sections) -> - lager:info("Rewrite unsubscribe: ~p", [Topics]), - {ok, [match_topic(Topic, Sections) || Topic <- Topics]}. +rewrite_unsubscribe({_ClientId, _Username}, {Topic, Opts}, Sections) -> + lager:info("Rewrite unsubscribe: ~p", [{Topic, Opts}]), + {ok, {match_topic(Topic, Sections), Opts}}. rewrite_publish(Message=#mqtt_message{topic = Topic}, Sections) -> %%TODO: this will not work if the client is always online. diff --git a/src/emqttd_mod_subscription.erl b/src/emqttd_mod_subscription.erl index c23ab6848..a545dcfaf 100644 --- a/src/emqttd_mod_subscription.erl +++ b/src/emqttd_mod_subscription.erl @@ -25,32 +25,22 @@ -export([load/1, on_client_connected/3, unload/1]). --record(state, {topics, backend = false}). - load(Opts) -> Topics = [{iolist_to_binary(Topic), QoS} || {Topic, QoS} <- Opts, ?IS_QOS(QoS)], - State = #state{topics = Topics, backend = lists:member(backend, Opts)}, - emqttd:hook('client.connected', fun ?MODULE:on_client_connected/3, [State]). + emqttd:hook('client.connected', fun ?MODULE:on_client_connected/3, [Topics]). on_client_connected(?CONNACK_ACCEPT, Client = #mqtt_client{client_id = ClientId, client_pid = ClientPid, - username = Username}, - #state{topics = Topics, backend = Backend}) -> + username = Username}, Topics) -> Replace = fun(Topic) -> rep(<<"$u">>, Username, rep(<<"$c">>, ClientId, Topic)) end, - TopicTable = [{Replace(Topic), Qos} || {Topic, Qos} <- with_backend(Backend, ClientId, Topics)], + TopicTable = [{Replace(Topic), Qos} || {Topic, Qos} <- Topics], emqttd_client:subscribe(ClientPid, TopicTable), {ok, Client}; on_client_connected(_ConnAck, _Client, _State) -> ok. -with_backend(false, _ClientId, TopicTable) -> - TopicTable; -with_backend(true, ClientId, TopicTable) -> - Fun = fun(#mqtt_subscription{topic = Topic, qos = Qos}) -> {Topic, Qos} end, - emqttd_opts:merge([Fun(Sub) || Sub <- emqttd_backend:lookup_subscriptions(ClientId)], TopicTable). - unload(_Opts) -> emqttd:unhook('client.connected', fun ?MODULE:on_client_connected/3). diff --git a/src/emqttd_plugins.erl b/src/emqttd_plugins.erl index 271b9bd08..360e828f0 100644 --- a/src/emqttd_plugins.erl +++ b/src/emqttd_plugins.erl @@ -27,14 +27,18 @@ %% @doc Load all plugins when the broker started. -spec(load() -> list() | {error, any()}). load() -> - case env(loaded_file) of + case emqttd:conf(plugins_loaded_file) of {ok, File} -> + ensure_file(File), with_loaded_file(File, fun(Names) -> load_plugins(Names, false) end); undefined -> %% No plugins available ignore end. +ensure_file(File) -> + case filelib:is_file(File) of false -> write_loaded([]); true -> ok end. + with_loaded_file(File, SuccFun) -> case read_loaded(File) of {ok, Names} -> @@ -56,7 +60,7 @@ load_plugins(Names, Persistent) -> %% @doc Unload all plugins before broker stopped. -spec(unload() -> list() | {error, any()}). unload() -> - case env(loaded_file) of + case emqttd:conf(plugins_loaded_file) of {ok, File} -> with_loaded_file(File, fun stop_plugins/1); undefined -> @@ -70,10 +74,10 @@ stop_plugins(Names) -> %% @doc List all available plugins -spec(list() -> [mqtt_plugin()]). list() -> - case env(plugins_dir) of - {ok, PluginsDir} -> - AppFiles = filelib:wildcard("*/ebin/*.app", PluginsDir), - Plugins = [plugin(PluginsDir, AppFile) || AppFile <- AppFiles], + case emqttd:conf(plugins_etc_dir) of + {ok, PluginsEtc} -> + CfgFiles = filelib:wildcard("*.conf", PluginsEtc), + Plugins = [plugin(CfgFile) || CfgFile <- CfgFiles], StartedApps = names(started_app), lists:map(fun(Plugin = #mqtt_plugin{name = Name}) -> case lists:member(Name, StartedApps) of @@ -85,21 +89,12 @@ list() -> [] end. -plugin(PluginsDir, AppFile0) -> - AppFile = filename:join(PluginsDir, AppFile0), - {ok, [{application, Name, Attrs}]} = file:consult(AppFile), - CfgFile = filename:join([PluginsDir, Name, "etc/plugin.config"]), - AppsEnv1 = - case filelib:is_file(CfgFile) of - true -> - {ok, [AppsEnv]} = file:consult(CfgFile), - AppsEnv; - false -> - [] - end, +plugin(CfgFile) -> + [AppName | _] = string:tokens(CfgFile, "."), + {ok, Attrs} = application:get_all_key(list_to_atom(AppName)), Ver = proplists:get_value(vsn, Attrs, "0"), Descr = proplists:get_value(description, Attrs, ""), - #mqtt_plugin{name = Name, version = Ver, config = AppsEnv1, descr = Descr}. + #mqtt_plugin{name = list_to_atom(AppName), version = Ver, descr = Descr}. %% @doc Load a Plugin -spec(load(atom()) -> ok | {error, any()}). @@ -118,31 +113,24 @@ load(PluginName) when is_atom(PluginName) -> end end. -load_plugin(#mqtt_plugin{name = Name, config = Config}, Persistent) -> - case load_app(Name, Config) of +load_plugin(#mqtt_plugin{name = Name}, Persistent) -> + case load_app(Name) of ok -> start_app(Name, fun(App) -> plugin_loaded(App, Persistent) end); {error, Error} -> {error, Error} end. -load_app(App, Config) -> +load_app(App) -> case application:load(App) of ok -> - set_config(Config); + ok; {error, {already_loaded, App}} -> - set_config(Config); + ok; {error, Error} -> {error, Error} end. -%% This trick is awesome:) -set_config([]) -> - ok; -set_config([{AppName, Envs} | Config]) -> - [application:set_env(AppName, Par, Val) || {Par, Val} <- Envs], - set_config(Config). - start_app(App, SuccFun) -> case application:ensure_all_started(App) of {ok, Started} -> @@ -238,14 +226,15 @@ plugin_unloaded(Name, true) -> end. read_loaded() -> - {ok, File} = env(loaded_file), - read_loaded(File). + case emqttd:conf(plugins_loaded_file) of + {ok, File} -> read_loaded(File); + undefined -> {error, not_found} + end. -read_loaded(File) -> - file:consult(File). +read_loaded(File) -> file:consult(File). write_loaded(AppNames) -> - {ok, File} = env(loaded_file), + {ok, File} = emqttd:conf(plugins_loaded_file), case file:open(File, [binary, write]) of {ok, Fd} -> lists:foreach(fun(Name) -> @@ -256,16 +245,3 @@ write_loaded(AppNames) -> {error, Error} end. -env(Name) -> - case application:get_env(emqttd, plugins) of - {ok, PluginsEnv} -> - case proplists:get_value(Name, PluginsEnv) of - undefined -> - undefined; - Val -> - {ok, Val} - end; - undefined -> - undefined - end. - diff --git a/src/emqttd_pmon.erl b/src/emqttd_pmon.erl new file mode 100644 index 000000000..dc9554015 --- /dev/null +++ b/src/emqttd_pmon.erl @@ -0,0 +1,50 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2016 Feng Lee . All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqttd_pmon). + +-author("Feng Lee "). + +-type(pmon() :: {?MODULE, map()}). + +-export([new/0, monitor/2, demonitor/2, erase/2]). + +new() -> {?MODULE, [maps:new()]}. + +-spec(monitor(pid(), pmon()) -> pmon()). +monitor(Pid, PM = {?MODULE, [M]}) -> + case maps:is_key(Pid, M) of + true -> + PM; + false -> + Ref = erlang:monitor(process, Pid), + {?MODULE, [maps:put(Pid, Ref, M)]} + end. + +-spec(demonitor(pid(), pmon()) -> pmon()). +demonitor(Pid, PM = {?MODULE, [M]}) -> + case maps:find(Pid, M) of + {ok, Ref} -> + erlang:demonitor(Ref, [flush]), + {?MODULE, [maps:remove(Pid, M)]}; + error -> + PM + end. + +-spec(erase(pid(), pmon()) -> pmon()). +erase(Pid, {?MODULE, [M]}) -> + {?MODULE, [maps:remove(Pid, M)]}. + diff --git a/src/emqttd_pool_sup.erl b/src/emqttd_pool_sup.erl index 0f16df67c..b47199cb0 100644 --- a/src/emqttd_pool_sup.erl +++ b/src/emqttd_pool_sup.erl @@ -41,10 +41,10 @@ start_link(Pool, Type, MFA) -> -spec(start_link(atom(), atom(), pos_integer(), mfa()) -> {ok, pid()} | {error, any()}). start_link(Pool, Type, Size, MFA) -> - supervisor:start_link({local, sup_name(Pool)}, ?MODULE, [Pool, Type, Size, MFA]). + supervisor:start_link(?MODULE, [Pool, Type, Size, MFA]). -sup_name(Pool) when is_atom(Pool) -> - list_to_atom(atom_to_list(Pool) ++ "_pool_sup"). +%% sup_name(Pool) when is_atom(Pool) -> +%% list_to_atom(atom_to_list(Pool) ++ "_pool_sup"). init([Pool, Type, Size, {M, F, Args}]) -> ensure_pool(Pool, Type, [{size, Size}]), diff --git a/src/emqttd_protocol.erl b/src/emqttd_protocol.erl index 7c8a87146..3448ce387 100644 --- a/src/emqttd_protocol.erl +++ b/src/emqttd_protocol.erl @@ -146,10 +146,10 @@ process(Packet = ?CONNECT_PACKET(Var), State0) -> State2 = maybe_set_clientid(State1), %% Start session - case emqttd_sm:start_session(CleanSess, clientid(State2)) of + case emqttd_sm:start_session(CleanSess, {clientid(State2), Username}) of {ok, Session, SP} -> %% Register the client - emqttd_cm:register(client(State2)), + emqttd_cm:reg(client(State2)), %% Start keepalive start_keepalive(KeepAlive), %% ACCEPT @@ -236,8 +236,8 @@ publish(Packet = ?PUBLISH_PACKET(?QOS_2, _PacketId), State) -> with_puback(Type, Packet = ?PUBLISH_PACKET(_Qos, PacketId), State = #proto_state{client_id = ClientId, - username = Username, - session = Session}) -> + username = Username, + session = Session}) -> Msg = emqttd_message:from_packet(Username, ClientId, Packet), case emqttd_session:publish(Session, Msg) of ok -> @@ -247,19 +247,16 @@ with_puback(Type, Packet = ?PUBLISH_PACKET(_Qos, PacketId), end. -spec(send(mqtt_message() | mqtt_packet(), proto_state()) -> {ok, proto_state()}). -send(Msg, State = #proto_state{client_id = ClientId}) +send(Msg, State = #proto_state{client_id = ClientId, username = Username}) when is_record(Msg, mqtt_message) -> - emqttd:run_hooks('message.delivered', [ClientId], Msg), + emqttd:run_hooks('message.delivered', [{ClientId, Username}], Msg), send(emqttd_message:to_packet(Msg), State); send(Packet, State = #proto_state{sendfun = SendFun}) when is_record(Packet, mqtt_packet) -> trace(send, Packet, State), emqttd_metrics:sent(Packet), - Data = emqttd_serializer:serialize(Packet), - ?LOG(debug, "SEND ~p", [Data], State), - emqttd_metrics:inc('bytes/sent', size(Data)), - SendFun(Data), + SendFun(Packet), {ok, State}. trace(recv, Packet, ProtoState) -> @@ -277,15 +274,16 @@ shutdown(_Error, #proto_state{client_id = undefined}) -> shutdown(conflict, #proto_state{client_id = _ClientId}) -> %% let it down - %% emqttd_cm:unregister(ClientId); + %% emqttd_cm:unreg(ClientId); ignore; -shutdown(Error, State = #proto_state{client_id = ClientId, will_msg = WillMsg}) -> +shutdown(Error, State = #proto_state{will_msg = WillMsg}) -> ?LOG(info, "Shutdown for ~p", [Error], State), - send_willmsg(ClientId, WillMsg), - emqttd:run_hooks('client.disconnected', [Error], ClientId), + Client = client(State), + send_willmsg(Client, WillMsg), + emqttd:run_hooks('client.disconnected', [Error], Client), %% let it down - %% emqttd_cm:unregister(ClientId). + %% emqttd_cm:unreg(ClientId). ok. willmsg(Packet) when is_record(Packet, mqtt_packet_connect) -> @@ -301,10 +299,10 @@ maybe_set_clientid(State = #proto_state{client_id = NullId}) maybe_set_clientid(State) -> State. -send_willmsg(_ClientId, undefined) -> +send_willmsg(_Client, undefined) -> ignore; -send_willmsg(ClientId, WillMsg) -> - emqttd:publish(WillMsg#mqtt_message{from = ClientId}). +send_willmsg(#mqtt_client{client_id = ClientId, username = Username}, WillMsg) -> + emqttd:publish(WillMsg#mqtt_message{from = {ClientId, Username}}). start_keepalive(0) -> ignore; @@ -397,13 +395,16 @@ validate_qos(_) -> %% PUBLISH ACL is cached in process dictionary. check_acl(publish, Topic, Client) -> - case get({acl, publish, Topic}) of - undefined -> + IfCache = emqttd:conf(cache_acl, true), + case {IfCache, get({acl, publish, Topic})} of + {true, undefined} -> AllowDeny = emqttd_access_control:check_acl(Client, publish, Topic), put({acl, publish, Topic}, AllowDeny), AllowDeny; - AllowDeny -> - AllowDeny + {true, AllowDeny} -> + AllowDeny; + {false, _} -> + emqttd_access_control:check_acl(Client, publish, Topic) end; check_acl(subscribe, Topic, Client) -> diff --git a/src/emqttd_pubsub.erl b/src/emqttd_pubsub.erl index 5ccb6bc40..1af6274bf 100644 --- a/src/emqttd_pubsub.erl +++ b/src/emqttd_pubsub.erl @@ -20,125 +20,83 @@ -include("emqttd.hrl"). --include("emqttd_protocol.hrl"). - -include("emqttd_internal.hrl"). -%% Mnesia Callbacks --export([mnesia/1]). - --boot_mnesia({mnesia, [boot]}). --copy_mnesia({mnesia, [copy]}). - %% API Exports --export([start_link/3, create_topic/1, lookup_topic/1]). - --export([subscribe/2, unsubscribe/2, publish/2, dispatch/2, +-export([start_link/3, subscribe/2, unsubscribe/2, publish/2, async_subscribe/2, async_unsubscribe/2]). +-export([subscribers/1]). + %% gen_server. -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). -record(state, {pool, id, env}). -%%-------------------------------------------------------------------- -%% Mnesia callbacks -%%-------------------------------------------------------------------- +-define(PUBSUB, ?MODULE). -mnesia(boot) -> - ok = emqttd_mnesia:create_table(topic, [ - {ram_copies, [node()]}, - {record_name, mqtt_topic}, - {attributes, record_info(fields, mqtt_topic)}]); - -mnesia(copy) -> - ok = emqttd_mnesia:copy_table(topic). - -%%-------------------------------------------------------------------- -%% Start PubSub -%%-------------------------------------------------------------------- - -%% @doc Start one pubsub --spec(start_link(Pool, Id, Env) -> {ok, pid()} | ignore | {error, any()} when - Pool :: atom(), - Id :: pos_integer(), - Env :: list(tuple())). +-spec(start_link(atom(), pos_integer(), [tuple()]) -> {ok, pid()} | ignore | {error, any()}). start_link(Pool, Id, Env) -> gen_server2:start_link({local, ?PROC_NAME(?MODULE, Id)}, ?MODULE, [Pool, Id, Env], []). -%% @doc Create a Topic. --spec(create_topic(binary()) -> ok | {error, any()}). -create_topic(Topic) when is_binary(Topic) -> - case mnesia:transaction(fun add_topic_/2, [Topic, [static]]) of - {atomic, ok} -> ok; - {aborted, Error} -> {error, Error} - end. +-spec(subscribe(binary(), emqttd:subscriber()) -> ok). +subscribe(Topic, Subscriber) -> + call(pick(Topic), {subscribe, Topic, Subscriber}). -%% @doc Lookup a Topic. --spec(lookup_topic(binary()) -> list(mqtt_topic())). -lookup_topic(Topic) when is_binary(Topic) -> - mnesia:dirty_read(topic, Topic). +-spec(async_subscribe(binary(), emqttd:subscriber()) -> ok). +async_subscribe(Topic, Subscriber) -> + cast(pick(Topic), {subscribe, Topic, Subscriber}). -%%-------------------------------------------------------------------- -%% PubSub API -%%-------------------------------------------------------------------- - -%% @doc Subscribe a Topic --spec(subscribe(binary(), pid()) -> ok). -subscribe(Topic, SubPid) when is_binary(Topic) -> - call(pick(Topic), {subscribe, Topic, SubPid}). - -%% @doc Asynchronous Subscribe --spec(async_subscribe(binary(), pid()) -> ok). -async_subscribe(Topic, SubPid) when is_binary(Topic) -> - cast(pick(Topic), {subscribe, Topic, SubPid}). - -%% @doc Publish message to Topic. --spec(publish(binary(), any()) -> any()). +-spec(publish(binary(), any()) -> {ok, mqtt_delivery()} | ignore). publish(Topic, Msg) -> - lists:foreach( - fun(#mqtt_route{topic = To, node = Node}) when Node =:= node() -> - ?MODULE:dispatch(To, Msg); - (#mqtt_route{topic = To, node = Node}) -> - rpc:cast(Node, ?MODULE, dispatch, [To, Msg]) - end, emqttd_router:lookup(Topic)). + route(emqttd_router:match(Topic), delivery(Msg)). + +route([], _Delivery) -> + ignore; + +%% Dispatch on the local node +route([#mqtt_route{topic = To, node = Node}], + Delivery = #mqtt_delivery{flows = Flows}) when Node =:= node() -> + dispatch(To, Delivery#mqtt_delivery{flows = [{route, Node, To} | Flows]}); + +%% Forward to other nodes +route([#mqtt_route{topic = To, node = Node}], Delivery = #mqtt_delivery{flows = Flows}) -> + forward(Node, To, Delivery#mqtt_delivery{flows = [{route, Node, To}|Flows]}); + +route(Routes, Delivery) -> + {ok, lists:foldl(fun(Route, DelAcc) -> + {ok, DelAcc1} = route([Route], DelAcc), DelAcc1 + end, Delivery, Routes)}. + +delivery(Msg) -> #mqtt_delivery{sender = self(), message = Msg, flows = []}. + +%% @doc Forward message to another node... +forward(Node, To, Delivery) -> + rpc:cast(Node, ?PUBSUB, dispatch, [To, Delivery]), {ok, Delivery}. %% @doc Dispatch Message to Subscribers --spec(dispatch(binary(), mqtt_message()) -> ok). -dispatch(Queue = <<"$queue/", _Q/binary>>, Msg) -> - case subscribers(Queue) of - [] -> - dropped(Queue); - [SubPid] -> - SubPid ! {dispatch, Queue, Msg}; - SubPids -> - Idx = crypto:rand_uniform(1, length(SubPids) + 1), - SubPid = lists:nth(Idx, SubPids), - SubPid ! {dispatch, Queue, Msg} - end; - -dispatch(Topic, Msg) -> +-spec(dispatch(binary(), mqtt_delivery()) -> mqtt_delivery()). +dispatch(Topic, Delivery = #mqtt_delivery{message = Msg, flows = Flows}) -> case subscribers(Topic) of [] -> - dropped(Topic); - [SubPid] -> - SubPid ! {dispatch, Topic, Msg}; - SubPids -> - lists:foreach(fun(SubPid) -> - SubPid ! {dispatch, Topic, Msg} - end, SubPids) + dropped(Topic), {ok, Delivery}; + [Sub] -> %% optimize? + dispatch(Sub, Topic, Msg), + {ok, Delivery#mqtt_delivery{flows = [{dispatch, Topic, 1} | Flows]}}; + Subscribers -> + Flows1 = [{dispatch, Topic, length(Subscribers)} | Flows], + lists:foreach(fun(Sub) -> dispatch(Sub, Topic, Msg) end, Subscribers), + {ok, Delivery#mqtt_delivery{flows = Flows1}} end. -%% @private -%% @doc Find all subscribers +dispatch(Pid, Topic, Msg) when is_pid(Pid) -> + Pid ! {dispatch, Topic, Msg}; +dispatch(SubId, Topic, Msg) when is_binary(SubId) -> + emqttd_sm:dispatch(SubId, Topic, Msg). + subscribers(Topic) -> - case ets:member(subscriber, Topic) of - true -> %% faster then lookup? - try ets:lookup_element(subscriber, Topic, 2) catch error:badarg -> [] end; - false -> - [] - end. + try ets:lookup_element(mqtt_subscriber, Topic, 2) catch error:badarg -> [] end. %% @private %% @doc Ingore $SYS Messages. @@ -147,51 +105,45 @@ dropped(<<"$SYS/", _/binary>>) -> dropped(_Topic) -> emqttd_metrics:inc('messages/dropped'). -%% @doc Unsubscribe --spec(unsubscribe(binary(), pid()) -> ok). -unsubscribe(Topic, SubPid) when is_binary(Topic) -> - call(pick(Topic), {unsubscribe, Topic, SubPid}). +-spec(unsubscribe(binary(), emqttd:subscriber()) -> ok). +unsubscribe(Topic, Subscriber) -> + call(pick(Topic), {unsubscribe, Topic, Subscriber}). -%% @doc Asynchronous Unsubscribe --spec(async_unsubscribe(binary(), pid()) -> ok). -async_unsubscribe(Topic, SubPid) when is_binary(Topic) -> - cast(pick(Topic), {unsubscribe, Topic, SubPid}). +-spec(async_unsubscribe(binary(), emqttd:subscriber()) -> ok). +async_unsubscribe(Topic, Subscriber) -> + cast(pick(Topic), {unsubscribe, Topic, Subscriber}). -call(PubSub, Req) when is_pid(PubSub) -> - gen_server2:call(PubSub, Req, infinity). +call(Server, Req) -> + gen_server2:call(Server, Req, infinity). -cast(PubSub, Msg) when is_pid(PubSub) -> - gen_server2:cast(PubSub, Msg). +cast(Server, Msg) -> + gen_server2:cast(Server, Msg). pick(Topic) -> gproc_pool:pick_worker(pubsub, Topic). -%%-------------------------------------------------------------------- -%% gen_server Callbacks -%%-------------------------------------------------------------------- - init([Pool, Id, Env]) -> ?GPROC_POOL(join, Pool, Id), {ok, #state{pool = Pool, id = Id, env = Env}}. -handle_call({subscribe, Topic, SubPid}, _From, State) -> - add_subscriber_(Topic, SubPid), - {reply, ok, setstats(State)}; +handle_call({subscribe, Topic, Subscriber}, _From, State) -> + add_subscriber_(Topic, Subscriber), + {reply, ok, setstats(State)}; -handle_call({unsubscribe, Topic, SubPid}, _From, State) -> - del_subscriber_(Topic, SubPid), - {reply, ok, setstats(State)}; +handle_call({unsubscribe, Topic, Subscriber}, _From, State) -> + del_subscriber_(Topic, Subscriber), + {reply, ok, setstats(State)}; handle_call(Req, _From, State) -> ?UNEXPECTED_REQ(Req, State). -handle_cast({subscribe, Topic, SubPid}, State) -> - add_subscriber_(Topic, SubPid), - {noreply, setstats(State)}; +handle_cast({subscribe, Topic, Subscriber}, State) -> + add_subscriber_(Topic, Subscriber), + {noreply, setstats(State)}; -handle_cast({unsubscribe, Topic, SubPid}, State) -> - del_subscriber_(Topic, SubPid), - {noreply, setstats(State)}; +handle_cast({unsubscribe, Topic, Subscriber}, State) -> + del_subscriber_(Topic, Subscriber), + {noreply, setstats(State)}; handle_cast(Msg, State) -> ?UNEXPECTED_MSG(Msg, State). @@ -203,68 +155,23 @@ terminate(_Reason, #state{pool = Pool, id = Id}) -> ?GPROC_POOL(leave, Pool, Id). code_change(_OldVsn, State, _Extra) -> - {ok, State}. + {ok, State}. %%-------------------------------------------------------------------- -%% Internal Functions +%% Internel Functions %%-------------------------------------------------------------------- -add_subscriber_(Topic, SubPid) -> - case ets:member(subscriber, Topic) of - false -> - mnesia:transaction(fun add_topic_route_/2, [Topic, node()]), - setstats(topic); - true -> - ok - end, - ets:insert(subscriber, {Topic, SubPid}). +add_subscriber_(Topic, Subscriber) -> + (not ets:member(mqtt_subscriber, Topic)) + andalso emqttd_router:add_route(Topic), + ets:insert(mqtt_subscriber, {Topic, Subscriber}). -del_subscriber_(Topic, SubPid) -> - ets:delete_object(subscriber, {Topic, SubPid}), - case ets:lookup(subscriber, Topic) of - [] -> - mnesia:transaction(fun del_topic_route_/2, [Topic, node()]), - setstats(topic); - [_|_] -> - ok - end. +del_subscriber_(Topic, Subscriber) -> + ets:delete_object(mqtt_subscriber, {Topic, Subscriber}), + (not ets:member(mqtt_subscriber, Topic)) + andalso emqttd_router:del_route(Topic). -add_topic_route_(Topic, Node) -> - add_topic_(Topic), emqttd_router:add_route(Topic, Node). - -add_topic_(Topic) -> - add_topic_(Topic, []). - -add_topic_(Topic, Flags) -> - Record = #mqtt_topic{topic = Topic, flags = Flags}, - case mnesia:wread({topic, Topic}) of - [] -> mnesia:write(topic, Record, write); - [_] -> ok - end. - -del_topic_route_(Topic, Node) -> - emqttd_router:del_route(Topic, Node), del_topic_(Topic). - -del_topic_(Topic) -> - case emqttd_router:has_route(Topic) of - true -> ok; - false -> do_del_topic_(Topic) - end. - -do_del_topic_(Topic) -> - case mnesia:wread({topic, Topic}) of - [#mqtt_topic{flags = []}] -> - mnesia:delete(topic, Topic, write); - _ -> - ok - end. - -setstats(State) when is_record(State, state) -> - setstats(subscriber), State; - -setstats(topic) -> - emqttd_stats:setstats('topics/count', 'topics/max', mnesia:table_info(topic, size)); - -setstats(subscriber) -> - emqttd_stats:setstats('subscribers/count', 'subscribers/max', ets:info(subscriber, size)). +setstats(State) -> + emqttd_stats:setstats('subscribers/count', 'subscribers/max', + ets:info(mqtt_subscriber, size)), State. diff --git a/src/emqttd_pubsub_sup.erl b/src/emqttd_pubsub_sup.erl index 28ff8033b..f87f87d48 100644 --- a/src/emqttd_pubsub_sup.erl +++ b/src/emqttd_pubsub_sup.erl @@ -19,58 +19,66 @@ -behaviour(supervisor). --include("emqttd.hrl"). - --define(CONCURRENCY_OPTS, [{read_concurrency, true}, {write_concurrency, true}]). - %% API -export([start_link/0, pubsub_pool/0]). %% Supervisor callbacks -export([init/1]). +-define(CONCURRENCY_OPTS, [{read_concurrency, true}, {write_concurrency, true}]). + +%%-------------------------------------------------------------------- +%% API +%%-------------------------------------------------------------------- + start_link() -> - supervisor:start_link({local, ?MODULE}, ?MODULE, [emqttd_broker:env(pubsub)]). + supervisor:start_link({local, ?MODULE}, ?MODULE, [emqttd_conf:pubsub()]). pubsub_pool() -> - hd([Pid|| {pubsub_pool, Pid, _, _} <- supervisor:which_children(?MODULE)]). + hd([Pid || {pubsub_pool, Pid, _, _} <- supervisor:which_children(?MODULE)]). + +%%-------------------------------------------------------------------- +%% Supervisor callbacks +%%-------------------------------------------------------------------- init([Env]) -> + %% Create ETS Tables + [create_tab(Tab) || Tab <- [mqtt_subproperty, mqtt_subscriber, mqtt_subscription]], - %% Create ETS Tabs - create_tab(subscriber), create_tab(subscribed), + {ok, { {one_for_all, 10, 3600}, [pool_sup(pubsub, Env), pool_sup(server, Env)]} }. - %% Router - Router = {router, {emqttd_router, start_link, []}, - permanent, 5000, worker, [emqttd_router]}, - - %% PubSub Pool Sup - PubSubMFA = {emqttd_pubsub, start_link, [Env]}, - PubSubPoolSup = emqttd_pool_sup:spec(pubsub_pool, [pubsub, hash, pool_size(Env), PubSubMFA]), - - %% Server Pool Sup - ServerMFA = {emqttd_server, start_link, [Env]}, - ServerPoolSup = emqttd_pool_sup:spec(server_pool, [server, hash, pool_size(Env), ServerMFA]), - - {ok, {{one_for_all, 5, 60}, [Router, PubSubPoolSup, ServerPoolSup]}}. +%%-------------------------------------------------------------------- +%% Pool +%%-------------------------------------------------------------------- pool_size(Env) -> Schedulers = erlang:system_info(schedulers), proplists:get_value(pool_size, Env, Schedulers). -create_tab(subscriber) -> - %% subscriber: Topic -> Pid1, Pid2, ..., PidN - %% duplicate_bag: o(1) insert - ensure_tab(subscriber, [public, named_table, duplicate_bag | ?CONCURRENCY_OPTS]); +pool_sup(Name, Env) -> + Pool = list_to_atom(atom_to_list(Name) ++ "_pool"), + MFA = {emqttd:adapter(Name), start_link, [Env]}, + emqttd_pool_sup:spec(Pool, [Name, hash, pool_size(Env), MFA]). -create_tab(subscribed) -> - %% subscribed: Pid -> Topic1, Topic2, ..., TopicN + +%%-------------------------------------------------------------------- +%% Create PubSub Tables +%%-------------------------------------------------------------------- + +create_tab(mqtt_subproperty) -> + %% Subproperty: {Topic, Sub} -> [{qos, 1}] + ensure_tab(mqtt_subproperty, [public, named_table, set | ?CONCURRENCY_OPTS]); + +create_tab(mqtt_subscriber) -> + %% Subscriber: Topic -> Sub1, Sub2, Sub3, ..., SubN + %% duplicate_bag: o(1) insert + ensure_tab(mqtt_subscriber, [public, named_table, duplicate_bag | ?CONCURRENCY_OPTS]); + +create_tab(mqtt_subscription) -> + %% Subscription: Sub -> Topic1, Topic2, Topic3, ..., TopicN %% bag: o(n) insert - ensure_tab(subscribed, [public, named_table, bag | ?CONCURRENCY_OPTS]). + ensure_tab(mqtt_subscription, [public, named_table, bag | ?CONCURRENCY_OPTS]). ensure_tab(Tab, Opts) -> - case ets:info(Tab, name) of - undefined -> ets:new(Tab, Opts); - _ -> ok - end. + case ets:info(Tab, name) of undefined -> ets:new(Tab, Opts); _ -> ok end. diff --git a/src/emqttd_retainer.erl b/src/emqttd_retainer.erl index e0bb631ba..435bd7abc 100644 --- a/src/emqttd_retainer.erl +++ b/src/emqttd_retainer.erl @@ -23,8 +23,16 @@ -include("emqttd_internal.hrl"). +-include_lib("stdlib/include/ms_transform.hrl"). + +%% Mnesia Callbacks +-export([mnesia/1]). + +-boot_mnesia({mnesia, [boot]}). +-copy_mnesia({mnesia, [copy]}). + %% API Function Exports --export([retain/1, dispatch/2]). +-export([retain/1, read_messages/1, dispatch/2]). %% API Function Exports -export([start_link/0]). @@ -33,8 +41,26 @@ -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). +-record(retained_message, {topic, msg}). + -record(state, {stats_fun, expired_after, stats_timer, expire_timer}). +%%-------------------------------------------------------------------- +%% Mnesia callbacks +%%-------------------------------------------------------------------- + +mnesia(boot) -> + ok = emqttd_mnesia:create_table(retained_message, [ + {type, ordered_set}, + {disc_copies, [node()]}, + {record_name, retained_message}, + {attributes, record_info(fields, retained_message)}, + {storage_properties, [{ets, [compressed]}, + {dets, [{auto_save, 1000}]}]}]); + +mnesia(copy) -> + ok = emqttd_mnesia:copy_table(retained_message). + %%-------------------------------------------------------------------- %% API %%-------------------------------------------------------------------- @@ -50,14 +76,14 @@ retain(#mqtt_message{retain = false}) -> ignore; %% RETAIN flag set to 1 and payload containing zero bytes retain(#mqtt_message{retain = true, topic = Topic, payload = <<>>}) -> - emqttd_backend:delete_message(Topic); + delete_message(Topic); retain(Msg = #mqtt_message{topic = Topic, retain = true, payload = Payload}) -> - TabSize = emqttd_backend:retained_count(), + TabSize = retained_count(), case {TabSize < limit(table), size(Payload) < limit(payload)} of {true, true} -> - emqttd_backend:retain_message(Msg), - emqttd_metrics:set('messages/retained', emqttd_backend:retained_count()); + retain_message(Msg), + emqttd_metrics:set('messages/retained', retained_count()); {false, _}-> lager:error("Cannot retain message(topic=~s) for table is full!", [Topic]); {_, false}-> @@ -71,7 +97,7 @@ limit(payload) -> env(max_playload_size). env(Key) -> case get({retained, Key}) of undefined -> - Env = emqttd_broker:env(retained), + Env = emqttd_conf:retained(), Val = proplists:get_value(Key, Env), put({retained, Key}, Val), Val; Val -> @@ -82,8 +108,8 @@ env(Key) -> -spec(dispatch(Topic :: binary(), CPid :: pid()) -> any()). dispatch(Topic, CPid) when is_binary(Topic) -> Msgs = case emqttd_topic:wildcard(Topic) of - false -> emqttd_backend:read_messages(Topic); - true -> emqttd_backend:match_messages(Topic) + false -> read_messages(Topic); + true -> match_messages(Topic) end, lists:foreach(fun(Msg) -> CPid ! {dispatch, Topic, Msg} end, lists:reverse(Msgs)). @@ -113,7 +139,7 @@ handle_cast(Msg, State) -> ?UNEXPECTED_MSG(Msg, State). handle_info(stats, State = #state{stats_fun = StatsFun}) -> - StatsFun(emqttd_backend:retained_count()), + StatsFun(retained_count()), {noreply, State, hibernate}; handle_info(expire, State = #state{expired_after = Never}) @@ -121,7 +147,7 @@ handle_info(expire, State = #state{expired_after = Never}) {noreply, State, hibernate}; handle_info(expire, State = #state{expired_after = ExpiredAfter}) -> - emqttd_backend:expire_messages(emqttd_time:now_to_secs() - ExpiredAfter), + expire_messages(emqttd_time:now_to_secs() - ExpiredAfter), {noreply, State, hibernate}; handle_info(Info, State) -> @@ -134,3 +160,47 @@ terminate(_Reason, _State = #state{stats_timer = TRef1, expire_timer = TRef2}) - code_change(_OldVsn, State, _Extra) -> {ok, State}. +%%-------------------------------------------------------------------- +%% Internal Functions +%%-------------------------------------------------------------------- + +-spec(retain_message(mqtt_message()) -> ok). +retain_message(Msg = #mqtt_message{topic = Topic}) -> + mnesia:dirty_write(#retained_message{topic = Topic, msg = Msg}). + +-spec(read_messages(binary()) -> [mqtt_message()]). +read_messages(Topic) -> + [Msg || #retained_message{msg = Msg} <- mnesia:dirty_read(retained_message, Topic)]. + +-spec(match_messages(binary()) -> [mqtt_message()]). +match_messages(Filter) -> + %% TODO: optimize later... + Fun = fun(#retained_message{topic = Name, msg = Msg}, Acc) -> + case emqttd_topic:match(Name, Filter) of + true -> [Msg|Acc]; + false -> Acc + end + end, + mnesia:async_dirty(fun mnesia:foldl/3, [Fun, [], retained_message]). + +-spec(delete_message(binary()) -> ok). +delete_message(Topic) -> + mnesia:dirty_delete(retained_message, Topic). + +-spec(expire_messages(pos_integer()) -> any()). +expire_messages(Time) when is_integer(Time) -> + mnesia:transaction( + fun() -> + Match = ets:fun2ms( + fun(#retained_message{topic = Topic, msg = #mqtt_message{timestamp = Ts}}) + when Time > Ts -> Topic + end), + Topics = mnesia:select(retained_message, Match, write), + lists:foreach(fun(<<"$SYS/", _/binary>>) -> ok; %% ignore $SYS/# messages + (Topic) -> mnesia:delete({retained_message, Topic}) + end, Topics) + end). + +-spec(retained_count() -> non_neg_integer()). +retained_count() -> mnesia:table_info(retained_message, size). + diff --git a/src/emqttd_router.erl b/src/emqttd_router.erl index 732cb448d..a58e599e0 100644 --- a/src/emqttd_router.erl +++ b/src/emqttd_router.erl @@ -16,6 +16,8 @@ -module(emqttd_router). +-author("Feng Lee "). + -behaviour(gen_server). -include("emqttd.hrl"). @@ -27,55 +29,73 @@ -copy_mnesia({mnesia, [copy]}). %% Start/Stop --export([start_link/0, stop/0]). +-export([start_link/0, topics/0, local_topics/0, stop/0]). %% Route APIs --export([add_route/1, add_route/2, add_routes/1, lookup/1, print/1, +-export([add_route/1, add_route/2, add_routes/1, match/1, print/1, del_route/1, del_route/2, del_routes/1, has_route/1]). +%% Local Route API +-export([add_local_route/1, del_local_route/1, match_local/1]). + %% gen_server Function Exports -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). +-export([dump/0]). + -record(state, {stats_timer}). +-define(ROUTER, ?MODULE). + %%-------------------------------------------------------------------- %% Mnesia Bootstrap %%-------------------------------------------------------------------- mnesia(boot) -> - ok = emqttd_mnesia:create_table(route, [ + ok = emqttd_mnesia:create_table(mqtt_topic, [ + {ram_copies, [node()]}, + {record_name, mqtt_topic}, + {attributes, record_info(fields, mqtt_topic)}]), + ok = emqttd_mnesia:create_table(mqtt_route, [ {type, bag}, {ram_copies, [node()]}, {record_name, mqtt_route}, {attributes, record_info(fields, mqtt_route)}]); mnesia(copy) -> - ok = emqttd_mnesia:copy_table(route, ram_copies). + ok = emqttd_mnesia:copy_table(topic), + ok = emqttd_mnesia:copy_table(mqtt_route, ram_copies). %%-------------------------------------------------------------------- %% Start the Router %%-------------------------------------------------------------------- start_link() -> - gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). + gen_server:start_link({local, ?ROUTER}, ?MODULE, [], []). %%-------------------------------------------------------------------- %% API %%-------------------------------------------------------------------- -%% @doc Lookup Routes. --spec(lookup(Topic:: binary()) -> [mqtt_route()]). -lookup(Topic) when is_binary(Topic) -> +topics() -> + mnesia:dirty_all_keys(mqtt_route). + +local_topics() -> + ets:select(mqtt_local_route, [{{'$1', '_'}, [], ['$1']}]). + +%% @doc Match Routes. +-spec(match(Topic:: binary()) -> [mqtt_route()]). +match(Topic) when is_binary(Topic) -> Matched = mnesia:async_dirty(fun emqttd_trie:match/1, [Topic]), %% Optimize: route table will be replicated to all nodes. - lists:append([ets:lookup(route, To) || To <- [Topic | Matched]]). + lists:append([ets:lookup(mqtt_route, To) || To <- [Topic | Matched]]). %% @doc Print Routes. -spec(print(Topic :: binary()) -> [ok]). print(Topic) -> [io:format("~s -> ~s~n", [To, Node]) || - #mqtt_route{topic = To, node = Node} <- lookup(Topic)]. + #mqtt_route{topic = To, node = Node} <- match(Topic)]. %% @doc Add Route -spec(add_route(binary() | mqtt_route()) -> ok | {error, Reason :: any()}). @@ -99,17 +119,18 @@ add_routes(Routes) -> %% @private add_route_(Route = #mqtt_route{topic = Topic}) -> - case mnesia:wread({route, Topic}) of + case mnesia:wread({mqtt_route, Topic}) of [] -> case emqttd_topic:wildcard(Topic) of true -> emqttd_trie:insert(Topic); false -> ok end, - mnesia:write(route, Route, write); + mnesia:write(Route), + mnesia:write(#mqtt_topic{topic = Topic}); Records -> case lists:member(Route, Records) of true -> ok; - false -> mnesia:write(route, Route, write) + false -> mnesia:write(Route) end end. @@ -134,27 +155,28 @@ del_routes(Routes) -> end. del_route_(Route = #mqtt_route{topic = Topic}) -> - case mnesia:wread({route, Topic}) of + case mnesia:wread({mqtt_route, Topic}) of [] -> ok; [Route] -> %% Remove route and trie - mnesia:delete_object(route, Route, write), + mnesia:delete_object(Route), case emqttd_topic:wildcard(Topic) of true -> emqttd_trie:delete(Topic); false -> ok - end; + end, + mnesia:delete({mqtt_topic, Topic}); _More -> %% Remove route only - mnesia:delete_object(route, Route, write) + mnesia:delete_object(Route) end. %% @doc Has Route? -spec(has_route(binary()) -> boolean()). has_route(Topic) -> Routes = case mnesia:is_transaction() of - true -> mnesia:read(route, Topic); - false -> mnesia:dirty_read(route, Topic) + true -> mnesia:read(mqtt_route, Topic); + false -> mnesia:dirty_read(mqtt_route, Topic) end, length(Routes) > 0. @@ -166,7 +188,28 @@ trans(Fun) -> {aborted, Error} -> {error, Error} end. -stop() -> gen_server:call(?MODULE, stop). +%%-------------------------------------------------------------------- +%% Local Route API +%%-------------------------------------------------------------------- + +-spec(add_local_route(binary()) -> ok). +add_local_route(Topic) -> + gen_server:cast(?ROUTER, {add_local_route, Topic}). + +-spec(del_local_route(binary()) -> ok). +del_local_route(Topic) -> + gen_server:cast(?ROUTER, {del_local_route, Topic}). + +-spec(match_local(binary()) -> [mqtt_route()]). +match_local(Name) -> + [#mqtt_route{topic = {local, Filter}, node = Node} + || {Filter, Node} <- ets:tab2list(mqtt_local_route), + emqttd_topic:match(Name, Filter)]. + +dump() -> + [{route, ets:tab2list(mqtt_route)}, {local_route, ets:tab2list(mqtt_local_route)}]. + +stop() -> gen_server:call(?ROUTER, stop). %%-------------------------------------------------------------------- %% gen_server Callbacks @@ -174,6 +217,7 @@ stop() -> gen_server:call(?MODULE, stop). init([]) -> mnesia:subscribe(system), + ets:new(mqtt_local_route, [set, named_table, protected]), {ok, TRef} = timer:send_interval(timer:seconds(1), stats), {ok, #state{stats_timer = TRef}}. @@ -183,6 +227,15 @@ handle_call(stop, _From, State) -> handle_call(_Req, _From, State) -> {reply, ignore, State}. +handle_cast({add_local_route, Topic}, State) -> + %% why node()...? + ets:insert(mqtt_local_route, {Topic, node()}), + {noreply, State}; + +handle_cast({del_local_route, Topic}, State) -> + ets:delete(mqtt_local_route, Topic), + {noreply, State}; + handle_cast(_Msg, State) -> {noreply, State}. diff --git a/src/emqttd_server.erl b/src/emqttd_server.erl index 1466d8a7a..e9b2d0eb5 100644 --- a/src/emqttd_server.erl +++ b/src/emqttd_server.erl @@ -16,6 +16,8 @@ -module(emqttd_server). +-author("Feng Lee "). + -behaviour(gen_server2). -include("emqttd.hrl"). @@ -24,49 +26,31 @@ -include("emqttd_internal.hrl"). -%% Mnesia Callbacks --export([mnesia/1]). - --boot_mnesia({mnesia, [boot]}). --copy_mnesia({mnesia, [copy]}). - -%% API Exports -export([start_link/3]). -%% PubSub API --export([subscribe/1, subscribe/3, publish/1, unsubscribe/1, unsubscribe/3, - lookup_subscription/1, update_subscription/4]). +%% PubSub API. +-export([subscribe/1, subscribe/2, subscribe/3, publish/1, + unsubscribe/1, unsubscribe/2]). -%% gen_server Function Exports +%% Async PubSub API. +-export([async_subscribe/1, async_subscribe/2, async_subscribe/3, + async_unsubscribe/1, async_unsubscribe/2]). + +%% Management API. +-export([setqos/3, subscriptions/1, subscribers/1, is_subscribed/2, + subscriber_down/1]). + +%% Debug API +-export([dump/0]). + +%% gen_server. -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2, code_change/3]). --record(state, {pool, id, env, monitors}). +-record(state, {pool, id, env, submon :: emqttd_pmon:pmon()}). -%%-------------------------------------------------------------------- -%% Mnesia callbacks -%%-------------------------------------------------------------------- - -mnesia(boot) -> - ok = emqttd_mnesia:create_table(subscription, [ - {type, bag}, - {ram_copies, [node()]}, - {local_content, true}, %% subscription table is local - {record_name, mqtt_subscription}, - {attributes, record_info(fields, mqtt_subscription)}]); - -mnesia(copy) -> - ok = emqttd_mnesia:copy_table(subscription). - -%%-------------------------------------------------------------------- -%% Start server -%%-------------------------------------------------------------------- - -%% @doc Start a Server --spec(start_link(Pool, Id, Env) -> {ok, pid()} | ignore | {error, any()} when - Pool :: atom(), - Id :: pos_integer(), - Env :: list(tuple())). +%% @doc Start server +-spec(start_link(atom(), pos_integer(), list()) -> {ok, pid()} | ignore | {error, any()}). start_link(Pool, Id, Env) -> gen_server2:start_link({local, ?PROC_NAME(?MODULE, Id)}, ?MODULE, [Pool, Id, Env], []). @@ -75,30 +59,37 @@ start_link(Pool, Id, Env) -> %%-------------------------------------------------------------------- %% @doc Subscribe a Topic --spec(subscribe(binary()) -> ok). +-spec(subscribe(binary()) -> ok | emqttd:pubsub_error()). subscribe(Topic) when is_binary(Topic) -> - From = self(), call(server(From), {subscribe, From, Topic}). + subscribe(Topic, self()). -%% @doc Subscribe from a MQTT session. --spec(subscribe(binary(), binary(), mqtt_qos()) -> ok). -subscribe(ClientId, Topic, Qos) -> - From = self(), call(server(From), {subscribe, From, ClientId, Topic, ?QOS_I(Qos)}). +-spec(subscribe(binary(), emqttd:subscriber()) -> ok | emqttd:pubsub_error()). +subscribe(Topic, Subscriber) when is_binary(Topic) -> + subscribe(Topic, Subscriber, []). -%% @doc Lookup subscriptions. --spec(lookup_subscription(binary()) -> [#mqtt_subscription{}]). -lookup_subscription(ClientId) -> - mnesia:dirty_read(subscription, ClientId). +-spec(subscribe(binary(), emqttd:subscriber(), [emqttd:suboption()]) -> + ok | emqttd:pubsub_error()). +subscribe(Topic, Subscriber, Options) when is_binary(Topic) -> + call(pick(Subscriber), {subscribe, Topic, Subscriber, Options}). -%% @doc Update a subscription. --spec(update_subscription(binary(), binary(), mqtt_qos(), mqtt_qos()) -> ok). -update_subscription(ClientId, Topic, OldQos, NewQos) -> - call(server(self()), {update_subscription, ClientId, Topic, ?QOS_I(OldQos), ?QOS_I(NewQos)}). +%% @doc Subscribe a Topic Asynchronously +-spec(async_subscribe(binary()) -> ok). +async_subscribe(Topic) when is_binary(Topic) -> + async_subscribe(Topic, self()). -%% @doc Publish a Message --spec(publish(Msg :: mqtt_message()) -> any()). +-spec(async_subscribe(binary(), emqttd:subscriber()) -> ok). +async_subscribe(Topic, Subscriber) when is_binary(Topic) -> + async_subscribe(Topic, Subscriber, []). + +-spec(async_subscribe(binary(), emqttd:subscriber(), [emqttd:suboption()]) -> ok). +async_subscribe(Topic, Subscriber, Options) when is_binary(Topic) -> + cast(pick(Subscriber), {subscribe, Topic, Subscriber, Options}). + +%% @doc Publish message to Topic. +-spec(publish(mqtt_message()) -> {ok, mqtt_delivery()} | ignore). publish(Msg = #mqtt_message{from = From}) -> trace(publish, From, Msg), - case emqttd:run_hooks('message.publish', [], Msg) of + case emqttd_hook:run('message.publish', [], Msg) of {ok, Msg1 = #mqtt_message{topic = Topic}} -> %% Retain message first. Don't create retained topic. Msg2 = case emqttd_retainer:retain(Msg1) of @@ -107,155 +98,10 @@ publish(Msg = #mqtt_message{from = From}) -> end, emqttd_pubsub:publish(Topic, Msg2); {stop, Msg1} -> - lager:warning("Stop publishing: ~s", [emqttd_message:format(Msg1)]) + lager:warning("Stop publishing: ~s", [emqttd_message:format(Msg1)]), + ignore end. -%% @doc Unsubscribe a Topic --spec(unsubscribe(binary()) -> ok). -unsubscribe(Topic) when is_binary(Topic) -> - From = self(), call(server(From), {unsubscribe, From, Topic}). - -%% @doc Unsubscribe a Topic from a MQTT session --spec(unsubscribe(binary(), binary(), mqtt_qos()) -> ok). -unsubscribe(ClientId, Topic, Qos) -> - From = self(), call(server(From), {unsubscribe, From, ClientId, Topic, Qos}). - -call(Server, Req) -> - gen_server2:call(Server, Req, infinity). - -server(From) -> - gproc_pool:pick_worker(server, From). - -%%-------------------------------------------------------------------- -%% gen_server Callbacks -%%-------------------------------------------------------------------- - -init([Pool, Id, Env]) -> - ?GPROC_POOL(join, Pool, Id), - {ok, #state{pool = Pool, id = Id, env = Env, monitors = dict:new()}}. - -handle_call({subscribe, SubPid, ClientId, Topic, Qos}, _From, State) -> - pubsub_subscribe_(SubPid, Topic), - if_subsciption(State, fun() -> - add_subscription_(ClientId, Topic, Qos), - set_subscription_stats() - end), - ok(monitor_subscriber_(ClientId, SubPid, State)); - -handle_call({subscribe, SubPid, Topic}, _From, State) -> - pubsub_subscribe_(SubPid, Topic), - ok(monitor_subscriber_(undefined, SubPid, State)); - -handle_call({update_subscription, ClientId, Topic, OldQos, NewQos}, _From, State) -> - if_subsciption(State, fun() -> - OldSub = #mqtt_subscription{subid = ClientId, topic = Topic, qos = OldQos}, - NewSub = #mqtt_subscription{subid = ClientId, topic = Topic, qos = NewQos}, - mnesia:transaction(fun update_subscription_/2, [OldSub, NewSub]), - set_subscription_stats() - end), ok(State); - -handle_call({unsubscribe, SubPid, ClientId, Topic, Qos}, _From, State) -> - pubsub_unsubscribe_(SubPid, Topic), - if_subsciption(State, fun() -> - del_subscription_(ClientId, Topic, Qos), - set_subscription_stats() - end), ok(State); - -handle_call({unsubscribe, SubPid, Topic}, _From, State) -> - pubsub_unsubscribe_(SubPid, Topic), ok(State); - -handle_call(Req, _From, State) -> - ?UNEXPECTED_REQ(Req, State). - -handle_cast(Msg, State) -> - ?UNEXPECTED_MSG(Msg, State). - -handle_info({'DOWN', _MRef, process, DownPid, _Reason}, State = #state{monitors = Monitors}) -> - %% unsubscribe - lists:foreach(fun({_, Topic}) -> - emqttd_pubsub:async_unsubscribe(Topic, DownPid) - end, ets:lookup(subscribed, DownPid)), - ets:delete(subscribed, DownPid), - - %% clean subscriptions - case dict:find(DownPid, Monitors) of - {ok, {undefined, _}} -> ok; - {ok, {ClientId, _}} -> mnesia:dirty_delete(subscription, ClientId); - error -> ok - end, - {noreply, State#state{monitors = dict:erase(DownPid, Monitors)}, hibernate}; - -handle_info(Info, State) -> - ?UNEXPECTED_INFO(Info, State). - -terminate(_Reason, #state{pool = Pool, id = Id}) -> - ?GPROC_POOL(leave, Pool, Id). - -code_change(_OldVsn, State, _Extra) -> - {ok, State}. - -%%-------------------------------------------------------------------- -%% Internal Functions -%%-------------------------------------------------------------------- - -if_subsciption(#state{env = Env}, Fun) -> - case proplists:get_value(subscription, Env, true) of - false -> ok; - _true -> Fun() - end. - -%% @private -%% @doc Add a subscription. --spec(add_subscription_(binary(), binary(), mqtt_qos()) -> ok). -add_subscription_(ClientId, Topic, Qos) -> - add_subscription_(#mqtt_subscription{subid = ClientId, topic = Topic, qos = Qos}). - --spec(add_subscription_(mqtt_subscription()) -> ok). -add_subscription_(Subscription) when is_record(Subscription, mqtt_subscription) -> - mnesia:dirty_write(subscription, Subscription). - -update_subscription_(OldSub, NewSub) -> - mnesia:delete_object(subscription, OldSub, write), - mnesia:write(subscription, NewSub, write). - -%% @private -%% @doc Delete a subscription --spec(del_subscription_(binary(), binary(), mqtt_qos()) -> ok). -del_subscription_(ClientId, Topic, Qos) -> - del_subscription_(#mqtt_subscription{subid = ClientId, topic = Topic, qos = Qos}). - -del_subscription_(Subscription) when is_record(Subscription, mqtt_subscription) -> - mnesia:dirty_delete_object(subscription, Subscription). - -%% @private -%% @doc Call pubsub to subscribe -pubsub_subscribe_(SubPid, Topic) -> - case ets:match(subscribed, {SubPid, Topic}) of - [] -> - emqttd_pubsub:async_subscribe(Topic, SubPid), - ets:insert(subscribed, {SubPid, Topic}); - [_] -> - false - end. - -%% @private -pubsub_unsubscribe_(SubPid, Topic) -> - emqttd_pubsub:async_unsubscribe(Topic, SubPid), - ets:delete_object(subscribed, {SubPid, Topic}). - -monitor_subscriber_(ClientId, SubPid, State = #state{monitors = Monitors}) -> - case dict:find(SubPid, Monitors) of - {ok, _} -> - State; - error -> - MRef = erlang:monitor(process, SubPid), - State#state{monitors = dict:store(SubPid, {ClientId, MRef}, Monitors)} - end. - -%%-------------------------------------------------------------------- -%% Trace Functions -%%-------------------------------------------------------------------- - trace(publish, From, _Msg) when is_atom(From) -> %% Dont' trace '$SYS' publish ignore; @@ -264,15 +110,181 @@ trace(publish, From, #mqtt_message{topic = Topic, payload = Payload}) -> lager:info([{client, From}, {topic, Topic}], "~s PUBLISH to ~s: ~p", [From, Topic, Payload]). +%% @doc Unsubscribe +-spec(unsubscribe(binary()) -> ok | emqttd:pubsub_error()). +unsubscribe(Topic) when is_binary(Topic) -> + unsubscribe(Topic, self()). + +%% @doc Unsubscribe +-spec(unsubscribe(binary(), emqttd:subscriber()) -> ok | emqttd:pubsub_error()). +unsubscribe(Topic, Subscriber) when is_binary(Topic) -> + call(pick(Subscriber), {unsubscribe, Topic, Subscriber}). + +%% @doc Async Unsubscribe +-spec(async_unsubscribe(binary()) -> ok). +async_unsubscribe(Topic) when is_binary(Topic) -> + async_unsubscribe(Topic, self()). + +-spec(async_unsubscribe(binary(), emqttd:subscriber()) -> ok). +async_unsubscribe(Topic, Subscriber) when is_binary(Topic) -> + cast(pick(Subscriber), {unsubscribe, Topic, Subscriber}). + +setqos(Topic, Subscriber, Qos) when is_binary(Topic) -> + call(pick(Subscriber), {setqos, Topic, Subscriber, Qos}). + +-spec(subscriptions(emqttd:subscriber()) -> [{binary(), list(emqttd:suboption())}]). +subscriptions(Subscriber) -> + lists:map(fun({_, Topic}) -> + subscription(Topic, Subscriber) + end, ets:lookup(mqtt_subscription, Subscriber)). + +subscription(Topic, Subscriber) -> + {Topic, Subscriber, ets:lookup_element(mqtt_subproperty, {Topic, Subscriber}, 2)}. + +subscribers(Topic) -> emqttd_pubsub:subscribers(Topic). + +-spec(is_subscribed(binary(), emqttd:subscriber()) -> boolean()). +is_subscribed(Topic, Subscriber) when is_binary(Topic) -> + ets:member(mqtt_subproperty, {Topic, Subscriber}). + +-spec(subscriber_down(emqttd:subscriber()) -> ok). +subscriber_down(Subscriber) -> + cast(pick(Subscriber), {subscriber_down, Subscriber}). + +call(Server, Req) -> + gen_server2:call(Server, Req, infinity). + +cast(Server, Msg) when is_pid(Server) -> + gen_server2:cast(Server, Msg). + +pick(Subscriber) -> + gproc_pool:pick_worker(server, Subscriber). + +dump() -> + [{Tab, ets:tab2list(Tab)} || Tab <- [mqtt_subproperty, mqtt_subscription, mqtt_subscriber]]. + %%-------------------------------------------------------------------- -%% Subscription Statistics +%% gen_server Callbacks %%-------------------------------------------------------------------- -set_subscription_stats() -> +init([Pool, Id, Env]) -> + ?GPROC_POOL(join, Pool, Id), + {ok, #state{pool = Pool, id = Id, env = Env, submon = emqttd_pmon:new()}}. + +handle_call({subscribe, Topic, Subscriber, Options}, _From, State) -> + case do_subscribe_(Topic, Subscriber, Options, State) of + {ok, NewState} -> {reply, ok, setstats(NewState)}; + {error, Error} -> {reply, {error, Error}, State} + end; + +handle_call({unsubscribe, Topic, Subscriber}, _From, State) -> + case do_unsubscribe_(Topic, Subscriber, State) of + {ok, NewState} -> {reply, ok, setstats(NewState), hibernate}; + {error, Error} -> {reply, {error, Error}, State} + end; + +handle_call({setqos, Topic, Subscriber, Qos}, _From, State) -> + Key = {Topic, Subscriber}, + case ets:lookup(mqtt_subproperty, Key) of + [{_, Opts}] -> + Opts1 = lists:ukeymerge(1, [{qos, Qos}], Opts), + ets:insert(mqtt_subproperty, {Key, Opts1}), + {reply, ok, State}; + [] -> + {reply, {error, {subscription_not_found, Topic}}, State} + end; + +handle_call(Req, _From, State) -> + ?UNEXPECTED_REQ(Req, State). + +handle_cast({subscribe, Topic, Subscriber, Options}, State) -> + case do_subscribe_(Topic, Subscriber, Options, State) of + {ok, NewState} -> {noreply, setstats(NewState)}; + {error, _Error} -> {noreply, State} + end; + +handle_cast({unsubscribe, Topic, Subscriber}, State) -> + case do_unsubscribe_(Topic, Subscriber, State) of + {ok, NewState} -> {noreply, setstats(NewState), hibernate}; + {error, _Error} -> {noreply, State} + end; + +handle_cast({subscriber_down, Subscriber}, State) -> + subscriber_down_(Subscriber), + {noreply, setstats(State)}; + +handle_cast(Msg, State) -> + ?UNEXPECTED_MSG(Msg, State). + +handle_info({'DOWN', _MRef, process, DownPid, _Reason}, State = #state{submon = PM}) -> + subscriber_down_(DownPid), + {noreply, setstats(State#state{submon = PM:erase(DownPid)}), hibernate}; + +handle_info(Info, State) -> + ?UNEXPECTED_INFO(Info, State). + +terminate(_Reason, #state{pool = Pool, id = Id}) -> + ?GPROC_POOL(leave, Pool, Id). + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%%-------------------------------------------------------------------- +%% Internal Functions +%%-------------------------------------------------------------------- + +do_subscribe_(Topic, Subscriber, Options, State) -> + case ets:lookup(mqtt_subproperty, {Topic, Subscriber}) of + [] -> + emqttd_pubsub:async_subscribe(Topic, Subscriber), + ets:insert(mqtt_subscription, {Subscriber, Topic}), + ets:insert(mqtt_subproperty, {{Topic, Subscriber}, Options}), + {ok, monitor_subpid(Subscriber, State)}; + [_] -> + {error, {already_subscribed, Topic}} + end. + +monitor_subpid(SubPid, State = #state{submon = PMon}) when is_pid(SubPid) -> + State#state{submon = PMon:monitor(SubPid)}; +monitor_subpid(_SubPid, State) -> + State. + +do_unsubscribe_(Topic, Subscriber, State) -> + case ets:lookup(mqtt_subproperty, {Topic, Subscriber}) of + [_] -> + emqttd_pubsub:async_unsubscribe(Topic, Subscriber), + ets:delete_object(mqtt_subscription, {Subscriber, Topic}), + ets:delete(mqtt_subproperty, {Topic, Subscriber}), + {ok, case ets:member(mqtt_subscription, Subscriber) of + true -> State; + false -> demonitor_subpid(Subscriber, State) + end}; + [] -> + {error, {subscription_not_found, Topic}} + end. + +demonitor_subpid(SubPid, State = #state{submon = PMon}) when is_pid(SubPid) -> + State#state{submon = PMon:demonitor(SubPid)}; +demonitor_subpid(_SubPid, State) -> + State. + +subscriber_down_(Subscriber) -> + lists:foreach(fun({_, Topic}) -> + subscriber_down_(Subscriber, Topic) + end, ets:lookup(mqtt_subscription, Subscriber)), + ets:delete(mqtt_subscription, Subscriber). + +subscriber_down_(Subscriber, Topic) -> + case ets:lookup(mqtt_subproperty, {Topic, Subscriber}) of + [] -> + %% here? + emqttd_pubsub:async_unsubscribe(Topic, Subscriber); + [_] -> + emqttd_pubsub:async_unsubscribe(Topic, Subscriber), + ets:delete(mqtt_subproperty, {Topic, Subscriber}) + end. + +setstats(State) -> emqttd_stats:setstats('subscriptions/count', 'subscriptions/max', - mnesia:table_info(subscription, size)). - -%%-------------------------------------------------------------------- - -ok(State) -> {reply, ok, State}. + ets:info(mqtt_subscription, size)), State. diff --git a/src/emqttd_session.erl b/src/emqttd_session.erl index ac7af465b..bf475e4f6 100644 --- a/src/emqttd_session.erl +++ b/src/emqttd_session.erl @@ -77,6 +77,9 @@ %% Old Client Pid that has been kickout old_client_pid :: pid(), + %% Username + username :: binary() | undefined, + %% Last packet id of the session packet_id = 1, @@ -136,9 +139,9 @@ "Session(~s): " ++ Format, [State#session.client_id | Args])). %% @doc Start a session. --spec(start_link(boolean(), mqtt_client_id(), pid()) -> {ok, pid()} | {error, any()}). -start_link(CleanSess, ClientId, ClientPid) -> - gen_server2:start_link(?MODULE, [CleanSess, ClientId, ClientPid], []). +-spec(start_link(boolean(), {mqtt_client_id(), mqtt_username()}, pid()) -> {ok, pid()} | {error, any()}). +start_link(CleanSess, {ClientId, Username}, ClientPid) -> + gen_server2:start_link(?MODULE, [CleanSess, {ClientId, Username}, ClientPid], []). %% @doc Resume a session. -spec(resume(pid(), mqtt_client_id(), pid()) -> ok). @@ -175,11 +178,11 @@ subscribe(SessPid, PacketId, TopicTable) -> -spec(publish(pid(), mqtt_message()) -> ok | {error, any()}). publish(_SessPid, Msg = #mqtt_message{qos = ?QOS_0}) -> %% publish qos0 directly - emqttd:publish(Msg); + emqttd:publish(Msg), ok; publish(_SessPid, Msg = #mqtt_message{qos = ?QOS_1}) -> %% publish qos1 directly, and client will puback automatically - emqttd:publish(Msg); + emqttd:publish(Msg), ok; publish(SessPid, Msg = #mqtt_message{qos = ?QOS_2}) -> %% publish qos2 by session @@ -208,22 +211,22 @@ unsubscribe(SessPid, Topics) -> gen_server2:cast(SessPid, {unsubscribe, Topics}). %%-------------------------------------------------------------------- -%% gen_server callbacks +%% gen_server Callbacks %%-------------------------------------------------------------------- -init([CleanSess, ClientId, ClientPid]) -> +init([CleanSess, {ClientId, Username}, ClientPid]) -> process_flag(trap_exit, true), true = link(ClientPid), - QEnv = emqttd:env(mqtt, queue), - SessEnv = emqttd:env(mqtt, session), + SessEnv = emqttd_conf:session(), Session = #session{ clean_sess = CleanSess, client_id = ClientId, client_pid = ClientPid, + username = Username, subscriptions = dict:new(), inflight_queue = [], max_inflight = get_value(max_inflight, SessEnv, 0), - message_queue = emqttd_mqueue:new(ClientId, QEnv, emqttd_alarm:alarm_fun()), + message_queue = emqttd_mqueue:new(ClientId, emqttd_conf:queue(), emqttd_alarm:alarm_fun()), awaiting_rel = #{}, awaiting_ack = #{}, awaiting_comp = #{}, @@ -233,8 +236,8 @@ init([CleanSess, ClientId, ClientPid]) -> expired_after = get_value(expired_after, SessEnv) * 60, collect_interval = get_value(collect_interval, SessEnv, 0), timestamp = os:timestamp()}, - emqttd_sm:register_session(CleanSess, ClientId, sess_info(Session)), - %% start statistics + emqttd_sm:reg_session(ClientId, CleanSess, sess_info(Session)), + %% Start statistics {ok, start_collector(Session), hibernate}. prioritise_call(Msg, _From, _Len, _State) -> @@ -285,62 +288,68 @@ handle_call({publish, Msg = #mqtt_message{qos = ?QOS_2, pktid = PktId}}, handle_call(Req, _From, State) -> ?UNEXPECTED_REQ(Req, State). -handle_cast({subscribe, TopicTable0, AckFun}, Session = #session{client_id = ClientId, - subscriptions = Subscriptions}) -> +%%TODO: 2.0 FIX - case emqttd:run_hooks('client.subscribe', [ClientId], TopicTable0) of - {ok, TopicTable} -> - ?LOG(info, "Subscribe ~p", [TopicTable], Session), - Subscriptions1 = lists:foldl( - fun({Topic, Qos}, SubDict) -> - case dict:find(Topic, SubDict) of - {ok, Qos} -> - ?LOG(warning, "duplicated subscribe: ~s, qos = ~w", [Topic, Qos], Session), - SubDict; - {ok, OldQos} -> - emqttd_server:update_subscription(ClientId, Topic, OldQos, Qos), - ?LOG(warning, "duplicated subscribe ~s, old_qos=~w, new_qos=~w", [Topic, OldQos, Qos], Session), - dict:store(Topic, Qos, SubDict); - error -> - emqttd:subscribe(ClientId, Topic, Qos), - %%TODO: the design is ugly... - %% : 3.8.4 - %% Where the Topic Filter is not identical to any existing Subscription’s filter, - %% a new Subscription is created and all matching retained messages are sent. - emqttd_retainer:dispatch(Topic, self()), +handle_cast({subscribe, TopicTable, AckFun}, Session = #session{client_id = ClientId, + username = Username, + subscriptions = Subscriptions}) -> + ?LOG(info, "Subscribe ~p", [TopicTable], Session), + {GrantedQos, Subscriptions1} = + lists:foldl(fun({RawTopic, Qos}, {QosAcc, SubDict}) -> + {Topic, Opts} = emqttd_topic:strip(RawTopic), + case emqttd:run_hooks('client.subscribe', [{ClientId, Username}], {Topic, Opts}) of + {ok, {Topic1, Opts1}} -> + NewQos = proplists:get_value(qos, Opts1, Qos), + {[NewQos | QosAcc], case dict:find(Topic, SubDict) of + {ok, NewQos} -> + ?LOG(warning, "duplicated subscribe: ~s, qos = ~w", [Topic, NewQos], Session), + SubDict; + {ok, OldQos} -> + emqttd:setqos(Topic, ClientId, NewQos), + ?LOG(warning, "duplicated subscribe ~s, old_qos=~w, new_qos=~w", [Topic, OldQos, NewQos], Session), + dict:store(Topic, NewQos, SubDict); + error -> + emqttd:subscribe(Topic1, ClientId, Opts1), + %%TODO: the design is ugly... + %% : 3.8.4 + %% Where the Topic Filter is not identical to any existing Subscription’s filter, + %% a new Subscription is created and all matching retained messages are sent. + emqttd_retainer:dispatch(Topic1, self()), + emqttd:run_hooks('client.subscribe.after', [{ClientId, Username}], {Topic1, Opts1}), - dict:store(Topic, Qos, SubDict) - end - end, Subscriptions, TopicTable), - AckFun([Qos || {_, Qos} <- TopicTable]), - emqttd:run_hooks('client.subscribe.after', [ClientId], TopicTable), - hibernate(Session#session{subscriptions = Subscriptions1}); - {stop, TopicTable} -> - ?LOG(error, "Cannot subscribe: ~p", [TopicTable], Session), - hibernate(Session) - end; + dict:store(Topic1, NewQos, SubDict) + end}; + {stop, _} -> + ?LOG(error, "Cannot subscribe: ~p", [Topic], Session), + {[128 | QosAcc], SubDict} + end + end, {[], Subscriptions}, TopicTable), + AckFun(lists:reverse(GrantedQos)), + hibernate(Session#session{subscriptions = Subscriptions1}); -handle_cast({unsubscribe, Topics0}, Session = #session{client_id = ClientId, - subscriptions = Subscriptions}) -> +%%TODO: 2.0 FIX - case emqttd:run_hooks('client.unsubscribe', [ClientId], Topics0) of - {ok, Topics} -> - ?LOG(info, "unsubscribe ~p", [Topics], Session), - Subscriptions1 = lists:foldl( - fun(Topic, SubDict) -> - case dict:find(Topic, SubDict) of - {ok, Qos} -> - emqttd:unsubscribe(ClientId, Topic, Qos), - dict:erase(Topic, SubDict); - error -> +handle_cast({unsubscribe, Topics}, Session = #session{client_id = ClientId, + username = Username, + subscriptions = Subscriptions}) -> + ?LOG(info, "unsubscribe ~p", [Topics], Session), + Subscriptions1 = + lists:foldl(fun(RawTopic, SubDict) -> + {Topic0, _Opts} = emqttd_topic:strip(RawTopic), + case emqttd:run_hooks('client.unsubscribe', [ClientId, Username], Topic0) of + {ok, Topic1} -> + case dict:find(Topic1, SubDict) of + {ok, _Qos} -> + emqttd:unsubscribe(Topic1, ClientId), + dict:erase(Topic1, SubDict); + error -> + SubDict + end; + {stop, _} -> SubDict end end, Subscriptions, Topics), - hibernate(Session#session{subscriptions = Subscriptions1}); - {stop, Topics} -> - ?LOG(info, "Cannot unsubscribe: ~p", [Topics], Session), - hibernate(Session) - end; + hibernate(Session#session{subscriptions = Subscriptions1}); handle_cast({destroy, ClientId}, Session = #session{client_id = ClientId}) -> ?LOG(warning, "destroyed", [], Session), @@ -386,8 +395,7 @@ handle_cast({resume, ClientId, ClientPid}, Session = #session{client_id = C if CleanSess =:= true -> ?LOG(warning, "CleanSess changed to false.", [], Session), - emqttd_sm:unregister_session(CleanSess, ClientId), - emqttd_sm:register_session(false, ClientId, sess_info(Session1)); + emqttd_sm:reg_session(ClientId, false, sess_info(Session1)); CleanSess =:= false -> ok end, @@ -501,7 +509,7 @@ handle_info({timeout, awaiting_comp, PktId}, Session = #session{awaiting_comp = end; handle_info(collect_info, Session = #session{clean_sess = CleanSess, client_id = ClientId}) -> - emqttd_sm:register_session(CleanSess, ClientId, sess_info(Session)), + emqttd_sm:reg_session(ClientId, CleanSess, sess_info(Session)), hibernate(start_collector(Session)); handle_info({'EXIT', ClientPid, _Reason}, Session = #session{clean_sess = true, @@ -532,8 +540,9 @@ handle_info(expired, Session) -> handle_info(Info, Session) -> ?UNEXPECTED_INFO(Info, Session). -terminate(_Reason, #session{clean_sess = CleanSess, client_id = ClientId}) -> - emqttd_sm:unregister_session(CleanSess, ClientId). +terminate(_Reason, #session{client_id = ClientId}) -> + emqttd:subscriber_down(ClientId), + emqttd_sm:unreg_session(ClientId). code_change(_OldVsn, Session, _Extra) -> {ok, Session}. @@ -650,11 +659,12 @@ await(#mqtt_message{pktid = PktId}, Session = #session{awaiting_ack = Awaiting Session#session{awaiting_ack = Awaiting1}. acked(PktId, Session = #session{client_id = ClientId, + username = Username, inflight_queue = InflightQ, awaiting_ack = Awaiting}) -> case lists:keyfind(PktId, 1, InflightQ) of {_, Msg} -> - emqttd:run_hooks('message.acked', [ClientId], Msg); + emqttd:run_hooks('message.acked', [{ClientId, Username}], Msg); false -> ?LOG(error, "Cannot find acked pktid: ~p", [PktId], Session) end, diff --git a/src/emqttd_session_sup.erl b/src/emqttd_session_sup.erl index 2b9ee9496..394cb84d0 100644 --- a/src/emqttd_session_sup.erl +++ b/src/emqttd_session_sup.erl @@ -29,9 +29,9 @@ start_link() -> supervisor:start_link({local, ?MODULE}, ?MODULE, []). %% @doc Start a session --spec(start_session(boolean(), binary(), pid()) -> {ok, pid()}). -start_session(CleanSess, ClientId, ClientPid) -> - supervisor:start_child(?MODULE, [CleanSess, ClientId, ClientPid]). +-spec(start_session(boolean(), {binary(), binary() | undefined} , pid()) -> {ok, pid()}). +start_session(CleanSess, {ClientId, Username}, ClientPid) -> + supervisor:start_child(?MODULE, [CleanSess, {ClientId, Username}, ClientPid]). %%-------------------------------------------------------------------- %% Supervisor callbacks diff --git a/src/emqttd_sm.erl b/src/emqttd_sm.erl index 05aed7b42..3a978b3fd 100644 --- a/src/emqttd_sm.erl +++ b/src/emqttd_sm.erl @@ -32,9 +32,9 @@ %% API Function Exports -export([start_link/2]). --export([start_session/2, lookup_session/1]). +-export([start_session/2, lookup_session/1, reg_session/3, unreg_session/1]). --export([register_session/3, unregister_session/2]). +-export([dispatch/3]). %% gen_server Function Exports -export([init/1, handle_call/3, handle_cast/2, handle_info/2, @@ -58,14 +58,14 @@ mnesia(boot) -> %% Global Session Table - ok = emqttd_mnesia:create_table(session, [ + ok = emqttd_mnesia:create_table(mqtt_session, [ {type, set}, {ram_copies, [node()]}, {record_name, mqtt_session}, {attributes, record_info(fields, mqtt_session)}]); mnesia(copy) -> - ok = emqttd_mnesia:copy_table(session). + ok = emqttd_mnesia:copy_table(mqtt_session). %%-------------------------------------------------------------------- %% API @@ -77,36 +77,35 @@ start_link(Pool, Id) -> gen_server2:start_link({local, ?PROC_NAME(?MODULE, Id)}, ?MODULE, [Pool, Id], []). %% @doc Start a session --spec(start_session(boolean(), binary()) -> {ok, pid(), boolean()} | {error, any()}). -start_session(CleanSess, ClientId) -> +-spec(start_session(boolean(), {binary(), binary() | undefined}) -> {ok, pid(), boolean()} | {error, any()}). +start_session(CleanSess, {ClientId, Username}) -> SM = gproc_pool:pick_worker(?POOL, ClientId), - call(SM, {start_session, {CleanSess, ClientId, self()}}). + call(SM, {start_session, CleanSess, {ClientId, Username}, self()}). %% @doc Lookup a Session -spec(lookup_session(binary()) -> mqtt_session() | undefined). lookup_session(ClientId) -> - case mnesia:dirty_read(session, ClientId) of + case mnesia:dirty_read(mqtt_session, ClientId) of [Session] -> Session; [] -> undefined end. %% @doc Register a session with info. --spec(register_session(CleanSess, ClientId, Info) -> ok when - CleanSess :: boolean(), - ClientId :: binary(), - Info :: [tuple()]). -register_session(CleanSess, ClientId, Info) -> - ets:insert(sesstab(CleanSess), {{ClientId, self()}, Info}). +-spec(reg_session(binary(), boolean(), [tuple()]) -> true). +reg_session(ClientId, CleanSess, Properties) -> + ets:insert(mqtt_local_session, {ClientId, self(), CleanSess, Properties}). %% @doc Unregister a session. --spec(unregister_session(CleanSess, ClientId) -> ok when - CleanSess :: boolean(), - ClientId :: binary()). -unregister_session(CleanSess, ClientId) -> - ets:delete(sesstab(CleanSess), {ClientId, self()}). +-spec(unreg_session(binary()) -> true). +unreg_session(ClientId) -> + ets:delete(mqtt_local_session, ClientId). -sesstab(true) -> mqtt_transient_session; -sesstab(false) -> mqtt_persistent_session. +dispatch(ClientId, Topic, Msg) -> + try ets:lookup_element(mqtt_local_session, ClientId, 2) of + Pid -> Pid ! {dispatch, Topic, Msg} + catch + error:badarg -> io:format("Session Not Found: ~p~n", [ClientId]), ok %%TODO: How?? + end. call(SM, Req) -> gen_server2:call(SM, Req, ?TIMEOUT). %%infinity). @@ -129,11 +128,11 @@ prioritise_info(_Msg, _Len, _State) -> 2. %% Persistent Session -handle_call({start_session, Client = {false, ClientId, ClientPid}}, _From, State) -> +handle_call({start_session, false, {ClientId, Username}, ClientPid}, _From, State) -> case lookup_session(ClientId) of undefined -> %% Create session locally - create_session(Client, State); + create_session({false, {ClientId, Username}, ClientPid}, State); Session -> case resume_session(Session, ClientPid) of {ok, SessPid} -> @@ -144,7 +143,8 @@ handle_call({start_session, Client = {false, ClientId, ClientPid}}, _From, State end; %% Transient Session -handle_call({start_session, Client = {true, ClientId, _ClientPid}}, _From, State) -> +handle_call({start_session, true, {ClientId, Username}, ClientPid}, _From, State) -> + Client = {true, {ClientId, Username}, ClientPid}, case lookup_session(ClientId) of undefined -> create_session(Client, State); @@ -167,11 +167,13 @@ handle_info({'DOWN', MRef, process, DownPid, _Reason}, State) -> case dict:find(MRef, State#state.monitors) of {ok, ClientId} -> mnesia:transaction(fun() -> - case mnesia:wread({session, ClientId}) of - [] -> ok; + case mnesia:wread({mqtt_session, ClientId}) of + [] -> + ok; [Sess = #mqtt_session{sess_pid = DownPid}] -> - mnesia:delete_object(session, Sess, write); - [_Sess] -> ok + mnesia:delete_object(mqtt_session, Sess, write); + [_Sess] -> + ok end end), {noreply, erase_monitor(MRef, State), hibernate}; @@ -194,8 +196,8 @@ code_change(_OldVsn, State, _Extra) -> %%-------------------------------------------------------------------- %% Create Session Locally -create_session({CleanSess, ClientId, ClientPid}, State) -> - case create_session(CleanSess, ClientId, ClientPid) of +create_session({CleanSess, {ClientId, Username}, ClientPid}, State) -> + case create_session(CleanSess, {ClientId, Username}, ClientPid) of {ok, SessPid} -> {reply, {ok, SessPid, false}, monitor_session(ClientId, SessPid, State)}; @@ -203,12 +205,10 @@ create_session({CleanSess, ClientId, ClientPid}, State) -> {reply, {error, Error}, State} end. -create_session(CleanSess, ClientId, ClientPid) -> - case emqttd_session_sup:start_session(CleanSess, ClientId, ClientPid) of +create_session(CleanSess, {ClientId, Username}, ClientPid) -> + case emqttd_session_sup:start_session(CleanSess, {ClientId, Username}, ClientPid) of {ok, SessPid} -> - Session = #mqtt_session{client_id = ClientId, - sess_pid = SessPid, - persistent = not CleanSess}, + Session = #mqtt_session{client_id = ClientId, sess_pid = SessPid, persistent = not CleanSess}, case insert_session(Session) of {aborted, {conflict, ConflictPid}} -> %% Conflict with othe node? @@ -224,17 +224,16 @@ create_session(CleanSess, ClientId, ClientPid) -> insert_session(Session = #mqtt_session{client_id = ClientId}) -> mnesia:transaction( fun() -> - case mnesia:wread({session, ClientId}) of + case mnesia:wread({mqtt_session, ClientId}) of [] -> - mnesia:write(session, Session, write); + mnesia:write(mqtt_session, Session, write); [#mqtt_session{sess_pid = SessPid}] -> mnesia:abort({conflict, SessPid}) end end). %% Local node -resume_session(Session = #mqtt_session{client_id = ClientId, - sess_pid = SessPid}, ClientPid) +resume_session(Session = #mqtt_session{client_id = ClientId, sess_pid = SessPid}, ClientPid) when node(SessPid) =:= node() -> case is_process_alive(SessPid) of @@ -284,7 +283,7 @@ destroy_session(Session = #mqtt_session{client_id = ClientId, end. remove_session(Session) -> - case mnesia:transaction(fun mnesia:delete_object/3, [session, Session, write]) of + case mnesia:transaction(fun mnesia:delete_object/1, [Session]) of {atomic, ok} -> ok; {aborted, Error} -> {error, Error} end. diff --git a/src/emqttd_sm_helper.erl b/src/emqttd_sm_helper.erl index 1c90acc32..aa6a7e365 100644 --- a/src/emqttd_sm_helper.erl +++ b/src/emqttd_sm_helper.erl @@ -54,10 +54,9 @@ handle_info({mnesia_system_event, {mnesia_down, Node}}, State) -> lager:error("!!!Mnesia node down: ~s", [Node]), Fun = fun() -> ClientIds = - mnesia:select(session, [{#mqtt_session{client_id = '$1', sess_pid = '$2', _ = '_'}, - [{'==', {node, '$2'}, Node}], - ['$1']}]), - lists:foreach(fun(ClientId) -> mnesia:delete({session, ClientId}) end, ClientIds) + mnesia:select(mqtt_session, [{#mqtt_session{client_id = '$1', sess_pid = '$2', _ = '_'}, + [{'==', {node, '$2'}, Node}], ['$1']}]), + lists:foreach(fun(ClientId) -> mnesia:delete({mqtt_session, ClientId}) end, ClientIds) end, mnesia:async_dirty(Fun), {noreply, State}; @@ -83,5 +82,5 @@ code_change(_OldVsn, State, _Extra) -> %%-------------------------------------------------------------------- setstats(State = #state{stats_fun = StatsFun}) -> - StatsFun(ets:info(mqtt_persistent_session, size)), State. + StatsFun(ets:info(mqtt_local_session, size)), State. diff --git a/src/emqttd_sm_sup.erl b/src/emqttd_sm_sup.erl index 556d9540f..1935bcfde 100644 --- a/src/emqttd_sm_sup.erl +++ b/src/emqttd_sm_sup.erl @@ -25,8 +25,6 @@ -define(HELPER, emqttd_sm_helper). --define(TABS, [mqtt_transient_session, mqtt_persistent_session]). - %% API -export([start_link/0]). @@ -38,7 +36,7 @@ start_link() -> init([]) -> %% Create session tables - create_session_tabs(), + ets:new(mqtt_local_session, [public, ordered_set, named_table, {write_concurrency, true}]), %% Helper StatsFun = emqttd_stats:statsfun('sessions/count', 'sessions/max'), @@ -50,9 +48,4 @@ init([]) -> PoolSup = emqttd_pool_sup:spec([?SM, hash, erlang:system_info(schedulers), MFA]), {ok, {{one_for_all, 10, 3600}, [Helper, PoolSup]}}. - -create_session_tabs() -> - Opts = [ordered_set, named_table, public, - {write_concurrency, true}], - [ets:new(Tab, Opts) || Tab <- ?TABS]. diff --git a/src/emqttd_stats.erl b/src/emqttd_stats.erl index 478313fd0..9ffc08299 100644 --- a/src/emqttd_stats.erl +++ b/src/emqttd_stats.erl @@ -122,7 +122,7 @@ init([]) -> Topics = ?SYSTOP_CLIENTS ++ ?SYSTOP_SESSIONS ++ ?SYSTOP_PUBSUB ++ ?SYSTOP_RETAINED, ets:insert(?STATS_TAB, [{Topic, 0} || Topic <- Topics]), % Create $SYS Topics - [ok = emqttd:create(topic, stats_topic(Topic)) || Topic <- Topics], + % [ok = emqttd:create(topic, stats_topic(Topic)) || Topic <- Topics], % Tick to publish stats {ok, #state{tick_tref = emqttd_broker:start_tick(tick)}, hibernate}. diff --git a/src/emqttd_sysmon.erl b/src/emqttd_sysmon.erl index a262ffc5e..43db85de6 100644 --- a/src/emqttd_sysmon.erl +++ b/src/emqttd_sysmon.erl @@ -162,10 +162,10 @@ publish(Sysmon, WarnMsg) -> topic(Sysmon) -> emqttd_topic:systop(list_to_binary(lists:concat(['sysmon/', Sysmon]))). -start_tracelog(undefined) -> - {ok, undefined}; -start_tracelog(LogFile) -> - lager:trace_file(LogFile, [{sysmon, true}], info, ?LOG_FMT). +%% start_tracelog(undefined) -> +%% {ok, undefined}; +%% start_tracelog(LogFile) -> +%% lager:trace_file(LogFile, [{sysmon, true}], info, ?LOG_FMT). cancel_tracelog(undefined) -> ok; diff --git a/src/emqttd_sysmon_sup.erl b/src/emqttd_sysmon_sup.erl index 02d60530a..3ed8e36a5 100644 --- a/src/emqttd_sysmon_sup.erl +++ b/src/emqttd_sysmon_sup.erl @@ -28,7 +28,15 @@ start_link() -> supervisor:start_link({local, ?MODULE}, ?MODULE, []). init([]) -> - Sysmon = {sysmon, {emqttd_sysmon, start_link, [emqttd:env(sysmon)]}, - permanent, 5000, worker, [emqttd_sysmon]} , + Sysmon = {sysmon, {emqttd_sysmon, start_link, [opts()]}, + permanent, 5000, worker, [emqttd_sysmon]}, {ok, {{one_for_one, 10, 100}, [Sysmon]}}. +opts() -> + Opts = [{long_gc, emqttd:conf(sysmon_long_gc)}, + {long_schedule, emqttd:conf(sysmon_long_schedule)}, + {large_heap, emqttd:conf(sysmon_large_heap)}, + {busy_port, emqttd:conf(busy_port)}, + {busy_dist_port, emqttd:conf(sysmon_busy_dist_port)}], + [{Key, Val} || {Key, {ok, Val}} <- Opts]. + diff --git a/src/emqttd_topic.erl b/src/emqttd_topic.erl index 5e91ffa26..ebd16714d 100644 --- a/src/emqttd_topic.erl +++ b/src/emqttd_topic.erl @@ -16,17 +16,21 @@ -module(emqttd_topic). +-import(lists, [reverse/1]). + -export([match/2, validate/1, triples/1, words/1, wildcard/1]). --export([join/1, feed_var/3, is_queue/1, systop/1]). +-export([join/1, feed_var/3, systop/1]). --type topic() :: binary(). +-export([strip/1, strip/2]). --type word() :: '' | '+' | '#' | binary(). +-type(topic() :: binary()). --type words() :: list(word()). +-type(word() :: '' | '+' | '#' | binary()). --type triple() :: {root | binary(), word(), binary()}. +-type(words() :: list(word())). + +-type(triple() :: {root | binary(), word(), binary()}). -export_type([topic/0, word/0, triple/0]). @@ -111,7 +115,7 @@ triples(Topic) when is_binary(Topic) -> triples(words(Topic), root, []). triples([], _Parent, Acc) -> - lists:reverse(Acc); + reverse(Acc); triples([W|Words], Parent, Acc) -> Node = join(Parent, W), @@ -137,13 +141,6 @@ word(<<"+">>) -> '+'; word(<<"#">>) -> '#'; word(Bin) -> Bin. -%% @doc Queue is a special topic name that starts with "$queue/" --spec(is_queue(topic()) -> boolean()). -is_queue(<<"$queue/", _Queue/binary>>) -> - true; -is_queue(_) -> - false. - %% @doc '$SYS' Topic. systop(Name) when is_atom(Name) -> list_to_binary(lists:concat(["$SYS/brokers/", node(), "/", Name])); @@ -155,7 +152,7 @@ systop(Name) when is_binary(Name) -> feed_var(Var, Val, Topic) -> feed_var(Var, Val, words(Topic), []). feed_var(_Var, _Val, [], Acc) -> - join(lists:reverse(Acc)); + join(reverse(Acc)); feed_var(Var, Val, [Var|Words], Acc) -> feed_var(Var, Val, Words, [Val|Acc]); feed_var(Var, Val, [W|Words], Acc) -> @@ -175,3 +172,28 @@ join(Words) -> end, {true, <<>>}, [bin(W) || W <- Words]), Bin. +-spec(strip(topic()) -> {topic(), [local | {share, binary()}]}). +strip(Topic) when is_binary(Topic) -> + strip(Topic, []). + +strip(Topic = <<"$local/", Topic1/binary>>, Options) -> + case lists:member(local, Options) of + true -> error({invalid_topic, Topic}); + false -> strip(Topic1, [local | Options]) + end; + +strip(Topic = <<"$queue/", Topic1/binary>>, Options) -> + case lists:keyfind(share, 1, Options) of + {share, _} -> error({invalid_topic, Topic}); + false -> strip(Topic1, [{share, '$queue'} | Options]) + end; + +strip(Topic = <<"$share/", Topic1/binary>>, Options) -> + case lists:keyfind(share, 1, Options) of + {share, _} -> error({invalid_topic, Topic}); + false -> [Share, Topic2] = binary:split(Topic1, <<"/">>), + {Topic2, [{share, Share} | Options]} + end; + +strip(Topic, Options) -> {Topic, Options}. + diff --git a/src/emqttd_trie.erl b/src/emqttd_trie.erl index c0ed8e064..7266ea4b6 100644 --- a/src/emqttd_trie.erl +++ b/src/emqttd_trie.erl @@ -38,21 +38,21 @@ -spec(mnesia(boot | copy) -> ok). mnesia(boot) -> %% Trie Table - ok = emqttd_mnesia:create_table(trie, [ + ok = emqttd_mnesia:create_table(mqtt_trie, [ {ram_copies, [node()]}, {record_name, trie}, {attributes, record_info(fields, trie)}]), %% Trie Node Table - ok = emqttd_mnesia:create_table(trie_node, [ + ok = emqttd_mnesia:create_table(mqtt_trie_node, [ {ram_copies, [node()]}, {record_name, trie_node}, {attributes, record_info(fields, trie_node)}]); mnesia(copy) -> %% Copy Trie Table - ok = emqttd_mnesia:copy_table(trie), + ok = emqttd_mnesia:copy_table(mqtt_trie), %% Copy Trie Node Table - ok = emqttd_mnesia:copy_table(trie_node). + ok = emqttd_mnesia:copy_table(mqtt_trie_node). %%-------------------------------------------------------------------- %% Trie API @@ -61,16 +61,16 @@ mnesia(copy) -> %% @doc Insert topic to trie -spec(insert(Topic :: binary()) -> ok). insert(Topic) when is_binary(Topic) -> - case mnesia:read(trie_node, Topic) of + case mnesia:read(mqtt_trie_node, Topic) of [#trie_node{topic=Topic}] -> ok; [TrieNode=#trie_node{topic=undefined}] -> - mnesia:write(TrieNode#trie_node{topic=Topic}); + write_trie_node(TrieNode#trie_node{topic=Topic}); [] -> - %add trie path + % Add trie path lists:foreach(fun add_path/1, emqttd_topic:triples(Topic)), - %add last node - mnesia:write(#trie_node{node_id=Topic, topic=Topic}) + % Add last node + write_trie_node(#trie_node{node_id=Topic, topic=Topic}) end. %% @doc Find trie nodes that match topic @@ -82,19 +82,19 @@ match(Topic) when is_binary(Topic) -> %% @doc Lookup a Trie Node -spec(lookup(NodeId :: binary()) -> [#trie_node{}]). lookup(NodeId) -> - mnesia:read(trie_node, NodeId). + mnesia:read(mqtt_trie_node, NodeId). %% @doc Delete topic from trie -spec(delete(Topic :: binary()) -> ok). delete(Topic) when is_binary(Topic) -> - case mnesia:read(trie_node, Topic) of - [#trie_node{edge_count=0}] -> - mnesia:delete({trie_node, Topic}), + case mnesia:read(mqtt_trie_node, Topic) of + [#trie_node{edge_count=0}] -> + mnesia:delete({mqtt_trie_node, Topic}), delete_path(lists:reverse(emqttd_topic:triples(Topic))); [TrieNode] -> - mnesia:write(TrieNode#trie_node{topic = undefined}); + write_trie_node(TrieNode#trie_node{topic = undefined}); [] -> - ok + ok end. %%-------------------------------------------------------------------- @@ -105,18 +105,18 @@ delete(Topic) when is_binary(Topic) -> %% @doc Add path to trie tree. add_path({Node, Word, Child}) -> Edge = #trie_edge{node_id=Node, word=Word}, - case mnesia:read(trie_node, Node) of + case mnesia:read(mqtt_trie_node, Node) of [TrieNode = #trie_node{edge_count=Count}] -> - case mnesia:wread({trie, Edge}) of + case mnesia:wread({mqtt_trie, Edge}) of [] -> - mnesia:write(TrieNode#trie_node{edge_count=Count+1}), - mnesia:write(#trie{edge=Edge, node_id=Child}); + write_trie_node(TrieNode#trie_node{edge_count=Count+1}), + write_trie(#trie{edge=Edge, node_id=Child}); [_] -> ok end; [] -> - mnesia:write(#trie_node{node_id=Node, edge_count=1}), - mnesia:write(#trie{edge=Edge, node_id=Child}) + write_trie_node(#trie_node{node_id=Node, edge_count=1}), + write_trie(#trie{edge=Edge, node_id=Child}) end. %% @private @@ -128,11 +128,11 @@ match_node(NodeId, Words) -> match_node(NodeId, Words, []). match_node(NodeId, [], ResAcc) -> - mnesia:read(trie_node, NodeId) ++ 'match_#'(NodeId, ResAcc); + mnesia:read(mqtt_trie_node, NodeId) ++ 'match_#'(NodeId, ResAcc); match_node(NodeId, [W|Words], ResAcc) -> lists:foldl(fun(WArg, Acc) -> - case mnesia:read(trie, #trie_edge{node_id=NodeId, word=WArg}) of + case mnesia:read(mqtt_trie, #trie_edge{node_id=NodeId, word=WArg}) of [#trie{node_id=ChildId}] -> match_node(ChildId, Words, Acc); [] -> Acc end @@ -141,9 +141,9 @@ match_node(NodeId, [W|Words], ResAcc) -> %% @private %% @doc Match node with '#'. 'match_#'(NodeId, ResAcc) -> - case mnesia:read(trie, #trie_edge{node_id=NodeId, word = '#'}) of + case mnesia:read(mqtt_trie, #trie_edge{node_id=NodeId, word = '#'}) of [#trie{node_id=ChildId}] -> - mnesia:read(trie_node, ChildId) ++ ResAcc; + mnesia:read(mqtt_trie_node, ChildId) ++ ResAcc; [] -> ResAcc end. @@ -153,16 +153,24 @@ match_node(NodeId, [W|Words], ResAcc) -> delete_path([]) -> ok; delete_path([{NodeId, Word, _} | RestPath]) -> - mnesia:delete({trie, #trie_edge{node_id=NodeId, word=Word}}), - case mnesia:read(trie_node, NodeId) of + mnesia:delete({mqtt_trie, #trie_edge{node_id=NodeId, word=Word}}), + case mnesia:read(mqtt_trie_node, NodeId) of [#trie_node{edge_count=1, topic=undefined}] -> - mnesia:delete({trie_node, NodeId}), + mnesia:delete({mqtt_trie_node, NodeId}), delete_path(RestPath); [TrieNode=#trie_node{edge_count=1, topic=_}] -> - mnesia:write(TrieNode#trie_node{edge_count=0}); + write_trie_node(TrieNode#trie_node{edge_count=0}); [TrieNode=#trie_node{edge_count=C}] -> - mnesia:write(TrieNode#trie_node{edge_count=C-1}); + write_trie_node(TrieNode#trie_node{edge_count=C-1}); [] -> throw({notfound, NodeId}) end. +%% @private +write_trie(Trie) -> + mnesia:write(mqtt_trie, Trie, write). + +%% @private +write_trie_node(TrieNode) -> + mnesia:write(mqtt_trie_node, TrieNode, write). + diff --git a/src/emqttd_vm.erl b/src/emqttd_vm.erl index bad4d7c96..bdc6cef40 100644 --- a/src/emqttd_vm.erl +++ b/src/emqttd_vm.erl @@ -152,7 +152,7 @@ schedulers() -> erlang:system_info(schedulers). microsecs() -> - {Mega, Sec, Micro} = erlang:now(), + {Mega, Sec, Micro} = os:timestamp(), (Mega * 1000000 + Sec) * 1000000 + Micro. loads() -> diff --git a/src/emqttd_ws.erl b/src/emqttd_ws.erl index 60371457f..56a4d92d9 100644 --- a/src/emqttd_ws.erl +++ b/src/emqttd_ws.erl @@ -31,7 +31,7 @@ %% @doc Handle WebSocket Request. handle_request(Req) -> Peer = Req:get(peer), - PktOpts = emqttd:env(mqtt, packet), + PktOpts = emqttd_conf:mqtt(), ParserFun = emqttd_parser:new(PktOpts), {ReentryWs, ReplyChannel} = upgrade(Req), {ok, ClientPid} = emqttd_ws_client_sup:start_client(self(), Req, ReplyChannel), diff --git a/src/emqttd_ws_client.erl b/src/emqttd_ws_client.erl index ff4c16e79..d8f30d309 100644 --- a/src/emqttd_ws_client.erl +++ b/src/emqttd_ws_client.erl @@ -66,17 +66,20 @@ init([MqttEnv, WsPid, Req, ReplyChannel]) -> {ok, Peername} = Req:get(peername), Headers = mochiweb_headers:to_list( mochiweb_request:get(headers, Req)), - PktOpts = proplists:get_value(packet, MqttEnv), - SendFun = fun(Payload) -> ReplyChannel({binary, Payload}) end, + %% SendFun = fun(Payload) -> ReplyChannel({binary, Payload}) end, + SendFun = fun(Packet) -> + Data = emqttd_serializer:serialize(Packet), + emqttd_metrics:inc('bytes/sent', size(Data)), + ReplyChannel({binary, Data}) + end, ProtoState = emqttd_protocol:init(Peername, SendFun, - [{ws_initial_headers, Headers} | PktOpts]), + [{ws_initial_headers, Headers} | MqttEnv]), {ok, #wsclient_state{ws_pid = WsPid, peer = Req:get(peer), connection = Req:get(connection), proto_state = ProtoState}, idle_timeout(MqttEnv)}. idle_timeout(MqttEnv) -> - ClientOpts = proplists:get_value(client, MqttEnv), - timer:seconds(proplists:get_value(idle_timeout, ClientOpts, 10)). + timer:seconds(proplists:get_value(client_idle_timeout, MqttEnv, 10)). handle_call(session, _From, State = #wsclient_state{proto_state = ProtoState}) -> {reply, emqttd_protocol:session(ProtoState), State}; diff --git a/src/emqttd_ws_client_sup.erl b/src/emqttd_ws_client_sup.erl index 3577527a6..33983fd8c 100644 --- a/src/emqttd_ws_client_sup.erl +++ b/src/emqttd_ws_client_sup.erl @@ -27,7 +27,7 @@ %% @doc Start websocket client supervisor -spec(start_link() -> {ok, pid()}). start_link() -> - supervisor:start_link({local, ?MODULE}, ?MODULE, [emqttd:env(mqtt)]). + supervisor:start_link({local, ?MODULE}, ?MODULE, [emqttd_conf:mqtt()]). %% @doc Start a WebSocket Client -spec(start_client(pid(), mochiweb_request:request(), fun()) -> {ok, pid()}). diff --git a/src/gen_server2.erl b/src/gen_server2.erl index 5c04f3f9e..d5127f8d6 100644 --- a/src/gen_server2.erl +++ b/src/gen_server2.erl @@ -624,7 +624,7 @@ unregister_name(_Name) -> ok. extend_backoff(undefined) -> undefined; extend_backoff({backoff, InitialTimeout, MinimumTimeout, DesiredHibPeriod}) -> - {backoff, InitialTimeout, MinimumTimeout, DesiredHibPeriod, now()}. + {backoff, InitialTimeout, MinimumTimeout, DesiredHibPeriod, rand:seed(exsplus)}. %%%======================================================================== %%% Internal functions @@ -636,8 +636,10 @@ loop(GS2State = #gs2_state { time = hibernate, timeout_state = undefined, queue = Queue }) -> case priority_queue:is_empty(Queue) of - true -> pre_hibernate(GS2State); - false -> process_next_msg(GS2State) + true -> + pre_hibernate(GS2State); + false -> + process_next_msg(GS2State) end; loop(GS2State) -> @@ -693,7 +695,9 @@ wake_hib(GS2State = #gs2_state { timeout_state = TS }) -> undefined -> undefined; {SleptAt, TimeoutState} -> - adjust_timeout_state(SleptAt, now(), TimeoutState) + adjust_timeout_state(SleptAt, + erlang:monotonic_time(), + TimeoutState) end, post_hibernate( drain(GS2State #gs2_state { timeout_state = TimeoutState1 })). @@ -701,7 +705,8 @@ wake_hib(GS2State = #gs2_state { timeout_state = TS }) -> hibernate(GS2State = #gs2_state { timeout_state = TimeoutState }) -> TS = case TimeoutState of undefined -> undefined; - {backoff, _, _, _, _} -> {now(), TimeoutState} + {backoff, _, _, _, _} -> {erlang:monotonic_time(), + TimeoutState} end, proc_lib:hibernate(?MODULE, wake_hib, [GS2State #gs2_state { timeout_state = TS }]). @@ -746,7 +751,8 @@ post_hibernate(GS2State = #gs2_state { state = State, adjust_timeout_state(SleptAt, AwokeAt, {backoff, CurrentTO, MinimumTO, DesiredHibPeriod, RandomState}) -> - NapLengthMicros = timer:now_diff(AwokeAt, SleptAt), + NapLengthMicros = erlang:convert_time_unit(AwokeAt - SleptAt, + native, micro_seconds), CurrentMicros = CurrentTO * 1000, MinimumMicros = MinimumTO * 1000, DesiredHibMicros = DesiredHibPeriod * 1000, @@ -758,7 +764,7 @@ adjust_timeout_state(SleptAt, AwokeAt, {backoff, CurrentTO, MinimumTO, true -> lists:max([MinimumTO, CurrentTO div 2]); false -> CurrentTO end, - {Extra, RandomState1} = random:uniform_s(Base, RandomState), + {Extra, RandomState1} = rand:uniform_s(Base, RandomState), CurrentTO1 = Base + Extra, {backoff, CurrentTO1, MinimumTO, DesiredHibPeriod, RandomState1}. diff --git a/test/emqttd_SUITE.erl b/test/emqttd_SUITE.erl index 3c507540d..06923ee8e 100644 --- a/test/emqttd_SUITE.erl +++ b/test/emqttd_SUITE.erl @@ -22,6 +22,8 @@ -include_lib("eunit/include/eunit.hrl"). +-define(CONTENT_TYPE, "application/x-www-form-urlencoded"). + all() -> [{group, protocol}, {group, pubsub}, @@ -32,19 +34,17 @@ all() -> {group, metrics}, {group, stats}, {group, hook}, - {group, backend}, + {group, http}, + %%{group, backend}, {group, cli}]. groups() -> [{protocol, [sequence], [mqtt_connect]}, {pubsub, [sequence], - [create_topic, - create_subscription, - subscribe_unsubscribe, + [subscribe_unsubscribe, publish, pubsub, - 'pubsub#', 'pubsub+', - pubsub_queue]}, + 'pubsub#', 'pubsub+']}, {router, [sequence], [router_add_del, router_print, @@ -61,11 +61,13 @@ groups() -> [add_delete_hook, run_hooks]}, {retainer, [sequence], - [retain_messages, - dispatch_retained_messages, - expire_retained_messages]}, + [dispatch_retained_messages]}, {backend, [sequence], - [backend_subscription]}, + []}, + {http, [sequence], + [request_status, + request_publish + ]}, {cli, [sequence], [ctl_register_cmd, cli_status, @@ -82,6 +84,8 @@ groups() -> init_per_suite(Config) -> application:start(lager), + DataDir = proplists:get_value(data_dir, Config), + application:set_env(emqttd, conf, filename:join([DataDir, "emqttd.conf"])), application:ensure_all_started(emqttd), Config. @@ -113,26 +117,13 @@ connect_broker_(Packet, RecvSize) -> %% PubSub Test %%-------------------------------------------------------------------- -create_topic(_) -> - ok = emqttd:create(topic, <<"topic/create">>), - ok = emqttd:create(topic, <<"topic/create2">>), - [#mqtt_topic{topic = <<"topic/create">>, flags = [static]}] - = emqttd:lookup(topic, <<"topic/create">>). - -create_subscription(_) -> - ok = emqttd:create(subscription, {<<"clientId">>, <<"topic/sub">>, qos2}), - [#mqtt_subscription{subid = <<"clientId">>, topic = <<"topic/sub">>, qos = 2}] - = emqttd_backend:lookup_subscriptions(<<"clientId">>), - ok = emqttd_backend:del_subscriptions(<<"clientId">>), - ?assertEqual([], emqttd_backend:lookup_subscriptions(<<"clientId">>)). - subscribe_unsubscribe(_) -> - ok = emqttd:subscribe(<<"topic/subunsub">>), - ok = emqttd:subscribe(<<"clientId">>, <<"topic/subunsub1">>, 1), - ok = emqttd:subscribe(<<"clientId">>, <<"topic/subunsub2">>, 2), - ok = emqttd:unsubscribe(<<"topic/subunsub">>), - ok = emqttd:unsubscribe(<<"clientId">>, <<"topic/subunsub1">>, 1), - ok = emqttd:unsubscribe(<<"clientId">>, <<"topic/subunsub2">>, 2). + ok = emqttd:subscribe(<<"topic">>, <<"clientId">>), + ok = emqttd:subscribe(<<"topic/1">>, <<"clientId">>, [{qos, 1}]), + ok = emqttd:subscribe(<<"topic/2">>, <<"clientId">>, [{qos, 2}]), + ok = emqttd:unsubscribe(<<"topic">>, <<"clientId">>), + ok = emqttd:unsubscribe(<<"topic/1">>, <<"clientId">>), + ok = emqttd:unsubscribe(<<"topic/2">>, <<"clientId">>). publish(_) -> Msg = emqttd_message:make(ct, <<"test/pubsub">>, <<"hello">>), @@ -143,11 +134,11 @@ publish(_) -> pubsub(_) -> Self = self(), - emqttd:subscribe({<<"clientId">>, <<"a/b/c">>, 1}), - emqttd:subscribe({<<"clientId">>, <<"a/b/c">>, 2}), + ok = emqttd:subscribe(<<"a/b/c">>, Self, [{qos, 1}]), + ?assertMatch({error, _}, emqttd:subscribe(<<"a/b/c">>, Self, [{qos, 2}])), timer:sleep(10), - [{Self, <<"a/b/c">>}] = ets:lookup(subscribed, Self), - [{<<"a/b/c">>, Self}] = ets:lookup(subscriber, <<"a/b/c">>), + [{Self, <<"a/b/c">>}] = ets:lookup(mqtt_subscription, Self), + [{<<"a/b/c">>, Self}] = ets:lookup(mqtt_subscriber, <<"a/b/c">>), emqttd:publish(emqttd_message:make(ct, <<"a/b/c">>, <<"hello">>)), ?assert(receive {dispatch, <<"a/b/c">>, _} -> true after 2 -> false end), spawn(fun() -> @@ -173,22 +164,6 @@ pubsub(_) -> ?assert(receive {dispatch, <<"a/+/+">>, _} -> true after 1 -> false end), emqttd:unsubscribe(<<"a/+/+">>). -pubsub_queue(_) -> - Self = self(), Q = <<"$queue/abc">>, - SubFun = fun() -> - emqttd:subscribe(Q), - timer:sleep(1), - {ok, Msgs} = loop_recv(Q, 10), - Self ! {recv, self(), Msgs} - end, - Sub1 = spawn(SubFun), Sub2 = spawn(SubFun), - timer:sleep(5), - emqttd:publish(emqttd_message:make(ct, Q, <<"1", Q/binary>>)), - emqttd:publish(emqttd_message:make(ct, Q, <<"2", Q/binary>>)), - emqttd:publish(emqttd_message:make(ct, Q, <<"3", Q/binary>>)), - ?assert(receive {recv, Sub1, Msgs1} -> length(Msgs1) < 3 end), - ?assert(receive {recv, Sub2, Msgs2} -> length(Msgs2) < 3 end). - loop_recv(Topic, Timeout) -> loop_recv(Topic, Timeout, []). @@ -213,15 +188,15 @@ router_add_del(_) -> #mqtt_route{topic = <<"#">>, node = node()}, #mqtt_route{topic = <<"+/#">>, node = node()}, #mqtt_route{topic = <<"a/b/c">>, node = node()}], - Routes = lists:sort(emqttd_router:lookup(<<"a/b/c">>)), + Routes = lists:sort(emqttd_router:match(<<"a/b/c">>)), %% Batch Add emqttd_router:add_routes(Routes), - Routes = lists:sort(emqttd_router:lookup(<<"a/b/c">>)), + Routes = lists:sort(emqttd_router:match(<<"a/b/c">>)), %% Del emqttd_router:del_route(<<"a/b/c">>), - [R1, R2] = lists:sort(emqttd_router:lookup(<<"a/b/c">>)), + [R1, R2] = lists:sort(emqttd_router:match(<<"a/b/c">>)), {atomic, []} = mnesia:transaction(fun emqttd_trie:lookup/1, [<<"a/b/c">>]), %% Batch Del @@ -229,7 +204,7 @@ router_add_del(_) -> emqttd_router:add_route(R3), emqttd_router:del_routes([R1, R2]), emqttd_router:del_route(R3), - [] = lists:sort(emqttd_router:lookup(<<"a/b/c">>)). + [] = lists:sort(emqttd_router:match(<<"a/b/c">>)). router_print(_) -> Routes = [#mqtt_route{topic = <<"a/b/c">>, node = node()}, @@ -332,14 +307,6 @@ hook_fun5(arg1, arg2, Acc, init) -> {stop, [r3 | Acc]}. %% Retainer Test %%-------------------------------------------------------------------- -retain_messages(_) -> - Msg = emqttd_message:make(<<"clientId">>, <<"topic">>, <<"payload">>), - emqttd_backend:retain_message(Msg), - [Msg] = emqttd_backend:read_messages(<<"topic">>), - [Msg] = emqttd_backend:match_messages(<<"topic/#">>), - emqttd_backend:delete_message(<<"topic">>), - 0 = emqttd_backend:retained_count(). - dispatch_retained_messages(_) -> Msg = #mqtt_message{retain = true, topic = <<"a/b/c">>, payload = <<"payload">>}, @@ -347,34 +314,53 @@ dispatch_retained_messages(_) -> emqttd_retainer:dispatch(<<"a/b/+">>, self()), ?assert(receive {dispatch, <<"a/b/+">>, Msg} -> true after 10 -> false end), emqttd_retainer:retain(#mqtt_message{retain = true, topic = <<"a/b/c">>, payload = <<>>}), - [] = emqttd_backend:read_messages(<<"a/b/c">>). + [] = emqttd_retainer:read_messages(<<"a/b/c">>). -expire_retained_messages(_) -> - Msg1 = emqttd_message:make(<<"clientId1">>, qos1, <<"topic/1">>, <<"payload1">>), - Msg2 = emqttd_message:make(<<"clientId2">>, qos2, <<"topic/2">>, <<"payload2">>), - emqttd_backend:retain_message(Msg1), - emqttd_backend:retain_message(Msg2), - timer:sleep(2000), - emqttd_backend:expire_messages(emqttd_time:now_to_secs()), - 0 = emqttd_backend:retained_count(). %%-------------------------------------------------------------------- -%% Backend Test +%% HTTP Request Test %%-------------------------------------------------------------------- -backend_subscription(_) -> - Sub1 = #mqtt_subscription{subid = <<"clientId">>, topic = <<"topic">>, qos = 2}, - Sub2 = #mqtt_subscription{subid = <<"clientId">>, topic = <<"#">>, qos = 2}, - emqttd_backend:add_subscription(Sub1), - emqttd_backend:add_subscription(Sub2), - [Sub1, Sub2] = emqttd_backend:lookup_subscriptions(<<"clientId">>), - emqttd_backend:del_subscription(<<"clientId">>, <<"topic">>), - [Sub2] = emqttd_backend:lookup_subscriptions(<<"clientId">>), - emqttd_backend:del_subscriptions(<<"clientId">>), - [] = emqttd_backend:lookup_subscriptions(<<"clientId">>). +request_status(_) -> + {InternalStatus, _ProvidedStatus} = init:get_status(), + AppStatus = + case lists:keysearch(emqttd, 1, application:which_applications()) of + false -> not_running; + {value, _Val} -> running + end, + Status = iolist_to_binary(io_lib:format("Node ~s is ~s~nemqttd is ~s", + [node(), InternalStatus, AppStatus])), + Url = "http://127.0.0.1:8083/status", + {ok, {{"HTTP/1.1", 200, "OK"}, _, Return}} = + httpc:request(get, {Url, []}, [], []), + ?assertEqual(binary_to_list(Status), Return). + +request_publish(_) -> + ok = emqttd:subscribe(<<"a/b/c">>, self(), [{qos, 1}]), + Params = "qos=1&retain=0&topic=a/b/c&message=hello", + ?assert(connect_emqttd_publish_(post, "mqtt/publish", Params, auth_header_("", ""))), + ?assert(receive {dispatch, <<"a/b/c">>, _} -> true after 2 -> false end), + emqttd:unsubscribe(<<"a/b/c">>). + +connect_emqttd_publish_(Method, Api, Params, Auth) -> + Url = "http://127.0.0.1:8083/" ++ Api, + case httpc:request(Method, {Url, [Auth], ?CONTENT_TYPE, Params}, [], []) of + {error, socket_closed_remotely} -> + false; + {ok, {{"HTTP/1.1", 200, "OK"}, _, _Return} } -> + true; + {ok, {{"HTTP/1.1", 400, _}, _, []}} -> + false; + {ok, {{"HTTP/1.1", 404, _}, _, []}} -> + false + end. + +auth_header_(User, Pass) -> + Encoded = base64:encode_to_string(lists:append([User,":",Pass])), + {"Authorization","Basic " ++ Encoded}. %%-------------------------------------------------------------------- -%% CLI Group +%% Cli group %%-------------------------------------------------------------------- ctl_register_cmd(_) -> diff --git a/test/emqttd_SUITE_data/emqttd.conf b/test/emqttd_SUITE_data/emqttd.conf new file mode 100644 index 000000000..562bd908f --- /dev/null +++ b/test/emqttd_SUITE_data/emqttd.conf @@ -0,0 +1,266 @@ +%%=================================================================== +%% +%% Config file for emqttd 2.0 +%% +%% Erlang Term Syntax: +%% +%% {}: Tuple, usually {Key, Value} +%% []: List, seperated by comma +%% %%: Comment +%% +%%=================================================================== + +%%-------------------------------------------------------------------- +%% MQTT Protocol +%%-------------------------------------------------------------------- + +%% Max ClientId Length Allowed. +{mqtt_max_clientid_len, 512}. + +%% Max Packet Size Allowed, 64K by default. +{mqtt_max_packet_size, 65536}. + +%% Client Idle Timeout. +{mqtt_client_idle_timeout, 30}. % Second + +%%-------------------------------------------------------------------- +%% Authentication +%%-------------------------------------------------------------------- + +%% Anonymous: Allow all +{auth, anonymous, []}. + +%% Authentication with username, password +{auth, username, []}. + +%% Authentication with clientId +{auth, clientid, [{password, no}]}. + +%%-------------------------------------------------------------------- +%% ACL +%%-------------------------------------------------------------------- + +{acl, anonymous, []}. + +{acl, internal, [{nomatch, allow}]}. + +%% Cache ACL result for PUBLISH +{cache_acl, true}. + +%%-------------------------------------------------------------------- +%% Broker +%%-------------------------------------------------------------------- + +%% System interval of publishing broker $SYS messages +{broker_sys_interval, 60}. + +%%-------------------------------------------------------------------- +%% Retained message +%%-------------------------------------------------------------------- + +%% Expired after seconds, never expired if 0 +{retained_expired_after, 0}. + +%% Max number of retained messages +{retained_max_message_num, 100000}. + +%% Max Payload Size of retained message +{retained_max_playload_size, 65536}. + +%%-------------------------------------------------------------------- +%% Session +%%-------------------------------------------------------------------- + +%% Max number of QoS 1 and 2 messages that can be “inflight” at one time. +%% 0 means no limit +{session_max_inflight, 100}. + +%% Retry interval for redelivering QoS1/2 messages. +{session_unack_retry_interval, 60}. + +%% Awaiting PUBREL Timeout +{session_await_rel_timeout, 20}. + +%% Max Packets that Awaiting PUBREL, 0 means no limit +{session_max_awaiting_rel, 0}. + +%% Statistics Collection Interval(seconds) +{session_collect_interval, 0}. + +%% Expired after 2 day (unit: minute) +{session_expired_after, 2880}. + +%%-------------------------------------------------------------------- +%% Queue +%%-------------------------------------------------------------------- + +%% Type: simple | priority +{queue_type, simple}. + +%% Topic Priority: 0~255, Default is 0 +%% {queue_priority, [{"topic/1", 10}, {"topic/2", 8}]}. + +%% Max queue length. Enqueued messages when persistent client disconnected, +%% or inflight window is full. +{queue_max_length, infinity}. + +%% Low-water mark of queued messages +{queue_low_watermark, 0.2}. + +%% High-water mark of queued messages +{queue_high_watermark, 0.6}. + +%% Queue Qos0 messages? +{queue_qos0, true}. + +%%-------------------------------------------------------------------- +%% Zone +%%-------------------------------------------------------------------- + +{zone, admin, []}. + +%%-------------------------------------------------------------------- +%% Listener +%%-------------------------------------------------------------------- + +%% Plain MQTT +{listener, mqtt, 1883, [ + %% Size of acceptor pool + {acceptors, 16}, + + %% Maximum number of concurrent clients + {max_clients, 512}, + + %% Mount point prefix + %% {mount_point, "prefix/"}, + + %% Socket Access Control + {access, [{allow, all}]}, + + %% Connection Options + {connopts, [ + %% Rate Limit. Format is 'burst, rate', Unit is KB/Sec + %% {rate_limit, "100,10"} %% 100K burst, 10K rate + ]}, + + %% Socket Options + {sockopts, [ + %Set buffer if hight thoughtput + %{recbuf, 4096}, + %{sndbuf, 4096}, + %{buffer, 4096}, + %{nodelay, true}, + {backlog, 1024} + ]} +]}. + +%% MQTT/SSL +{listener, mqtts, 8883, [ + %% Size of acceptor pool + {acceptors, 4}, + + %% Maximum number of concurrent clients + {max_clients, 512}, + + %% Socket Access Control + {access, [{allow, all}]}, + + %% SSL certificate and key files + {ssl, [{certfile, "etc/ssl/ssl.crt"}, + {keyfile, "etc/ssl/ssl.key"}]}, + + %% Socket Options + {sockopts, [ + {backlog, 1024} + %{buffer, 4096}, + ]} +]}. + +%% HTTP and WebSocket Listener +{listener, http, 8083, [ + %% Size of acceptor pool + {acceptors, 4}, + + %% Maximum number of concurrent clients + {max_clients, 64}, + + %% Socket Access Control + {access, [{allow, all}]}, + + %% Socket Options + {sockopts, [ + {backlog, 1024} + %{buffer, 4096}, + ]} +]}. + +%%-------------------------------------------------------------------- +%% PubSub +%%-------------------------------------------------------------------- + +%% PubSub and Router. Default should be scheduler numbers. +{pubsub_pool_size, 8}. + +%%-------------------------------------------------------------------- +%% Routing +%%-------------------------------------------------------------------- + +%% Route aging time(seconds) +{routing_age, 5}. + +%%-------------------------------------------------------------------- +%% Bridge +%%-------------------------------------------------------------------- + +%% TODO: Bridge Queue Size +{bridge_max_queue_len, 10000}. + +%% Ping Interval of bridge node +{bridge_ping_down_interval, 1}. % second + +%%------------------------------------------------------------------- +%% Plugins +%%------------------------------------------------------------------- + +%% Dir of plugins' config +{plugins_etc_dir, "etc/plugins/"}. + +%% File to store loaded plugin names. +{plugins_loaded_file, "data/loaded_plugins"}. + +%%-------------------------------------------------------------------- +%% Modules +%%-------------------------------------------------------------------- + +%% Client presence management module. Publish presence messages when +%% client connected or disconnected. +{module, presence, [{qos, 0}]}. + +%% Subscribe topics automatically when client connected +{module, subscription, [{"$queue/clients/$c", 1}, backend]}. + +%% [Rewrite](https://github.com/emqtt/emqttd/wiki/Rewrite) +{module, rewrite, []}. + +%%------------------------------------------------------------------- +%% Erlang System Monitor +%%------------------------------------------------------------------- + +%% Long GC, don't monitor in production mode for: +%% https://github.com/erlang/otp/blob/feb45017da36be78d4c5784d758ede619fa7bfd3/erts/emulator/beam/erl_gc.c#L421 + +{sysmon_long_gc, false}. + +%% Long Schedule(ms) +{sysmon_long_schedule, 240}. + +%% 8M words. 32MB on 32-bit VM, 64MB on 64-bit VM. +%% 8 * 1024 * 1024 +{sysmon_large_heap, 8388608}. + +%% Busy Port +{sysmon_busy_port, false}. + +%% Busy Dist Port +{sysmon_busy_dist_port, true}. + diff --git a/test/emqttd_access_SUITE.erl b/test/emqttd_access_SUITE.erl index 8a8d05766..074fd15df 100644 --- a/test/emqttd_access_SUITE.erl +++ b/test/emqttd_access_SUITE.erl @@ -38,9 +38,34 @@ groups() -> [compile_rule, match_rule]}]. +init_per_group(access_control, Config) -> + application:load(emqttd), + prepare_config(), + gen_conf:init(emqttd), + Config; + init_per_group(_Group, Config) -> Config. +prepare_config() -> + Rules = [{allow, {ipaddr, "127.0.0.1"}, subscribe, ["$SYS/#", "#"]}, + {allow, {user, "testuser"}, subscribe, ["a/b/c", "d/e/f/#"]}, + {allow, {user, "admin"}, pubsub, ["a/b/c", "d/e/f/#"]}, + {allow, {client, "testClient"}, subscribe, ["testTopics/testClient"]}, + {allow, all, subscribe, ["clients/$c"]}, + {allow, all, pubsub, ["users/$u/#"]}, + {deny, all, subscribe, ["$SYS/#", "#"]}, + {deny, all}], + write_config("access_SUITE_acl.conf", Rules), + Config = [{auth, anonymous, []}, + {acl, internal, [{config, "access_SUITE_acl.conf"}, + {nomatch, allow}]}], + write_config("access_SUITE_emqttd.conf", Config), + application:set_env(emqttd, conf, "access_SUITE_emqttd.conf"). + +write_config(Filename, Terms) -> + file:write_file(Filename, [io_lib:format("~tp.~n", [Term]) || Term <- Terms]). + end_per_group(_Group, Config) -> Config. @@ -48,14 +73,7 @@ init_per_testcase(TestCase, Config) when TestCase =:= reload_acl; TestCase =:= register_mod; TestCase =:= unregister_mod; TestCase =:= check_acl -> - DataDir = proplists:get_value(data_dir, Config), - AclOpts = [ - {auth, [{anonymous, []}]}, - {acl, [{internal, [{file, filename:join([DataDir, "test_acl.config"])}, - {nomatch, allow}]}]} - ], - {ok, _Pid} = ?AC:start_link(AclOpts), - Config; + {ok, _Pid} = ?AC:start_link(), Config; init_per_testcase(_TestCase, Config) -> Config. diff --git a/test/emqttd_access_SUITE_data/test_acl.config b/test/emqttd_access_SUITE_data/test_acl.config deleted file mode 100644 index 4a2c6ea44..000000000 --- a/test/emqttd_access_SUITE_data/test_acl.config +++ /dev/null @@ -1,16 +0,0 @@ -{allow, {ipaddr, "127.0.0.1"}, subscribe, ["$SYS/#", "#"]}. - -{allow, {user, "testuser"}, subscribe, ["a/b/c", "d/e/f/#"]}. - -{allow, {user, "admin"}, pubsub, ["a/b/c", "d/e/f/#"]}. - -{allow, {client, "testClient"}, subscribe, ["testTopics/testClient"]}. - -{allow, all, subscribe, ["clients/$c"]}. - -{allow, all, pubsub, ["users/$u/#"]}. - -{deny, all, subscribe, ["$SYS/#", "#"]}. - -{deny, all}. - diff --git a/test/emqttd_backend_SUITE.erl b/test/emqttd_backend_SUITE.erl deleted file mode 100644 index ceb1b3d6e..000000000 --- a/test/emqttd_backend_SUITE.erl +++ /dev/null @@ -1,45 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2012-2016 Feng Lee . -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqttd_backend_SUITE). - --include("emqttd.hrl"). - --compile(export_all). - -all() -> [{group, subscription}]. - -groups() -> [{subscription, [], [add_del_subscription]}]. - -init_per_suite(Config) -> - ok = emqttd_mnesia:ensure_started(), - emqttd_backend:mnesia(boot), - emqttd_backend:mnesia(copy), - Config. - -end_per_suite(_Config) -> - emqttd_mnesia:ensure_stopped(). - -add_del_subscription(_) -> - Sub1 = #mqtt_subscription{subid = <<"clientId">>, topic = <<"topic">>, qos = 2}, - Sub2 = #mqtt_subscription{subid = <<"clientId">>, topic = <<"topic">>, qos = 1}, - ok = emqttd_backend:add_subscription(Sub1), - {error, already_existed} = emqttd_backend:add_subscription(Sub1), - ok = emqttd_backend:add_subscription(Sub2), - [Sub2] = emqttd_backend:lookup_subscriptions(<<"clientId">>), - emqttd_backend:del_subscription(<<"clientId">>, <<"topic">>), - [] = emqttd_backend:lookup_subscriptions(<<"clientId">>). - diff --git a/test/emqttd_lib_SUITE.erl b/test/emqttd_lib_SUITE.erl index 1a3b1aef6..51dd499ba 100644 --- a/test/emqttd_lib_SUITE.erl +++ b/test/emqttd_lib_SUITE.erl @@ -16,6 +16,8 @@ -module(emqttd_lib_SUITE). +-include_lib("eunit/include/eunit.hrl"). + -compile(export_all). -define(SOCKOPTS, [ @@ -35,7 +37,7 @@ all() -> [{group, guid}, {group, opts}, {group, node}, {group, base62}]. groups() -> - [{guid, [], [guid_gen]}, + [{guid, [], [guid_gen, guid_hexstr, guid_base62]}, {opts, [], [opts_merge]}, {?PQ, [], [priority_queue_plen, priority_queue_out2]}, @@ -56,6 +58,14 @@ guid_gen(_) -> Ts2 = emqttd_guid:timestamp(emqttd_guid:gen()), true = Ts2 > Ts1. +guid_hexstr(_) -> + Guid = emqttd_guid:gen(), + ?assertEqual(Guid, emqttd_guid:from_hexstr(emqttd_guid:to_hexstr(Guid))). + +guid_base62(_) -> + Guid = emqttd_guid:gen(), + ?assertEqual(Guid, emqttd_guid:from_base62(emqttd_guid:to_base62(Guid))). + %%-------------------------------------------------------------------- %% emqttd_opts %%-------------------------------------------------------------------- diff --git a/test/emqttd_mock_client.erl b/test/emqttd_mock_client.erl index 7a9f90010..f4d26fa30 100644 --- a/test/emqttd_mock_client.erl +++ b/test/emqttd_mock_client.erl @@ -39,7 +39,7 @@ init([ClientId]) -> {ok, #state{clientid = ClientId}}. handle_call(start_session, _From, State = #state{clientid = ClientId}) -> - {ok, SessPid, _} = emqttd_sm:start_session(true, ClientId), + {ok, SessPid, _} = emqttd_sm:start_session(true, {ClientId, undefined}), {reply, {ok, SessPid}, State#state{session = SessPid}}; handle_call(stop, _From, State) -> diff --git a/test/emqttd_protocol_SUITE.erl b/test/emqttd_protocol_SUITE.erl index d9344786d..96b969278 100644 --- a/test/emqttd_protocol_SUITE.erl +++ b/test/emqttd_protocol_SUITE.erl @@ -337,9 +337,8 @@ packet_format(_) -> message_make(_) -> Msg = emqttd_message:make(<<"clientid">>, <<"topic">>, <<"payload">>), 0 = Msg#mqtt_message.qos, - undefined = Msg#mqtt_message.msgid, Msg1 = emqttd_message:make(<<"clientid">>, qos2, <<"topic">>, <<"payload">>), - true = is_binary(Msg1#mqtt_message.msgid), + true = is_binary(Msg1#mqtt_message.id), 2 = Msg1#mqtt_message.qos. message_from_packet(_) -> @@ -356,8 +355,7 @@ message_from_packet(_) -> Msg2 = emqttd_message:from_packet(<<"username">>, <<"clientid">>, ?PUBLISH_PACKET(1, <<"topic">>, 20, <<"payload">>)), - <<"clientid">> = Msg2#mqtt_message.from, - <<"username">> = Msg2#mqtt_message.sender, + {<<"clientid">>, <<"username">>} = Msg2#mqtt_message.from, io:format("~s", [emqttd_message:format(Msg2)]). message_flag(_) -> diff --git a/test/emqttd_topic_SUITE.erl b/test/emqttd_topic_SUITE.erl index abcf50cf9..5e9608e00 100644 --- a/test/emqttd_topic_SUITE.erl +++ b/test/emqttd_topic_SUITE.erl @@ -16,18 +16,20 @@ -module(emqttd_topic_SUITE). +-include_lib("eunit/include/eunit.hrl"). + %% CT -compile(export_all). -import(emqttd_topic, [wildcard/1, match/2, validate/1, triples/1, join/1, - words/1, systop/1, is_queue/1, feed_var/3]). + words/1, systop/1, feed_var/3, strip/1, strip/2]). -define(N, 10000). all() -> [t_wildcard, t_match, t_match2, t_validate, t_triples, t_join, - t_words, t_systop, t_is_queue, t_feed_var, t_sys_match, 't_#_match', + t_words, t_systop, t_feed_var, t_sys_match, 't_#_match', t_sigle_level_validate, t_sigle_level_match, t_match_perf, - t_triples_perf]. + t_triples_perf, t_strip]. t_wildcard(_) -> true = wildcard(<<"a/b/#">>), @@ -155,21 +157,25 @@ t_join(_) -> <<"/ab/cd/ef/">> = join(words(<<"/ab/cd/ef/">>)), <<"ab/+/#">> = join(words(<<"ab/+/#">>)). -t_is_queue(_) -> - true = is_queue(<<"$queue/queue">>), - false = is_queue(<<"xyz/queue">>). - t_systop(_) -> SysTop1 = iolist_to_binary(["$SYS/brokers/", atom_to_list(node()), "/xyz"]), - SysTop1 = systop('xyz'), + ?assertEqual(SysTop1, systop('xyz')), SysTop2 = iolist_to_binary(["$SYS/brokers/", atom_to_list(node()), "/abc"]), - SysTop2 = systop(<<"abc">>). + ?assertEqual(SysTop2,systop(<<"abc">>)). t_feed_var(_) -> - <<"$queue/client/clientId">> = feed_var(<<"$c">>, <<"clientId">>, <<"$queue/client/$c">>), - <<"username/test/client/x">> = feed_var(<<"%u">>, <<"test">>, <<"username/%u/client/x">>), - <<"username/test/client/clientId">> = feed_var(<<"%c">>, <<"clientId">>, <<"username/test/client/%c">>). + ?assertEqual(<<"$queue/client/clientId">>, feed_var(<<"$c">>, <<"clientId">>, <<"$queue/client/$c">>)), + ?assertEqual(<<"username/test/client/x">>, feed_var(<<"%u">>, <<"test">>, <<"username/%u/client/x">>)), + ?assertEqual(<<"username/test/client/clientId">>, feed_var(<<"%c">>, <<"clientId">>, <<"username/test/client/%c">>)). long_topic() -> iolist_to_binary([[integer_to_list(I), "/"] || I <- lists:seq(0, 10000)]). +t_strip(_) -> + ?assertEqual({<<"a/b/+/#">>, []}, strip(<<"a/b/+/#">>)), + ?assertEqual({<<"topic">>, [{share, '$queue'}]}, strip(<<"$queue/topic">>)), + ?assertEqual({<<"topic">>, [{share, <<"group">>}]}, strip(<<"$share/group/topic">>)), + ?assertEqual({<<"topic">>, [local]}, strip(<<"$local/topic">>)), + ?assertEqual({<<"topic">>, [{share, '$queue'}, local]}, strip(<<"$local/$queue/topic">>)), + ?assertEqual({<<"/a/b/c">>, [{share, <<"group">>}, local]}, strip(<<"$local/$share/group//a/b/c">>)). + diff --git a/test/emqttd_trie_SUITE.erl b/test/emqttd_trie_SUITE.erl index 7ee9d6616..ceda3abcb 100644 --- a/test/emqttd_trie_SUITE.erl +++ b/test/emqttd_trie_SUITE.erl @@ -22,6 +22,8 @@ -define(TRIE, emqttd_trie). +-include_lib("eunit/include/eunit.hrl"). + all() -> [t_insert, t_match, t_match2, t_match3, t_delete, t_delete2, t_delete3]. @@ -81,9 +83,9 @@ t_match3(_) -> Topics = [<<"d/#">>, <<"a/b/c">>, <<"a/b/+">>, <<"a/#">>, <<"#">>, <<"$SYS/#">>], mnesia:transaction(fun() -> [emqttd_trie:insert(Topic) || Topic <- Topics] end), Matched = mnesia:async_dirty(fun emqttd_trie:match/1, [<<"a/b/c">>]), - 4 = length(Matched), + ?assertEqual(4, length(Matched)), SysMatched = mnesia:async_dirty(fun emqttd_trie:match/1, [<<"$SYS/a/b/c">>]), - [<<"$SYS/#">>] = SysMatched. + ?assertEqual([<<"$SYS/#">>], SysMatched). t_delete(_) -> TN = #trie_node{node_id = <<"sensor/1">>, @@ -129,5 +131,5 @@ t_delete3(_) -> end). clear_tables() -> - lists:foreach(fun mnesia:clear_table/1, [trie, trie_node]). + lists:foreach(fun mnesia:clear_table/1, [mqtt_trie, mqtt_trie_node]).