Merge pull request #470 from emqtt/0.17

0.17.0 - Documents and Improve the design of Hooks, PubSub and Router

CHANGELOG.md

@@ -1,6 +1,14 @@
 
 emqttd ChangeLog
-=================
+================
+
+0.17.0-beta(2016-03-12)
+------------------------
+
+Full Documents released on https://docs.emqtt.com
+
+Improve the design of Hook, PubSub and Router
+
 
 0.16.0-beta(2016-02-16)
 ------------------------
@@ -725,7 +733,7 @@ Change: rename project from 'emqtt' to 'emqttd'
 
 Change: lager:debug to dump RECV/SENT packets
 
-Feature: emqttd_bridg, emqttd_bridge_sup to support broker bridge
+Feature: emqttd_bridge, emqttd_bridge_sup to support broker bridge
 
 Feature: emqtt_event to publish client connected/disconnected message to $SYS topics
 
@@ -890,7 +898,7 @@ Bugfix: fix "mosquitto_sub -q 2 ......" bug
 
 Bugfix: fix keep alive bug
 
-0.1.3 (2012-01-04)
+0.1.3 (2013-01-04)
 -------------------
 
 Feature: support QOS2 PUBREC, PUBREL,PUBCOMP messages

Makefile

@@ -26,7 +26,7 @@ clean:
 	@$(REBAR) clean
 
 test:
-	ERL_FLAGS="-config rel/files/test.config" $(REBAR) -v skip_deps=true ct
+	ERL_FLAGS="-config rel/files/emqttd.test.config" $(REBAR) -v skip_deps=true ct
 	#$(REBAR) skip_deps=true eunit
 
 edoc:

README.md

@@ -1,7 +1,9 @@
 
 ## Overview [![Build Status](https://travis-ci.org/emqtt/emqttd.svg?branch=master)](https://travis-ci.org/emqtt/emqttd)  [![Join the chat at https://gitter.im/emqtt/emqttd](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/emqtt/emqttd?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 
-emqttd is a massively scalable and clusterable MQTT V3.1/V3.1.1 broker written in Erlang/OTP. emqttd support both MQTT V3.1/V3.1.1 protocol specification with extended features.
+emqttd is a massively scalable and clusterable MQTT V3.1/V3.1.1 broker written in Erlang/OTP.
+
+emqttd is fully open source and licensed under the Apache Version 2.0. emqttd implements both MQTT V3.1 and V3.1.1 protocol specifications, and supports WebSocket, STOMP, SockJS, CoAP and MQTT-SN at the same time.
 
 emqttd requires Erlang R17+ to build.
 
@@ -11,7 +13,7 @@ Follow us on Twitter: [@emqtt](https://twitter.com/emqtt)
 
 ## Goals
 
-emqttd is aimed to provide a solid, enterprise grade, extensible open-source MQTT broker for IoT, M2M and Mobile applications that need to support ten millions of concurrent MQTT clients.
+The emqttd project is aimed to implement a scalable, distributed, extensible open-source MQTT broker for IoT, M2M and Mobile applications that hope to handle millions of concurrent MQTT clients.
 
 * Easy to install
 * Massively scalable
@@ -23,50 +25,59 @@ emqttd is aimed to provide a solid, enterprise grade, extensible open-source MQT
 * Full MQTT V3.1/V3.1.1 protocol specification support
 * QoS0, QoS1, QoS2 Publish and Subscribe
 * Session Management and Offline Messages
-* Retained Messages Support
+* Retained Message
-* Last Will Message Support
+* Last Will Message
-* TCP/SSL Connection Support
+* TCP/SSL Connection
-* MQTT Over Websocket(SSL) Support
+* MQTT Over WebSocket(SSL)
-* HTTP Publish API Support
+* HTTP Publish API
-* [$SYS/brokers/#](https://github.com/emqtt/emqtt/wiki/$SYS-Topics-of-Broker) Support
+* STOMP protocol
-* Client Authentication with clientId, ipaddress
+* STOMP over SockJS
-* Client Authentication with username, password.
+* $SYS/# Topics
-* Client ACL control with ipaddress, clientid, username.
+* ClientID Authentication
-* Cluster brokers on several servers.
+* IpAddress Authentication
-* [Bridge](https://github.com/emqtt/emqttd/wiki/Bridge) brokers locally or remotely
+* Username and Password Authentication
-* 500K+ concurrent clients connections per server
+* Access control based on IpAddress, ClientID, Username
+* Authentication with LDAP, Redis, MySQL, PostgreSQL
+* Cluster brokers on several servers
+* Bridge brokers locally or remotely
+* mosquitto, RSMB bridge
 * Extensible architecture with Hooks, Modules and Plugins
 * Passed eclipse paho interoperability tests
 
 ## Modules
 
-* [emqttd_auth_clientid](https://github.com/emqtt/emqttd/wiki/Authentication) - Authentication with ClientIds
+Module                   | Description
-* [emqttd_auth_username](https://github.com/emqtt/emqttd/wiki/Authentication) - Authentication with Username and Password
+-------------------------|------------------------------
-* [emqttd_auth_ldap](https://github.com/emqtt/emqttd/wiki/Authentication) - Authentication with LDAP
+emqttd_auth_clientid     | Authentication with ClientIds
-* [emqttd_mod_presence](https://github.com/emqtt/emqttd/wiki/Presence) - Publish presence message to $SYS topics when client connected or disconnected
+emqttd_auth_username     | Authentication with Username and Password
-* emqttd_mod_autosub - Subscribe topics when client connected
+emqttd_auth_ldap         | Authentication with LDAP
-* [emqttd_mod_rewrite](https://github.com/emqtt/emqttd/wiki/Rewrite) - Topics rewrite like HTTP rewrite module
+emqttd_mod_presence      | Publish presence message when client connected or disconnected
+emqttd_mod_subscription  | Subscribe topics when client connected
+emqttd_mod_rewrite       | Topic path rewrite like HTTP rewrite module
 
 ## Plugins
 
-* [emqttd_plugin_template](https://github.com/emqtt/emqttd_plugin_template) - Plugin template and demo
+Plugin                                                                    | Description
-* [emqttd_dashboard](https://github.com/emqtt/emqttd_dashboard) - Web Dashboard
+--------------------------------------------------------------------------|--------------------------------------
-* [emqttd_plugin_mysql](https://github.com/emqtt/emqttd_plugin_mysql) - Authentication with MySQL
+[emqttd_plugin_template](https://github.com/emqtt/emqttd_plugin_template) | Plugin template and demo
-* [emqttd_plugin_pgsql](https://github.com/emqtt/emqttd_plugin_pgsql) - Authentication with PostgreSQL
+[emqttd_dashboard](https://github.com/emqtt/emqttd_dashboard)             | Web Dashboard
-* [emqttd_plugin_kafka](https://github.com/emqtt/emqtt_kafka) - Publish MQTT Messages to Kafka
+[emqttd_plugin_mysql](https://github.com/emqtt/emqttd_plugin_mysql)       | MySQL Authentication/ACL Plugin
-* [emqttd_plugin_redis](https://github.com/emqtt/emqttd_plugin_redis) - Redis Plugin
+[emqttd_plugin_pgsql](https://github.com/emqtt/emqttd_plugin_pgsql)       | PostgreSQL Authentication/ACL Plugin
-* [emqttd_plugin_mongo](https://github.com/emqtt/emqttd_plugin_mongo) - MongoDB Plugin
+[emqttd_plugin_redis](https://github.com/emqtt/emqttd_plugin_redis)       | Redis Authentication/ACL Plugin
-* [emqttd_stomp](https://github.com/emqtt/emqttd_stomp) - Stomp Protocol Plugin
+[emqttd_plugin_mongo](https://github.com/emqtt/emqttd_plugin_mongo)       | MongoDB Authentication/ACL Plugin
-* [emqttd_sockjs](https://github.com/emqtt/emqttd_sockjs) - SockJS(Stomp) Plugin
+[emqttd_stomp](https://github.com/emqtt/emqttd_stomp)                     | Stomp Protocol Plugin
-* [emqttd_recon](https://github.com/emqtt/emqttd_recon) - Recon Plugin
+[emqttd_sockjs](https://github.com/emqtt/emqttd_sockjs)                   | SockJS(Stomp) Plugin
+[emqttd_recon](https://github.com/emqtt/emqttd_recon)                     | Recon Plugin
 
 ## Dashboard
 
-The broker released a simple web dashboard in 0.10.0 version.
+A Web Dashboard will be loaded when the emqttd broker started successfully.
 
-Address: http://localhost:18083
+The Dashboard helps monitor broker's running status, statistics and metrics of MQTT packets.
-Username: admin
+
-Password: public
+Default Address: http://localhost:18083
+
+Default Login/Password: admin/public
 
 ## Design
 
@@ -74,12 +85,12 @@ Password: public
 
 ## QuickStart
 
-Download binary packeges for linux, mac and freebsd from [http://emqtt.io/downloads](http://emqtt.io/downloads).
+Download binary package for Linux, Mac and Freebsd from [http://emqtt.io/downloads](http://emqtt.io/downloads).
 
-For example:
+Installing on Ubuntu64, for example:
 
 ```sh
-unzip emqttd-ubuntu64-0.12.0-beta-20151008.zip && cd emqttd
+unzip emqttd-macosx-0.16.0-beta-20160216.zip && cd emqttd
 
 # start console
 ./bin/emqttd console
@@ -94,21 +105,25 @@ unzip emqttd-ubuntu64-0.12.0-beta-20151008.zip && cd emqttd
 ./bin/emqttd stop
 ``` 
 
-Build from source:
+Installing from source:
 
 ```
 git clone https://github.com/emqtt/emqttd.git
 
 cd emqttd && make && make dist
+
+cd rel/emqttd && ./bin/emqttd console
 ```
 
-## GetStarted
+## Documents
 
-Read [emqtt wiki](https://github.com/emqtt/emqttd/wiki) for detailed installation and configuration guide.
+Read Documents on [emqttd-docs.rtfd.org](http://emqttd-docs.rtfd.org) for installation and configuration guide.
 
 ## Benchmark
 
-Benchmark 0.12.0-beta on a centos6 server with 8 Core, 32G memory from QingCloud:
+Latest release of emqttd broker is scaling to 1.3 million MQTT connections on a 12 Core, 32G CentOS server.
+
+Benchmark 0.12.0-beta on a CentOS6 server with 8 Core, 32G memory from QingCloud:
 
 250K Connections, 250K Topics, 250K Subscriptions, 4K Qos1 Messages/Sec In, 20K Qos1 Messages/Sec Out, 8M+(bps) In, 40M+(bps) Out Traffic
 

docs/Makefile

@@ -0,0 +1,177 @@
+# Makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS    =
+SPHINXBUILD   = sphinx-build
+PAPER         =
+BUILDDIR      = build
+
+# User-friendly check for sphinx-build
+ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1)
+$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/)
+endif
+
+# Internal variables.
+PAPEROPT_a4     = -D latex_paper_size=a4
+PAPEROPT_letter = -D latex_paper_size=letter
+ALLSPHINXOPTS   = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source
+# the i18n builder cannot share the environment and doctrees with the others
+I18NSPHINXOPTS  = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source
+
+.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext
+
+help:
+	@echo "Please use \`make <target>' where <target> is one of"
+	@echo "  html       to make standalone HTML files"
+	@echo "  dirhtml    to make HTML files named index.html in directories"
+	@echo "  singlehtml to make a single large HTML file"
+	@echo "  pickle     to make pickle files"
+	@echo "  json       to make JSON files"
+	@echo "  htmlhelp   to make HTML files and a HTML help project"
+	@echo "  qthelp     to make HTML files and a qthelp project"
+	@echo "  devhelp    to make HTML files and a Devhelp project"
+	@echo "  epub       to make an epub"
+	@echo "  latex      to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
+	@echo "  latexpdf   to make LaTeX files and run them through pdflatex"
+	@echo "  latexpdfja to make LaTeX files and run them through platex/dvipdfmx"
+	@echo "  text       to make text files"
+	@echo "  man        to make manual pages"
+	@echo "  texinfo    to make Texinfo files"
+	@echo "  info       to make Texinfo files and run them through makeinfo"
+	@echo "  gettext    to make PO message catalogs"
+	@echo "  changes    to make an overview of all changed/added/deprecated items"
+	@echo "  xml        to make Docutils-native XML files"
+	@echo "  pseudoxml  to make pseudoxml-XML files for display purposes"
+	@echo "  linkcheck  to check all external links for integrity"
+	@echo "  doctest    to run all doctests embedded in the documentation (if enabled)"
+
+clean:
+	rm -rf $(BUILDDIR)/*
+
+html:
+	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
+	@echo
+	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
+
+dirhtml:
+	$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
+	@echo
+	@echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."
+
+singlehtml:
+	$(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml
+	@echo
+	@echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml."
+
+pickle:
+	$(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle
+	@echo
+	@echo "Build finished; now you can process the pickle files."
+
+json:
+	$(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json
+	@echo
+	@echo "Build finished; now you can process the JSON files."
+
+htmlhelp:
+	$(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp
+	@echo
+	@echo "Build finished; now you can run HTML Help Workshop with the" \
+	      ".hhp project file in $(BUILDDIR)/htmlhelp."
+
+qthelp:
+	$(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp
+	@echo
+	@echo "Build finished; now you can run "qcollectiongenerator" with the" \
+	      ".qhcp project file in $(BUILDDIR)/qthelp, like this:"
+	@echo "# qcollectiongenerator $(BUILDDIR)/qthelp/ErlangMQTTBroker.qhcp"
+	@echo "To view the help file:"
+	@echo "# assistant -collectionFile $(BUILDDIR)/qthelp/ErlangMQTTBroker.qhc"
+
+devhelp:
+	$(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp
+	@echo
+	@echo "Build finished."
+	@echo "To view the help file:"
+	@echo "# mkdir -p $$HOME/.local/share/devhelp/ErlangMQTTBroker"
+	@echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/ErlangMQTTBroker"
+	@echo "# devhelp"
+
+epub:
+	$(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub
+	@echo
+	@echo "Build finished. The epub file is in $(BUILDDIR)/epub."
+
+latex:
+	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
+	@echo
+	@echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex."
+	@echo "Run \`make' in that directory to run these through (pdf)latex" \
+	      "(use \`make latexpdf' here to do that automatically)."
+
+latexpdf:
+	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
+	@echo "Running LaTeX files through pdflatex..."
+	$(MAKE) -C $(BUILDDIR)/latex all-pdf
+	@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
+
+latexpdfja:
+	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
+	@echo "Running LaTeX files through platex and dvipdfmx..."
+	$(MAKE) -C $(BUILDDIR)/latex all-pdf-ja
+	@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
+
+text:
+	$(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text
+	@echo
+	@echo "Build finished. The text files are in $(BUILDDIR)/text."
+
+man:
+	$(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man
+	@echo
+	@echo "Build finished. The manual pages are in $(BUILDDIR)/man."
+
+texinfo:
+	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
+	@echo
+	@echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo."
+	@echo "Run \`make' in that directory to run these through makeinfo" \
+	      "(use \`make info' here to do that automatically)."
+
+info:
+	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
+	@echo "Running Texinfo files through makeinfo..."
+	make -C $(BUILDDIR)/texinfo info
+	@echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo."
+
+gettext:
+	$(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale
+	@echo
+	@echo "Build finished. The message catalogs are in $(BUILDDIR)/locale."
+
+changes:
+	$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
+	@echo
+	@echo "The overview file is in $(BUILDDIR)/changes."
+
+linkcheck:
+	$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
+	@echo
+	@echo "Link check complete; look for any errors in the above output " \
+	      "or in $(BUILDDIR)/linkcheck/output.txt."
+
+doctest:
+	$(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
+	@echo "Testing of doctests in the sources finished, look at the " \
+	      "results in $(BUILDDIR)/doctest/output.txt."
+
+xml:
+	$(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml
+	@echo
+	@echo "Build finished. The XML files are in $(BUILDDIR)/xml."
+
+pseudoxml:
+	$(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml
+	@echo
+	@echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml."

docs/README

@@ -0,0 +1,7 @@
+
+http://docs.emqtt.com/
+
+or
+
+http://emqttd-docs.rtfd.org
+

docs/make.bat

@@ -0,0 +1,242 @@
+@ECHO OFF
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+	set SPHINXBUILD=sphinx-build
+)
+set BUILDDIR=build
+set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% source
+set I18NSPHINXOPTS=%SPHINXOPTS% source
+if NOT "%PAPER%" == "" (
+	set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS%
+	set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS%
+)
+
+if "%1" == "" goto help
+
+if "%1" == "help" (
+	:help
+	echo.Please use `make ^<target^>` where ^<target^> is one of
+	echo.  html       to make standalone HTML files
+	echo.  dirhtml    to make HTML files named index.html in directories
+	echo.  singlehtml to make a single large HTML file
+	echo.  pickle     to make pickle files
+	echo.  json       to make JSON files
+	echo.  htmlhelp   to make HTML files and a HTML help project
+	echo.  qthelp     to make HTML files and a qthelp project
+	echo.  devhelp    to make HTML files and a Devhelp project
+	echo.  epub       to make an epub
+	echo.  latex      to make LaTeX files, you can set PAPER=a4 or PAPER=letter
+	echo.  text       to make text files
+	echo.  man        to make manual pages
+	echo.  texinfo    to make Texinfo files
+	echo.  gettext    to make PO message catalogs
+	echo.  changes    to make an overview over all changed/added/deprecated items
+	echo.  xml        to make Docutils-native XML files
+	echo.  pseudoxml  to make pseudoxml-XML files for display purposes
+	echo.  linkcheck  to check all external links for integrity
+	echo.  doctest    to run all doctests embedded in the documentation if enabled
+	goto end
+)
+
+if "%1" == "clean" (
+	for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i
+	del /q /s %BUILDDIR%\*
+	goto end
+)
+
+
+%SPHINXBUILD% 2> nul
+if errorlevel 9009 (
+	echo.
+	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+	echo.installed, then set the SPHINXBUILD environment variable to point
+	echo.to the full path of the 'sphinx-build' executable. Alternatively you
+	echo.may add the Sphinx directory to PATH.
+	echo.
+	echo.If you don't have Sphinx installed, grab it from
+	echo.http://sphinx-doc.org/
+	exit /b 1
+)
+
+if "%1" == "html" (
+	%SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Build finished. The HTML pages are in %BUILDDIR%/html.
+	goto end
+)
+
+if "%1" == "dirhtml" (
+	%SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml.
+	goto end
+)
+
+if "%1" == "singlehtml" (
+	%SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml.
+	goto end
+)
+
+if "%1" == "pickle" (
+	%SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Build finished; now you can process the pickle files.
+	goto end
+)
+
+if "%1" == "json" (
+	%SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Build finished; now you can process the JSON files.
+	goto end
+)
+
+if "%1" == "htmlhelp" (
+	%SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Build finished; now you can run HTML Help Workshop with the ^
+.hhp project file in %BUILDDIR%/htmlhelp.
+	goto end
+)
+
+if "%1" == "qthelp" (
+	%SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Build finished; now you can run "qcollectiongenerator" with the ^
+.qhcp project file in %BUILDDIR%/qthelp, like this:
+	echo.^> qcollectiongenerator %BUILDDIR%\qthelp\ErlangMQTTBroker.qhcp
+	echo.To view the help file:
+	echo.^> assistant -collectionFile %BUILDDIR%\qthelp\ErlangMQTTBroker.ghc
+	goto end
+)
+
+if "%1" == "devhelp" (
+	%SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Build finished.
+	goto end
+)
+
+if "%1" == "epub" (
+	%SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Build finished. The epub file is in %BUILDDIR%/epub.
+	goto end
+)
+
+if "%1" == "latex" (
+	%SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Build finished; the LaTeX files are in %BUILDDIR%/latex.
+	goto end
+)
+
+if "%1" == "latexpdf" (
+	%SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex
+	cd %BUILDDIR%/latex
+	make all-pdf
+	cd %BUILDDIR%/..
+	echo.
+	echo.Build finished; the PDF files are in %BUILDDIR%/latex.
+	goto end
+)
+
+if "%1" == "latexpdfja" (
+	%SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex
+	cd %BUILDDIR%/latex
+	make all-pdf-ja
+	cd %BUILDDIR%/..
+	echo.
+	echo.Build finished; the PDF files are in %BUILDDIR%/latex.
+	goto end
+)
+
+if "%1" == "text" (
+	%SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Build finished. The text files are in %BUILDDIR%/text.
+	goto end
+)
+
+if "%1" == "man" (
+	%SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Build finished. The manual pages are in %BUILDDIR%/man.
+	goto end
+)
+
+if "%1" == "texinfo" (
+	%SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo.
+	goto end
+)
+
+if "%1" == "gettext" (
+	%SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Build finished. The message catalogs are in %BUILDDIR%/locale.
+	goto end
+)
+
+if "%1" == "changes" (
+	%SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.The overview file is in %BUILDDIR%/changes.
+	goto end
+)
+
+if "%1" == "linkcheck" (
+	%SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Link check complete; look for any errors in the above output ^
+or in %BUILDDIR%/linkcheck/output.txt.
+	goto end
+)
+
+if "%1" == "doctest" (
+	%SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Testing of doctests in the sources finished, look at the ^
+results in %BUILDDIR%/doctest/output.txt.
+	goto end
+)
+
+if "%1" == "xml" (
+	%SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Build finished. The XML files are in %BUILDDIR%/xml.
+	goto end
+)
+
+if "%1" == "pseudoxml" (
+	%SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml
+	if errorlevel 1 exit /b 1
+	echo.
+	echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml.
+	goto end
+)
+
+:end

docs/source/bridge.rst

@@ -0,0 +1,129 @@
+
+.. _bridge::
+
+======
+Bridge
+======
+
+-------------
+emqttd Bridge
+-------------
+
+Two or more emqttd brokers could be bridged together. Bridges forward MQTT messages from one broker node to another::
+
+                  ---------                     ---------                     ---------
+    Publisher --> | node1 | --Bridge Forward--> | node2 | --Bridge Forward--> | node3 | --> Subscriber
+                  ---------                     ---------                     ---------
+
+Configure Bridge
+----------------
+
+Suppose that we create two emqttd brokers on localhost:
+
++---------+---------------------+-----------+
+| Name    | Node                | MQTT Port |
++---------+---------------------+-----------+
+| emqttd1 | emqttd1@127.0.0.1   | 1883      |
++---------+---------------------+-----------+
+| emqttd2 | emqttd2@127.0.0.1   | 2883      |
++---------+---------------------+-----------+
+
+Create a bridge that forwards all the 'sensor/#' messages from emqttd1 to emqttd2.
+
+1. Start Brokers
+................
+
+.. code:: console
+
+    cd emqttd1/ && ./bin/emqttd start
+    cd emqttd2/ && ./bin/emqttd start
+
+2. Create bridge: emqttd1--sensor/#-->emqttd2
+.............................................
+
+.. code:: console
+
+    $ cd emqttd1 && ./bin/emqttd_ctl bridges start emqttd2@127.0.0.1 sensor/#
+
+    bridge is started.
+    
+    $ ./bin/emqttd_ctl bridges list
+
+    bridge: emqttd1@127.0.0.1--sensor/#-->emqttd2@127.0.0.1
+
+3. Test the bridge
+...................
+
+.. code:: console
+
+    #emqttd2
+    mosquitto_sub -t sensor/# -p 2883 -d
+
+    #emqttd1
+    mosquitto_pub -t sensor/1/temperature -m "37.5" -d 
+
+4. Delete the bridge
+.....................
+
+.. code:: console
+
+    ./bin/emqttd_ctl bridges stop emqttd2@127.0.0.1 sensor/#
+
+-----------------
+emqttd Bridge CLI
+-----------------
+
+.. code:: console
+
+    #query bridges
+    ./bin/emqttd_ctl bridges list
+
+    #start bridge                       
+    ./bin/emqttd_ctl bridges start <Node> <Topic>
+
+    #start bridge with options
+    ./bin/emqttd_ctl bridges start <Node> <Topic> <Options>
+
+    #stop bridge  
+    ./bin/emqttd_ctl bridges stop <Node> <Topic>
+
+-----------------
+mosquitto Bridge
+-----------------
+
+Bridge mosquitto to emqttd broker::
+
+                 -------------             -----------------
+    Sensor ----> | mosquitto | --Bridge--> |               |
+                 -------------             |    emqttd     |
+                 -------------             |    Cluster    |
+    Sensor ----> | mosquitto | --Bridge--> |               |
+                 -------------             -----------------
+
+mosquitto.conf
+--------------
+
+Suppose that we start an emqttd broker on localost:2883, and mosquitto on localhost:1883.
+
+A bridge configured in mosquitto.conf::
+
+    connection emqttd
+    address 127.0.0.1:2883
+    topic sensor/# out 2
+
+    # Set the version of the MQTT protocol to use with for this bridge. Can be one
+    # of mqttv31 or mqttv311. Defaults to mqttv31.
+    bridge_protocol_version mqttv311
+
+-------------
+rsmb Bridge
+-------------
+
+Bridge RSMB to emqttd broker, same settings as mosquitto.
+
+broker.cfg::
+
+    connection emqttd
+    addresses 127.0.0.1:2883
+    topic sensor/#
+

docs/source/cluster.rst

@@ -0,0 +1,252 @@
+
+.. _cluster:
+
+==========
+Clustering
+==========
+
+----------------------
+Distributed Erlang/OTP
+----------------------
+
+Erlang/OTP is a concurrent, fault-tolerant, distributed programming platform. A distributed Erlang/OTP system consists of a number of Erlang runtime systems called 'node'. Nodes connect to each other with TCP/IP sockets and communite by Message Passing.
+
+.. code::
+
+    ---------         ---------
+    | Node1 | --------| Node2 |
+    ---------         ---------
+        |     \     /    |
+        |       \ /      |
+        |       / \      |
+        |     /     \    |
+    ---------         ---------
+    | Node3 | --------| Node4 |
+    ---------         ---------
+
+Node
+----
+
+An erlang runtime system called 'node' is identified by a unique name like email addreass. Erlang nodes communicate with each other by the name.
+
+Suppose we start four Erlang nodes on localhost:
+
+.. code:: console
+
+    erl -name node1@127.0.0.1
+    erl -name node2@127.0.0.1
+    erl -name node3@127.0.0.1
+    erl -name node4@127.0.0.1
+
+connect all the nodes::
+
+    (node1@127.0.0.1)1> net_kernel:connect_node('node2@127.0.0.1').
+    true
+    (node1@127.0.0.1)2> net_kernel:connect_node('node3@127.0.0.1').
+    true
+    (node1@127.0.0.1)3> net_kernel:connect_node('node4@127.0.0.1').
+    true
+    (node1@127.0.0.1)4> nodes().
+    ['node2@127.0.0.1','node3@127.0.0.1','node4@127.0.0.1']
+
+epmd
+----
+
+epmd(Erlang Port Mapper Daemon) is a daemon service that is responsible for mapping node names to machine addresses(TCP sockets). The daemon is started automatically on every host where an Erlang node started.
+
+.. code:: console
+
+    (node1@127.0.0.1)6> net_adm:names().
+    {ok,[{"node1",62740},
+         {"node2",62746},
+         {"node3",62877},
+         {"node4",62895}]}
+
+Cookie
+------
+
+Erlang nodes authenticate each other by a magic cookie when communicating. The cookie could be configured by::
+
+    1. $HOME/.erlang.cookie
+
+    2. erl -setcookie <Cookie>
+
+.. NOTE:: Content of this chapter is from: http://erlang.org/doc/reference_manual/distributed.html
+
+---------------------
+emqttd Cluster Design
+---------------------
+
+The cluster architecture of emqttd broker is based on distrubuted Erlang/OTP and Mnesia database.
+
+The cluster design could be summarized by the following two rules::
+
+1. When a MQTT client SUBSCRIBE a Topic on a node, the node will tell all the other nodes in the cluster: I subscribed a Topic.
+
+2. When a MQTT Client PUBLISH a message to a node, the node will lookup the Topic table and forard the message to nodes that subscribed the Topic.
+
+Finally there will be a global route table(Topic -> Node) that replicated to all nodes in the cluster::
+
+    topic1 -> node1, node2
+    topic2 -> node3
+    topic3 -> node2, node4
+
+Topic Trie and Route Table
+--------------------------
+
+Every node in the cluster will store a topic trie and route table in mnesia database. 
+
+Suppose that we create subscriptions:
+
++----------------+-------------+----------------------------+
+| Client         | Node        |  Topics                    |
++================+=============+============================+
+| client1        | node1       | t/+/x, t/+/y               |
++----------------+-------------+----------------------------+
+| client2        | node2       | t/#                        |
++----------------+-------------+----------------------------+
+| client3        | node3       | t/+/x, t/a                 |
++----------------+-------------+----------------------------+
+
+Finally the topic trie and route table in the cluster::
+
+    --------------------------
+    |          t             |
+    |         / \            |
+    |        +   #           |
+    |      /  \              |
+    |    x      y            |
+    --------------------------
+    | t/+/x -> node1, node3  |
+    | t/+/y -> node1         |
+    | t/#   -> node2         |
+    | t/a   -> node3         |
+    --------------------------
+
+Message Route and Deliver
+--------------------------
+
+The brokers in the cluster route messages by topic trie and route table, deliver messages to MQTT clients by subscriptions. Subscriptions are mapping from topic to subscribers, are stored only in the local node, will not be replicated to other nodes.
+
+Suppose client1 PUBLISH a message to the topic 't/a', the message Route and Deliver process::
+
+    title: Message Route and Deliver
+
+    client1->node1: Publish[t/a]
+    node1-->node2: Route[t/#]
+    node1-->node3: Route[t/a]
+    node2-->client2: Deliver[t/#]
+    node3-->client3: Deliver[t/a]
+
+.. image:: _static/images/route.png
+
+-------------
+Cluster Setup
+-------------
+
+Suppose we deploy two nodes cluster on host1, host2:
+
++----------------+-----------+---------------------+
+| Node           | Host      |  IP and Port        |
++----------------+-----------+---------------------+
+| emqttd@host1   | host1     | 192.168.1.10:1883   |
++----------------+-----------+---------------------+
+| emqttd@host2   | host2     | 192.168.1.20:1883   |
++----------------+-----------+---------------------+
+
+emqttd@host1 setting
+--------------------
+
+emqttd/etc/vm.args::
+
+    -name emqttd@host1
+
+    or
+
+    -name emqttd@192.168.0.10
+
+.. WARNING:: The name cannot be changed after node joined the cluster.
+
+emqttd@host2 setting
+--------------------
+
+emqttd/etc/vm.args::
+
+    -name emqttd@host2
+
+    or
+
+    -name emqttd@192.168.0.20
+
+Join the cluster
+----------------
+
+Start the two broker nodes, and 'cluster join ' on emqttd@host2::
+
+    $ ./bin/emqttd_ctl cluster join emqttd@host1
+
+    Join the cluster successfully.
+    Cluster status: [{running_nodes,['emqttd@host1','emqttd@host2']}]
+
+Or 'cluster join' on emqttd@host1::
+
+    $ ./bin/emqttd_ctl cluster join emqttd@host2
+
+    Join the cluster successfully.
+    Cluster status: [{running_nodes,['emqttd@host1','emqttd@host2']}]
+
+Query the cluster status::
+
+    $ ./bin/emqttd_ctl cluster status
+
+    Cluster status: [{running_nodes,['emqttd@host1','emqttd@host2']}]
+
+Leave the cluster
+-----------------
+
+Two ways to leave the cluster:
+
+1. leave: this node leaves the cluster
+
+2. remove: remove other nodes from the cluster
+
+emqttd@host2 node tries to leave the cluster::
+
+    $ ./bin/emqttd_ctl cluster leave
+
+Or remove emqttd@host2 node from the cluster on emqttd@host1::
+
+    $ ./bin/emqttd_ctl cluster remove emqttd@host2
+
+
+--------------------
+Session across Nodes
+--------------------
+
+The persistent MQTT sessions (clean session = false) are across nodes in the cluster.
+
+If a persistent MQTT client connected to node1 first, then disconnected and connects to node2, the MQTT connection and session will be located on different nodes::
+
+                                      node1
+                                   -----------
+                               |-->| session |
+                               |   -----------
+                 node2         |
+              --------------   |
+     client-->| connection |<--|
+              --------------
+
+----------------
+Notice: NetSplit
+----------------
+
+The emqttd cluster does not support deployment across IDC, and the cluster will not handle NetSplit automatically. If NetSplit occures, nodes have to be rebooted manually.
+
+
+-----------------------
+Consistent Hash and DHT
+-----------------------
+
+Consistent Hash and DHT are popular in the design of NoSQL databases. Cluster of emqttd broker could support 10 million size of global routing table now. We could use the Consistent Hash or DHT to partition the routing table, and evolve the cluster to larger size.
+
+

docs/source/coap.rst

@@ -0,0 +1,3 @@
+==============
+CoAP Protocol
+==============

docs/source/commands.rst

@@ -0,0 +1,698 @@
+
+.. _commands::
+
+============
+Command Line
+============
+
+The './bin/emqttd_ctl' command line could be used to query and administrate emqttd broker.
+
+.. WARNING:: Cannot work on Windows
+
+------
+status
+------
+
+Show running status of the broker::
+
+    $ ./bin/emqttd_ctl status
+
+    Node 'emqttd@127.0.0.1' is started
+    emqttd 0.16.0 is running
+
+------
+broker
+------
+
+Query basic information,  statistics and metrics of the broker.
+
++----------------+-------------------------------------------------+
+| broker         | Show version, description, uptime of the broker |
++----------------+-------------------------------------------------+
+| broker pubsub  | Show status of the core pubsub process          |
++----------------+-------------------------------------------------+
+| broker stats   | Show statistics of client, session, topic,      |
+|                | subscription and route of the broker            |
++----------------+-------------------------------------------------+
+| broker metrics | Show metrics of MQTT bytes, packets, messages   |
+|                | sent/received.                                  |
++----------------+-------------------------------------------------+
+
+Query version, description and uptime of the broker::
+
+    $ ./bin/emqttd_ctl broker
+
+    sysdescr  : Erlang MQTT Broker
+    version   : 0.15.0
+    uptime    : 1 hours, 25 minutes, 24 seconds
+    datetime  : 2016-01-16 13:17:32
+
+broker stats
+------------
+
+Query statistics of MQTT Client, Session, Topic, Subscription and Route::
+
+    $ ./bin/emqttd_ctl broker stats
+
+    clients/count       : 1
+    clients/max         : 1
+    queues/count        : 0
+    queues/max          : 0
+    retained/count      : 2
+    retained/max        : 2
+    routes/count        : 2
+    routes/reverse      : 2
+    sessions/count      : 0
+    sessions/max        : 0
+    subscriptions/count : 1
+    subscriptions/max   : 1
+    topics/count        : 54
+    topics/max          : 54
+
+broker metrics
+--------------
+
+Query metrics of Bytes, MQTT Packets and Messages(sent/received)::
+
+    $ ./bin/emqttd_ctl broker metrics
+
+    bytes/received          : 297
+    bytes/sent              : 40
+    messages/dropped        : 348
+    messages/qos0/received  : 0
+    messages/qos0/sent      : 0
+    messages/qos1/received  : 0
+    messages/qos1/sent      : 0
+    messages/qos2/received  : 0
+    messages/qos2/sent      : 0
+    messages/received       : 0
+    messages/retained       : 2
+    messages/sent           : 0
+    packets/connack         : 5
+    packets/connect         : 5
+    packets/disconnect      : 0
+    packets/pingreq         : 0
+    packets/pingresp        : 0
+    packets/puback/received : 0
+    packets/puback/sent     : 0
+    packets/pubcomp/received: 0
+    packets/pubcomp/sent    : 0
+    packets/publish/received: 0
+    packets/publish/sent    : 0
+    packets/pubrec/received : 0
+    packets/pubrec/sent     : 0
+    packets/pubrel/received : 0
+    packets/pubrel/sent     : 0
+    packets/received        : 9
+    packets/sent            : 9
+    packets/suback          : 4
+    packets/subscribe       : 4
+    packets/unsuback        : 0
+    packets/unsubscribe     : 0
+
+
+-------
+cluster
+-------
+
+Cluster two or more emqttd brokers.
+
++-----------------------+--------------------------------+
+| cluster join <Node>   | Join the cluster               |
++-----------------------+--------------------------------+
+| cluster leave         | Leave the cluster              |
++-----------------------+--------------------------------+
+| cluster remove <Node> | Remove a node from the cluster |
++-----------------------+--------------------------------+
+| cluster status        | Query cluster status and nodes |
++-----------------------+--------------------------------+
+
+Suppose we create two emqttd nodes on localhost and cluster them:
+
++-----------+---------------------+-------------+
+| Folder    | Node                | MQTT Port   |
++-----------+---------------------+-------------+
+| emqttd1   | emqttd1@127.0.0.1   | 1883        |
++-----------+---------------------+-------------+
+| emqttd2   | emqttd2@127.0.0.1   | 2883        |
++-----------+---------------------+-------------+
+
+Start emqttd1 node::
+
+    cd emqttd1 && ./bin/emqttd start
+
+Start emqttd2 node::
+
+    cd emqttd2 && ./bin/emqttd start
+
+Under emqttd2 folder:: 
+
+    $ ./bin/emqttd_ctl cluster join emqttd1@127.0.0.1
+
+    Join the cluster successfully.
+    Cluster status: [{running_nodes,['emqttd1@127.0.0.1','emqttd2@127.0.0.1']}]
+
+Query cluster status::
+
+    $ ./bin/emqttd_ctl cluster status
+
+    Cluster status: [{running_nodes,['emqttd2@127.0.0.1','emqttd1@127.0.0.1']}]
+
+Message Route between nodes::
+
+    # Subscribe topic 'x' on emqttd1 node
+    mosquitto_sub -t x -q 1 -p 1883
+
+    # Publish to topic 'x' on emqttd2 node
+    mosquitto_pub -t x -q 1 -p 2883 -m hello
+
+emqttd2 leaves the cluster::
+
+    cd emqttd2 && ./bin/emqttd_ctl cluster leave
+
+Or remove emqttd2 from the cluster on emqttd1 node::
+
+    cd emqttd1 && ./bin/emqttd_ctl cluster remove emqttd2@127.0.0.1
+
+-------
+clients
+-------
+
+Query MQTT clients connected to the broker:
+
++-------------------------+----------------------------------+
+| clients list            | List all MQTT clients            |
++-------------------------+----------------------------------+
+| clients show <ClientId> | Show a MQTT Client               |
++-------------------------+----------------------------------+
+| clients kick <ClientId> | Kick out a MQTT client           |
++-------------------------+----------------------------------+
+
+clients lists
+-------------
+
+Query All MQTT clients connected to the broker::
+
+    $ ./bin/emqttd_ctl clients list
+
+    Client(mosqsub/43832-airlee.lo, clean_sess=true, username=test, peername=127.0.0.1:64896, connected_at=1452929113)
+    Client(mosqsub/44011-airlee.lo, clean_sess=true, username=test, peername=127.0.0.1:64961, connected_at=1452929275)
+    ...
+
+Properties of the Client:
+
++--------------+---------------------------------------------------+
+| clean_sess   | Clean Session Flag                                |
++--------------+---------------------------------------------------+
+| username     | Username of the client                            |
++--------------+---------------------------------------------------+
+| peername     | Peername of the TCP connection                    |
++--------------+---------------------------------------------------+
+| connected_at | The timestamp when client connected to the broker |
++--------------+---------------------------------------------------+
+
+clients show <ClientId>
+-----------------------
+
+Show a specific MQTT Client::
+
+    ./bin/emqttd_ctl clients show "mosqsub/43832-airlee.lo"
+
+    Client(mosqsub/43832-airlee.lo, clean_sess=true, username=test, peername=127.0.0.1:64896, connected_at=1452929113)
+
+clients kick <ClientId>
+-----------------------
+        
+Kick out a MQTT Client::
+
+    ./bin/emqttd_ctl clients kick "clientid"
+
+
+--------
+sessions
+--------
+
+Query all MQTT sessions. The broker will create a session for each MQTT client. Persistent Session if clean_session flag is true, transient session otherwise.
+
++--------------------------+-------------------------------+
+| sessions list            | List all Sessions             |
++--------------------------+-------------------------------+
+| sessions list persistent | Query all persistent Sessions |
++--------------------------+-------------------------------+
+| sessions list transient  | Query all transient Sessions  |
++--------------------------+-------------------------------+
+| sessions show <ClientId> | Show a session                |
++--------------------------+-------------------------------+
+
+sessions list
+-------------
+
+Query all sessions::
+
+    $ ./bin/emqttd_ctl sessions list
+
+    Session(clientid, clean_sess=false, max_inflight=100, inflight_queue=0, message_queue=0, message_dropped=0, awaiting_rel=0, awaiting_ack=0, awaiting_comp=0, created_at=1452935508)
+    Session(mosqsub/44101-airlee.lo, clean_sess=true, max_inflight=100, inflight_queue=0, message_queue=0, message_dropped=0, awaiting_rel=0, awaiting_ack=0, awaiting_comp=0, created_at=1452935401)
+
+Properties of Session:
+
+TODO:??
+
++-------------------+----------------------------------------------------------------+
+| clean_sess        | clean sess flag. false: persistent, true: transient            |
++-------------------+----------------------------------------------------------------+
+| max_inflight      | Inflight window (Max number of messages delivering)            |
++-------------------+----------------------------------------------------------------+
+| inflight_queue    | Inflight Queue Size                                            |
++-------------------+----------------------------------------------------------------+
+| message_queue     | Message Queue Size                                             |
++-------------------+----------------------------------------------------------------+
+| message_dropped   | Number of Messages Dropped for queue is full                   |
++-------------------+----------------------------------------------------------------+
+| awaiting_rel      | The number of QoS2 messages received and waiting for PUBREL    |
++-------------------+----------------------------------------------------------------+
+| awaiting_ack      | The number of QoS1/2 messages delivered and waiting for PUBACK |
++-------------------+----------------------------------------------------------------+
+| awaiting_comp     | The number of QoS2 messages delivered and waiting for PUBCOMP  |
++-------------------+----------------------------------------------------------------+
+| created_at        | Timestamp when the session is created                          |
++-------------------+----------------------------------------------------------------+
+
+sessions list persistent
+------------------------
+
+Query all persistent sessions::
+
+    $ ./bin/emqttd_ctl sessions list persistent
+
+    Session(clientid, clean_sess=false, max_inflight=100, inflight_queue=0, message_queue=0, message_dropped=0, awaiting_rel=0, awaiting_ack=0, awaiting_comp=0, created_at=1452935508)
+
+sessions list transient
+-----------------------
+
+Query all transient sessions::
+
+    $ ./bin/emqttd_ctl sessions list transient
+
+    Session(mosqsub/44101-airlee.lo, clean_sess=true, max_inflight=100, inflight_queue=0, message_queue=0, message_dropped=0, awaiting_rel=0, awaiting_ack=0, awaiting_comp=0, created_at=1452935401)
+
+sessions show <ClientId>
+------------------------
+
+Show a session::
+
+    $ ./bin/emqttd_ctl sessions show clientid
+
+    Session(clientid, clean_sess=false, max_inflight=100, inflight_queue=0, message_queue=0, message_dropped=0, awaiting_rel=0, awaiting_ack=0, awaiting_comp=0, created_at=1452935508)
+
+
+------
+topics
+------
+
+Query topic table of the broker.
+
+topics list
+-----------
+
+Query all the topics::
+
+    $ ./bin/emqttd_ctl topics list
+
+    topic1: ['emqttd2@127.0.0.1']
+    topic2: ['emqttd1@127.0.0.1','emqttd2@127.0.0.1']
+
+topics show <Topic>
+-------------------
+
+Show a topic::
+
+    $ ./bin/emqttd_ctl topics show topic2
+
+    topic2: ['emqttd1@127.0.0.1','emqttd2@127.0.0.1']
+
+The result will show which nodes the topic is on.
+
+-------------
+subscriptions
+-------------
+
+Query the subscription table of the broker:
+
++--------------------------------------------+--------------------------------------+
+| subscriptions list                         | List all subscriptions               |
++--------------------------------------------+--------------------------------------+
+| subscriptions show <ClientId>              | Show a subscription                  |
++--------------------------------------------+--------------------------------------+
+| subscriptions add <ClientId> <Topic> <Qos> | Add a static subscription manually   |
++--------------------------------------------+--------------------------------------+
+| subscriptions del <ClientId> <Topic>       | Remove a static subscription manually|
++--------------------------------------------+--------------------------------------+
+
+subscriptions list
+------------------
+
+Query all subscriptions::
+
+    $ ./bin/emqttd_ctl subscriptions list
+
+    mosqsub/45744-airlee.lo: [{<<"y">>,0},{<<"x">>,0}]
+
+subscriptions show <ClientId>
+-----------------------------
+
+Show the subscriptions of a MQTT client::
+
+    $ ./bin/emqttd_ctl subscriptions show clientid
+
+    clientid: [{<<"x">>,1},{<<"topic2">>,1},{<<"topic3">>,1}]
+
+subscriptions add <ClientId> <Topic> <QoS>
+------------------------------------------
+
+Add a static subscription manually::
+
+    $ ./bin/emqttd_ctl subscriptions add clientid new_topic 1
+    ok
+
+subscriptions del <ClientId> <Topic>
+------------------------------------
+
+Remove a static subscription manually::
+
+    $ ./bin/emqttd_ctl subscriptions del clientid new_topic
+    ok
+
+-------
+plugins
+-------
+
+List, load or unload plugins of emqttd broker.
+
++---------------------------+-------------------------+
+| plugins list              | List all plugins        |
++---------------------------+-------------------------+
+| plugins load <Plugin>     | Load Plugin             |
++---------------------------+-------------------------+
+| plugins unload <Plugin>   | Unload (Plugin)        |
++---------------------------+-------------------------+
+
+plugins list
+------------
+
+List all plugins::
+
+    $ ./bin/emqttd_ctl plugins list
+
+    Plugin(emqttd_dashboard, version=0.16.0, description=emqttd web dashboard, active=true)
+    Plugin(emqttd_plugin_mysql, version=0.16.0, description=emqttd Authentication/ACL with MySQL, active=false)
+    Plugin(emqttd_plugin_pgsql, version=0.16.0, description=emqttd PostgreSQL Plugin, active=false)
+    Plugin(emqttd_plugin_redis, version=0.16.0, description=emqttd Redis Plugin, active=false)
+    Plugin(emqttd_plugin_template, version=0.16.0, description=emqttd plugin template, active=false)
+    Plugin(emqttd_recon, version=0.16.0, description=emqttd recon plugin, active=false)
+    Plugin(emqttd_stomp, version=0.16.0, description=Stomp Protocol Plugin for emqttd broker, active=false)
+
+Properties of a plugin:
+
++-------------+--------------------------+
+| version     | Plugin Version           |
++-------------+--------------------------+
+| description | Plugin Description       |
++-------------+--------------------------+
+| active      | If the plugin is Loaded  | 
++-------------+--------------------------+
+
+load <Plugin>
+-------------
+
+Load a Plugin::
+
+    $ ./bin/emqttd_ctl plugins load emqttd_recon
+
+    Start apps: [recon,emqttd_recon]
+    Plugin emqttd_recon loaded successfully.
+
+unload <Plugin>
+---------------
+
+Unload a Plugin::
+
+    $ ./bin/emqttd_ctl plugins unload emqttd_recon
+
+    Plugin emqttd_recon unloaded successfully.
+
+-------
+bridges
+-------
+
+Bridge two or more emqttd brokers::
+
+                  ---------                     ---------
+    Publisher --> | node1 | --Bridge Forward--> | node2 | --> Subscriber
+                  ---------                     ---------
+
+commands for bridge:
+
++----------------------------------------+------------------------------+
+| bridges list                           | List all bridges             |
++----------------------------------------+------------------------------+
+| bridges options                        | Show bridge options          |
++----------------------------------------+------------------------------+
+| bridges start <Node> <Topic>           | Create a bridge              |
++----------------------------------------+------------------------------+
+| bridges start <Node> <Topic> <Options> | Create a bridge with options |
++----------------------------------------+------------------------------+
+| bridges stop <Node> <Topic>            | Delete a bridge              |
++----------------------------------------+------------------------------+
+
+Suppose we create a bridge between emqttd1 and emqttd2 on localhost:
+
++---------+---------------------+-----------+
+| Name    | Node                | MQTT Port |
++---------+---------------------+-----------+
+| emqttd1 | emqttd1@127.0.0.1   | 1883      |
++---------+---------------------+-----------+
+| emqttd2 | emqttd2@127.0.0.1   | 2883      |
++---------+---------------------+-----------+
+
+The bridge will forward all the the 'sensor/#' messages from emqttd1 to emqttd2:: 
+
+    $ ./bin/emqttd_ctl bridges start emqttd2@127.0.0.1 sensor/#
+
+    bridge is started.
+    
+    $ ./bin/emqttd_ctl bridges list
+
+    bridge: emqttd1@127.0.0.1--sensor/#-->emqttd2@127.0.0.1
+
+The the 'emqttd1--sensor/#-->emqttd2' bridge:: 
+
+    #emqttd2 node
+
+    mosquitto_sub -t sensor/# -p 2883 -d
+
+    #emqttd1节点上
+
+    mosquitto_pub -t sensor/1/temperature -m "37.5" -d 
+
+bridges options
+---------------
+
+Show bridge options::
+
+    $ ./bin/emqttd_ctl bridges options
+
+    Options:
+      qos     = 0 | 1 | 2
+      prefix  = string
+      suffix  = string
+      queue   = integer
+    Example:
+      qos=2,prefix=abc/,suffix=/yxz,queue=1000
+
+bridges stop <Node> <Topic>
+---------------------------
+
+Delete the emqttd1--sensor/#-->emqttd2 bridge::
+
+    $ ./bin/emqttd_ctl bridges stop emqttd2@127.0.0.1 sensor/#
+
+    bridge is stopped.
+
+--
+vm
+--
+
+Query the load, cpu, memory, processes and IO information of the Erlang VM.
+
++-------------+-----------------------------------+
+| vm all      | Query all                         |
++-------------+-----------------------------------+
+| vm load     | Query VM Load                     |
++-------------+-----------------------------------+
+| vm memory   | Query Memory Usage                |
++-------------+-----------------------------------+
+| vm process  | Query Number of Erlang Processes  |
++-------------+-----------------------------------+
+| vm io       | Query Max Fds of VM               |
++-------------+-----------------------------------+
+
+vm load
+-------
+
+Query load::
+
+    $ ./bin/emqttd_ctl vm load
+
+    cpu/load1               : 2.21
+    cpu/load5               : 2.60
+    cpu/load15              : 2.36
+
+vm memory
+---------
+
+Query memory::
+
+    $ ./bin/emqttd_ctl vm memory
+
+    memory/total            : 23967736
+    memory/processes        : 3594216
+    memory/processes_used   : 3593112
+    memory/system           : 20373520
+    memory/atom             : 512601
+    memory/atom_used        : 491955
+    memory/binary           : 51432
+    memory/code             : 13401565
+    memory/ets              : 1082848
+
+vm process
+----------
+
+Query number of erlang processes::
+
+    $ ./bin/emqttd_ctl vm process
+
+    process/limit           : 8192
+    process/count           : 221
+
+vm io
+-----
+
+Query max, active file descriptors of IO::
+
+    $ ./bin/emqttd_ctl vm io
+
+    io/max_fds              : 2560
+    io/active_fds           : 1
+
+-----
+trace
+-----
+
+Trace MQTT packets, messages(sent/received) by ClientId or Topic.
+
++-----------------------------------+-----------------------------------+
+| trace list                        | List all the traces               |
++-----------------------------------+-----------------------------------+
+| trace client <ClientId> <LogFile> | Trace a client                    |
++-----------------------------------+-----------------------------------+
+| trace client <ClientId> off       | Stop tracing the client           |
++-----------------------------------+-----------------------------------+
+| trace topic <Topic> <LogFile>     | Trace a topic                     |
++-----------------------------------+-----------------------------------+
+| trace topic <Topic> off           | Stop tracing the topic            |
++-----------------------------------+-----------------------------------+
+
+trace client <ClientId> <LogFile>
+---------------------------------
+
+Start to trace a client::
+
+    $ ./bin/emqttd_ctl trace client clientid log/clientid_trace.log
+
+    trace client clientid successfully.
+
+trace client <ClientId> off
+---------------------------
+
+Stop tracing the client::
+
+    $ ./bin/emqttd_ctl trace client clientid off
+    
+    stop tracing client clientid successfully.
+
+trace topic <Topic> <LogFile>
+-----------------------------
+
+Start to trace a topic::
+
+    $ ./bin/emqttd_ctl trace topic topic log/topic_trace.log
+
+    trace topic topic successfully.
+
+trace topic <Topic> off
+-----------------------
+
+Stop tracing the topic::
+
+    $ ./bin/emqttd_ctl trace topic topic off
+
+    stop tracing topic topic successfully.
+
+trace list
+----------
+
+List all traces::
+
+    $ ./bin/emqttd_ctl trace list
+
+    trace client clientid -> log/clientid_trace.log
+    trace topic topic -> log/topic_trace.log
+
+---------
+listeners
+---------
+
+Show all the TCP listeners::
+
+    $ ./bin/emqttd_ctl listeners
+
+    listener on http:8083
+      acceptors       : 4
+      max_clients     : 64
+      current_clients : 0
+      shutdown_count  : []
+    listener on mqtts:8883
+      acceptors       : 4
+      max_clients     : 512
+      current_clients : 0
+      shutdown_count  : []
+    listener on mqtt:1883
+      acceptors       : 16
+      max_clients     : 8192
+      current_clients : 1
+      shutdown_count  : [{closed,1}]
+    listener on http:18083
+      acceptors       : 4
+      max_clients     : 512
+      current_clients : 0
+      shutdown_count  : []
+
+listener parameters:
+
++-----------------+--------------------------------------+
+| acceptors       | TCP Acceptor Pool                    |
++-----------------+--------------------------------------+
+| max_clients     | Max number of clients                |
++-----------------+--------------------------------------+
+| current_clients | Count of current clients             |
++-----------------+--------------------------------------+
+| shutdown_count  | Statistics of client shutdown reason |
++-----------------+---------------------------------------+
+
+------
+mnesia
+------
+
+Query system_info of mnesia database.
+

docs/source/conf.py

@@ -0,0 +1,337 @@
+# -*- coding: utf-8 -*-
+#
+# Erlang MQTT Broker documentation build configuration file, created by
+# sphinx-quickstart on Mon Feb 22 00:46:47 2016.
+#
+# This file is execfile()d with the current directory set to its
+# containing dir.
+#
+# Note that not all possible configuration values are present in this
+# autogenerated file.
+#
+# All configuration values have a default; values that are commented out
+# serve to show the default.
+
+import sys
+import os
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#sys.path.insert(0, os.path.abspath('.'))
+
+# -- General configuration ------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+    'sphinx.ext.autodoc',
+    'sphinx.ext.intersphinx',
+    'sphinx.ext.todo',
+    'sphinx.ext.coverage',
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix of source filenames.
+source_suffix = '.rst'
+
+# The encoding of source files.
+#source_encoding = 'utf-8-sig'
+
+# The master toctree document.
+master_doc = 'index'
+
+# General information about the project.
+project = u'Erlang MQTT Broker'
+copyright = u'2016, Feng Lee'
+
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+#
+# The short X.Y version.
+version = '1.0'
+# The full version, including alpha/beta/rc tags.
+release = '1.0'
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#language = None
+
+# There are two options for replacing |today|: either, you set today to some
+# non-false value, then it is used:
+#today = ''
+# Else, today_fmt is used as the format for a strftime call.
+#today_fmt = '%B %d, %Y'
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+exclude_patterns = []
+
+# The reST default role (used for this markup: `text`) to use for all
+# documents.
+#default_role = None
+
+# If true, '()' will be appended to :func: etc. cross-reference text.
+#add_function_parentheses = True
+
+# If true, the current module name will be prepended to all description
+# unit titles (such as .. function::).
+#add_module_names = True
+
+# If true, sectionauthor and moduleauthor directives will be shown in the
+# output. They are ignored by default.
+#show_authors = False
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+# A list of ignored prefixes for module index sorting.
+#modindex_common_prefix = []
+
+# If true, keep warnings as "system message" paragraphs in the built documents.
+#keep_warnings = False
+
+
+# -- Options for HTML output ----------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+html_theme = 'default'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+#html_theme_options = {}
+
+# Add any paths that contain custom themes here, relative to this directory.
+#html_theme_path = []
+
+# The name for this set of Sphinx documents.  If None, it defaults to
+# "<project> v<release> documentation".
+#html_title = None
+
+# A shorter title for the navigation bar.  Default is the same as html_title.
+#html_short_title = None
+
+# The name of an image file (relative to this directory) to place at the top
+# of the sidebar.
+#html_logo = None
+
+# The name of an image file (within the static path) to use as favicon of the
+# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
+# pixels large.
+#html_favicon = None
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+# Add any extra paths that contain custom files (such as robots.txt or
+# .htaccess) here, relative to this directory. These files are copied
+# directly to the root of the documentation.
+#html_extra_path = []
+
+# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
+# using the given strftime format.
+#html_last_updated_fmt = '%b %d, %Y'
+
+# If true, SmartyPants will be used to convert quotes and dashes to
+# typographically correct entities.
+#html_use_smartypants = True
+
+# Custom sidebar templates, maps document names to template names.
+#html_sidebars = {}
+
+# Additional templates that should be rendered to pages, maps page names to
+# template names.
+#html_additional_pages = {}
+
+# If false, no module index is generated.
+#html_domain_indices = True
+
+# If false, no index is generated.
+#html_use_index = True
+
+# If true, the index is split into individual pages for each letter.
+#html_split_index = False
+
+# If true, links to the reST sources are added to the pages.
+#html_show_sourcelink = True
+
+# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
+#html_show_sphinx = True
+
+# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
+#html_show_copyright = True
+
+# If true, an OpenSearch description file will be output, and all pages will
+# contain a <link> tag referring to it.  The value of this option must be the
+# base URL from which the finished HTML is served.
+#html_use_opensearch = ''
+
+# This is the file name suffix for HTML files (e.g. ".xhtml").
+#html_file_suffix = None
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'ErlangMQTTBrokerdoc'
+
+
+# -- Options for LaTeX output ---------------------------------------------
+
+latex_elements = {
+# The paper size ('letterpaper' or 'a4paper').
+#'papersize': 'letterpaper',
+
+# The font size ('10pt', '11pt' or '12pt').
+#'pointsize': '10pt',
+
+# Additional stuff for the LaTeX preamble.
+#'preamble': '',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+#  author, documentclass [howto, manual, or own class]).
+latex_documents = [
+  ('index', 'ErlangMQTTBroker.tex', u'Erlang MQTT Broker Documentation',
+   u'Feng Lee', 'manual'),
+]
+
+# The name of an image file (relative to this directory) to place at the top of
+# the title page.
+#latex_logo = None
+
+# For "manual" documents, if this is true, then toplevel headings are parts,
+# not chapters.
+#latex_use_parts = False
+
+# If true, show page references after internal links.
+#latex_show_pagerefs = False
+
+# If true, show URL addresses after external links.
+#latex_show_urls = False
+
+# Documents to append as an appendix to all manuals.
+#latex_appendices = []
+
+# If false, no module index is generated.
+#latex_domain_indices = True
+
+
+# -- Options for manual page output ---------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+    ('index', 'erlangmqttbroker', u'Erlang MQTT Broker Documentation',
+     [u'Feng Lee'], 1)
+]
+
+# If true, show URL addresses after external links.
+#man_show_urls = False
+
+
+# -- Options for Texinfo output -------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+  ('index', 'ErlangMQTTBroker', u'Erlang MQTT Broker Documentation',
+   u'Feng Lee', 'ErlangMQTTBroker', 'One line description of project.',
+   'Miscellaneous'),
+]
+
+# Documents to append as an appendix to all manuals.
+#texinfo_appendices = []
+
+# If false, no module index is generated.
+#texinfo_domain_indices = True
+
+# How to display URL addresses: 'footnote', 'no', or 'inline'.
+#texinfo_show_urls = 'footnote'
+
+# If true, do not generate a @detailmenu in the "Top" node's menu.
+#texinfo_no_detailmenu = False
+
+
+# -- Options for Epub output ----------------------------------------------
+
+# Bibliographic Dublin Core info.
+epub_title = u'Erlang MQTT Broker'
+epub_author = u'Feng Lee'
+epub_publisher = u'Feng Lee'
+epub_copyright = u'2016, Feng Lee'
+
+# The basename for the epub file. It defaults to the project name.
+#epub_basename = u'Erlang MQTT Broker'
+
+# The HTML theme for the epub output. Since the default themes are not optimized
+# for small screen space, using the same theme for HTML and epub output is
+# usually not wise. This defaults to 'epub', a theme designed to save visual
+# space.
+#epub_theme = 'epub'
+
+# The language of the text. It defaults to the language option
+# or en if the language is not set.
+#epub_language = ''
+
+# The scheme of the identifier. Typical schemes are ISBN or URL.
+#epub_scheme = ''
+
+# The unique identifier of the text. This can be a ISBN number
+# or the project homepage.
+#epub_identifier = ''
+
+# A unique identification for the text.
+#epub_uid = ''
+
+# A tuple containing the cover image and cover page html template filenames.
+#epub_cover = ()
+
+# A sequence of (type, uri, title) tuples for the guide element of content.opf.
+#epub_guide = ()
+
+# HTML files that should be inserted before the pages created by sphinx.
+# The format is a list of tuples containing the path and title.
+#epub_pre_files = []
+
+# HTML files shat should be inserted after the pages created by sphinx.
+# The format is a list of tuples containing the path and title.
+#epub_post_files = []
+
+# A list of files that should not be packed into the epub file.
+epub_exclude_files = ['search.html']
+
+# The depth of the table of contents in toc.ncx.
+#epub_tocdepth = 3
+
+# Allow duplicate toc entries.
+#epub_tocdup = True
+
+# Choose between 'default' and 'includehidden'.
+#epub_tocscope = 'default'
+
+# Fix unsupported image types using the PIL.
+#epub_fix_images = False
+
+# Scale large images.
+#epub_max_image_width = 0
+
+# How to display URL addresses: 'footnote', 'no', or 'inline'.
+#epub_show_urls = 'inline'
+
+# If false, no index is generated.
+#epub_use_index = True
+
+
+# Example configuration for intersphinx: refer to the Python standard library.
+intersphinx_mapping = {'http://docs.python.org/': None}

docs/source/config.rst

@@ -0,0 +1,701 @@
+
+.. _configuration:
+
+=============
+Configuration
+=============
+
+Configuration files of the broker are under 'etc/' folder, including:
+
++-------------------+-----------------------------------+
+| File              | Description                       |
++-------------------+-----------------------------------+
+| etc/vm.args       | Erlang VM Arguments               |
++-------------------+-----------------------------------+
+| etc/emqttd.config | emqttd broker Config              |
++-------------------+-----------------------------------+
+| etc/acl.config    | ACL Config                        |
++-------------------+-----------------------------------+
+| etc/clients.config| ClientId Authentication           |
++-------------------+-----------------------------------+
+| etc/rewrite.config| Rewrite Rules                     |
++-------------------+-----------------------------------+
+| etc/ssl/*         | SSL certificate and key files     |
++-------------------+-----------------------------------+
+
+-----------
+etc/vm.args
+-----------
+
+Configure and Optimize Erlang VM::
+
+    ##-------------------------------------------------------------------------
+    ## Name of the node
+    ##-------------------------------------------------------------------------
+    -name emqttd@127.0.0.1
+
+    ## Cookie for distributed erlang
+    -setcookie emqttdsecretcookie
+
+    ##-------------------------------------------------------------------------
+    ## Flags
+    ##-------------------------------------------------------------------------
+
+    ## Heartbeat management; auto-restarts VM if it dies or becomes unresponsive
+    ## (Disabled by default..use with caution!)
+    ##-heart
+    -smp true
+
+    ## Enable kernel poll and a few async threads
+    +K true
+
+    ## 12 threads/core.
+    +A 48
+
+    ## max process numbers
+    +P 8192
+
+    ## Sets the maximum number of simultaneously existing ports for this system
+    +Q 8192
+
+    ## max atom number
+    ## +t
+
+    ## Set the distribution buffer busy limit (dist_buf_busy_limit) in kilobytes.
+    ## Valid range is 1-2097151. Default is 1024.
+    ## +zdbbl 8192
+
+    ## CPU Schedulers
+    ## +sbt db
+
+    ##-------------------------------------------------------------------------
+    ## Env
+    ##-------------------------------------------------------------------------
+
+    ## Increase number of concurrent ports/sockets, deprecated in R17
+    -env ERL_MAX_PORTS 8192
+
+    -env ERTS_MAX_PORTS 8192
+
+    -env ERL_MAX_ETS_TABLES 1024
+
+    ## Tweak GC to run more often
+    -env ERL_FULLSWEEP_AFTER 1000
+
+The two most important parameters in etc/vm.args:
+
++-------+---------------------------------------------------------------------------+
+| +P    | Max number of Erlang proccesses. A MQTT client consumes two proccesses.   |
+|       | The value should be larger than max_clients * 2                           | 
++-------+---------------------------------------------------------------------------+
+| +Q    | Max number of Erlang Ports. A MQTT client consumes one port.              |
+|       | The value should be larger than max_clients.                              |
++-------+---------------------------------------------------------------------------+
+
+The name and cookie of Erlang Node should be configured when clustering::
+
+    -name emqttd@host_or_ip
+
+    ## Cookie for distributed erlang
+    -setcookie emqttdsecretcookie
+
+-----------------
+etc/emqttd.config
+-----------------
+
+This is the main emqttd broker configuration file.
+
+File Syntax
+-----------
+
+The file users the standard Erlang config syntax, consists of a list of erlang applications and their environments.
+
+.. code:: erlang
+
+    [{kernel, [
+        {start_timer, true},
+        {start_pg2, true}
+     ]},
+     {sasl, [
+        {sasl_error_logger, {file, "log/emqttd_sasl.log"}}
+     ]},
+
+     ...
+
+     {emqttd, [
+        ...
+     ]}
+    ].
+
+The file adopts Erlang Term Syntax:
+
+1. [ ]: List, seperated by comma
+2. { }: Tuple, Usually {Env, Value}
+3. %  : comment
+
+Log Level and File
+------------------
+
+Logger of emqttd broker is implemented by 'lager' application::
+
+  {lager, [
+    ...
+  ]},
+
+Configure log handlers::
+
+    {handlers, [
+        {lager_console_backend, info},
+
+        {lager_file_backend, [
+            {formatter_config, [time, " ", pid, " [",severity,"] ", message, "\n"]},
+            {file, "log/emqttd_info.log"},
+            {level, info},
+            {size, 104857600},
+            {date, "$D0"},
+            {count, 30}
+        ]},
+
+        {lager_file_backend, [
+            {formatter_config, [time, " ", pid, " [",severity,"] ", message, "\n"]},
+            {file, "log/emqttd_error.log"},
+            {level, error},
+            {size, 104857600},
+            {date, "$D0"},
+            {count, 30}
+        ]}
+    ]}
+
+emqttd Application
+------------------
+
+The MQTT broker is implemented by erlang 'emqttd' application::
+
+ {emqttd, [
+    %% Authentication and Authorization
+    {access, [
+        ...
+    ]},
+    %% MQTT Protocol Options
+    {mqtt, [
+        ...
+    ]},
+    %% Broker Options
+    {broker, [
+        ...
+    ]},
+    %% Modules
+    {modules, [
+        ...
+    ]},
+    %% Plugins
+    {plugins, [
+        ...
+    ]},
+
+    %% Listeners
+    {listeners, [
+        ...
+    ]},
+
+    %% Erlang System Monitor
+    {sysmon, [
+    ]}
+ ]}
+
+Pluggable Authentication
+------------------------
+
+The emqttd broker supports pluggable authentication mechanism with a list of modules and plugins.
+
+The broker provides Username, ClientId, LDAP and anonymous authentication modules by default::
+
+    %% Authetication. Anonymous Default
+    {auth, [
+        %% Authentication with username, password
+        %% Add users: ./bin/emqttd_ctl users add Username Password
+        %% {username, [{"test", "public"}]},
+        
+        %% Authentication with clientid
+        % {clientid, [{password, no}, {file, "etc/clients.config"}]},
+
+        %% Authentication with LDAP
+        % {ldap, [
+        %    {servers, ["localhost"]},
+        %    {port, 389},
+        %    {timeout, 30},
+        %    {user_dn, "uid=$u,ou=People,dc=example,dc=com"},
+        %    {ssl, fasle},
+        %    {sslopts, [
+        %        {"certfile", "ssl.crt"},
+        %        {"keyfile", "ssl.key"}]}
+        % ]},
+
+        %% Allow all
+        {anonymous, []}
+    ]},
+
+The modules enabled at the same time compose an authentication chain:
+
+               ----------------           ----------------           -------------
+    Client --> |   Username   | -ignore-> |   ClientID   | -ignore-> | Anonymous |
+               ----------------           ----------------           -------------
+                      |                         |                         |
+                     \|/                       \|/                       \|/
+                allow | deny              allow | deny              allow | deny
+ 
+.. NOTE:: There are also MySQL、PostgreSQL、Redis、MongoDB Authentication Plugins.
+
+Username Authentication
+.......................
+
+.. code:: erlang
+
+    {username, [{client1, "passwd1"}, {client2, "passwd2"}]},
+
+Two ways to configure users:
+
+1. Configure username and plain password directly::
+
+    {username, [{client1, "passwd1"}, {client2, "passwd2"}]},
+
+2. Add user by './bin/emqttd_ctl users' command::
+
+   $ ./bin/emqttd_ctl users add <Username> <Password>
+
+ClientID Authentication
+.......................
+
+.. code:: erlang
+
+    {clientid, [{password, no}, {file, "etc/clients.config"}]},
+
+Configure ClientIDs in etc/clients.config::
+
+    testclientid0
+    testclientid1 127.0.0.1
+    testclientid2 192.168.0.1/24
+
+LDAP Authentication
+...................
+
+.. code:: erlang
+
+    {ldap, [
+       {servers, ["localhost"]},
+       {port, 389},
+       {timeout, 30},
+       {user_dn, "uid=$u,ou=People,dc=example,dc=com"},
+       {ssl, fasle},
+       {sslopts, [
+           {"certfile", "ssl.crt"},
+           {"keyfile", "ssl.key"}]}
+    ]},
+
+
+Anonymous Authentication
+........................
+
+Allow any client to connect to the broker::
+
+    {anonymous, []}
+
+
+ACL
+---
+
+Enable the default ACL module::
+
+    {acl, [
+        %% Internal ACL module
+        {internal,  [{file, "etc/acl.config"}, {nomatch, allow}]}
+    ]}
+
+MQTT Packet and ClientID
+------------------------
+
+.. code::
+
+    {packet, [
+
+        %% Max ClientId Length Allowed
+        {max_clientid_len, 1024},
+
+        %% Max Packet Size Allowed, 64K default
+        {max_packet_size,  65536}
+    ]},
+
+MQTT Client Idle Timeout
+------------------------
+
+.. code::
+
+    {client, [
+        %% Socket is connected, but no 'CONNECT' packet received
+        {idle_timeout, 10}
+    ]},
+
+MQTT Session
+------------
+
+.. code::
+
+    {session, [
+        %% Max number of QoS 1 and 2 messages that can be “in flight” at one time.
+        %% 0 means no limit
+        {max_inflight, 100},
+
+        %% Retry interval for unacked QoS1/2 messages.
+        {unack_retry_interval, 20},
+
+        %% Awaiting PUBREL Timeout
+        {await_rel_timeout, 20},
+
+        %% Max Packets that Awaiting PUBREL, 0 means no limit
+        {max_awaiting_rel, 0},
+
+        %% Interval of Statistics Collection(seconds)
+        {collect_interval, 20},
+
+        %% Expired after 2 days
+        {expired_after, 48}
+
+    ]},
+
+Session parameters:
+
++----------------------+----------------------------------------------------------+
+| max_inflight         | Max number of QoS1/2 messages that can be delivered in   |
+|                      | the same time                                            |
++----------------------+----------------------------------------------------------+
+| unack_retry_interval | Retry interval for unacked QoS1/2 messages.              |
++----------------------+----------------------------------------------------------+
+| await_rel_timeout    | Awaiting PUBREL Timeout                                  |
++----------------------+----------------------------------------------------------+
+| max_awaiting_rel     | Max number of Packets that Awaiting PUBREL               |
++----------------------+----------------------------------------------------------+
+| collect_interval     | Interval of Statistics Collection                        |
++----------------------+----------------------------------------------------------+
+| expired_after        | Expired after                                            |
++----------------------+----------------------------------------------------------+
+
+MQTT Message Queue
+------------------
+
+The message queue of session stores:
+
+1. Offline messages for persistent session.
+
+2. Pending messages for inflight window is full
+
+Queue parameters::
+
+    {queue, [
+        %% simple | priority
+        {type, simple},
+
+        %% Topic Priority: 0~255, Default is 0
+        %% {priority, [{"topic/1", 10}, {"topic/2", 8}]},
+
+        %% Max queue length. Enqueued messages when persistent client disconnected,
+        %% or inflight window is full.
+        {max_length, infinity},
+
+        %% Low-water mark of queued messages
+        {low_watermark, 0.2},
+
+        %% High-water mark of queued messages
+        {high_watermark, 0.6},
+
+        %% Queue Qos0 messages?
+        {queue_qos0, true}
+    ]}
+
++----------------------+---------------------------------------------------+
+| type                 | Queue type: simple or priority                    |
++----------------------+---------------------------------------------------+
+| priority             | Topic priority                                    |
++----------------------+---------------------------------------------------+
+| max_length           | Max Queue size, infinity means no limit           |
++----------------------+---------------------------------------------------+
+| low_watermark        | Low watermark                                     |
++----------------------+---------------------------------------------------+
+| high_watermark       | High watermark                                    |
++----------------------+---------------------------------------------------+
+| queue_qos0           | If Qos0 message queued?                           |
++----------------------+---------------------------------------------------+
+
+Sys Interval of Broker
+-----------------------
+
+.. code::
+
+    %% System interval of publishing $SYS messages
+    {sys_interval, 60},
+
+Retained messages
+-----------------
+
+.. code::
+
+    {retained, [
+        %% Expired after seconds, never expired if 0
+        {expired_after, 0},
+
+        %% Maximum number of retained messages
+        {max_message_num, 100000},
+
+        %% Max Payload Size of retained message
+        {max_playload_size, 65536}
+    ]},
+
+PubSub and Router
+-----------------
+
+.. code:: erlang
+
+    {pubsub, [
+        %% PubSub Pool
+        {pool_size, 8},
+        
+        %% Subscription: disc | ram | false
+        {subscription, ram},
+
+        %% Route aging time(seconds)
+        {route_aging, 5}
+    ]},
+
+Bridge Parameters
+-----------------
+
+.. code:: erlang
+
+    {bridge, [
+        %% Bridge Queue Size
+        {max_queue_len, 10000},
+
+        %% Ping Interval of bridge node
+        {ping_down_interval, 1}
+    ]}
+
+
+Enable Modules
+--------------
+
+'presence' module will publish presence message to $SYS topic when a client connected or disconnected::
+
+        {presence, [{qos, 0}]},
+
+'subscription' module forces the client to subscribe some topics when connected to the broker::
+
+        %% Subscribe topics automatically when client connected
+        {subscription, [
+            %% Subscription from stored table
+            stored,
+
+            %% $u will be replaced with username
+            {"$Q/username/$u", 1},
+
+            %% $c will be replaced with clientid
+            {"$Q/client/$c", 1}
+        ]}
+
+'rewrite' module supports to rewrite the topic path::
+
+        %% Rewrite rules
+        {rewrite, [{file, "etc/rewrite.config"}]}
+
+Plugins Folder
+--------------
+
+.. code:: erlang
+
+    {plugins, [
+        %% Plugin App Library Dir
+        {plugins_dir, "./plugins"},
+
+        %% File to store loaded plugin names.
+        {loaded_file, "./data/loaded_plugins"}
+    ]},
+
+
+TCP Listeners
+-------------
+
+Congfigure the TCP listeners for MQTT, MQTT(SSL) and HTTP Protocols.
+
+The most important parameter is 'max_clients' - max concurrent clients allowed.
+
+The TCP Ports occupied by emqttd broker by default:
+
++-----------+-----------------------------------+
+| 1883      | MQTT Port                         |
++-----------+-----------------------------------+
+| 8883      | MQTT(SSL) Port                    |
++-----------+-----------------------------------+
+| 8083      | MQTT(WebSocket), HTTP API Port    |
++-----------+-----------------------------------+
+
+.. code:: erlang
+
+    {listeners, [
+
+        {mqtt, 1883, [
+            %% Size of acceptor pool
+            {acceptors, 16},
+
+            %% Maximum number of concurrent clients
+            {max_clients, 8192},
+
+            %% Socket Access Control
+            {access, [{allow, all}]},
+
+            %% Connection Options
+            {connopts, [
+                %% Rate Limit. Format is 'burst, rate', Unit is KB/Sec
+                %% {rate_limit, "100,10"} %% 100K burst, 10K rate
+            ]},
+
+            %% Socket Options
+            {sockopts, [
+                %Set buffer if hight thoughtput
+                %{recbuf, 4096},
+                %{sndbuf, 4096},
+                %{buffer, 4096},
+                %{nodelay, true},
+                {backlog, 1024}
+            ]}
+        ]},
+
+        {mqtts, 8883, [
+            %% Size of acceptor pool
+            {acceptors, 4},
+
+            %% Maximum number of concurrent clients
+            {max_clients, 512},
+
+            %% Socket Access Control
+            {access, [{allow, all}]},
+
+            %% SSL certificate and key files
+            {ssl, [{certfile, "etc/ssl/ssl.crt"},
+                   {keyfile,  "etc/ssl/ssl.key"}]},
+
+            %% Socket Options
+            {sockopts, [
+                {backlog, 1024}
+                %{buffer, 4096},
+            ]}
+        ]},
+        %% WebSocket over HTTPS Listener
+        %% {https, 8083, [
+        %%  %% Size of acceptor pool
+        %%  {acceptors, 4},
+        %%  %% Maximum number of concurrent clients
+        %%  {max_clients, 512},
+        %%  %% Socket Access Control
+        %%  {access, [{allow, all}]},
+        %%  %% SSL certificate and key files
+        %%  {ssl, [{certfile, "etc/ssl/ssl.crt"},
+        %%         {keyfile,  "etc/ssl/ssl.key"}]},
+        %%  %% Socket Options
+        %%  {sockopts, [
+        %%      %{buffer, 4096},
+        %%      {backlog, 1024}
+        %%  ]}
+        %%]},
+
+        %% HTTP and WebSocket Listener
+        {http, 8083, [
+            %% Size of acceptor pool
+            {acceptors, 4},
+            %% Maximum number of concurrent clients
+            {max_clients, 64},
+            %% Socket Access Control
+            {access, [{allow, all}]},
+            %% Socket Options
+            {sockopts, [
+                {backlog, 1024}
+                %{buffer, 4096},
+            ]}
+        ]}
+    ]},
+
+Listener Parameters:
+
++-------------+----------------------------------------------------------------+
+| acceptors   | TCP Acceptor Pool                                              |
++-------------+----------------------------------------------------------------+
+| max_clients | Maximum number of concurrent TCP connections allowed           |
++-------------+----------------------------------------------------------------+
+| access      | Access Control by IP, for example: [{allow, "192.168.1.0/24"}] |
++-------------+----------------------------------------------------------------+
+| connopts    | Rate Limit Control, for example: {rate_limit, "100,10"}        |
++-------------+----------------------------------------------------------------+
+| sockopts    | TCP Socket parameters                                          |
++-------------+----------------------------------------------------------------+
+
+.. _config_acl:
+
+--------------
+etc/acl.config
+--------------
+
+The 'etc/acl.config' is the default ACL config for emqttd broker. The rules by default::
+
+    %% Allow 'dashboard' to subscribe '$SYS/#'
+    {allow, {user, "dashboard"}, subscribe, ["$SYS/#"]}.
+
+    %% Allow clients from localhost to subscribe any topics
+    {allow, {ipaddr, "127.0.0.1"}, pubsub, ["$SYS/#", "#"]}.
+
+    %% Deny clients to subscribe '$SYS#' and '#'
+    {deny, all, subscribe, ["$SYS/#", {eq, "#"}]}.
+
+    %% Allow all by default
+    {allow, all}.
+
+An ACL rule is an Erlang tuple. The Access control module of emqttd broker matches the rule one by one from top to bottom::
+
+              ---------              ---------              ---------   
+    Client -> | Rule1 | --nomatch--> | Rule2 | --nomatch--> | Rule3 | --> Default
+              ---------              ---------              ---------
+                  |                      |                      |
+                match                  match                  match
+                 \|/                    \|/                    \|/
+            allow | deny           allow | deny           allow | deny
+                
+.. _config_rewrite:
+
+------------------
+etc/clients.config
+------------------
+
+Enable ClientId Authentication in 'etc/emqttd.config'::
+
+    {auth, [
+        %% Authentication with clientid
+        {clientid, [{password, no}, {file, "etc/clients.config"}]}
+    ]},
+
+Configure all allowed ClientIDs, IP Addresses in etc/clients.config::
+
+    testclientid0
+    testclientid1 127.0.0.1
+    testclientid2 192.168.0.1/24
+
+------------------
+etc/rewrite.config
+------------------
+
+The Rewrite Rules for emqttd_mod_rewrite::
+
+    {topic, "x/#", [
+        {rewrite, "^x/y/(.+)$", "z/y/$1"},
+        {rewrite, "^x/(.+)$", "y/$1"}
+    ]}.
+
+    {topic, "y/+/z/#", [
+        {rewrite, "^y/(.+)/z/(.+)$", "y/z/$2"}
+    ]}.
+

docs/source/design.rst

@@ -0,0 +1,330 @@
+==============
+Design Guide
+==============
+
+---------------
+Pubsub Sequence
+---------------
+
+## PubSub Sequence
+
+### Clean Session = 1
+
+```
+
+title PubSub Sequence(Clean Session = 1)
+
+ClientA-->PubSub: Publish Message
+PubSub-->ClientB: Dispatch Message
+```
+
+![PubSub_CleanSess_1](http://emqtt.io/static/img/design/PubSub_CleanSess_1.png)
+
+### Clean Session = 0
+
+```
+title PubSub Sequence(Clean Session = 0)
+
+ClientA-->SessionA: Publish Message
+SessionA-->PubSub: Publish Message
+PubSub-->SessionB: Dispatch Message
+SessionB-->ClientB: Dispatch Message
+
+```
+![PubSub_CleanSess_0](http://emqtt.io/static/img/design/PubSub_CleanSess_0.png)
+
+
+## Qos
+
+PubQos | SubQos | In Message | Out Message
+-------|--------|------------|-------------
+0   |   0    |   0        | 0
+0   |   1    |   0        | 0
+0   |   2    |   0        | 0
+1   |   0    |   1        | 0
+1   |   1    |   1        | 1
+1   |   2    |   1        | 1
+2   |   0    |   2        | 0
+2   |   1    |   2        | 1
+2   |   2    |   2        | 2
+
+
+## Topic Functions Benchmark
+
+Mac Air(11):
+
+Function     | Time(microseconds)
+-------------|--------------------
+match        | 6.25086
+triples      | 13.86881
+words        | 3.41177
+binary:split | 3.03776
+
+iMac:
+
+Function     | Time(microseconds)
+-------------|--------------------
+match        | 3.2348
+triples      | 6.93524
+words        | 1.89616
+binary:split | 1.65243
+
+
+--------------
+Cluster Design
+--------------
+
+## Cluster Design
+
+1. One 'disc_copies' node and many 'ram_copies' nodes.
+
+   2. Topic trie tree will be copied to every clusterd node.
+
+   3. Subscribers to topic will be stored in each node and will not be copied.
+
+   ## Cluster Strategy
+
+   TODO:...
+
+   1. A message only gets forwarded to other cluster nodes if a cluster node is interested in it. this reduces the network traffic tremendously, because it prevents nodes from forwarding unnecessary messages.
+
+   2. As soon as a client on a node subscribes to a topic it becomes known within the cluster. If one of the clients somewhere in the cluster is publishing to this topic, the message will be delivered to its subscriber no matter to which cluster node it is connected.
+
+   ....
+
+## Cluster Architecture
+
+![Cluster Design](http://emqtt.io/static/img/Cluster.png)
+## Cluster Command
+
+```sh
+./bin/emqttd_ctl cluster DiscNode
+```
+
+## Mnesia Example
+
+```
+(emqttd3@127.0.0.1)3> mnesia:info().
+---> Processes holding locks <---
+---> Processes waiting for locks <---
+---> Participant transactions <---
+---> Coordinator transactions <---
+---> Uncertain transactions <---
+---> Active tables <---
+mqtt_retained : with 6 records occupying 221 words of mem
+topic_subscriber: with 0 records occupying 305 words of mem
+topic_trie_node: with 129 records occupying 3195 words of mem
+topic_trie : with 128 records occupying 3986 words of mem
+topic : with 93 records occupying 1797 words of mem
+schema : with 6 records occupying 1081 words of mem
+===> System info in version "4.12.4", debug level = none <===
+opt_disc. Directory "/Users/erylee/Projects/emqttd/rel/emqttd3/data/mnesia" is NOT used.
+use fallback at restart = false
+running db nodes = ['emqttd2@127.0.0.1','emqttd@127.0.0.1','emqttd3@127.0.0.1']
+stopped db nodes = []
+master node tables = []
+remote = []
+ram_copies = [mqtt_retained,schema,topic,topic_subscriber,topic_trie,
+topic_trie_node]
+disc_copies = []
+disc_only_copies = []
+[{'emqttd2@127.0.0.1',ram_copies},
+{'emqttd3@127.0.0.1',ram_copies},
+{'emqttd@127.0.0.1',disc_copies}] = [schema]
+[{'emqttd2@127.0.0.1',ram_copies},
+{'emqttd3@127.0.0.1',ram_copies},
+{'emqttd@127.0.0.1',ram_copies}] = [topic,topic_trie,topic_trie_node,
+mqtt_retained]
+[{'emqttd3@127.0.0.1',ram_copies}] = [topic_subscriber]
+44 transactions committed, 5 aborted, 0 restarted, 0 logged to disc
+   0 held locks, 0 in queue; 0 local transactions, 0 remote
+   0 transactions waits for other nodes: []
+   ```
+
+   ## Cluster vs Bridge
+
+   Cluster will copy topic trie tree between nodes, Bridge will not.
+
+
+
+-------------
+Hooks Design
+-------------
+
+## Overview 
+
+emqttd supported a simple hooks mechanism in 0.8.0 release to extend the broker. The designed is improved in 0.9.0 release.
+
+## API
+
+emqttd_broker Hook API:
+
+```
+-export([hook/3, unhook/2, foreach_hooks/2, foldl_hooks/3]).
+```
+
+### Hook 
+
+``` 
+-spec hook(Hook :: atom(), Name :: any(), MFA :: mfa()) -> ok | {error, any()}.
+hook(Hook, Name, MFA) ->
+    ...
+    ```
+
+ ### Unhook
+
+ ```
+ -spec unhook(Hook :: atom(), Name :: any()) -> ok | {error, any()}.
+ unhook(Hook, Name) ->
+     ...
+     ```
+
+  ### Foreach Hooks
+
+  ```
+  -spec foreach_hooks(Hook :: atom(), Args :: list()) -> any().
+  foreach_hooks(Hook, Args) ->
+      ...
+      ```
+
+   ### Foldl Hooks
+
+   ```
+   -spec foldl_hooks(Hook :: atom(), Args :: list(), Acc0 :: any()) -> any().
+   foldl_hooks(Hook, Args, Acc0) ->
+       ...
+       ```
+
+    ## Hooks 
+
+    Name             | Type      | Description
+    ---------------  | ----------| --------------
+    client.connected | foreach   | Run when client connected successfully
+    client.subscribe | foldl     | Run before client subscribe topics
+    client.subscribe.after | foreach | Run After client subscribe topics
+    client.unsubscribe | foldl   | Run when client unsubscribe topics
+    message.publish   | foldl     | Run when message is published
+    message.acked   | foreach     | Run when message is acked
+    client.disconnected | foreach | Run when client is disconnnected
+
+    ## End-to-End Message Pub/Ack
+
+    Could use 'message.publish', 'message.acked' hooks to implement end-to-end message pub/ack:
+
+    ```
+     PktId <-- --> MsgId <-- --> MsgId <-- --> PktId
+          |<--- Qos --->|<---PubSub--->|<-- Qos -->|
+          ```
+## Limit
+
+The design is experimental.
+
+
+--------------
+Plugin Design
+--------------
+
+## Overview
+
+**Notice that 0.11.0 release use rebar to manage plugin's deps.**
+
+A plugin is just an erlang application that extends emqttd broker.
+
+The plugin application should be put in "emqttd/plugins/" folder to build. 
+
+
+## Plugin Project
+
+You could create a standalone plugin project outside emqttd, and then add it to "emqttd/plugins/" folder by "git submodule". 
+
+Git submodule to compile emqttd_dashboard plugin with the broker, For example:
+
+```
+git submodule add https://github.com/emqtt/emqttd_dashboard.git plugins/emqttd_dashboard
+make && make dist
+```
+
+## plugin.config
+
+**Each plugin should have a 'etc/plugin.config' file**
+
+For example, project structure of emqttd_dashboard plugin:
+
+```
+LICENSE
+README.md
+ebin
+etc
+priv
+rebar.config
+src
+```
+
+etc/plugin.config for emqttd_dashboard plugin:
+
+```
+[
+{emqttd_dashboard, [
+{listener,
+{emqttd_dashboard, 18083, [
+{acceptors, 4},
+{max_clients, 512}]}}
+]}
+].
+```
+
+## rebar.config
+
+**Plugin should use 'rebar.config' to manage depencies**
+
+emqttd_plugin_pgsql plugin's rebar.config, for example:
+
+```
+%% -*- erlang -*-
+
+{deps, [
+{epgsql, ".*",{git, "https://github.com/epgsql/epgsql.git", {branch, "master"}}}
+]}.
+```
+
+## Build emqttd with plugins
+
+Put all the plugins you required in 'plugins/' folder of emqttd project, and then:
+
+```
+make && make dist
+```
+
+## Load Plugin
+
+'./bin/emqttd_ctl' to load/unload plugin, when emqttd broker started.
+
+```
+./bin/emqttd_ctl plugins load emqttd_plugin_demo
+
+./bin/emqttd_ctl plugins unload emqttd_plugin_demo
+```
+
+## List Plugins
+
+```
+./bin/emqttd_ctl plugins list
+```
+
+## API
+
+```
+%% Load all active plugins after broker started
+emqttd_plugins:load() 
+
+%% Load new plugin
+emqttd_plugins:load(Name)
+
+%% Unload all active plugins before broker stopped
+emqttd_plugins:unload()
+
+%% Unload a plugin
+emqttd_plugins:unload(Name)
+```
+
+

docs/source/faq.rst

@@ -0,0 +1,77 @@
+===
+FAQ
+===
+
+
+##### Q1. Is port 4369 and another random port secure?
+
+```
+HI, when start emqttd , I found the port 4369 and another random port(63703) is open, are this security ?
+
+Example: 
+tcp 0 0 0.0.0.0:4369 0.0.0.0:* LISTEN 13736/epmd
+tcp 0 0 0.0.0.0:8083 0.0.0.0:* LISTEN 16745/beam.smp
+tcp 0 0 0.0.0.0:8883 0.0.0.0:* LISTEN 16745/beam.smp
+tcp 0 0 0.0.0.0:63703 0.0.0.0:* LISTEN 16745/beam.smp
+tcp 0 0 0.0.0.0:1883 0.0.0.0:* LISTEN 16745/beam.smp
+
+1883: mqtt no ssl
+8883: mqtt with ssl
+8083: websocket
+```
+
+4369 and some random ports are opened by erlang node for internal communication. Configure your firewall to allow 1883, 8883, 8083 ports to be accessed from outside for security.
+
+Access control of emqttd broker has two layers:
+
+eSockd TCP Acceptor, Ipaddress based Access Control, Example:
+
+```
+{access, [{deny, "192.168.1.1"},
+{allow, "192.168.1.0/24"},
+{deny, all}]}
+```
+
+MQTT Subscribe/Publish Access Control by etc/acl.config, Example:
+
+```
+{allow, {ipaddr, "127.0.0.1"}, pubsub, ["$SYS/#", "#"]}.
+
+{deny, all, subscribe, ["$SYS/#", {eq, "#"}]}.
+
+{allow, all}.
+```
+
+##### Q2. cannot compile emqttd under Chinese folder?
+
+It seems that rebar cannot support Chinese folder name.
+
+##### Q3. emqttd is ready for production ? 
+
+The core features are solid and scalable. A small full-time team with many contributors are developing this project. You could submit issues if any feature requests or bug reports.
+
+##### Q4. Benchmark and performance issue
+
+Wiki: https://github.com/emqtt/emqttd/wiki/One-Million-Connections
+
+##### Q5. 'session' identified by clientID？when session expired what will happen？All queued messages will be deleted and subscribed topics will be deleted too？when reconnected， need redo subscription？(#150)
+
+When a client connected to broker with 'clean session' flag 0, a session identified by clientId will be created. The session will expire after 48 hours(configured in etc/emqttd.config) if no client connections bind with it, and all queued messages and subscriptions will be dropped.
+
+##### Q6. "{max_queued_messages, 100}" in 0.8 release or "{queue, {max_length, 1000},..." means queue for one session or one topic？If it stands for session，while one topic has lots of offline messages(100), the user's other topic offline messages will be flushed? (#150)
+
+For session. Topic just dispatch messages to clients or sessions that matched the subscriptions. Will Flood.
+
+##### Q7. About the retained message, how to config one topic only keep the latest retained message, the older retained messages will be auto deleted?(#150)
+
+By default, the broker only keep the latest retained message of one topic.
+
+##### Q8. When the persistent client with 'clean session' flag 0 is offline but not expired, will the broker put session's subscribed topic new messages to session's queue?(#150)
+
+Yes
+
+##### Q9. If max_length of queue is 100, when the session subscribed topic1 and topic2, what will happen when topic1 fill 70 messages, then topic2 fill 80 messages? After the reconnection, will the session lose first 50 message?(#150)
+
+Lose the oldest 50 messages.
+
+

docs/source/getstarted.rst

@@ -0,0 +1,293 @@
+
+.. _getstarted:
+
+===========
+Get Started
+===========
+
+--------
+Overview
+--------
+
+emqttd(Erlang MQTT Broker) is an open source MQTT broker written in Erlang/OTP. Erlang/OTP is a concurrent, fault-tolerant, soft-realtime and distributed programming platform. MQTT is an extremely lightweight publish/subscribe messaging protocol powering IoT, M2M and Mobile applications.
+
+The emqttd project is aimed to implement a scalable, distributed, extensible open-source MQTT broker for IoT, M2M and Mobile applications that hope to handle millions of concurrent MQTT clients.
+
+Highlights of the emqttd broker:
+
+* Full MQTT V3.1/3.1.1 Protocol Specifications Support
+* Easy to Install - Quick Install on Linux, FreeBSD, Mac and Windows
+* Massively scalable - Scaling to 1 million connections on a single server
+* Cluster and Bridge Support
+* Easy to extend - Hooks and plugins to customize or extend the broker
+* Pluggable Authentication - LDAP, MySQL, PostgreSQL, Redis Authentication Plugins
+
+--------
+Features
+--------
+
+* Full MQTT V3.1/V3.1.1 protocol specification support
+* QoS0, QoS1, QoS2 Publish and Subscribe
+* Session Management and Offline Messages
+* Retained Message
+* Last Will Message
+* TCP/SSL Connection
+* MQTT Over WebSocket(SSL)
+* HTTP Publish API
+* STOMP protocol
+* STOMP over SockJS
+* $SYS/# Topics
+* ClientID Authentication
+* IpAddress Authentication
+* Username and Password Authentication
+* Access control based on IpAddress, ClientID, Username
+* Authentication with LDAP, Redis, MySQL, PostgreSQL
+* Cluster brokers on several servers
+* Bridge brokers locally or remotely
+* mosquitto, RSMB bridge
+* Extensible architecture with Hooks, Modules and Plugins
+* Passed eclipse paho interoperability tests
+
+-----------
+Quick Start
+-----------
+
+Download and Install
+--------------------
+
+The emqttd broker is cross-platform, could be deployed on Linux, Mac, FreeBSD, Windows and Raspberry Pi.
+
+Download binary package from: http://emqtt.io/downloads.
+
+Installing on Mac, For example:
+
+.. code:: console
+
+    unzip emqttd-macosx-0.16.0-beta-20160216.zip && cd emqttd
+
+    # Start emqttd
+    ./bin/emqttd start
+
+    # Check Status
+    ./bin/emqttd_ctl status
+
+    # Stop emqttd
+    ./bin/emqttd stop
+
+Installing from Source
+----------------------
+
+.. NOTE:: emqttd broker requires Erlang R17+ to build.
+
+.. code:: console
+
+    git clone https://github.com/emqtt/emqttd.git
+
+    cd emqttd && make && make dist
+
+    cd rel/emqttd && ./bin/emqttd console
+
+-------------
+Web Dashboard
+-------------
+
+A Web Dashboard will be loaded when the emqttd broker is started successfully.
+
+The Dashboard helps check running status of the broker, monitor statistics and metrics of MQTT packets, query clients, sessions, topics and subscriptions.
+
++------------------+---------------------------+
+| Default Address  | http://localhost:18083    |
++------------------+---------------------------+
+| Default User     | admin                     |
++------------------+---------------------------+
+| Default Password | public                    |
++------------------+---------------------------+
+
+.. image:: ./_static/images/dashboard.png
+
+-------------------
+Modules and Plugins
+-------------------
+
+The Authentication and Authorization(ACL) are usually implemented by a Module or Plugin.
+
+Modules
+-------
+
++-------------------------+--------------------------------------------+
+| emqttd_auth_clientid    | Authentication with ClientId               |
++-------------------------+--------------------------------------------+
+| emqttd_auth_username    | Authentication with Username and Password  |
++-------------------------+--------------------------------------------+
+| emqttd_auth_ldap        | Authentication with LDAP                   |
++-------------------------+--------------------------------------------+
+| emqttd_mod_presence     | Publish presence message to $SYS topics    |
+|                         | when client connected or disconnected      |
++-------------------------+--------------------------------------------+
+| emqttd_mod_subscription | Subscribe topics automatically when client |
+|                         | connected                                  |
++-------------------------+--------------------------------------------+
+| emqttd_mod_rewrite      | Topics rewrite like HTTP rewrite module    |
++-------------------------+--------------------------------------------+
+
+Configure the 'auth', 'module' paragraph in 'etc/emqttd.config' to enable a module.
+
+Enable 'emqttd_auth_username' module::
+
+    {access, [
+        %% Authetication. Anonymous Default
+        {auth, [
+            %% Authentication with username, password
+            {username, []},
+
+            ...
+
+Enable 'emqttd_mod_presence' module::
+
+    {modules, [
+        %% Client presence management module.
+        %% Publish messages when client connected or disconnected
+        {presence, [{qos, 0}]}
+
+Plugins
+-------
+
+A plugin is an Erlang application to extend the emqttd broker.
+
++----------------------------+-----------------------------------+
+| `emqttd_plugin_template`_  | Plugin template and demo          |
++----------------------------+-----------------------------------+
+| `emqttd_dashboard`_        | Web Dashboard                     |
++----------------------------+-----------------------------------+
+| `emqttd_plugin_mysql`_     | Authentication with MySQL         |
++----------------------------+-----------------------------------+
+| `emqttd_plugin_pgsql`_     | Authentication with PostgreSQL    |
++----------------------------+-----------------------------------+
+| `emqttd_plugin_redis`_     | Authentication with Redis         |
++----------------------------+-----------------------------------+
+| `emqttd_plugin_mongo`_     | Authentication with MongoDB       |
++----------------------------+-----------------------------------+
+| `emqttd_stomp`_            |  STOMP Protocol Plugin            |
++----------------------------+-----------------------------------+
+| `emqttd_sockjs`_           | SockJS(Stomp) Plugin              |
++----------------------------+-----------------------------------+
+| `emqttd_recon`_            | Recon Plugin                      |
++----------------------------+-----------------------------------+
+
+A plugin could be enabled by 'bin/emqttd_ctl plugins load' command.
+
+For example, enable 'emqttd_plugin_pgsql' plugin::
+
+    ./bin/emqttd_ctl plugins load emqttd_plugin_pgsql
+
+-----------------------
+One Million Connections
+-----------------------
+
+Latest release of emqttd broker is scaling to 1.3 million MQTT connections on a 12 Core, 32G CentOS server.
+
+.. NOTE::
+
+    The emqttd broker only allows 512 concurrent connections by default, for 'ulimit -n' limit is 1024 on most platform.
+
+We need tune the OS Kernel, TCP Stack, Erlang VM and emqttd broker for one million connections benchmark.
+
+Linux Kernel Parameters
+-----------------------
+
+.. code::
+
+    # 2M:
+    sysctl -w fs.file-max=2097152
+    sysctl -w fs.nr_open=2097152
+    echo 2097152 > /proc/sys/fs/nr_open
+
+    # 1M:
+    ulimit -n 1048576
+
+TCP Stack Parameters
+--------------------
+
+.. code::
+
+    # backlog
+    sysctl -w net.core.somaxconn=65536
+
+Erlang VM
+---------
+
+emqttd/etc/vm.args::
+
+    ## max process numbers
+    +P 2097152
+
+    ## Sets the maximum number of simultaneously existing ports for this system
+    +Q 1048576
+
+    ## Increase number of concurrent ports/sockets
+    -env ERL_MAX_PORTS 1048576
+
+    -env ERTS_MAX_PORTS 1048576
+
+emqttd broker
+-------------
+
+emqttd/etc/emqttd.config::
+
+        {mqtt, 1883, [
+            %% Size of acceptor pool
+            {acceptors, 64},
+
+            %% Maximum number of concurrent clients
+            {max_clients, 1000000},
+
+            %% Socket Access Control
+            {access, [{allow, all}]},
+
+            %% Connection Options
+            {connopts, [
+                %% Rate Limit. Format is 'burst, rate', Unit is KB/Sec
+                %% {rate_limit, "100,10"} %% 100K burst, 10K rate
+            ]},
+            ...
+
+Test Client
+-----------
+
+.. code::
+
+    sysctl -w net.ipv4.ip_local_port_range="500 65535"
+    echo 1000000 > /proc/sys/fs/nr_open
+    ulimit -n 100000
+
+---------------------
+MQTT Client Libraries
+---------------------
+
+GitHub: https://github.com/emqtt
+
++--------------------+----------------------+
+| `emqttc`_          | Erlang MQTT Client   |
++--------------------+----------------------+
+| `emqtt_benchmark`_ | MQTT benchmark Tool  |
++--------------------+----------------------+
+| `CocoaMQTT`_       | Swift MQTT Client    |
++--------------------+----------------------+
+| `QMQTT`_           | QT MQTT Client       |
++--------------------+----------------------+
+
+.. _emqttc:          https://github.com/emqtt/emqttc
+.. _emqtt_benchmark: https://github.com/emqtt/emqtt_benchmark
+.. _CocoaMQTT:       https://github.com/emqtt/CocoaMQTT
+.. _QMQTT:           https://github.com/emqtt/qmqtt
+
+.. _emqttd_plugin_template: https://github.com/emqtt/emqttd_plugin_template
+.. _emqttd_dashboard:       https://github.com/emqtt/emqttd_dashboard
+.. _emqttd_plugin_mysql:    https://github.com/emqtt/emqttd_plugin_mysql
+.. _emqttd_plugin_pgsql:    https://github.com/emqtt/emqttd_plugin_pgsql
+.. _emqttd_plugin_redis:    https://github.com/emqtt/emqttd_plugin_redis
+.. _emqttd_plugin_mongo:    https://github.com/emqtt/emqttd_plugin_mongo
+.. _emqttd_stomp:           https://github.com/emqtt/emqttd_stomp
+.. _emqttd_sockjs:          https://github.com/emqtt/emqttd_sockjs
+.. _emqttd_recon:           https://github.com/emqtt/emqttd_recon
+

docs/source/guide.rst

@@ -0,0 +1,760 @@
+
+.. _guide:
+
+==========
+User Guide
+==========
+
+--------------
+Authentication
+--------------
+
+The emqttd broker supports to authenticate MQTT clients with ClientID, Username/Password, IpAddress and even HTTP Cookies.
+
+The authentication is provided by a list of extended modules, or MySQL, PostgreSQL and Redis Plugins.
+
+Enable an authentication module in etc/emqttd.config::
+
+    %% Authentication and Authorization
+    {access, [
+        %% Authetication. Anonymous Default
+        {auth, [
+            %% Authentication with username, password
+            %{username, []},
+            
+            %% Authentication with clientid
+            %{clientid, [{password, no}, {file, "etc/clients.config"}]},
+
+            %% Authentication with LDAP
+            % {ldap, [
+            %    {servers, ["localhost"]},
+            %    {port, 389},
+            %    {timeout, 30},
+            %    {user_dn, "uid=$u,ou=People,dc=example,dc=com"},
+            %    {ssl, fasle},
+            %    {sslopts, [
+            %        {"certfile", "ssl.crt"},
+            %        {"keyfile", "ssl.key"}]}
+            % ]},
+
+            %% Allow all
+            {anonymous, []}
+        ]},
+
+.. NOTE:: "%" comments the line.
+
+If we enable several modules at the same time, the authentication process::
+
+               ----------------           ----------------           -------------
+    Client --> |   Username   | -ignore-> |   ClientID   | -ignore-> | Anonymous |
+               ----------------           ----------------           -------------
+                      |                         |                         |
+                     \|/                       \|/                       \|/
+                allow | deny              allow | deny              allow | deny
+
+The authentication plugins developed by emqttd:
+
++---------------------------+---------------------------+
+| Plugin                    | Description               |
++===========================+===========================+
+| `emqttd_plugin_mysql`_    | MySQL Auth/ACL Plugin     |
++---------------------------+---------------------------+
+| `emqttd_plugin_pgsql`_    | PostgreSQL Auth/ACL Plugin|
++---------------------------+---------------------------+
+| `emqttd_plugin_redis`_    | Redis Auth/ACL Plugin     |
++---------------------------+---------------------------+
+
+.. NOTE:: If we load an authentication plugin, the authentication modules will be disabled.
+
+Username
+--------
+
+Authenticate MQTT client with Username/Password::
+
+    {username, [{client1, "passwd1"}, {client1, "passwd2"}]},
+
+Two ways to add users:
+
+1. Configure username and plain password directly::
+
+    {username, [{client1, "passwd1"}, {client1, "passwd2"}]},
+
+2. Add user by './bin/emqttd_ctl users' command::
+
+   $ ./bin/emqttd_ctl users add <Username> <Password>
+
+ClientId
+--------
+
+.. code:: erlang
+
+    {clientid, [{password, no}, {file, "etc/clients.config"}]},
+
+Configure ClientIDs in etc/clients.config::
+
+    testclientid0
+    testclientid1 127.0.0.1
+    testclientid2 192.168.0.1/24
+
+LDAP
+----
+
+.. code:: erlang
+
+    {ldap, [
+       {servers, ["localhost"]},
+       {port, 389},
+       {timeout, 30},
+       {user_dn, "uid=$u,ou=People,dc=example,dc=com"},
+       {ssl, fasle},
+       {sslopts, [
+           {"certfile", "ssl.crt"},
+           {"keyfile", "ssl.key"}]}
+    ]},
+
+Anonymous
+---------
+
+Allow any client to connect to the broker::
+
+    {anonymous, []}
+
+MySQL
+-----
+
+Authenticate against MySQL database. Support we create a mqtt_user table::
+
+    CREATE TABLE `mqtt_user` (
+      `id` int(11) unsigned NOT NULL AUTO_INCREMENT,
+      `username` varchar(100) DEFAULT NULL,
+      `password` varchar(100) DEFAULT NULL,
+      `salt` varchar(20) DEFAULT NULL,
+      `created` datetime DEFAULT NULL,
+      PRIMARY KEY (`id`),
+      UNIQUE KEY `mqtt_username` (`username`)
+    ) ENGINE=MyISAM DEFAULT CHARSET=utf8;
+
+Configure the 'authquery' and 'password_hash' in emqttd_plugin_mysql/etc/plugin.config::
+
+    [
+
+    {emqttd_plugin_mysql, [
+
+        ...
+
+        %% select password only
+        {authquery, "select password from mqtt_user where username = '%u' limit 1"},
+
+        %% hash algorithm: md5, sha, sha256, pbkdf2?
+        {password_hash, sha256},
+
+        ...
+
+    ]}
+    ].
+
+Load the plugin::
+
+    ./bin/emqttd_ctl plugins load emqttd_plugin_mysql
+
+
+PostgreSQL
+----------
+
+Authenticate against PostgreSQL database. Create a mqtt_user table::
+
+    CREATE TABLE mqtt_user (
+      id SERIAL primary key,
+      username character varying(100),
+      password character varying(100),
+      salt character varying(40)
+    );
+
+Configure the 'authquery' and 'password_hash' in emqttd_plugin_pgsql/etc/plugin.config::
+
+    [
+
+      {emqttd_plugin_pgsql, [
+
+        ...
+
+        %% select password only
+        {authquery, "select password from mqtt_user where username = '%u' limit 1"},
+
+        %% hash algorithm: md5, sha, sha256, pbkdf2?
+        {password_hash, sha256},
+        
+        ...
+
+      ]}
+    ].
+
+Load the plugin::
+
+    ./bin/emqttd_ctl plugins load emqttd_plugin_pgsql
+
+Redis
+-----
+
+Authenticate against Redis. MQTT users could be stored in redis HASH, the key is "mqtt_user:<Username>".
+
+Configure 'authcmd' and 'password_hash' in emqttd_plugin_redis/etc/plugin.config::
+
+    [
+      {emqttd_plugin_redis, [
+
+        ...
+
+        %% HMGET mqtt_user:%u password
+        {authcmd, ["HGET", "mqtt_user:%u", "password"]},
+
+        %% Password hash algorithm: plain, md5, sha, sha256, pbkdf2?
+        {password_hash, sha256},
+
+        ...
+
+      ]}
+    ].
+
+Load the plugin::
+
+    ./bin/emqttd_ctl plugins load emqttd_plugin_redis
+
+---
+ACL
+---
+
+The ACL of emqttd broker is responsbile for authorizing MQTT clients to publish/subscribe topics.
+
+The ACL rules define::
+
+    Allow|Deny Who Publish|Subscribe Topics
+
+Access Control Module of emqttd broker will match the rules one by one::
+
+              ---------              ---------              ---------   
+    Client -> | Rule1 | --nomatch--> | Rule2 | --nomatch--> | Rule3 | --> Default
+              ---------              ---------              ---------
+                  |                      |                      |
+                match                  match                  match
+                 \|/                    \|/                    \|/
+            allow | deny           allow | deny           allow | deny
+
+Internal
+--------
+
+The default ACL of emqttd broker is implemented by an 'internal' module.
+
+Enable the 'internal' ACL module in etc/emqttd.config::
+
+    {acl, [
+        %% Internal ACL module
+        {internal,  [{file, "etc/acl.config"}, {nomatch, allow}]}
+    ]}
+
+The ACL rules of 'internal' module are defined in 'etc/acl.config' file::
+
+    %% Allow 'dashboard' to subscribe '$SYS/#'
+    {allow, {user, "dashboard"}, subscribe, ["$SYS/#"]}.
+
+    %% Allow clients from localhost to subscribe any topics
+    {allow, {ipaddr, "127.0.0.1"}, pubsub, ["$SYS/#", "#"]}.
+
+    %% Deny clients to subscribe '$SYS#' and '#'
+    {deny, all, subscribe, ["$SYS/#", {eq, "#"}]}.
+
+    %% Allow all by default
+    {allow, all}.
+
+MySQL
+-----
+
+ACL against MySQL database. The mqtt_acl table and default data::
+
+    CREATE TABLE `mqtt_acl` (
+      `id` int(11) unsigned NOT NULL AUTO_INCREMENT,
+      `allow` int(1) DEFAULT NULL COMMENT '0: deny, 1: allow',
+      `ipaddr` varchar(60) DEFAULT NULL COMMENT 'IpAddress',
+      `username` varchar(100) DEFAULT NULL COMMENT 'Username',
+      `clientid` varchar(100) DEFAULT NULL COMMENT 'ClientId',
+      `access` int(2) NOT NULL COMMENT '1: subscribe, 2: publish, 3: pubsub',
+      `topic` varchar(100) NOT NULL DEFAULT '' COMMENT 'Topic Filter',
+      PRIMARY KEY (`id`)
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+
+    INSERT INTO mqtt_acl (id, allow, ipaddr, username, clientid, access, topic)
+    VALUES
+        (1,1,NULL,'$all',NULL,2,'#'),
+        (2,0,NULL,'$all',NULL,1,'$SYS/#'),
+        (3,0,NULL,'$all',NULL,1,'eq #'),
+        (5,1,'127.0.0.1',NULL,NULL,2,'$SYS/#'),
+        (6,1,'127.0.0.1',NULL,NULL,2,'#'),
+        (7,1,NULL,'dashboard',NULL,1,'$SYS/#');
+
+Configure 'aclquery' and 'acl_nomatch' in emqttd_plugin_mysql/etc/plugin.config::
+
+    [
+
+      {emqttd_plugin_mysql, [
+
+        ...
+
+        %% comment this query, the acl will be disabled
+        {aclquery, "select * from mqtt_acl where ipaddr = '%a' or username = '%u' or username = '$all' or clientid = '%c'"},
+
+        %% If no rules matched, return...
+        {acl_nomatch, allow}
+
+      ]}
+    ].
+
+PostgreSQL
+----------
+
+ACL against PostgreSQL database. The mqtt_acl table and default data::
+
+    CREATE TABLE mqtt_acl (
+      id SERIAL primary key,
+      allow integer,
+      ipaddr character varying(60),
+      username character varying(100),
+      clientid character varying(100),
+      access  integer,
+      topic character varying(100)
+    );
+
+    INSERT INTO mqtt_acl (id, allow, ipaddr, username, clientid, access, topic)
+    VALUES
+        (1,1,NULL,'$all',NULL,2,'#'),
+        (2,0,NULL,'$all',NULL,1,'$SYS/#'),
+        (3,0,NULL,'$all',NULL,1,'eq #'),
+        (5,1,'127.0.0.1',NULL,NULL,2,'$SYS/#'),
+        (6,1,'127.0.0.1',NULL,NULL,2,'#'),
+        (7,1,NULL,'dashboard',NULL,1,'$SYS/#');
+
+Configure 'aclquery' and 'acl_nomatch' in emqttd_plugin_pgsql/etc/plugin.config::
+
+    [
+
+      {emqttd_plugin_pgsql, [
+
+        ...
+
+        %% Comment this query, the acl will be disabled. Notice: don't edit this query!
+        {aclquery, "select allow, ipaddr, username, clientid, access, topic from mqtt_acl
+                     where ipaddr = '%a' or username = '%u' or username = '$all' or clientid = '%c'"},
+
+        %% If no rules matched, return...
+        {acl_nomatch, allow}
+
+        ...
+
+      ]}
+    ].
+
+Redis
+-----
+
+ACL against Redis. We store ACL rules for each MQTT client in a Redis List by defualt. The key is "mqtt_acl:<Username>", the value is a list of "publish <Topic>", "subscribe <Topic>" or "pubsub <Topic>".
+
+Configure 'aclcmd' and 'acl_nomatch' in emqttd_plugin_redis/etc/plugin.config::
+
+    [
+      {emqttd_plugin_redis, [
+
+        ...
+
+        %% SMEMBERS mqtt_acl:%u
+        {aclcmd, ["SMEMBERS", "mqtt_acl:%u"]},
+
+        %% If no rules matched, return...
+        {acl_nomatch, deny},
+
+        ...
+
+      ]}
+    ].
+
+----------------------
+MQTT Publish/Subscribe
+----------------------
+
+MQTT is a an extremely lightweight publish/subscribe messaging protocol desgined for IoT, M2M and Mobile applications.
+
+.. image:: _static/images/pubsub_concept.png
+
+Install and start the emqttd broker, and then any MQTT client could connect to the broker, subscribe topics and publish messages.
+
+MQTT Client Libraries: https://github.com/mqtt/mqtt.github.io/wiki/libraries
+
+For example, we use mosquitto_sub/pub commands::
+
+    mosquitto_sub -t topic -q 2
+    mosquitto_pub -t topic -q 1 -m "Hello, MQTT!"
+
+MQTT V3.1.1 Protocol Specification: http://docs.oasis-open.org/mqtt/mqtt/v3.1.1/mqtt-v3.1.1.html
+
+MQTT Listener of emqttd broker is configured in etc/emqttd.config::
+
+        {mqtt, 1883, [
+            %% Size of acceptor pool
+            {acceptors, 16},
+
+            %% Maximum number of concurrent clients
+            {max_clients, 512},
+
+            %% Socket Access Control
+            {access, [{allow, all}]},
+
+            %% Connection Options
+            {connopts, [
+                %% Rate Limit. Format is 'burst, rate', Unit is KB/Sec
+                %% {rate_limit, "100,10"} %% 100K burst, 10K rate
+            ]},
+
+            %% Socket Options
+            {sockopts, [
+                %Set buffer if hight thoughtput
+                %{recbuf, 4096},
+                %{sndbuf, 4096},
+                %{buffer, 4096},
+                %{nodelay, true},
+                {backlog, 512}
+            ]}
+        ]},
+
+MQTT(SSL) Listener, Default Port is 8883::
+
+        {mqtts, 8883, [
+            %% Size of acceptor pool
+            {acceptors, 4},
+
+            %% Maximum number of concurrent clients
+            {max_clients, 512},
+
+            %% Socket Access Control
+            {access, [{allow, all}]},
+
+            %% SSL certificate and key files
+            {ssl, [{certfile, "etc/ssl/ssl.crt"},
+                   {keyfile,  "etc/ssl/ssl.key"}]},
+
+            %% Socket Options
+            {sockopts, [
+                {backlog, 1024}
+                %{buffer, 4096},
+            ]}
+        ]},
+
+----------------
+HTTP Publish API
+----------------
+
+The emqttd broker provides a HTTP API to help application servers to publish messages to MQTT clients.
+
+HTTP API: POST http://host:8083/mqtt/publish
+
+Web servers such as PHP, Java, Python, NodeJS and Ruby on Rails could use HTTP POST to publish MQTT messages to the broker::
+
+    curl -v --basic -u user:passwd -d "qos=1&retain=0&topic=/a/b/c&message=hello from http..." -k http://localhost:8083/mqtt/publish
+
+Parameters of the HTTP API:
+
++---------+----------------+
+| Name    | Description    |
++=========+================+
+| client  | clientid       |
++---------+----------------+
+| qos     | QoS(0, 1, 2)   |
++---------+----------------+
+| retain  | Retain(0, 1)   |
++---------+----------------+
+| topic   | Topic          |
++---------+----------------+
+| message | Payload        |
++---------+----------------+
+
+.. NOTE:: The API uses HTTP Basic Authentication.
+
+-------------------
+MQTT Over WebSocket
+-------------------
+
+Web browsers could connect to the emqttd broker directly by MQTT Over WebSocket.
+
++-------------------------+----------------------------+
+| WebSocket URI:          | ws(s)://host:8083/mqtt     |
++-------------------------+----------------------------+
+| Sec-WebSocket-Protocol: | 'mqttv3.1' or 'mqttv3.1.1' |
++-------------------------+----------------------------+
+
+The Dashboard plugin provides a test page for WebSocket::
+
+    http://127.0.0.1:18083/websocket.html
+
+Listener of WebSocket and HTTP Publish API is configured in etc/emqttd.config::
+
+    %% HTTP and WebSocket Listener
+    {http, 8083, [
+        %% Size of acceptor pool
+        {acceptors, 4},
+        %% Maximum number of concurrent clients
+        {max_clients, 64},
+        %% Socket Access Control
+        {access, [{allow, all}]},
+        %% Socket Options
+        {sockopts, [
+            {backlog, 1024}
+            %{buffer, 4096},
+        ]}
+    ]}
+
+-----------
+$SYS Topics
+-----------
+
+The emqttd broker periodically publishes internal status, MQTT statistics, metrics and client online/offline status to $SYS/# topics.
+
+For emqttd broker is clustered, the $SYS topic path is started with::
+
+    $SYS/brokers/${node}/
+
+'${node}' is the erlang node name of emqttd broker. For example::
+
+    $SYS/brokers/emqttd@127.0.0.1/version
+
+    $SYS/brokers/emqttd@host2/uptime
+
+.. NOTE:: The broker only allows clients from localhost to subscribe $SYS topics by default. 
+
+Sys Interval of publishing $SYS messages, could be configured in etc/emqttd.config::
+
+    {broker, [
+        %% System interval of publishing broker $SYS messages
+        {sys_interval, 60},
+
+
+Broker Version, Uptime and Description
+---------------------------------------
+
++--------------------------------+-----------------------+
+| Topic                          | Description           |
++================================+=======================+
+| $SYS/brokers                   | Broker nodes          |
++--------------------------------+-----------------------+
+| $SYS/brokers/${node}/version   | Broker Version        |
++--------------------------------+-----------------------+
+| $SYS/brokers/${node}/uptime    | Broker Uptime         |
++--------------------------------+-----------------------+
+| $SYS/brokers/${node}/datetime  | Broker DateTime       |
++--------------------------------+-----------------------+
+| $SYS/brokers/${node}/sysdescr  | Broker Description    |
++--------------------------------+-----------------------+
+
+Online/Offline Status of MQTT Client
+------------------------------------
+
+The topic path started with: $SYS/brokers/${node}/clients/
+
++--------------------------+--------------------------------------------+------------------------------------+
+| Topic                    | Payload(JSON)                              | Description                        |
++==========================+============================================+====================================+
+| ${clientid}/connected    | {ipaddress: "127.0.0.1", username: "test", | Publish when a client connected    |
+|                          |  session: false, version: 3, connack: 0,   |                                    |
+|                          |  ts: 1432648482}                           |                                    |
++--------------------------+--------------------------------------------+------------------------------------+
+| ${clientid}/disconnected | {reason: "keepalive_timeout",              | Publish when a client disconnected |
+|                          |  ts: 1432749431}                           |                                    |
++--------------------------+--------------------------------------------+------------------------------------+
+
+Properties of 'connected' Payload::
+
+    ipaddress: "127.0.0.1", 
+    username:  "test", 
+    session:   false, 
+    protocol:  3, 
+    connack:   0, 
+    ts:        1432648482
+
+Properties of 'disconnected' Payload::
+
+    reason: normal,
+    ts:     1432648486
+
+Broker Statistics
+-----------------
+
+Topic path started with: $SYS/brokers/${node}/stats/
+
+Clients
+.......
+
++---------------------+---------------------------------------------+
+| Topic               | Description                                 |
++---------------------+---------------------------------------------+
+| clients/count       | Count of current connected clients          |
++---------------------+---------------------------------------------+
+| clients/max         | Max number of cocurrent connected clients   |
++---------------------+---------------------------------------------+
+
+Sessions
+........
+
++---------------------+---------------------------------------------+
+| Topic               | Description                                 |
++---------------------+---------------------------------------------+
+| sessions/count      | Count of current sessions                   |
++---------------------+---------------------------------------------+
+| sessions/max        | Max number of sessions                      |
++---------------------+---------------------------------------------+
+
+Subscriptions
+.............
+
++---------------------+---------------------------------------------+
+| Topic               | Description                                 |
++---------------------+---------------------------------------------+
+| subscriptions/count | Count of current subscriptions              | 
++---------------------+---------------------------------------------+
+| subscriptions/max   | Max number of subscriptions                 |
++---------------------+---------------------------------------------+
+
+Topics
+......
+
++---------------------+---------------------------------------------+
+| Topic               | Description                                 |
++---------------------+---------------------------------------------+
+| topics/count        | Count of current topics                     |
++---------------------+---------------------------------------------+
+| topics/max          | Max number of topics                        |
++---------------------+---------------------------------------------+
+
+Broker Metrics
+--------------
+
+Topic path started with: $SYS/brokers/${node}/metrics/
+
+Bytes Sent/Received
+...................
+
++---------------------+---------------------------------------------+
+| Topic               | Description                                 |
++---------------------+---------------------------------------------+
+| bytes/received      | MQTT Bytes Received since broker started    |
++---------------------+---------------------------------------------+
+| bytes/sent          | MQTT Bytes Sent since the broker started    |
++---------------------+---------------------------------------------+
+
+Packets Sent/Received
+.....................
+
++--------------------------+---------------------------------------------+
+| Topic                    | Description                                 |
++--------------------------+---------------------------------------------+
+| packets/received         | MQTT Packets received                       |
++--------------------------+---------------------------------------------+
+| packets/sent             | MQTT Packets sent                           |
++--------------------------+---------------------------------------------+
+| packets/connect          | MQTT CONNECT Packet received                |
++--------------------------+---------------------------------------------+
+| packets/connack          | MQTT CONNACK Packet sent                    |
++--------------------------+---------------------------------------------+
+| packets/publish/received | MQTT PUBLISH packets received               |
++--------------------------+---------------------------------------------+
+| packets/publish/sent     | MQTT PUBLISH packets sent                   |
++--------------------------+---------------------------------------------+
+| packets/subscribe        | MQTT SUBSCRIBE Packets received             |
++--------------------------+---------------------------------------------+
+| packets/suback           | MQTT SUBACK packets sent                    |
++--------------------------+---------------------------------------------+
+| packets/unsubscribe      | MQTT UNSUBSCRIBE Packets received           |
++--------------------------+---------------------------------------------+
+| packets/unsuback         | MQTT UNSUBACK Packets sent                  |
++--------------------------+---------------------------------------------+
+| packets/pingreq          | MQTT PINGREQ packets received               |
++--------------------------+---------------------------------------------+
+| packets/pingresp         | MQTT PINGRESP Packets sent                  |
++--------------------------+---------------------------------------------+
+| packets/disconnect       | MQTT DISCONNECT Packets received            |
++--------------------------+---------------------------------------------+
+
+Messages Sent/Received
+......................
+
++--------------------------+---------------------------------------------+
+| Topic                    | Description                                 |
++--------------------------+---------------------------------------------+
+| messages/received        | Messages Received                           |
++--------------------------+---------------------------------------------+
+| messages/sent            | Messages Sent                               |
++--------------------------+---------------------------------------------+
+| messages/retained        | Messages Retained                           |
++--------------------------+---------------------------------------------+
+| messages/stored          | TODO: Messages Stored                       |
++--------------------------+---------------------------------------------+
+| messages/dropped         | Messages Dropped                            |
++--------------------------+---------------------------------------------+
+
+Broker Alarms
+-------------
+
+Topic path started with: $SYS/brokers/${node}/alarms/
+
++------------------+------------------+
+| Topic            | Description      |
++------------------+------------------+
+| ${alarmId}/alert | New Alarm        |
++------------------+------------------+
+| ${alarmId}/clear | Clear Alarm      |
++------------------+------------------+
+
+Broker Sysmon
+-------------
+
+Topic path started with: '$SYS/brokers/${node}/sysmon/'
+
++------------------+--------------------+
+| Topic            | Description        |
++------------------+--------------------+
+| long_gc          | Long GC Warning    |
++------------------+--------------------+
+| long_schedule    | Long Schedule      |
++------------------+--------------------+
+| large_heap       | Large Heap Warning |
++------------------+--------------------+
+| busy_port        | Busy Port Warning  |
++------------------+--------------------+
+| busy_dist_port   | Busy Dist Port     |
++------------------+--------------------+
+
+
+-----
+Trace
+-----
+
+The emqttd broker supports to trace MQTT packets received/sent from/to a client, or trace MQTT messages published to a topic.
+
+Trace a client::
+
+    ./bin/emqttd_ctl trace client "clientid" "trace_clientid.log"
+
+Trace a topic::
+
+    ./bin/emqttd_ctl trace topic "topic" "trace_topic.log"
+
+Lookup Traces::
+
+    ./bin/emqttd_ctl trace list
+
+Stop a Trace::
+
+    ./bin/emqttd_ctl trace client "clientid" off
+
+    ./bin/emqttd_ctl trace topic "topic" off
+
+
+.. _emqttd_plugin_mysql:    https://github.com/emqtt/emqttd_plugin_mysql
+.. _emqttd_plugin_pgsql:    https://github.com/emqtt/emqttd_plugin_pgsql
+.. _emqttd_plugin_redis:    https://github.com/emqtt/emqttd_plugin_redis
+

docs/source/index.rst

@@ -0,0 +1,60 @@
+.. Erlang MQTT Broker documentation master file, created by
+   sphinx-quickstart on Mon Feb 22 00:46:47 2016.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+===========================
+emqttd - Erlang MQTT Broker
+===========================
+
+emqttd(Erlang MQTT Broker) is a massively scalable and clusterable MQTT V3.1/V3.1.1 broker written in Erlang/OTP.
+
+emqttd is fully open source and licensed under the Apache Version 2.0. emqttd implements both MQTT V3.1 and V3.1.1 protocol specifications, and supports WebSocket, STOMP, SockJS, CoAP and MQTT-SN at the same time.
+
+Latest release of the emqttd broker is scaling to 1.3 million MQTT connections on a 12 Core, 32G CentOS server.
+
+.. image:: ./_static/images/emqtt.png
+
+The emqttd project provides a scalable, enterprise grade, extensible open-source MQTT broker for IoT, M2M, Smart Hardware, Mobile Messaging and HTML5 Web Messaging Applications.
+
+Sensors, Mobiles, Web Browsers and Application Servers could be connected by emqttd brokers with asynchronous PUB/SUB MQTT messages.
+
++---------------+-----------------------------------------+
+| Homepage:     | http://emqtt.io                         |
++---------------+-----------------------------------------+
+| Downloads:    | http://emqtt.io/downloads               |
++---------------+-----------------------------------------+
+| GitHub:       | https://github.com/emqtt                |
++---------------+-----------------------------------------+
+| Twitter:      | @emqtt                                  |
++---------------+-----------------------------------------+
+| Forum:        | https://groups.google.com/d/forum/emqtt |
++---------------+-----------------------------------------+
+| Mailing List: | emqtt@googlegroups.com                  |
++---------------+-----------------------------------------+
+| Author:       | Feng Lee <feng@emqtt.io>                |
++---------------+-----------------------------------------+
+
+.. NOTE:: MQTT-SN，CoAP Protocols are planned to 1.x release.
+
+Contents:
+
+.. toctree::
+   :maxdepth: 2
+
+   getstarted
+   install
+   config
+   cluster
+   bridge
+   guide
+   commands
+   plugins
+   tune
+
+-------
+License
+-------
+
+Apache License Version 2.0
+

docs/source/install.rst

@@ -0,0 +1,342 @@
+
+=======================
+Installation
+=======================
+
+emqttd broker is cross-platform, could deploy on Linux, FreeBSD, Mac OS X and Windows.
+
+.. NOTE::
+
+    Linux, FreeBSD Recommended.
+
+-----------------
+Download  Package
+-----------------
+
+Download binary package from: http://emqtt.io/downloads
+
++-----------+-----------------------------------+
+| Ubuntu    | http://emqtt.io/downloads/ubuntu  |
++-----------+-----------------------------------+
+| CentOS    | http://emqtt.io/downloads/centos  |
++-----------+-----------------------------------+
+| FreeBSD   | http://emqtt.io/downloads/freebsd |
++-----------+-----------------------------------+
+| Mac OS X  | http://emqtt.io/downloads/macosx  |
++-----------+-----------------------------------+
+| Windows   | http://emqtt.io/downloads/windows |
++-----------+-----------------------------------+
+
+The package name consists of platform, version and release time.
+
+For example: emqttd-centos64-0.16.0-beta-20160216.zip
+
+--------------------
+Installing on Linux
+--------------------
+
+Download CentOS Package from: http://emqtt.io/downloads/centos, and then unzip:
+
+.. code:: console
+
+    unzip emqttd-centos64-0.16.0-beta-20160216.zip
+
+Start the broker in console mode::
+
+.. code:: console
+
+    cd emqttd && ./bin/emqttd console
+
+If the broker started successfully, console will print:
+
+.. code:: console
+
+    starting emqttd on node 'emqttd@127.0.0.1'
+    emqttd ctl is starting...[done]
+    emqttd trace is starting...[done]
+    emqttd pubsub is starting...[done]
+    emqttd stats is starting...[done]
+    emqttd metrics is starting...[done]
+    emqttd retainer is starting...[done]
+    emqttd pooler is starting...[done]
+    emqttd client manager is starting...[done]
+    emqttd session manager is starting...[done]
+    emqttd session supervisor is starting...[done]
+    emqttd broker is starting...[done]
+    emqttd alarm is starting...[done]
+    emqttd mod supervisor is starting...[done]
+    emqttd bridge supervisor is starting...[done]
+    emqttd access control is starting...[done]
+    emqttd system monitor is starting...[done]
+    http listen on 0.0.0.0:18083 with 4 acceptors.
+    mqtt listen on 0.0.0.0:1883 with 16 acceptors.
+    mqtts listen on 0.0.0.0:8883 with 4 acceptors.
+    http listen on 0.0.0.0:8083 with 4 acceptors.
+    Erlang MQTT Broker 0.16.0 is running now
+    Eshell V6.4  (abort with ^G)
+    (emqttd@127.0.0.1)1>
+
+CTRL+C to close the console and stop the broker.
+
+Start the broker in daemon mode:
+
+.. code:: console
+
+    ./bin/emqttd start
+
+The boot logs in log/emqttd_sasl.log file.
+
+Check the running status of the broker:
+
+.. code:: console
+
+    $ ./bin/emqttd_ctl status
+    Node 'emqttd@127.0.0.1' is started
+    emqttd 0.16.0 is running
+
+Or check the status by URL::
+
+    http://localhost:8083/status
+
+Stop the broker::
+
+    ./bin/emqttd stop
+
+
+---------------------
+Installing on FreeBSD
+---------------------
+
+Download FreeBSD Package from: http://emqtt.io/downloads/freebsd
+
+The installing process is same to Linux.
+
+
+----------------------
+Installing on Mac OS X
+----------------------
+
+We could install the broker on Mac OS X to develop and debug MQTT applications.
+
+Download Mac Package from: http://emqtt.io/downloads/macosx
+
+Configure 'lager' log level in 'etc/emqttd.config', all MQTT messages recevied/sent will be printed on console:
+
+.. code:: erlang
+
+    {lager, [
+        ...
+        {handlers, [
+            {lager_console_backend, info},
+            ...
+        ]}
+    ]},
+
+The install and boot process on Mac are same to Linux.
+
+---------------------
+Installing on Windows
+---------------------
+
+Download Package from: http://emqtt.io/downloads/windows.
+
+Unzip the package to install folder. Open the command line window and 'cd' to the folder.
+
+Start the broker in console mode::
+
+    .\bin\emqttd console
+
+If the broker started successfully, a Erlang console window will popup.
+
+Close the console window and stop the emqttd broker. Prepare to register emqttd as window service.
+
+Install emqttd serivce::
+    
+    .\bin\emqttd install
+
+Start emqttd serivce::
+
+    .\bin\emqttd start
+
+Stop emqttd serivce::
+
+    .\bin\emqttd stop
+
+Uninstall emqttd service::
+
+    .\bin\emqttd uninstall
+
+.. WARNING:: './bin/emqttd_ctl' command line cannot work on Windows.
+
+----------------------
+Installing From Source
+----------------------
+
+The emqttd broker requires Erlang/OTP R17+ and git client to build:
+
+Install Erlang: http://www.erlang.org/
+
+Install Git Client: http://www.git-scm.com/
+
+Could use apt-get on Ubuntu, yum on CentOS/RedHat and brew on Mac to install Erlang and Git.
+
+When all dependencies are ready, clone the emqttd project from github.com and build:
+
+.. code:: console
+
+    git clone https://github.com/emqtt/emqttd.git
+
+    cd emqttd
+
+    make && make dist
+
+The binary package output in folder::
+
+    rel/emqttd
+
+------------------
+TCP Ports Occupied
+------------------
+
++-----------+-----------------------------------+
+| 1883      | MQTT Port                         |
++-----------+-----------------------------------+
+| 8883      | MQTT Over SSL Port                |
++-----------+-----------------------------------+
+| 8083      | MQTT(WebSocket), HTTP API Port    |
++-----------+-----------------------------------+
+| 18083     | Dashboard Port                    |
++-----------+-----------------------------------+
+
+The TCP ports could be configured in etc/emqttd.config:
+
+.. code:: erlang
+
+    {listeners, [
+        {mqtt, 1883, [
+            ...
+        ]},
+
+        {mqtts, 8883, [
+            ...
+        ]},
+        %% HTTP and WebSocket Listener
+        {http, 8083, [
+            ...
+        ]}
+    ]},
+
+The 18083 port is used by Web Dashboard of the broker. Default login: admin, Password: public
+
+-----------
+Quick Setup
+-----------
+
+emqttd消息服务器主要配置文件:
+
++-------------------+-----------------------------------+
+| etc/vm.args       | Erlang VM的启动参数设置           |
++-------------------+-----------------------------------+
+| etc/emqttd.config | emqttd消息服务器参数设置          |
++-------------------+-----------------------------------+
+
+etc/vm.args中两个重要的启动参数:
+
++-------+------------------------------------------------------------------+
+| +P    | Erlang虚拟机允许的最大进程数，emqttd一个连接会消耗2个Erlang进程  |
++-------+------------------------------------------------------------------+
+| +Q    | Erlang虚拟机允许的最大Port数量，emqttd一个连接消耗1个Port        |
++-------+------------------------------------------------------------------+
+
++P 参数值 > 最大允许连接数 * 2
+
++Q 参数值 > 最大允许连接数
+
+.. WARNING:: 实际连接数量超过Erlang虚拟机参数设置，会引起emqttd消息服务器宕机!
+
+etc/emqttd.config文件listeners段落设置最大允许连接数:
+
+.. code:: erlang
+
+    {listeners, [
+        {mqtt, 1883, [
+            %% TCP Acceptor池设置
+            {acceptors, 16},
+
+            %% 最大允许连接数设置
+            {max_clients, 8192},
+
+            ...
+
+        ]},
+
+emqttd消息服务器详细设置，请参见文档: :ref:`config`
+
+
+-------------------
+/etc/init.d/emqttd
+-------------------
+
+.. code:: shell
+
+    #!/bin/sh
+    #
+    # emqttd       Startup script for emqttd.
+    #
+    # chkconfig: 2345 90 10
+    # description: emqttd is mqtt broker.
+
+    # source function library
+    . /etc/rc.d/init.d/functions
+
+    # export HOME=/root
+
+    start() {
+        echo "starting emqttd..."
+        cd /opt/emqttd && ./bin/emqttd start
+    }
+
+    stop() {
+        echo "stopping emqttd..."
+        cd /opt/emqttd && ./bin/emqttd stop
+    }
+
+    restart() {
+        stop
+        start
+    }
+
+    case "$1" in
+        start)
+            start
+            ;;
+        stop)
+            stop
+            ;;
+        restart)
+            restart
+            ;;
+        *)
+            echo $"Usage: $0 {start|stop}"
+            RETVAL=2
+    esac
+
+
+chkconfig::
+
+    chmod +x /etc/init.d/emqttd
+    chkconfig --add emqttd
+    chkconfig --list
+
+boot test::
+
+    service emqttd start
+
+.. NOTE::
+
+    ## erlexec: HOME must be set
+    uncomment '# export HOME=/root' if "HOME must be set" error.
+
+
+
+

docs/source/mqtt.rst

@@ -0,0 +1,101 @@
+
+.. _mqtt:
+
+TODO:...
+
+=============
+MQTT Protocol
+=============
+
+----------------------
+MQTT Protocol Tutorial
+----------------------
+
+MQTT.ORG docs: a publish/subscribe messaging protocol which is extremely lightweight, for IoT, M2M and mobile messaging
+
+.. image:: _static/images/pubsub_concept.png
+
+Publish/Subscribe Model
+-----------------------
+
+.. image:: _static/images/pubsub_concept.png
+
+
+MQTT Control Packets
+--------------------
+
+MQTT Packet Structure
+---------------------
+
+Compact: 1 byte header
+
+MQTT Packet Types
+-----------------
+
+MQTT Packet Flags
+-----------------
+
+MQTT Client Libraries
+---------------------
+
+MQTT Client Libraries
+---------------------
+
+mosquitto_pub mosquitto_sub co
+
+mqtt.org:
+
+TODO: LIST
+
+Maintained by emqtt.com:
+
+TODO: LIST
+
+
+-------------------------
+QoS0, QoS1, QoS2 Messages
+-------------------------
+
+C->S Sequence...
+
+
+----------------
+Retained Message
+----------------
+
+publish a retained message::
+
+    mosquitto_pub -t topic -m msg -q 1 -r
+
+subscribe to get the message::
+
+    mosquitto_sub -t topic -m msg -q 1 -r
+
+
+------------
+Will Message
+------------
+
+------------
+Keep Alive
+------------
+
+----------------------------------
+Clean Session and Offline Messages
+----------------------------------
+
+
+
+
+MQTT Client Libraries
+---------------------
+
+mosquitto_pub mosquitto_sub co
+
+mqtt.org:
+
+TODO: LIST
+
+Maintained by emqtt.com:
+
+TODO: LIST

docs/source/plugins.rst

@@ -0,0 +1,615 @@
+
+.. _plugins:
+
+=======
+Plugins
+=======
+
+The emqttd broker could be extended by plugins. Users could develop plugins to customize authentication, ACL and functions of the broker, or integrate the broker with other systems.
+
+The plugins that emqtt project released:
+
++---------------------------+---------------------------+
+| Plugin                    | Description               |
++===========================+===========================+
+| `emqttd_plugin_template`_ | Template Plugin           |
++---------------------------+---------------------------+
+| `emqttd_dashboard`_       | Web Dashboard             |
++---------------------------+---------------------------+
+| `emqttd_plugin_mysql`_    | MySQL Auth/ACL Plugin     |
++---------------------------+---------------------------+
+| `emqttd_plugin_pgsql`_    | PostgreSQL Auth/ACL Plugin|
++---------------------------+---------------------------+
+| `emqttd_plugin_redis`_    | Redis Auth/ACL Plugin     |
++---------------------------+---------------------------+
+| `emqttd_stomp`_           | STOMP Protocol Plugin     |
++---------------------------+---------------------------+
+| `emqttd_sockjs`_          | STOMP over SockJS Plugin  |
++---------------------------+---------------------------+
+| `emqttd_recon`_           | Recon Plugin              |
++---------------------------+---------------------------+
+
+----------------------------------------
+emqttd_plugin_template - Template Plugin
+----------------------------------------
+
+A plugin is just a normal Erlang application under the 'emqttd/plugins' folder. Each plugin has e configuration file: 'etc/plugin.config'.
+
+plugins/emqttd_plugin_template is a demo plugin. The folder structure:
+
++------------------------+---------------------------+
+| File                   | Description               |
++========================+===========================+
+| etc/plugin.config      | Plugin config file        |
++------------------------+---------------------------+
+| ebin/                  | Erlang program files      |
++------------------------+---------------------------+
+
+Load, unload Plugin
+-------------------
+
+Use 'bin/emqttd_ctl plugins' CLI to load, unload a plugin::
+
+    ./bin/emqttd_ctl plugins load <PluginName>
+
+    ./bin/emqttd_ctl plugins unload <PluginName>
+
+    ./bin/emqttd_ctl plugins list
+
+-----------------------------------
+emqttd_dashboard - Dashboard Plugin
+-----------------------------------
+
+The Web Dashboard for emqttd broker. The plugin will be loaded automatically when the broker started successfully.
+
++------------------+---------------------------+
+| Address          | http://localhost:18083    |
++------------------+---------------------------+
+| Default User     | admin                     |
++------------------+---------------------------+
+| Default Password | public                    |
++------------------+---------------------------+
+
+.. image:: _static/images/dashboard.png
+
+Configure Dashboard
+-------------------
+
+emqttd_dashboard/etc/plugin.config::
+
+    [
+      {emqttd_dashboard, [
+        {default_admin, [
+          {login, "admin"},
+          {password, "public"}
+        ]},
+        {listener,
+          {emqttd_dashboard, 18083, [
+            {acceptors, 4},
+            {max_clients, 512}]}
+        }
+      ]}
+    ].
+
+-------------------------------------------
+emqttd_plugin_mysql - MySQL Auth/ACL Plugin
+-------------------------------------------
+
+MQTT Authentication, ACL with MySQL database.
+
+MQTT User Table
+---------------
+
+.. code:: sql
+
+    CREATE TABLE `mqtt_user` (
+      `id` int(11) unsigned NOT NULL AUTO_INCREMENT,
+      `username` varchar(100) DEFAULT NULL,
+      `password` varchar(100) DEFAULT NULL,
+      `salt` varchar(20) DEFAULT NULL,
+      `created` datetime DEFAULT NULL,
+      PRIMARY KEY (`id`),
+      UNIQUE KEY `mqtt_username` (`username`)
+    ) ENGINE=MyISAM DEFAULT CHARSET=utf8;
+
+MQTT ACL Table
+--------------
+
+.. code:: sql
+
+    CREATE TABLE `mqtt_acl` (
+      `id` int(11) unsigned NOT NULL AUTO_INCREMENT,
+      `allow` int(1) DEFAULT NULL COMMENT '0: deny, 1: allow',
+      `ipaddr` varchar(60) DEFAULT NULL COMMENT 'IpAddress',
+      `username` varchar(100) DEFAULT NULL COMMENT 'Username',
+      `clientid` varchar(100) DEFAULT NULL COMMENT 'ClientId',
+      `access` int(2) NOT NULL COMMENT '1: subscribe, 2: publish, 3: pubsub',
+      `topic` varchar(100) NOT NULL DEFAULT '' COMMENT 'Topic Filter',
+      PRIMARY KEY (`id`)
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+
+
+Configure emqttd_plugin_mysql/etc/plugin.config
+-----------------------------------------------
+
+Configure MySQL host, username, password and database::
+
+    [
+
+    {emqttd_plugin_mysql, [
+
+        {mysql_pool, [
+            %% ecpool options
+            {pool_size, 4},
+            {auto_reconnect, 3},
+
+            %% mysql options
+            {host,     "localhost"},
+            {port,     3306},
+            {user,     ""},
+            {password, ""},
+            {database, "mqtt"},
+            {encoding, utf8}
+        ]},
+
+        %% select password only
+        {authquery, "select password from mqtt_user where username = '%u' limit 1"},
+
+        %% hash algorithm: md5, sha, sha256, pbkdf2?
+        {password_hash, sha256},
+
+        %% select password with salt
+        %% {authquery, "select password, salt from mqtt_user where username = '%u'"},
+
+        %% sha256 with salt prefix
+        %% {password_hash, {salt, sha256}},
+
+        %% sha256 with salt suffix
+        %% {password_hash, {sha256, salt}},
+
+        %% comment this query, the acl will be disabled
+        {aclquery, "select * from mqtt_acl where ipaddr = '%a' or username = '%u' or username = '$all' or clientid = '%c'"},
+
+        %% If no rules matched, return...
+        {acl_nomatch, allow}
+    ]}
+    ].
+
+Load emqttd_plugin_mysql plugin
+-------------------------------
+
+.. code::
+
+    ./bin/emqttd_ctl plugins load emqttd_plugin_mysql
+
+------------------------------------------------
+emqttd_plugin_pgsql - PostgreSQL Auth/ACL Plugin
+------------------------------------------------
+
+MQTT Authentication, ACL with PostgreSQL Database.
+
+MQTT User Table
+---------------
+
+.. code:: sql
+
+    CREATE TABLE mqtt_user (
+      id SERIAL primary key,
+      username character varying(100),
+      password character varying(100),
+      salt character varying(40)
+    );
+
+MQTT ACL Table
+--------------
+
+.. code:: sql
+
+    CREATE TABLE mqtt_acl (
+      id SERIAL primary key,
+      allow integer,
+      ipaddr character varying(60),
+      username character varying(100),
+      clientid character varying(100),
+      access  integer,
+      topic character varying(100)
+    );
+
+    INSERT INTO mqtt_acl (id, allow, ipaddr, username, clientid, access, topic)
+    VALUES
+        (1,1,NULL,'$all',NULL,2,'#'),
+        (2,0,NULL,'$all',NULL,1,'$SYS/#'),
+        (3,0,NULL,'$all',NULL,1,'eq #'),
+        (5,1,'127.0.0.1',NULL,NULL,2,'$SYS/#'),
+        (6,1,'127.0.0.1',NULL,NULL,2,'#'),
+        (7,1,NULL,'dashboard',NULL,1,'$SYS/#');
+
+Configure emqttd_plugin_pgsql/etc/plugin.config
+-----------------------------------------------
+
+Configure host, username, password and database of PostgreSQL::
+
+    [
+
+      {emqttd_plugin_pgsql, [
+
+        {pgsql_pool, [
+          %% ecpool options
+          {pool_size, 4},
+          {auto_reconnect, 3},
+
+          %% pgsql options
+          {host, "localhost"},
+          {port, 5432},
+          {username, "feng"},
+          {password, ""},
+          {database, "mqtt"},
+          {encoding,  utf8}
+        ]},
+
+        %% select password only
+        {authquery, "select password from mqtt_user where username = '%u' limit 1"},
+
+        %% hash algorithm: md5, sha, sha256, pbkdf2?
+        {password_hash, sha256},
+
+        %% select password with salt
+        %% {authquery, "select password, salt from mqtt_user where username = '%u'"},
+
+        %% sha256 with salt prefix
+        %% {password_hash, {salt, sha256}},
+
+        %% sha256 with salt suffix
+        %% {password_hash, {sha256, salt}},
+
+        %% Comment this query, the acl will be disabled. Notice: don't edit this query!
+        {aclquery, "select allow, ipaddr, username, clientid, access, topic from mqtt_acl
+                     where ipaddr = '%a' or username = '%u' or username = '$all' or clientid = '%c'"},
+
+        %% If no rules matched, return...
+        {acl_nomatch, allow}
+      ]}
+    ].
+
+Load emqttd_plugin_pgsql Plugin
+-------------------------------
+
+.. code:: shell
+
+    ./bin/emqttd_ctl plugins load emqttd_plugin_pgsql
+
+-------------------------------------------
+emqttd_plugin_redis - Redis Auth/ACL Plugin
+-------------------------------------------
+
+MQTT Authentication, ACL with Redis.
+
+Configure emqttd_plugin_redis/etc/plugin.config
+-----------------------------------------------
+
+.. code:: erlang
+
+    [
+      {emqttd_plugin_redis, [
+
+        {eredis_pool, [
+          %% ecpool options
+          {pool_size, 8},
+          {auto_reconnect, 2},
+
+          %% eredis options
+          {host, "127.0.0.1"},
+          {port, 6379},
+          {database, 0},
+          {password, ""}
+        ]},
+
+        %% HMGET mqtt_user:%u password
+        {authcmd, ["HGET", "mqtt_user:%u", "password"]},
+
+        %% Password hash algorithm: plain, md5, sha, sha256, pbkdf2?
+        {password_hash, sha256},
+
+        %% SMEMBERS mqtt_acl:%u
+        {aclcmd, ["SMEMBERS", "mqtt_acl:%u"]},
+
+        %% If no rules matched, return...
+        {acl_nomatch, deny},
+
+        %% Store subscriptions to redis when SUBSCRIBE packets received.
+        {subcmd, ["HMSET", "mqtt_subs:%u"]},
+
+        %% Load Subscriptions form Redis when client connected.
+        {loadsub, ["HGETALL", "mqtt_subs:%u"]},
+
+        %% Remove subscriptions from redis when UNSUBSCRIBE packets received.
+        {unsubcmd, ["HDEL", "mqtt_subs:%u"]}
+
+      ]}
+    ].
+
+Load emqttd_plugin_redis Plugin
+-------------------------------
+
+.. code:: console
+
+    ./bin/emqttd_ctl plugins load emqttd_plugin_redis
+
+-----------------------------
+emqttd_stomp - STOMP Protocol
+-----------------------------
+
+Support STOMP 1.0/1.1/1.2 clients to connect to emqttd broker and communicate with MQTT Clients.
+
+Configure emqttd_stomp/etc/plugin.config
+----------------------------------------
+
+.. NOTE:: Default Port for STOMP Protocol: 61613
+
+.. code:: erlang
+
+    [
+      {emqttd_stomp, [
+
+        {default_user, [
+            {login,    "guest"},
+            {passcode, "guest"}
+        ]},
+
+        {allow_anonymous, true},
+
+        %%TODO: unused...
+        {frame, [
+          {max_headers,       10},
+          {max_header_length, 1024},
+          {max_body_length,   8192}
+        ]},
+
+        {listeners, [
+          {emqttd_stomp, 61613, [
+            {acceptors,   4},
+            {max_clients, 512}
+          ]}
+        ]}
+
+      ]}
+    ].
+
+Load emqttd_stomp Plugin
+------------------------
+
+.. code::
+
+    ./bin/emqttd_ctl plugins load emqttd_stomp
+
+
+-----------------------------------
+emqttd_sockjs - STOMP/SockJS Plugin
+-----------------------------------
+
+emqttd_sockjs plugin enables web browser to connect to emqttd broker and communicate with MQTT clients.
+
+.. NOTE:: Default TCP Port: 61616
+
+Configure emqttd_sockjs
+-----------------------
+
+.. code:: erlang
+
+    [
+      {emqttd_sockjs, [
+
+        {sockjs, []},
+
+        {cowboy_listener, {stomp_sockjs, 61616, 4}},
+
+      ]}
+    ].
+
+Load emqttd_sockjs Plugin
+-------------------------
+
+.. NOTE:: emqttd_stomp Plugin required.
+
+.. code:: console
+
+    ./bin/emqttd_ctl plugins load emqttd_stomp
+
+    ./bin/emqttd_ctl plugins load emqttd_sockjs
+
+SockJS Demo Page
+----------------
+
+http://localhost:61616/index.html
+
+---------------------------
+emqttd_recon - Recon Plugin
+---------------------------
+
+The plugin loads `recon`_ library on a running emqttd broker. Recon libray helps to debug and optimize an Erlang application.
+
+Load emqttd_recon Plugin
+------------------------
+
+.. code:: console
+
+    ./bin/emqttd_ctl plugins load emqttd_recon
+
+Recon CLI
+---------
+
+.. code:: console
+
+    ./bin/emqttd_ctl recon
+
+    recon memory                 #recon_alloc:memory/2
+    recon allocated              #recon_alloc:memory(allocated_types, current|max)
+    recon bin_leak               #recon:bin_leak(100)
+    recon node_stats             #recon:node_stats(10, 1000)
+    recon remote_load Mod        #recon:remote_load(Mod)
+
+
+------------------------
+Plugin Development Guide
+------------------------
+
+Create a Plugin Project
+-----------------------
+
+Clone emqttd source from github.com::
+
+    git clone https://github.com/emqtt/emqttd.git
+
+Create a plugin project under 'plugins' folder::
+
+    cd plugins && mkdir emqttd_my_plugin
+
+    cd emqttd_my_plugin && rebar create-app appid=emqttd_my_plugin
+
+Template Plugin: https://github.com/emqtt/emqttd_plugin_template
+
+Register Auth/ACL Modules
+-------------------------
+
+emqttd_auth_demo.erl - demo authentication module:
+
+.. code:: erlang
+
+    -module(emqttd_auth_demo).
+
+    -behaviour(emqttd_auth_mod).
+
+    -include("../../../include/emqttd.hrl").
+
+    -export([init/1, check/3, description/0]).
+
+    init(Opts) -> {ok, Opts}.
+
+    check(#mqtt_client{client_id = ClientId, username = Username}, Password, _Opts) ->
+        io:format("Auth Demo: clientId=~p, username=~p, password=~p~n",
+                  [ClientId, Username, Password]),
+        ok.
+
+    description() -> "Demo Auth Module".
+
+emqttd_acl_demo.erl - demo ACL module:
+
+.. code:: erlang
+
+    -module(emqttd_acl_demo).
+
+    -include("../../../include/emqttd.hrl").
+
+    %% ACL callbacks
+    -export([init/1, check_acl/2, reload_acl/1, description/0]).
+
+    init(Opts) ->
+        {ok, Opts}.
+
+    check_acl({Client, PubSub, Topic}, Opts) ->
+        io:format("ACL Demo: ~p ~p ~p~n", [Client, PubSub, Topic]),
+        allow.
+
+    reload_acl(_Opts) ->
+        ok.
+
+    description() -> "ACL Module Demo".
+
+emqttd_plugin_template_app.erl - Register the auth/ACL modules:
+
+.. code:: erlang
+
+    ok = emqttd_access_control:register_mod(auth, emqttd_auth_demo, []),
+    ok = emqttd_access_control:register_mod(acl, emqttd_acl_demo, []),
+
+
+Register Handlers for Hooks
+---------------------------
+
+The plugin could register handlers for hooks. The hooks will be called by the broker when a client connected/disconnected, a topic subscribed/unsubscribed or a message published/delivered:
+
++------------------------+-------------+---------------------------------------+
+| Name                   | Type        | Description                           |
++------------------------+-------------+---------------------------------------+
+| client.connected       | foreach     | Run when a client connected to the    |
+|                        |             | broker successfully                   |
++------------------------+-------------+---------------------------------------+
+| client.subscribe       | foldl       | Run before a client subscribes topics |
++------------------------+-------------+---------------------------------------+
+| client.subscribe.after | foreach     | Run after a client subscribed topics  |
++------------------------+-------------+---------------------------------------+
+| client.unsubscribe     | foldl       | Run when a client unsubscribes topics |
++------------------------+-------------+---------------------------------------+
+| message.publish        | foldl       | Run when a message is published       |
++------------------------+-------------+---------------------------------------+
+| message.acked          | foreach     | Run when a message is delivered       |
++------------------------+-------------+---------------------------------------+
+| client.disconnected    | foreach     | Run when a client is disconnnected    |
++----------------------- +-------------+---------------------------------------+
+
+emqttd_plugin_template.erl for example::
+
+    %% Called when the plugin application start
+    load(Env) ->
+
+        emqttd_broker:hook('client.connected', {?MODULE, on_client_connected},
+                           {?MODULE, on_client_connected, [Env]}),
+
+        emqttd_broker:hook('client.disconnected', {?MODULE, on_client_disconnected},
+                           {?MODULE, on_client_disconnected, [Env]}),
+
+        emqttd_broker:hook('client.subscribe', {?MODULE, on_client_subscribe},
+                           {?MODULE, on_client_subscribe, [Env]}),
+
+        emqttd_broker:hook('client.subscribe.after', {?MODULE, on_client_subscribe_after},
+                           {?MODULE, on_client_subscribe_after, [Env]}),
+
+        emqttd_broker:hook('client.unsubscribe', {?MODULE, on_client_unsubscribe},
+                           {?MODULE, on_client_unsubscribe, [Env]}),
+
+        emqttd_broker:hook('message.publish', {?MODULE, on_message_publish},
+                           {?MODULE, on_message_publish, [Env]}),
+
+        emqttd_broker:hook('message.acked', {?MODULE, on_message_acked},
+                           {?MODULE, on_message_acked, [Env]}).
+
+Register CLI Modules
+--------------------
+
+emqttd_cli_demo.erl:
+
+.. code:: erlang
+
+    -module(emqttd_cli_demo).
+
+    -include("../../../include/emqttd_cli.hrl").
+
+    -export([cmd/1]).
+
+    cmd(["arg1", "arg2"]) ->
+        ?PRINT_MSG("ok");
+
+    cmd(_) ->
+        ?USAGE([{"cmd arg1 arg2", "cmd demo"}]).
+
+emqttd_plugin_template_app.erl - register the CLI module to emqttd broker:
+
+.. code:: erlang
+
+    emqttd_ctl:register_cmd(cmd, {emqttd_cli_demo, cmd}, []).
+
+There will be a new CLI after the plugin loaded::
+
+    ./bin/emqttd_ctl cmd arg1 arg2
+
+
+.. _emqttd_dashboard:       https://github.com/emqtt/emqttd_dashboard
+.. _emqttd_plugin_mysql:    https://github.com/emqtt/emqttd_plugin_mysql
+.. _emqttd_plugin_pgsql:    https://github.com/emqtt/emqttd_plugin_pgsql
+.. _emqttd_plugin_redis:    https://github.com/emqtt/emqttd_plugin_redis
+.. _emqttd_stomp:           https://github.com/emqtt/emqttd_stomp
+.. _emqttd_sockjs:          https://github.com/emqtt/emqttd_sockjs
+.. _emqttd_recon:           https://github.com/emqtt/emqttd_recon
+.. _emqttd_plugin_template: https://github.com/emqtt/emqttd_plugin_template
+.. _recon:                  http://ferd.github.io/recon/
+

docs/source/tune.rst

@@ -0,0 +1,144 @@
+
+.. _tune:
+
+============
+Tuning Guide
+============
+
+Tuning the Linux Kernel, Networking, Erlang VM and emqttd broker for one million concurrent MQTT connections.
+
+-------------------
+Linux Kernel Tuning
+-------------------
+
+The system-wide limit on max opened file handles::
+
+    # 2 million system-wide
+    sysctl -w fs.file-max=2097152
+    sysctl -w fs.nr_open=2097152
+    echo 2097152 > /proc/sys/fs/nr_open
+
+The limit on opened file handles for current session::
+
+    ulimit -n 1048576
+
+/etc/sysctl.conf
+----------------
+
+Add the 'fs.file-max' to /etc/sysctl.conf, make the changes permanent::
+
+    fs.file-max = 1048576
+
+/etc/security/limits.conf
+-------------------------
+
+Persist the limits on opened file handles for users in /etc/security/limits.conf::
+
+    *      soft   nofile      1048576
+    *      hard   nofile      1048576
+
+--------------
+Network Tuning
+--------------
+
+Increase number of incoming connections backlog::
+
+    sysctl -w net.core.somaxconn=32768
+    net.ipv4.tcp_max_syn_backlog=16384
+    sysctl -w net.core.netdev_max_backlog=16384
+
+Local Port Range::
+
+    sysctl -w net.ipv4.ip_local_port_range=1000 65535
+
+Read/Write Buffer for TCP connections::
+
+    sysctl -w net.core.rmem_default=262144
+    sysctl -w net.core.wmem_default=262144
+    sysctl -w net.core.rmem_max=16777216
+    sysctl -w net.core.wmem_max=16777216
+    sysctl -w net.core.optmem_max=16777216
+
+    #sysctl -w net.ipv4.tcp_mem='16777216 16777216 16777216'
+    sysctl -w net.ipv4.tcp_rmem='1024 4096 16777216'
+    sysctl -w net.ipv4.tcp_wmem='1024 4096 16777216'
+
+Connection Tracking::
+
+    sysctl -w net.nf_conntrack_max=1000000
+    sysctl -w net.netfilter.nf_conntrack_max=1000000
+    sysctl -w net.netfilter.nf_conntrack_tcp_timeout_time_wait=30
+
+The TIME-WAIT Buckets Pool, Recycling and Reuse::
+
+    net.ipv4.tcp_max_tw_buckets=1048576
+    net.ipv4.tcp_tw_recycle = 1
+    net.ipv4.tcp_tw_reuse = 1
+
+Timeout for FIN-WAIT-2 sockets::
+
+    net.ipv4.tcp_fin_timeout = 15
+
+----------------
+Erlang VM Tuning
+----------------
+
+Tuning and optimize the Erlang VM in etc/vm.args file::
+
+    ## max number of erlang processes
+    +P 2097152
+
+    ## Sets the maximum number of simultaneously existing ports for this system
+    +Q 1048576
+
+    ## Increase number of concurrent ports/sockets, deprecated in R17
+    -env ERL_MAX_PORTS 1048576
+
+    -env ERTS_MAX_PORTS 1048576
+
+    ## Mnesia and SSL will create temporary ets tables.
+    -env ERL_MAX_ETS_TABLES 1024
+
+    ## Tweak GC to run more often
+    -env ERL_FULLSWEEP_AFTER 1000
+
+-------------
+emqttd broker
+-------------
+
+Tune the acceptor pool, max_clients limit and sockopts for TCP listener in etc/emqttd.config::
+
+    {mqtt, 1883, [
+        %% Size of acceptor pool
+        {acceptors, 64},
+
+        %% Maximum number of concurrent clients
+        {max_clients, 1000000},
+
+        %% Socket Access Control
+        {access, [{allow, all}]},
+
+        %% Connection Options
+        {connopts, [
+            %% Rate Limit. Format is 'burst, rate', Unit is KB/Sec
+            %% {rate_limit, "100,10"} %% 100K burst, 10K rate
+        ]},
+        ...
+
+--------------
+Client Machine
+--------------
+
+Tune the client machine to benchmark emqttd broker::
+
+    sysctl -w net.ipv4.ip_local_port_range="500 65535"
+    sysctl -w fs.file-max=1000000
+    echo 1000000 > /proc/sys/fs/nr_open
+    ulimit -n 100000
+
+---------------
+emqtt_benchmark
+---------------
+
+Test tool for concurrent connections: http://github.com/emqtt/emqtt_benchmark
+

include/emqttd.hrl

@@ -47,7 +47,7 @@
 %%--------------------------------------------------------------------
 -record(mqtt_topic, {
     topic   :: binary(),
-    node    :: node()
+    flags   :: [retained | static]
 }).
 
 -type mqtt_topic() :: #mqtt_topic{}.
@@ -63,6 +63,16 @@
 
 -type mqtt_subscription() :: #mqtt_subscription{}.
 
+%%--------------------------------------------------------------------
+%% MQTT Route
+%%--------------------------------------------------------------------
+-record(mqtt_route, {
+    topic   :: binary(),
+    node    :: node()
+}).
+
+-type mqtt_route() :: #mqtt_route{}.
+
 %%--------------------------------------------------------------------
 %% MQTT Client
 %%--------------------------------------------------------------------

include/emqttd_cli.hrl

@@ -18,7 +18,7 @@
 
 -define(PRINT(Format, Args), io:format(Format, Args)).
 
--define(PRINT_CMD(Cmd, Descr), io:format("~-40s#~s~n", [Cmd, Descr])).
+-define(PRINT_CMD(Cmd, Descr), io:format("~-48s# ~s~n", [Cmd, Descr])).
 
 -define(USAGE(CmdList), [?PRINT_CMD(Cmd, Descr) || {Cmd, Descr} <- CmdList]).
 

include/emqttd_internal.hrl

@@ -52,7 +52,7 @@
             {noreply, State}
         end)).
 
--define(IF(Cond, TrueFun,FalseFun),
+-define(IF(Cond, TrueFun, FalseFun),
         (case (Cond) of
             true -> (TrueFun);
             false-> (FalseFun)

rebar.config

@@ -23,7 +23,7 @@
 
 {ct_log_dir, "logs"}.
 
-{ct_extra_params, "-name ct_emqttd@127.0.0.1 -config rel/files/test.config"}.
+{ct_extra_params, "-name ct_emqttd@127.0.0.1 -config rel/files/emqttd.test.config"}.
 
 {ct_use_short_names, false}.
 

rel/files/emqttd.config.development

@@ -79,7 +79,7 @@
             %% Max ClientId Length Allowed
             {max_clientid_len, 1024},
             %% Max Packet Size Allowed, 64K default
-            {max_packet_size,  65536}
+            {max_packet_size, 65536}
         ]},
         %% Client
         {client, [
@@ -152,15 +152,6 @@
             %% Default should be scheduler numbers
             {pool_size, 8},
 
-            %% Subscription: disc | ram | false
-            {subscription, ram},
-
-            %% Route shard
-            {route_shard, false},
-
-            %% Route delay, false | integer
-            {route_delay, false},
-
             %% Route aging time(seconds)
             {route_aging, 5}
         ]},
@@ -182,14 +173,11 @@
 
         %% Subscribe topics automatically when client connected
         {subscription, [
-            %% Subscription from stored table
+            %% Static subscriptions from backend
-            stored,
+            backend,
 
-            %% $u will be replaced with username
+            %% $c will be replaced by clientid
-            {"$Q/username/$u", 1},
+            {"$queue/clients/$c", 1}
-
-            %% $c will be replaced with clientid
-            {"$Q/client/$c", 1}
         ]}
 
         %% Rewrite rules

rel/files/emqttd.config.production

@@ -77,9 +77,6 @@
         {client, [
             %% Socket is connected, but no 'CONNECT' packet received
             {idle_timeout, 20} %% seconds
-            %TODO: Network ingoing limit
-            %{ingoing_rate_limit, '64KB/s'}
-            %TODO: Reconnet control
         ]},
         %% Session
         {session, [
@@ -147,15 +144,6 @@
             %% Default should be scheduler numbers
             {pool_size, 8},
 
-            %% Subscription: disc | ram | false
-            {subscription, ram},
-
-            %% Route shard
-            {route_shard, false},
-
-            %% Route delay, false | integer
-            {route_delay, false},
-
             %% Route aging time(seconds)
             {route_aging, 5}
         ]},
@@ -173,19 +161,16 @@
     {modules, [
         %% Client presence management module.
         %% Publish messages when client connected or disconnected
-        {presence, [{qos, 0}]}
+        {presence, [{qos, 0}]},
 
         %% Subscribe topics automatically when client connected
-        %% {subscription, [
+        {subscription, [
-        %%    %% Subscription from stored table
+            %% Static subscriptions from backend
-        %%    stored,
+            backend,
-        %%
+
-        %%   %% $u will be replaced with username
+            %% $c will be replaced by clientid
-        %%    {"$Q/username/$u", 1},
+            {"$queue/clients/$c", 1}
-        %%
+        ]}
-        %%   %% $c will be replaced with clientid
-        %%    {"$Q/client/$c", 1}
-        %% ]}
 
         %% Rewrite rules
         %% {rewrite, [{file, "etc/rewrite.config"}]}

rel/files/emqttd.test.config

@@ -152,15 +152,6 @@
             %% Default should be scheduler numbers
             {pool_size, 8},
 
-            %% Subscription: disc | ram | false
-            {subscription, ram},
-
-            %% Route shard
-            {route_shard, false},
-
-            %% Route delay, false | integer
-            {route_delay, false},
-
             %% Route aging time(seconds)
             {route_aging, 5}
         ]},
@@ -182,14 +173,14 @@
 
         %% Subscribe topics automatically when client connected
         {subscription, [
-            %% Subscription from stored table
+            %% Static subscriptions from backend
-            stored,
+            backend,
 
             %% $u will be replaced with username
-            {"$Q/username/$u", 1},
+            {"$queue/username/$u", 1},
 
             %% $c will be replaced with clientid
-            {"$Q/client/$c", 1}
+            {"$queue/clients/$c", 1}
         ]}
 
         %% Rewrite rules

src/emqttd.app.src

@@ -1,7 +1,7 @@
 {application, emqttd,
  [
   {description, "Erlang MQTT Broker"},
-  {vsn, "0.16.0"},
+  {vsn, "0.17.0"},
   {id, "emqttd"},
   {modules, []},
   {registered, []},

src/emqttd.erl

@@ -16,83 +16,39 @@
 
 -module(emqttd).
 
--export([start/0, env/1, env/2, start_listeners/0, stop_listeners/0,
+-include("emqttd.hrl").
-         load_all_mods/0, is_mod_enabled/1, is_running/1]).
 
--define(MQTT_SOCKOPTS, [
+-include("emqttd_protocol.hrl").
-        binary,
+
-        {packet,    raw},
+-export([start/0, env/1, env/2, is_running/1]).
-        {reuseaddr, true},
+
-        {backlog,   512},
+%% PubSub API
-        {nodelay,   true}]).
+-export([create/2, lookup/2, publish/1, subscribe/1, subscribe/3,
+         unsubscribe/1, unsubscribe/3]).
+
+%% Hooks API
+-export([hook/4, hook/3, unhook/2, run_hooks/3]).
 
 -define(APP, ?MODULE).
 
--type listener() :: {atom(), inet:port_number(), [esockd:option()]}.
+%%--------------------------------------------------------------------
+%% Bootstrap, environment, is_running...
+%%--------------------------------------------------------------------
 
 %% @doc Start emqttd application.
--spec start() -> ok | {error, any()}.
+-spec(start() -> ok | {error, any()}).
 start() -> application:start(?APP).
 
 %% @doc Group environment
--spec env(Group :: atom()) -> list().
+-spec(env(Group :: atom()) -> list()).
 env(Group) -> application:get_env(?APP, Group, []).
 
 %% @doc Get environment
--spec env(Group :: atom(), Name :: atom()) -> undefined | any().
+-spec(env(Group :: atom(), Name :: atom()) -> undefined | any()).
 env(Group, Name) -> proplists:get_value(Name, env(Group)).
 
-%% @doc Start Listeners of the broker.
--spec start_listeners() -> any().
-start_listeners() -> lists:foreach(fun start_listener/1, env(listeners)).
-
-%% Start mqtt listener
--spec start_listener(listener()) -> any().
-start_listener({mqtt, Port, Opts}) -> start_listener(mqtt, Port, Opts);
-
-%% Start mqtt(SSL) listener
-start_listener({mqtts, Port, Opts}) -> start_listener(mqtts, Port, Opts);
-
-%% Start http listener
-start_listener({http, Port, Opts}) ->
-    mochiweb:start_http(Port, Opts, {emqttd_http, handle_request, []});
-
-%% Start https listener
-start_listener({https, Port, Opts}) ->
-    mochiweb:start_http(Port, Opts, {emqttd_http, handle_request, []}).
-
-start_listener(Protocol, Port, Opts) ->
-    MFArgs = {emqttd_client, start_link, [env(mqtt)]},
-    esockd:open(Protocol, Port, merge_sockopts(Opts), MFArgs).
-
-merge_sockopts(Options) ->
-    SockOpts = emqttd_opts:merge(?MQTT_SOCKOPTS,
-                                 proplists:get_value(sockopts, Options, [])),
-    emqttd_opts:merge(Options, [{sockopts, SockOpts}]).
-
-%% @doc Stop Listeners
-stop_listeners() -> lists:foreach(fun stop_listener/1, env(listeners)).
-
-stop_listener({Protocol, Port, _Opts}) -> esockd:close({Protocol, Port}).
-
-%% @doc load all modules
-load_all_mods() ->
-    lists:foreach(fun load_mod/1, env(modules)).
-
-load_mod({Name, Opts}) ->
-    Mod = list_to_atom("emqttd_mod_" ++ atom_to_list(Name)),
-    case catch Mod:load(Opts) of
-        ok               -> lager:info("Load module ~s successfully", [Name]);
-        {error, Error}   -> lager:error("Load module ~s error: ~p", [Name, Error]);
-        {'EXIT', Reason} -> lager:error("Load module ~s error: ~p", [Name, Reason])
-    end.
-
-%% @doc Is module enabled?
--spec is_mod_enabled(Name :: atom()) -> boolean().
-is_mod_enabled(Name) -> env(modules, Name) =/= undefined.
-
 %% @doc Is running?
--spec is_running(node()) -> boolean().
+-spec(is_running(node()) -> boolean()).
 is_running(Node) ->
     case rpc:call(Node, erlang, whereis, [?APP]) of
         {badrpc, _}          -> false;
@@ -100,3 +56,71 @@ is_running(Node) ->
         Pid when is_pid(Pid) -> true
     end.
 
+%%--------------------------------------------------------------------
+%% PubSub APIs that wrap emqttd_server, emqttd_pubsub
+%%--------------------------------------------------------------------
+
+%% @doc Lookup Topic or Subscription
+-spec(lookup(topic, binary()) -> [mqtt_topic()];
+            (subscription, binary()) -> [mqtt_subscription()]).
+lookup(topic, Topic) when is_binary(Topic) ->
+    emqttd_pubsub:lookup_topic(Topic);
+
+lookup(subscription, ClientId) when is_binary(ClientId) ->
+    emqttd_server:lookup_subscription(ClientId).
+
+%% @doc Create a Topic or Subscription
+-spec(create(topic | subscription, binary()) -> ok | {error, any()}).
+create(topic, Topic) when is_binary(Topic) ->
+    emqttd_pubsub:create_topic(Topic);
+
+create(subscription, {ClientId, Topic, Qos}) ->
+    Subscription = #mqtt_subscription{subid = ClientId, topic = Topic, qos = ?QOS_I(Qos)},
+    emqttd_backend:add_subscription(Subscription).
+
+%% @doc Publish MQTT Message
+-spec(publish(mqtt_message()) -> ok).
+publish(Msg) when is_record(Msg, mqtt_message) ->
+    emqttd_server:publish(Msg), ok.
+
+%% @doc Subscribe
+-spec(subscribe(binary()) -> ok;
+               ({binary(), binary(), mqtt_qos()}) -> ok).
+subscribe(Topic) when is_binary(Topic) ->
+    emqttd_server:subscribe(Topic);
+subscribe({ClientId, Topic, Qos}) ->
+    subscribe(ClientId, Topic, Qos).
+
+-spec(subscribe(binary(), binary(), mqtt_qos()) -> {ok, mqtt_qos()}).
+subscribe(ClientId, Topic, Qos) ->
+    emqttd_server:subscribe(ClientId, Topic, Qos).
+
+%% @doc Unsubscribe
+-spec(unsubscribe(binary()) -> ok).
+unsubscribe(Topic) when is_binary(Topic) ->
+    emqttd_server:unsubscribe(Topic).
+
+-spec(unsubscribe(binary(), binary(), mqtt_qos()) -> ok).
+unsubscribe(ClientId, Topic, Qos) ->
+    emqttd_server:unsubscribe(ClientId, Topic, Qos).
+
+%%--------------------------------------------------------------------
+%% Hooks API
+%%--------------------------------------------------------------------
+
+-spec(hook(atom(), function(), list(any())) -> ok | {error, any()}).
+hook(Hook, Function, InitArgs) ->
+    emqttd_hook:add(Hook, Function, InitArgs).
+
+-spec(hook(atom(), function(), list(any()), integer()) -> ok | {error, any()}).
+hook(Hook, Function, InitArgs, Priority) ->
+    emqttd_hook:add(Hook, Function, InitArgs, Priority).
+
+-spec(unhook(atom(), function()) -> ok | {error, any()}).
+unhook(Hook, Function) ->
+    emqttd_hook:delete(Hook, Function).
+
+-spec(run_hooks(atom(), list(any()), any()) -> {ok | stop, any()}).
+run_hooks(Hook, Args, Acc) ->
+    emqttd_hook:run(Hook, Args, Acc).
+

src/emqttd_alarm.erl

@@ -91,12 +91,12 @@ handle_event({set_alarm, Alarm = #mqtt_alarm{id       = AlarmId,
                               {title, iolist_to_binary(Title)},
                               {summary, iolist_to_binary(Summary)},
                               {ts, emqttd_time:now_to_secs(Timestamp)}]),
-    emqttd_pubsub:publish(alarm_msg(alert, AlarmId, Json)),
+    emqttd:publish(alarm_msg(alert, AlarmId, Json)),
     {ok, [Alarm#mqtt_alarm{timestamp = Timestamp} | Alarms]};
 
 handle_event({clear_alarm, AlarmId}, Alarms) ->
     Json = mochijson2:encode([{id, AlarmId}, {ts, emqttd_time:now_to_secs()}]),
-    emqttd_pubsub:publish(alarm_msg(clear, AlarmId, Json)),
+    emqttd:publish(alarm_msg(clear, AlarmId, Json)),
     {ok, lists:keydelete(AlarmId, 2, Alarms), hibernate};
 
 handle_event(_, Alarms)->

src/emqttd_app.erl

@@ -16,13 +16,25 @@
 
 -module(emqttd_app).
 
--include("emqttd_cli.hrl").
-
 -behaviour(application).
 
+-include("emqttd_cli.hrl").
+
 %% Application callbacks
 -export([start/2, stop/1]).
 
+-export([start_listener/1, stop_listener/1, is_mod_enabled/1]).
+
+%% MQTT SockOpts
+-define(MQTT_SOCKOPTS, [
+        binary,
+        {packet,    raw},
+        {reuseaddr, true},
+        {backlog,   512},
+        {nodelay,   true}]).
+
+-type listener() :: {atom(), inet:port_number(), [esockd:option()]}.
+
 %%--------------------------------------------------------------------
 %% Application callbacks
 %%--------------------------------------------------------------------
@@ -38,13 +50,21 @@ start(_StartType, _StartArgs) ->
     {ok, Sup} = emqttd_sup:start_link(),
     start_servers(Sup),
     emqttd_cli:load(),
-    emqttd:load_all_mods(),
+    load_all_mods(),
     emqttd_plugins:load(),
-    emqttd:start_listeners(),
+    start_listeners(),
     register(emqttd, self()),
     print_vsn(),
     {ok, Sup}.
 
+-spec stop(State :: term()) -> term().
+stop(_State) ->
+    catch stop_listeners().
+
+%%--------------------------------------------------------------------
+%% Print Banner
+%%--------------------------------------------------------------------
+
 print_banner() ->
     ?PRINT("starting emqttd on node '~s'~n", [node()]).
 
@@ -53,14 +73,19 @@ print_vsn() ->
     {ok, Desc} = application:get_key(description),
     ?PRINT("~s ~s is running now~n", [Desc, Vsn]).
 
+%%--------------------------------------------------------------------
+%% Start Servers
+%%--------------------------------------------------------------------
+
 start_servers(Sup) ->
     Servers = [{"emqttd ctl", emqttd_ctl},
-               {"emqttd trace", {supervisor, emqttd_trace_sup}},
+               {"emqttd hook", emqttd_hook},
                {"emqttd pubsub", {supervisor, emqttd_pubsub_sup}},
                {"emqttd stats", emqttd_stats},
                {"emqttd metrics", emqttd_metrics},
                {"emqttd retainer", emqttd_retainer},
                {"emqttd pooler", {supervisor, emqttd_pooler}},
+               {"emqttd trace", {supervisor, emqttd_trace_sup}},
                {"emqttd client manager", {supervisor, emqttd_cm_sup}},
                {"emqttd session manager", {supervisor, emqttd_sm_sup}},
                {"emqttd session supervisor", {supervisor, emqttd_session_sup}},
@@ -117,7 +142,64 @@ worker_spec(Module, Opts) when is_atom(Module) ->
 worker_spec(M, F, A) ->
     {M, {M, F, A}, permanent, 10000, worker, [M]}.
 
--spec stop(State :: term()) -> term().
+%%--------------------------------------------------------------------
-stop(_State) ->
+%% Load Modules
-    catch emqttd:stop_listeners().
+%%--------------------------------------------------------------------
+
+%% @doc load all modules
+load_all_mods() ->
+    lists:foreach(fun load_mod/1, emqttd:env(modules)).
+
+load_mod({Name, Opts}) ->
+    Mod = list_to_atom("emqttd_mod_" ++ atom_to_list(Name)),
+    case catch Mod:load(Opts) of
+        ok               -> lager:info("Load module ~s successfully", [Name]);
+        {error, Error}   -> lager:error("Load module ~s error: ~p", [Name, Error]);
+        {'EXIT', Reason} -> lager:error("Load module ~s error: ~p", [Name, Reason])
+    end.
+
+%% @doc Is module enabled?
+-spec is_mod_enabled(Name :: atom()) -> boolean().
+is_mod_enabled(Name) -> emqttd:env(modules, Name) =/= undefined.
+
+%%--------------------------------------------------------------------
+%% Start Listeners
+%%--------------------------------------------------------------------
+
+%% @doc Start Listeners of the broker.
+-spec start_listeners() -> any().
+start_listeners() -> lists:foreach(fun start_listener/1, emqttd:env(listeners)).
+
+%% Start mqtt listener
+-spec start_listener(listener()) -> any().
+start_listener({mqtt, Port, Opts}) -> start_listener(mqtt, Port, Opts);
+
+%% Start mqtt(SSL) listener
+start_listener({mqtts, Port, Opts}) -> start_listener(mqtts, Port, Opts);
+
+%% Start http listener
+start_listener({http, Port, Opts}) ->
+    mochiweb:start_http(Port, Opts, {emqttd_http, handle_request, []});
+
+%% Start https listener
+start_listener({https, Port, Opts}) ->
+    mochiweb:start_http(Port, Opts, {emqttd_http, handle_request, []}).
+
+start_listener(Protocol, Port, Opts) ->
+    MFArgs = {emqttd_client, start_link, [emqttd:env(mqtt)]},
+    esockd:open(Protocol, Port, merge_sockopts(Opts), MFArgs).
+
+merge_sockopts(Options) ->
+    SockOpts = emqttd_opts:merge(?MQTT_SOCKOPTS,
+                                 proplists:get_value(sockopts, Options, [])),
+    emqttd_opts:merge(Options, [{sockopts, SockOpts}]).
+
+%%--------------------------------------------------------------------
+%% Stop Listeners
+%%--------------------------------------------------------------------
+
+%% @doc Stop Listeners
+stop_listeners() -> lists:foreach(fun stop_listener/1, emqttd:env(listeners)).
+
+stop_listener({Protocol, Port, _Opts}) -> esockd:close({Protocol, Port}).
 

src/emqttd_auth_username.erl

@@ -26,8 +26,9 @@
 
 -behaviour(emqttd_auth_mod).
 
--export([add_user/2, remove_user/1,
+-export([is_enabled/0]).
-         lookup_user/1, all_users/0]).
+
+-export([add_user/2, remove_user/1, lookup_user/1, all_users/0]).
 
 %% emqttd_auth callbacks
 -export([init/1, check/3, description/0]).
@@ -41,19 +42,35 @@
 %%--------------------------------------------------------------------
 
 cli(["add", Username, Password]) ->
-    ?PRINT("~p~n", [add_user(iolist_to_binary(Username), iolist_to_binary(Password))]);
+    if_enabled(fun() ->
+        ?PRINT("~p~n", [add_user(iolist_to_binary(Username), iolist_to_binary(Password))])
+    end);
 
 cli(["del", Username]) ->
-    ?PRINT("~p~n", [remove_user(iolist_to_binary(Username))]);
+    if_enabled(fun() ->
+        ?PRINT("~p~n", [remove_user(iolist_to_binary(Username))])
+    end);
 
 cli(_) ->
     ?USAGE([{"users add <Username> <Password>", "Add User"},
             {"users del <Username>", "Delete User"}]).
 
+if_enabled(Fun) ->
+    case is_enabled() of
+        true  -> Fun();
+        false -> hint()
+    end.
+
+hint() ->
+    ?PRINT_MSG("Please enable '{username, []}' authentication in etc/emqttd.config first.~n").
+
 %%--------------------------------------------------------------------
 %% API
 %%--------------------------------------------------------------------
 
+is_enabled() ->
+    lists:member(?AUTH_USERNAME_TAB, mnesia:system_info(tables)).
+
 %% @doc Add User
 -spec add_user(binary(), binary()) -> ok | {error, any()}.
 add_user(Username, Password) ->

src/emqttd_backend.erl

@@ -0,0 +1,91 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2012-2016 Feng Lee <feng@emqtt.io>.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqttd_backend).
+
+-include("emqttd.hrl").
+
+%% Mnesia Callbacks
+-export([mnesia/1]).
+
+-boot_mnesia({mnesia, [boot]}).
+-copy_mnesia({mnesia, [copy]}).
+
+%% API.
+-export([add_subscription/1, lookup_subscriptions/1, del_subscriptions/1,
+         del_subscription/2]).
+
+%%--------------------------------------------------------------------
+%% Mnesia callbacks
+%%--------------------------------------------------------------------
+
+mnesia(boot) ->
+    ok = emqttd_mnesia:create_table(backend_subscription, [
+                {type, bag},
+                {disc_copies, [node()]},
+                {record_name, mqtt_subscription},
+                {attributes, record_info(fields, mqtt_subscription)},
+                {storage_properties, [{ets, [compressed]},
+                                      {dets, [{auto_save, 5000}]}]}]);
+
+mnesia(copy) ->
+    ok = emqttd_mnesia:copy_table(backend_subscription).
+
+%%--------------------------------------------------------------------
+%% Static Subscriptions
+%%--------------------------------------------------------------------
+
+%% @doc Add a static subscription manually.
+-spec add_subscription(mqtt_subscription()) -> ok | {error, already_existed}.
+add_subscription(Subscription = #mqtt_subscription{subid = SubId, topic = Topic}) ->
+    Pattern = match_pattern(SubId, Topic),
+    return(mnesia:transaction(fun() ->
+                    case mnesia:match_object(backend_subscription, Pattern, write) of
+                        [] ->
+                            mnesia:write(backend_subscription, Subscription, write);
+                        [Subscription] ->
+                            mnesia:abort(already_existed);
+                        [Subscription1] -> %% QoS is different
+                            mnesia:delete_object(backend_subscription, Subscription1, write),
+                            mnesia:write(backend_subscription, Subscription, write)
+                    end
+            end)).
+
+%% @doc Lookup static subscriptions.
+-spec lookup_subscriptions(binary()) -> list(mqtt_subscription()).
+lookup_subscriptions(ClientId) when is_binary(ClientId) ->
+    mnesia:dirty_read(backend_subscription, ClientId).
+
+%% @doc Delete static subscriptions by ClientId manually.
+-spec del_subscriptions(binary()) -> ok.
+del_subscriptions(ClientId) when is_binary(ClientId) ->
+    return(mnesia:transaction(fun mnesia:delete/1, [{backend_subscription, ClientId}])).
+
+%% @doc Delete a static subscription manually.
+-spec del_subscription(binary(), binary()) -> ok.
+del_subscription(ClientId, Topic) when is_binary(ClientId) andalso is_binary(Topic) ->
+    return(mnesia:transaction(fun del_subscription_/1, [match_pattern(ClientId, Topic)])).
+
+del_subscription_(Pattern) ->
+    lists:foreach(fun(Subscription) ->
+                mnesia:delete_object(backend_subscription, Subscription, write)
+        end, mnesia:match_object(backend_subscription, Pattern, write)).
+
+match_pattern(SubId, Topic) ->
+    #mqtt_subscription{subid = SubId, topic = Topic, qos = '_'}.
+
+return({atomic, ok})      -> ok;
+return({aborted, Reason}) -> {error, Reason}.

src/emqttd_broker.erl

@@ -28,9 +28,6 @@
 %% Event API
 -export([subscribe/1, notify/2]).
 
-%% Hook API
--export([hook/3, unhook/2, foreach_hooks/2, foldl_hooks/3]).
-
 %% Broker API
 -export([env/1, version/0, uptime/0, datetime/0, sysdescr/0]).
 
@@ -100,40 +97,6 @@ datetime() ->
         io_lib:format(
             "~4..0w-~2..0w-~2..0w ~2..0w:~2..0w:~2..0w", [Y, M, D, H, MM, S])).
 
-%% @doc Hook
--spec hook(Hook :: atom(), Name :: any(), MFA :: mfa()) -> ok | {error, any()}.
-hook(Hook, Name, MFA) ->
-    gen_server:call(?SERVER, {hook, Hook, Name, MFA}).
-
-%% @doc Unhook
--spec unhook(Hook :: atom(), Name :: any()) -> ok | {error, any()}.
-unhook(Hook, Name) ->
-    gen_server:call(?SERVER, {unhook, Hook, Name}).
-
-%% @doc Foreach hooks
--spec foreach_hooks(Hook :: atom(), Args :: list()) -> any().
-foreach_hooks(Hook, Args) ->
-    case ets:lookup(?BROKER_TAB, {hook, Hook}) of
-        [{_, Hooks}] ->
-            lists:foreach(fun({_Name, {M, F, A}}) ->
-                    apply(M, F, Args++A)
-                end, Hooks);
-        [] ->
-            ok
-    end.
-
-%% @doc Foldl hooks
--spec foldl_hooks(Hook :: atom(), Args :: list(), Acc0 :: any()) -> any().
-foldl_hooks(Hook, Args, Acc0) ->
-    case ets:lookup(?BROKER_TAB, {hook, Hook}) of
-        [{_, Hooks}] -> 
-            lists:foldl(fun({_Name, {M, F, A}}, Acc) -> 
-                    apply(M, F, lists:append([Args, [Acc], A]))
-                end, Acc0, Hooks);
-        [] -> 
-            Acc0
-    end.
-
 %% @doc Start a tick timer
 start_tick(Msg) ->
     start_tick(timer:seconds(env(sys_interval)), Msg).
@@ -157,7 +120,7 @@ init([]) ->
     emqttd_time:seed(),
     ets:new(?BROKER_TAB, [set, public, named_table]),
     % Create $SYS Topics
-    emqttd_pubsub:create(topic, <<"$SYS/brokers">>),
+    emqttd:create(topic, <<"$SYS/brokers">>),
     [ok = create_topic(Topic) || Topic <- ?SYSTOP_BROKERS],
     % Tick
     {ok, #state{started_at = os:timestamp(),
@@ -167,31 +130,6 @@ init([]) ->
 handle_call(uptime, _From, State) ->
     {reply, uptime(State), State};
 
-handle_call({hook, Hook, Name, MFArgs}, _From, State) ->
-    Key = {hook, Hook}, Reply =
-    case ets:lookup(?BROKER_TAB, Key) of
-        [{Key, Hooks}] ->
-            case lists:keyfind(Name, 1, Hooks) of
-                {Name, _MFArgs} ->
-                    {error, existed};
-                false ->
-                    insert_hooks(Key, Hooks ++ [{Name, MFArgs}])
-            end;
-        [] ->
-            insert_hooks(Key, [{Name, MFArgs}])
-    end,
-    {reply, Reply, State};
-
-handle_call({unhook, Hook, Name}, _From, State) ->
-    Key = {hook, Hook}, Reply =
-    case ets:lookup(?BROKER_TAB, Key) of
-        [{Key, Hooks}] ->
-            insert_hooks(Key, lists:keydelete(Name, 1, Hooks));
-        [] ->
-            {error, not_found}
-    end,
-    {reply, Reply, State};
-
 handle_call(Req, _From, State) ->
     ?UNEXPECTED_REQ(Req, State).
 
@@ -224,25 +162,22 @@ code_change(_OldVsn, State, _Extra) ->
 %% Internal functions
 %%--------------------------------------------------------------------
 
-insert_hooks(Key, Hooks) ->
-    ets:insert(?BROKER_TAB, {Key, Hooks}), ok.
-
 create_topic(Topic) ->
-    emqttd_pubsub:create(topic, emqttd_topic:systop(Topic)).
+    emqttd:create(topic, emqttd_topic:systop(Topic)).
 
 retain(brokers) ->
     Payload = list_to_binary(string:join([atom_to_list(N) ||
                     N <- emqttd_mnesia:running_nodes()], ",")),
     Msg = emqttd_message:make(broker, <<"$SYS/brokers">>, Payload),
-    emqttd_pubsub:publish(emqttd_message:set_flag(sys, Msg)).
+    emqttd:publish(emqttd_message:set_flag(sys, Msg)).
 
 retain(Topic, Payload) when is_binary(Payload) ->
     Msg = emqttd_message:make(broker, emqttd_topic:systop(Topic), Payload),
-    emqttd_pubsub:publish(emqttd_message:set_flag(retain, Msg)).
+    emqttd:publish(emqttd_message:set_flag(retain, Msg)).
 
 publish(Topic, Payload) when is_binary(Payload) ->
     Msg = emqttd_message:make(broker, emqttd_topic:systop(Topic), Payload),
-    emqttd_pubsub:publish(Msg).
+    emqttd:publish(Msg).
 
 uptime(#state{started_at = Ts}) ->
     Secs = timer:now_diff(os:timestamp(), Ts) div 1000000,

src/emqttd_cli.erl

@@ -28,9 +28,9 @@
 
 -export([load/0]).
 
--export([status/1, broker/1, cluster/1, bridges/1,
+-export([status/1, broker/1, cluster/1, users/1, clients/1, sessions/1,
-         clients/1, sessions/1, topics/1, subscriptions/1,
+         routes/1, topics/1, subscriptions/1, plugins/1, bridges/1,
-         plugins/1, listeners/1, vm/1, mnesia/1, trace/1]).
+         listeners/1, vm/1, mnesia/1, trace/1]).
 
 -define(PROC_INFOKEYS, [status,
                         memory,
@@ -40,7 +40,7 @@
                         stack_size,
                         reductions]).
 
--define(MAX_LINES, 20000).
+-define(MAX_LINES, 10000).
 
 -define(APP, emqttd).
 
@@ -67,7 +67,7 @@ status([]) ->
             ?PRINT("emqttd ~s is running~n", [Vsn])
     end;
 status(_) ->
-     ?PRINT_CMD("status", "query broker status").
+     ?PRINT_CMD("status", "Show broker status").
 
 %%--------------------------------------------------------------------
 %% @doc Query broker
@@ -98,10 +98,10 @@ broker(["pubsub"]) ->
              end, lists:reverse(Pubsubs));
 
 broker(_) ->
-    ?USAGE([{"broker",         "query broker version, uptime and description"},
+    ?USAGE([{"broker",         "Show broker version, uptime and description"},
-            {"broker pubsub",  "query process_info of pubsub"},
+            {"broker pubsub",  "Show process_info of pubsub"},
-            {"broker stats",   "query broker statistics of clients, topics, subscribers"},
+            {"broker stats",   "Show broker statistics of clients, topics, subscribers"},
-            {"broker metrics", "query broker metrics"}]).
+            {"broker metrics", "Show broker metrics"}]).
 
 %%--------------------------------------------------------------------
 %% @doc Cluster with other nodes
@@ -141,10 +141,14 @@ cluster(_) ->
             {"cluster remove <Node>","Remove the node from cluster"},
             {"cluster status",       "Cluster status"}]).
 
+%%--------------------------------------------------------------------
+%% @doc Users usage
+users(Args) -> emqttd_auth_username:cli(Args).
+
 %%--------------------------------------------------------------------
 %% @doc Query clients
 clients(["list"]) ->
-    dump(ets, mqtt_client, fun print/1);
+    dump(mqtt_client);
 
 clients(["show", ClientId]) ->
     if_client(ClientId, fun print/1);
@@ -153,9 +157,9 @@ clients(["kick", ClientId]) ->
     if_client(ClientId, fun(#mqtt_client{client_pid = Pid}) -> emqttd_client:kick(Pid) end);
 
 clients(_) ->
-    ?USAGE([{"clients list",            "list all clients"},
+    ?USAGE([{"clients list",            "List all clients"},
-            {"clients show <ClientId>", "show a client"},
+            {"clients show <ClientId>", "Show a client"},
-            {"clients kick <ClientId>", "kick a client"}]).
+            {"clients kick <ClientId>", "Kick out a client"}]).
 
 if_client(ClientId, Fun) ->
     case emqttd_cm:lookup(bin(ClientId)) of
@@ -169,10 +173,10 @@ sessions(["list"]) ->
     [sessions(["list", Type]) || Type <- ["persistent", "transient"]];
 
 sessions(["list", "persistent"]) ->
-    dump(ets, mqtt_persistent_session, fun print/1);
+    dump(mqtt_persistent_session);
 
 sessions(["list", "transient"]) ->
-    dump(ets, mqtt_transient_session,  fun print/1);
+    dump(mqtt_transient_session);
 
 sessions(["show", ClientId]) ->
     MP = {{bin(ClientId), '_'}, '_'},
@@ -187,63 +191,78 @@ sessions(["show", ClientId]) ->
     end;
 
 sessions(_) ->
-    ?USAGE([{"sessions list",            "list all sessions"},
+    ?USAGE([{"sessions list",            "List all sessions"},
-            {"sessions list persistent", "list all persistent sessions"},
+            {"sessions list persistent", "List all persistent sessions"},
-            {"sessions list transient",  "list all transient sessions"},
+            {"sessions list transient",  "List all transient sessions"},
-            {"sessions show <ClientId>", "show a session"}]).
+            {"sessions show <ClientId>", "Show a session"}]).
+
+%%--------------------------------------------------------------------
+%% @doc Routes Command
+routes(["list"]) ->
+    if_could_print(route, fun print/1);
+
+routes(["show", Topic]) ->
+    print(mnesia:dirty_read(route, bin(Topic)));
+
+routes(_) ->
+    ?USAGE([{"routes list",         "List all routes"},
+            {"routes show <Topic>", "Show a route"}]).
 
 %%--------------------------------------------------------------------
 %% @doc Topics Command
 topics(["list"]) ->
-    Print = fun(Topic, Records) -> print(topic, Topic, Records) end,
+    if_could_print(topic, fun print/1);
-    if_could_print(topic, Print);
 
 topics(["show", Topic]) ->
-    print(topic, Topic, ets:lookup(topic, bin(Topic)));
+    print(mnesia:dirty_read(topic, bin(Topic)));
 
 topics(_) ->
-    ?USAGE([{"topics list",         "list all topics"},
+    ?USAGE([{"topics list",         "List all topics"},
-            {"topics show <Topic>", "show a topic"}]).
+            {"topics show <Topic>", "Show a topic"}]).
 
 subscriptions(["list"]) ->
-    Print = fun(ClientId, Records) -> print(subscription, ClientId, Records) end,
+    if_could_print(subscription, fun print/1);
-    if_subscription(fun() -> if_could_print(subscription, Print) end);
+
+subscriptions(["list", "static"]) ->
+    if_could_print(backend_subscription, fun print/1);
 
 subscriptions(["show", ClientId]) ->
-    if_subscription(fun() ->
+    case mnesia:dirty_read(subscription, bin(ClientId)) of
-            case emqttd_pubsub:lookup(subscription, bin(ClientId)) of
+        []      -> ?PRINT_MSG("Not Found.~n");
-                []      -> ?PRINT_MSG("Not Found.~n");
+        Records -> print(Records)
-                Records -> print(subscription, ClientId, Records)
+    end;
-            end
-        end);
 
 subscriptions(["add", ClientId, Topic, QoS]) ->
-    Create = fun(IntQos) ->
+    Add = fun(IntQos) ->
-                Subscription = {bin(ClientId), bin(Topic), IntQos},
+            Subscription = #mqtt_subscription{subid = bin(ClientId),
-                case emqttd_pubsub:create(subscription, Subscription) of
+                                              topic = bin(Topic),
-                    ok             -> ?PRINT_MSG("ok~n");
+                                              qos   = IntQos},
-                    {error, Error} -> ?PRINT("Error: ~p~n", [Error])
+            case emqttd_backend:add_subscription(Subscription) of
-                end
+                ok ->
-             end,
+                    ?PRINT_MSG("ok~n");
-    if_subscription(fun() -> if_valid_qos(QoS, Create) end);
+                {error, already_existed} ->
+                    ?PRINT_MSG("Error: already existed~n");
+                {error, Reason} ->
+                    ?PRINT("Error: ~p~n", [Reason])
+            end
+          end,
+    if_valid_qos(QoS, Add);
+
+subscriptions(["del", ClientId]) ->
+    Ok = emqttd_backend:del_subscriptions(bin(ClientId)),
+    ?PRINT("~p~n", [Ok]);
 
 subscriptions(["del", ClientId, Topic]) ->
-    if_subscription(fun() ->
+    Ok = emqttd_backend:del_subscription(bin(ClientId), bin(Topic)),
-            Ok = emqttd_pubsub:delete(subscription, {bin(ClientId), bin(Topic)}),
+    ?PRINT("~p~n", [Ok]);
-            ?PRINT("~p~n", [Ok])
-        end);
 
 subscriptions(_) ->
-    ?USAGE([{"subscriptions list",                         "list all subscriptions"},
+    ?USAGE([{"subscriptions list",                         "List all subscriptions"},
-            {"subscriptions show <ClientId>",              "show subscriptions of a client"},
+            {"subscriptions list static",                  "List all static subscriptions"},
-            {"subscriptions add <ClientId> <Topic> <QoS>", "add subscription"},
+            {"subscriptions show <ClientId>",              "Show subscriptions of a client"},
-            {"subscriptions del <ClientId> <Topic>",       "delete subscription"}]).
+            {"subscriptions add <ClientId> <Topic> <QoS>", "Add a static subscription manually"},
-
+            {"subscriptions del <ClientId>",               "Delete static subscriptions manually"},
-if_subscription(Fun) ->
+            {"subscriptions del <ClientId> <Topic>",       "Delete a static subscription manually"}]).
-    case ets:info(subscription, name) of
-        undefined -> ?PRINT_MSG("Error: subscription table not found!~n");
-        _         -> Fun()
-    end.
 
 if_could_print(Tab, Fun) ->
     case mnesia:table_info(Tab, size) of
@@ -251,7 +270,7 @@ if_could_print(Tab, Fun) ->
             ?PRINT("Could not list, too many ~ss: ~p~n", [Tab, Size]);
         _Size ->
             Keys = mnesia:dirty_all_keys(Tab),
-            foreach(fun(Key) -> Fun(Key, ets:lookup(Tab, Key)) end, Keys)
+            foreach(fun(Key) -> Fun(ets:lookup(Tab, Key)) end, Keys)
     end.
 
 if_valid_qos(QoS, Fun) ->
@@ -282,9 +301,9 @@ plugins(["unload", Name]) ->
     end;
 
 plugins(_) ->
-    ?USAGE([{"plugins list",            "show loaded plugins"},
+    ?USAGE([{"plugins list",            "Show loaded plugins"},
-            {"plugins load <Plugin>",   "load plugin"},
+            {"plugins load <Plugin>",   "Load plugin"},
-            {"plugins unload <Plugin>", "unload plugin"}]).
+            {"plugins unload <Plugin>", "Unload plugin"}]).
 
 %%--------------------------------------------------------------------
 %% @doc Bridges command
@@ -322,11 +341,11 @@ bridges(["stop", SNode, Topic]) ->
     end;
 
 bridges(_) ->
-    ?USAGE([{"bridges list",                 "query bridges"},
+    ?USAGE([{"bridges list",                 "List bridges"},
-            {"bridges options",              "bridge options"},
+            {"bridges options",              "Bridge options"},
-            {"bridges start <Node> <Topic>", "start bridge"},
+            {"bridges start <Node> <Topic>", "Start a bridge"},
-            {"bridges start <Node> <Topic> <Options>", "start bridge with options"},
+            {"bridges start <Node> <Topic> <Options>", "Start a bridge with options"},
-            {"bridges stop <Node> <Topic>", "stop bridge"}]).
+            {"bridges stop <Node> <Topic>", "Stop a bridge"}]).
 
 parse_opts(Cmd, OptStr) ->
     Tokens = string:tokens(OptStr, ","),
@@ -369,11 +388,11 @@ vm(["io"]) ->
             end, [max_fds, active_fds]);
 
 vm(_) ->
-    ?USAGE([{"vm all",     "query info of erlang vm"},
+    ?USAGE([{"vm all",     "Show info of erlang vm"},
-            {"vm load",    "query load of erlang vm"},
+            {"vm load",    "Show load of erlang vm"},
-            {"vm memory",  "query memory of erlang vm"},
+            {"vm memory",  "Show memory of erlang vm"},
-            {"vm process", "query process of erlang vm"},
+            {"vm process", "Show process of erlang vm"},
-            {"vm io",      "queue io of erlang vm"}]).
+            {"vm io",      "Show IO of erlang vm"}]).
 
 %%--------------------------------------------------------------------
 %% @doc mnesia Command
@@ -381,7 +400,7 @@ mnesia([]) ->
     mnesia:system_info();
 
 mnesia(_) ->
-    ?PRINT_CMD("mnesia", "mnesia system info").
+    ?PRINT_CMD("mnesia", "Mnesia system info").
 
 %%--------------------------------------------------------------------
 %% @doc Trace Command
@@ -403,11 +422,11 @@ trace(["topic", Topic, LogFile]) ->
     trace_on(topic, Topic, LogFile);
 
 trace(_) ->
-    ?USAGE([{"trace list",                       "query all traces"},
+    ?USAGE([{"trace list",                       "List all traces"},
-            {"trace client <ClientId> <LogFile>","trace client with ClientId"},
+            {"trace client <ClientId> <LogFile>","Trace a client"},
-            {"trace client <ClientId> off",      "stop to trace client"},
+            {"trace client <ClientId> off",      "Stop tracing a client"},
-            {"trace topic <Topic> <LogFile>",    "trace topic with Topic"},
+            {"trace topic <Topic> <LogFile>",    "Trace a topic"},
-            {"trace topic <Topic> off",          "stop to trace Topic"}]).
+            {"trace topic <Topic> off",          "Stop tracing a Topic"}]).
 
 trace_on(Who, Name, LogFile) ->
     case emqttd_trace:start_trace({Who, iolist_to_binary(Name)}, LogFile) of
@@ -420,9 +439,9 @@ trace_on(Who, Name, LogFile) ->
 trace_off(Who, Name) ->
     case emqttd_trace:stop_trace({Who, iolist_to_binary(Name)}) of
         ok -> 
-            ?PRINT("stop to trace ~s ~s successfully.~n", [Who, Name]);
+            ?PRINT("stop tracing ~s ~s successfully.~n", [Who, Name]);
         {error, Error} ->
-            ?PRINT("stop to trace ~s ~s error: ~p.~n", [Who, Name, Error])
+            ?PRINT("stop tracing ~s ~s error: ~p.~n", [Who, Name, Error])
     end.
 
 %%--------------------------------------------------------------------
@@ -440,22 +459,55 @@ listeners([]) ->
             end, esockd:listeners());
 
 listeners(_) ->
-    ?PRINT_CMD("listeners", "query broker listeners").
+    ?PRINT_CMD("listeners", "List listeners").
+
+%%--------------------------------------------------------------------
+%% Dump ETS
+%%--------------------------------------------------------------------
+
+dump(Table) ->
+    dump(Table, ets:first(Table)).
+
+dump(_Table, '$end_of_table') ->
+    ok;
+
+dump(Table, Key) ->
+    case ets:lookup(Table, Key) of
+        [Record] -> print(Record);
+        [] -> ok
+    end,
+    dump(Table, ets:next(Table, Key)).
+
+print([]) ->
+    ok;
+
+print(Routes = [#mqtt_route{topic = Topic} | _]) ->
+    Nodes = [atom_to_list(Node) || #mqtt_route{node = Node} <- Routes],
+    ?PRINT("~s -> ~s~n", [Topic, string:join(Nodes, ",")]);
+
+print(Subscriptions = [#mqtt_subscription{subid = ClientId} | _]) ->
+    TopicTable = [io_lib:format("~s:~w", [Topic, Qos])
+                  || #mqtt_subscription{topic = Topic, qos = Qos} <- Subscriptions],
+    ?PRINT("~s -> ~s~n", [ClientId, string:join(TopicTable, ",")]);
+
+print(Topics = [#mqtt_topic{}|_]) ->
+    foreach(fun print/1, Topics);
 
 print(#mqtt_plugin{name = Name, version = Ver, descr = Descr, active = Active}) ->
     ?PRINT("Plugin(~s, version=~s, description=~s, active=~s)~n",
-               [Name, Ver, Descr, Active]);
+           [Name, Ver, Descr, Active]);
 
-print(#mqtt_client{client_id = ClientId, clean_sess = CleanSess,
+print(#mqtt_client{client_id = ClientId, clean_sess = CleanSess, username = Username,
-                   username = Username, peername = Peername,
+                   peername = Peername, connected_at = ConnectedAt}) ->
-                   connected_at = ConnectedAt}) ->
     ?PRINT("Client(~s, clean_sess=~s, username=~s, peername=~s, connected_at=~p)~n",
-            [ClientId, CleanSess, Username,
+           [ClientId, CleanSess, Username, emqttd_net:format(Peername),
-             emqttd_net:format(Peername),
+            emqttd_time:now_to_secs(ConnectedAt)]);
-             emqttd_time:now_to_secs(ConnectedAt)]);
 
-print(#mqtt_topic{topic = Topic, node = Node}) ->
+print(#mqtt_topic{topic = Topic, flags = Flags}) ->
-    ?PRINT("~s on ~s~n", [Topic, Node]);
+    ?PRINT("~s: ~s~n", [Topic, string:join([atom_to_list(F) || F <- Flags], ",")]);
+
+print(#mqtt_route{topic = Topic, node = Node}) ->
+    ?PRINT("~s -> ~s~n", [Topic, Node]);
 
 print({{ClientId, _ClientPid}, SessInfo}) ->
     InfoKeys = [clean_sess, 
@@ -473,36 +525,11 @@ print({{ClientId, _ClientPid}, SessInfo}) ->
            "created_at=~w)~n",
             [ClientId | [format(Key, proplists:get_value(Key, SessInfo)) || Key <- InfoKeys]]).
 
-print(topic, Topic, Records) ->
-    Nodes = [Node || #mqtt_topic{node = Node} <- Records],
-    ?PRINT("~s: ~p~n", [Topic, Nodes]);
-
-print(subscription, ClientId, Subscriptions) ->
-    TopicTable = [{Topic, Qos} || #mqtt_subscription{topic = Topic, qos = Qos} <- Subscriptions],
-    ?PRINT("~s: ~p~n", [ClientId, TopicTable]).
-
 format(created_at, Val) ->
     emqttd_time:now_to_secs(Val);
 
-format(subscriptions, List) ->
-    string:join([io_lib:format("~s:~w", [Topic, Qos]) || {Topic, Qos} <- List], ",");
-
 format(_, Val) ->
     Val.
 
 bin(S) -> iolist_to_binary(S).
 
-%%TODO: ...
-dump(ets, Table, Fun) ->
-    dump(ets, Table, ets:first(Table), Fun).
-
-dump(ets, _Table, '$end_of_table', _Fun) ->
-    ok;
-
-dump(ets, Table, Key, Fun) ->
-    case ets:lookup(Table, Key) of
-        [Record] -> Fun(Record);
-        [] -> ignore
-    end,
-    dump(ets, Table, ets:next(Table, Key), Fun).
-

src/emqttd_ctl.erl

@@ -25,7 +25,8 @@
 -define(SERVER, ?MODULE).
 
 %% API Function Exports
--export([start_link/0, register_cmd/3, unregister_cmd/1, run/1]).
+-export([start_link/0, register_cmd/2, register_cmd/3, unregister_cmd/1,
+         lookup/1, run/1]).
 
 %% gen_server Function Exports
 -export([init/1, handle_call/3, handle_cast/2, handle_info/2,
@@ -43,6 +44,11 @@ start_link() ->
     gen_server:start_link({local, ?SERVER}, ?MODULE, [], []).
 
 %% @doc Register a command
+-spec register_cmd(atom(), {module(), atom()}) -> ok.
+register_cmd(Cmd, MF) ->
+    register_cmd(Cmd, MF, []).
+
+%% @doc Register a command with opts
 -spec register_cmd(atom(), {module(), atom()}, list()) -> ok.
 register_cmd(Cmd, MF, Opts) ->
     cast({register_cmd, Cmd, MF, Opts}).
@@ -55,17 +61,25 @@ unregister_cmd(Cmd) ->
 cast(Msg) -> gen_server:cast(?SERVER, Msg).
 
 %% @doc Run a command
+-spec run([string()]) -> any().
 run([]) -> usage();
 
 run(["help"]) -> usage();
 
 run([CmdS|Args]) ->
-    Cmd = list_to_atom(CmdS),
+    case lookup(list_to_atom(CmdS)) of
-    case ets:match(?CMD_TAB, {{'_', Cmd}, '$1', '_'}) of
+        [{Mod, Fun}] -> Mod:Fun(Args);
-        [[{Mod, Fun}]] -> Mod:Fun(Args);
         [] -> usage() 
     end.
 
+%% @doc Lookup a command
+-spec lookup(atom()) -> [{module(), atom()}].
+lookup(Cmd) ->
+    case ets:match(?CMD_TAB, {{'_', Cmd}, '$1', '_'}) of
+        [El] -> El;
+        []   -> []
+    end.
+
 %% @doc Usage
 usage() ->
     ?PRINT("Usage: ~s~n", [?MODULE]),

src/emqttd_hook.erl

@@ -0,0 +1,155 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2016 Feng Lee <feng@emqtt.io>.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqttd_hook).
+
+-author("Feng Lee <feng@emqtt.io>").
+
+-behaviour(gen_server).
+
+%% Start
+-export([start_link/0]).
+
+%% Hooks API
+-export([add/3, add/4, delete/2, run/3, lookup/1]).
+
+%% gen_server Function Exports
+-export([init/1, handle_call/3, handle_cast/2, handle_info/2,
+         terminate/2, code_change/3]).
+
+-record(state, {}).
+
+-record(callback, {function       :: function(),
+                   init_args = [] :: list(any()),
+                   priority  = 0  :: integer()}).
+
+-record(hook, {name :: atom(), callbacks = [] :: list(#callback{})}).
+
+-define(HOOK_TAB, mqtt_hook).
+
+%%--------------------------------------------------------------------
+%% Start API
+%%--------------------------------------------------------------------
+
+start_link() ->
+    gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
+
+%%--------------------------------------------------------------------
+%% Hooks API
+%%--------------------------------------------------------------------
+
+-spec(add(atom(), function(), list(any())) -> ok).
+add(HookPoint, Function, InitArgs) ->
+    add(HookPoint, Function, InitArgs, 0).
+
+-spec(add(atom(), function(), list(any()), integer()) -> ok).
+add(HookPoint, Function, InitArgs, Priority) ->
+    gen_server:call(?MODULE, {add, HookPoint, Function, InitArgs, Priority}).
+
+-spec(delete(atom(), function()) -> ok).
+delete(HookPoint, Function) ->
+    gen_server:call(?MODULE, {delete, HookPoint, Function}).
+
+-spec(run(atom(), list(any()), any()) -> any()).
+run(HookPoint, Args, Acc) ->
+    run_(lookup(HookPoint), Args, Acc).
+
+%% @private
+run_([#callback{function = Fun, init_args = InitArgs} | Callbacks], Args, Acc) ->
+    case apply(Fun, lists:append([Args, [Acc], InitArgs])) of
+        ok             -> run_(Callbacks, Args, Acc);
+        {ok, NewAcc}   -> run_(Callbacks, Args, NewAcc);
+        stop           -> {stop, Acc};
+        {stop, NewAcc} -> {stop, NewAcc}
+    end;
+
+run_([], _Args, Acc) ->
+    {ok, Acc}.
+
+-spec(lookup(atom()) -> [#callback{}]).
+lookup(HookPoint) ->
+    case ets:lookup(?HOOK_TAB, HookPoint) of
+        [] -> [];
+        [#hook{callbacks = Callbacks}] -> Callbacks
+    end.
+
+%%--------------------------------------------------------------------
+%% gen_server Callbacks
+%%--------------------------------------------------------------------
+
+init([]) ->
+    ets:new(?HOOK_TAB, [set, protected, named_table, {keypos, #hook.name}]),
+    {ok, #state{}}.
+
+handle_call({add, HookPoint, Function, InitArgs, Priority}, _From, State) ->
+    Reply =
+    case ets:lookup(?HOOK_TAB, HookPoint) of
+        [#hook{callbacks = Callbacks}] ->
+            case lists:keyfind(Function, #callback.function, Callbacks) of
+                false ->
+                    Callback = #callback{function  = Function,
+                                         init_args = InitArgs,
+                                         priority  = Priority},
+                    insert_hook_(HookPoint, add_callback_(Callback, Callbacks));
+                _Callback ->
+                    {error, already_hooked}
+            end;
+        [] ->
+            Callback = #callback{function  = Function,
+                                 init_args = InitArgs,
+                                 priority  = Priority},
+            insert_hook_(HookPoint, [Callback])
+    end,
+    {reply, Reply, State};
+
+handle_call({delete, HookPoint, Function}, _From, State) ->
+    Reply =
+    case ets:lookup(?HOOK_TAB, HookPoint) of
+        [#hook{callbacks = Callbacks}] ->
+            insert_hook_(HookPoint, del_callback_(Function, Callbacks));
+        [] ->
+            {error, not_found}
+    end,
+    {reply, Reply, State};
+
+handle_call(_Req, _From, State) ->
+    {reply, ignore, State}.
+
+handle_cast(_Msg, State) ->
+    {noreply, State}.
+
+handle_info(_Info, State) ->
+    {noreply, State}.
+
+terminate(_Reason, _State) ->
+    ok.
+
+code_change(_OldVsn, State, _Extra) ->
+    {ok, State}.
+
+%%--------------------------------------------------------------------
+%% Internal functions
+%%--------------------------------------------------------------------
+
+insert_hook_(HookPoint, Callbacks) ->
+    ets:insert(?HOOK_TAB, #hook{name = HookPoint, callbacks = Callbacks}), ok.
+
+add_callback_(Callback, Callbacks) ->
+    lists:keymerge(#callback.priority, Callbacks, [Callback]).
+
+del_callback_(Function, Callbacks) ->
+    lists:keydelete(Function, #callback.function, Callbacks).
+

src/emqttd_http.erl

@@ -44,27 +44,9 @@ handle_request('GET', "/status", Req) ->
 %%--------------------------------------------------------------------
 
 handle_request('POST', "/mqtt/publish", Req) ->
-    Params = mochiweb_request:parse_post(Req),
-    lager:info("HTTP Publish: ~p", [Params]),
     case authorized(Req) of
-    true ->
+        true  -> http_publish(Req);
-        ClientId = get_value("client", Params, http),
+        false -> Req:respond({401, [], <<"Fobbiden">>})
-        Qos      = int(get_value("qos", Params, "0")),
-        Retain   = bool(get_value("retain", Params,  "0")),
-        Topic    = list_to_binary(get_value("topic", Params)),
-        Payload  = list_to_binary(get_value("message", Params)),
-        case {validate(qos, Qos), validate(topic, Topic)} of
-            {true, true} ->
-                Msg = emqttd_message:make(ClientId, Qos, Topic, Payload),
-                emqttd_pubsub:publish(Msg#mqtt_message{retain  = Retain}),
-                Req:ok({"text/plain", <<"ok">>});
-           {false, _} ->
-                Req:respond({400, [], <<"Bad QoS">>});
-            {_, false} ->
-                Req:respond({400, [], <<"Bad Topic">>})
-        end;
-    false ->
-        Req:respond({401, [], <<"Fobbiden">>})
     end;
 
 %%--------------------------------------------------------------------
@@ -97,9 +79,53 @@ handle_request(Method, Path, Req) ->
     lager:error("Unexpected HTTP Request: ~s ~s", [Method, Path]),
     Req:not_found().
 
+%%--------------------------------------------------------------------
+%% HTTP Publish
+%%--------------------------------------------------------------------
+
+http_publish(Req) ->
+    Params = mochiweb_request:parse_post(Req),
+    lager:info("HTTP Publish: ~p", [Params]),
+    Topics   = topics(Params),
+    ClientId = get_value("client", Params, http),
+    Qos      = int(get_value("qos", Params, "0")),
+    Retain   = bool(get_value("retain", Params, "0")),
+    Payload  = list_to_binary(get_value("message", Params)),
+    case {validate(qos, Qos), validate(topics, Topics)} of
+        {true, true} ->
+            lists:foreach(fun(Topic) ->
+                Msg = emqttd_message:make(ClientId, Qos, Topic, Payload),
+                emqttd:publish(Msg#mqtt_message{retain  = Retain})
+            end, Topics),
+            Req:ok({"text/plain", <<"ok">>});
+       {false, _} ->
+            Req:respond({400, [], <<"Bad QoS">>});
+        {_, false} ->
+            Req:respond({400, [], <<"Bad Topics">>})
+    end.
+
+topics(Params) ->
+    Tokens = [get_value("topic", Params) | string:tokens(get_value("topics", Params, ""), ",")],
+    [list_to_binary(Token) || Token <- Tokens, Token =/= undefined].
+
+validate(qos, Qos) ->
+    (Qos >= ?QOS_0) and (Qos =< ?QOS_2);
+
+validate(topics, [Topic|Left]) ->
+    case validate(topic, Topic) of
+        true  -> validate(topics, Left);
+        false -> false
+    end;
+validate(topics, []) ->
+    true;
+
+validate(topic, Topic) ->
+    emqttd_topic:validate({name, Topic}).
+
 %%--------------------------------------------------------------------
 %% basic authorization
 %%--------------------------------------------------------------------
+
 authorized(Req) ->
     case Req:get_header_value("Authorization") of
     undefined ->
@@ -118,11 +144,6 @@ authorized(Req) ->
 user_passwd(BasicAuth) ->
     list_to_tuple(binary:split(base64:decode(BasicAuth), <<":">>)). 
 
-validate(qos, Qos) ->
-    (Qos >= ?QOS_0) and (Qos =< ?QOS_2); 
-
-validate(topic, Topic) ->
-    emqttd_topic:validate({name, Topic}).
 
 int(S) -> list_to_integer(S).
 

src/emqttd_metrics.erl

@@ -243,7 +243,7 @@ init([]) ->
     % Init metrics
     [create_metric(Metric) ||  Metric <- Metrics],
     % $SYS Topics for metrics
-    [ok = emqttd_pubsub:create(topic, metric_topic(Topic)) || {_, Topic} <- Metrics],
+    [ok = emqttd:create(topic, metric_topic(Topic)) || {_, Topic} <- Metrics],
     % Tick to publish metrics
     {ok, #state{tick_tref = emqttd_broker:start_tick(tick)}, hibernate}.
 
@@ -273,7 +273,7 @@ code_change(_OldVsn, State, _Extra) ->
 
 publish(Metric, Val) ->
     Msg = emqttd_message:make(metrics, metric_topic(Metric), bin(Val)),
-    emqttd_pubsub:publish(emqttd_message:set_flag(sys, Msg)).
+    emqttd:publish(emqttd_message:set_flag(sys, Msg)).
 
 create_metric({gauge, Name}) ->
     ets:insert(?METRIC_TAB, {{Name, 0}, 0});

src/emqttd_mod_presence.erl

@@ -23,57 +23,51 @@
 
 -export([load/1, unload/1]).
 
--export([client_connected/3, client_disconnected/3]).
+-export([on_client_connected/3, on_client_disconnected/3]).
 
 load(Opts) ->
-    emqttd_broker:hook('client.connected', {?MODULE, client_connected},
+    emqttd:hook('client.connected', fun ?MODULE:on_client_connected/3, [Opts]),
-                        {?MODULE, client_connected, [Opts]}),
+    emqttd:hook('client.disconnected', fun ?MODULE:on_client_disconnected/3, [Opts]).
-    emqttd_broker:hook('client.disconnected', {?MODULE, client_disconnected},
-                        {?MODULE, client_disconnected, [Opts]}),
-    ok.
 
-client_connected(ConnAck, #mqtt_client{client_id  = ClientId,
+on_client_connected(ConnAck, Client = #mqtt_client{client_id  = ClientId,
-                                       username   = Username,
+                                                   username   = Username,
-                                       peername   = {IpAddress, _},
+                                                   peername   = {IpAddr, _},
-                                       clean_sess = CleanSess,
+                                                   clean_sess = CleanSess,
-                                       proto_ver  = ProtoVer}, Opts) ->
+                                                   proto_ver  = ProtoVer}, Opts) ->
-    Sess = case CleanSess of
-        true -> false;
-        false -> true
-    end,
     Json = mochijson2:encode([{clientid, ClientId},
                               {username, Username},
-                              {ipaddress, list_to_binary(emqttd_net:ntoa(IpAddress))},
+                              {ipaddress, list_to_binary(emqttd_net:ntoa(IpAddr))},
-                              {session, Sess},
+                              {session, sess(CleanSess)},
                               {protocol, ProtoVer},
                               {connack, ConnAck},
                               {ts, emqttd_time:now_to_secs()}]),
-    Msg = emqttd_message:make(presence,
+    emqttd:publish(message(qos(Opts), topic(connected, ClientId), Json)),
-                              proplists:get_value(qos, Opts, 0),
+    {ok, Client}.
-                              topic(connected, ClientId),
-                              iolist_to_binary(Json)),
-    emqttd_pubsub:publish(Msg).
 
-client_disconnected(Reason, ClientId, Opts) ->
+on_client_disconnected(Reason, ClientId, Opts) ->
     Json = mochijson2:encode([{clientid, ClientId},
                               {reason, reason(Reason)},
                               {ts, emqttd_time:now_to_secs()}]),
-    Msg = emqttd_message:make(presence,
+    emqttd:publish(message(qos(Opts), topic(disconnected, ClientId), Json)).
-                              proplists:get_value(qos, Opts, 0),
-                              topic(disconnected, ClientId),
-                              iolist_to_binary(Json)),
-    emqttd_pubsub:publish(Msg).
 
 unload(_Opts) ->
-    emqttd_broker:unhook('client.connected', {?MODULE, client_connected}),
+    emqttd:unhook('client.connected', fun ?MODULE:on_client_connected/3),
-    emqttd_broker:unhook('client.disconnected', {?MODULE, client_disconnected}).
+    emqttd:unhook('client.disconnected', fun ?MODULE:on_client_disconnected/3).
+
+sess(false) -> true;
+sess(true)  -> false.
+
+qos(Opts) -> proplists:get_value(qos, Opts, 0).
+
+message(Qos, Topic, Json) ->
+    emqttd_message:make(presence, Qos, Topic, iolist_to_binary(Json)).
 
 topic(connected, ClientId) ->
     emqttd_topic:systop(list_to_binary(["clients/", ClientId, "/connected"]));
 topic(disconnected, ClientId) ->
     emqttd_topic:systop(list_to_binary(["clients/", ClientId, "/disconnected"])).
 
-reason(Reason) when is_atom(Reason)    -> Reason;
+reason(Reason) when is_atom(Reason) -> Reason;
 reason({Error, _}) when is_atom(Error) -> Error;
 reason(_) -> internal_error.
 

src/emqttd_mod_rewrite.erl

@@ -23,7 +23,7 @@
 
 -export([load/1, reload/1, unload/1]).
 
--export([rewrite/3, rewrite/4]).
+-export([rewrite_subscribe/3, rewrite_unsubscribe/3, rewrite_publish/2]).
 
 %%--------------------------------------------------------------------
 %% API
@@ -33,23 +33,19 @@ load(Opts) ->
     File = proplists:get_value(file, Opts),
     {ok, Terms} = file:consult(File),
     Sections = compile(Terms),
-    emqttd_broker:hook('client.subscribe', {?MODULE, rewrite_subscribe}, 
+    emqttd:hook('client.subscribe', fun ?MODULE:rewrite_subscribe/3, [Sections]),
-                        {?MODULE, rewrite, [subscribe, Sections]}),
+    emqttd:hook('client.unsubscribe', fun ?MODULE:rewrite_unsubscribe/3, [Sections]),
-    emqttd_broker:hook('client.unsubscribe', {?MODULE, rewrite_unsubscribe},
+    emqttd:hook('message.publish', fun ?MODULE:rewrite_publish/2, [Sections]).
-                        {?MODULE, rewrite, [unsubscribe, Sections]}),
-    emqttd_broker:hook('message.publish', {?MODULE, rewrite_publish},
-                        {?MODULE, rewrite, [publish, Sections]}),
-    ok.
 
-rewrite(_ClientId, TopicTable, subscribe, Sections) ->
+rewrite_subscribe(_ClientId, TopicTable, Sections) ->
-    lager:info("rewrite subscribe: ~p", [TopicTable]),
+    lager:info("Rewrite subscribe: ~p", [TopicTable]),
-    [{match_topic(Topic, Sections), Qos} || {Topic, Qos} <- TopicTable];
+    {ok, [{match_topic(Topic, Sections), Qos} || {Topic, Qos} <- TopicTable]}.
 
-rewrite(_ClientId, Topics, unsubscribe, Sections) ->
+rewrite_unsubscribe(_ClientId, Topics, Sections) ->
-    lager:info("rewrite unsubscribe: ~p", [Topics]),
+    lager:info("Rewrite unsubscribe: ~p", [Topics]),
-    [match_topic(Topic, Sections) || Topic <- Topics].
+    {ok, [match_topic(Topic, Sections) || Topic <- Topics]}.
 
-rewrite(Message=#mqtt_message{topic = Topic}, publish, Sections) ->
+rewrite_publish(Message=#mqtt_message{topic = Topic}, Sections) ->
     %%TODO: this will not work if the client is always online.
     RewriteTopic =
     case get({rewrite, Topic}) of
@@ -59,11 +55,11 @@ rewrite(Message=#mqtt_message{topic = Topic}, publish, Sections) ->
         DestTopic ->
             DestTopic
         end,
-    Message#mqtt_message{topic = RewriteTopic}.
+    {ok, Message#mqtt_message{topic = RewriteTopic}}.
 
 reload(File) ->
     %%TODO: The unload api is not right...
-    case emqttd:is_mod_enabled(rewrite) of
+    case emqttd_app:is_mod_enabled(rewrite) of
         true -> 
             unload(state),
             load([{file, File}]);
@@ -72,9 +68,9 @@ reload(File) ->
     end.
             
 unload(_) ->
-    emqttd_broker:unhook('client.subscribe',  {?MODULE, rewrite_subscribe}),
+    emqttd:unhook('client.subscribe',  fun ?MODULE:rewrite_subscribe/3),
-    emqttd_broker:unhook('client.unsubscribe',{?MODULE, rewrite_unsubscribe}),
+    emqttd:unhook('client.unsubscribe',fun ?MODULE:rewrite_unsubscribe/3),
-    emqttd_broker:unhook('message.publish',   {?MODULE, rewrite_publish}).
+    emqttd:unhook('message.publish',   fun ?MODULE:rewrite_publish/2).
 
 %%--------------------------------------------------------------------
 %% Internal functions

src/emqttd_mod_subscription.erl

@@ -23,35 +23,36 @@
 
 -include("emqttd_protocol.hrl").
 
--export([load/1, client_connected/3, unload/1]).
+-export([load/1, on_client_connected/3, unload/1]).
 
--record(state, {topics, stored = false}).
+-record(state, {topics, backend = false}).
 
 load(Opts) ->
     Topics = [{iolist_to_binary(Topic), QoS} || {Topic, QoS} <- Opts, ?IS_QOS(QoS)],
-    State = #state{topics = Topics, stored = lists:member(stored, Opts)},
+    State = #state{topics = Topics, backend = lists:member(backend, Opts)},
-    emqttd_broker:hook('client.connected', {?MODULE, client_connected},
+    emqttd:hook('client.connected', fun ?MODULE:on_client_connected/3, [State]).
-                       {?MODULE, client_connected, [State]}),
+
+on_client_connected(?CONNACK_ACCEPT, Client = #mqtt_client{client_id  = ClientId,
+                                                           client_pid = ClientPid,
+                                                           username   = Username},
+                    #state{topics = Topics, backend = Backend}) ->
+
+    Replace = fun(Topic) -> rep(<<"$u">>, Username, rep(<<"$c">>, ClientId, Topic)) end,
+    TopicTable = [{Replace(Topic), Qos} || {Topic, Qos} <- with_backend(Backend, ClientId, Topics)],
+    emqttd_client:subscribe(ClientPid, TopicTable),
+    {ok, Client};
+
+on_client_connected(_ConnAck, _Client, _State) ->
     ok.
 
-client_connected(?CONNACK_ACCEPT, #mqtt_client{client_id  = ClientId,
+with_backend(false, _ClientId, TopicTable) ->
-                                               client_pid = ClientPid,
-                                               username   = Username},
-                 #state{topics = Topics, stored = Stored}) ->
-    Replace = fun(Topic) -> rep(<<"$u">>, Username, rep(<<"$c">>, ClientId, Topic)) end,
-    TopicTable = with_stored(Stored, ClientId, [{Replace(Topic), Qos} || {Topic, Qos} <- Topics]),
-    emqttd_client:subscribe(ClientPid, TopicTable);
-
-client_connected(_ConnAck, _Client, _State) -> ok.
-
-with_stored(false, _ClientId, TopicTable) ->
     TopicTable;
-with_stored(true, ClientId, TopicTable) ->
+with_backend(true, ClientId, TopicTable) ->
     Fun = fun(#mqtt_subscription{topic = Topic, qos = Qos}) -> {Topic, Qos} end,
-    emqttd_opts:merge([Fun(Sub) || Sub <- emqttd_pubsub:lookup(subscription, ClientId)], TopicTable).
+    emqttd_opts:merge([Fun(Sub) || Sub <- emqttd_backend:lookup_subscriptions(ClientId)], TopicTable).
 
 unload(_Opts) ->
-    emqttd_broker:unhook('client.connected', {?MODULE, client_connected}).
+    emqttd:unhook('client.connected', fun ?MODULE:on_client_connected/3).
 
 rep(<<"$c">>, ClientId, Topic) ->
     emqttd_topic:feed_var(<<"$c">>, ClientId, Topic);

src/emqttd_parser.erl

@@ -74,7 +74,8 @@ parse_frame(Bin, #mqtt_packet_header{type = Type, qos  = Qos} = Header, Length)
     case {Type, Bin} of
         {?CONNECT, <<FrameBin:Length/binary, Rest/binary>>} ->
             {ProtoName, Rest1} = parse_utf(FrameBin),
-            <<ProtoVersion : 8, Rest2/binary>> = Rest1,
+            %% Fix mosquitto bridge: 0x83, 0x84
+            <<_Bridge:4, ProtoVersion:4, Rest2/binary>> = Rest1,
             <<UsernameFlag : 1,
               PasswordFlag : 1,
               WillRetain   : 1,

src/emqttd_protocol.erl

@@ -165,7 +165,7 @@ process(Packet = ?CONNECT_PACKET(Var), State0) ->
             {ReturnCode, false, State1}
     end,
     %% Run hooks
-    emqttd_broker:foreach_hooks('client.connected', [ReturnCode1, client(State3)]),
+    emqttd:run_hooks('client.connected', [ReturnCode1], client(State3)),
     %% Send connack
     send(?CONNACK_PACKET(ReturnCode1, sp(SessPresent)), State3);
 
@@ -247,7 +247,9 @@ with_puback(Type, Packet = ?PUBLISH_PACKET(_Qos, PacketId),
     end.
 
 -spec send(mqtt_message() | mqtt_packet(), proto_state()) -> {ok, proto_state()}.
-send(Msg, State) when is_record(Msg, mqtt_message) ->
+send(Msg, State = #proto_state{client_id = ClientId})
+        when is_record(Msg, mqtt_message) ->
+    emqttd:run_hooks('message.delivered', [ClientId], Msg),
     send(emqttd_message:to_packet(Msg), State);
 
 send(Packet, State = #proto_state{sendfun = SendFun})
@@ -281,7 +283,7 @@ shutdown(conflict, #proto_state{client_id = _ClientId}) ->
 shutdown(Error, State = #proto_state{client_id = ClientId, will_msg = WillMsg}) ->
     ?LOG(info, "Shutdown for ~p", [Error], State),
     send_willmsg(ClientId, WillMsg),
-    emqttd_broker:foreach_hooks('client.disconnected', [Error, ClientId]),
+    emqttd:run_hooks('client.disconnected', [Error], ClientId),
     %% let it down
     %% emqttd_cm:unregister(ClientId).
     ok.
@@ -302,12 +304,12 @@ maybe_set_clientid(State) ->
 send_willmsg(_ClientId, undefined) ->
     ignore;
 send_willmsg(ClientId, WillMsg) -> 
-    emqttd_pubsub:publish(WillMsg#mqtt_message{from = ClientId}).
+    emqttd:publish(WillMsg#mqtt_message{from = ClientId}).
 
 start_keepalive(0) -> ignore;
 
 start_keepalive(Sec) when Sec > 0 ->
-    self() ! {keepalive, start, round(Sec * 1.2)}.
+    self() ! {keepalive, start, round(Sec * 0.75)}.
 
 %%--------------------------------------------------------------------
 %% Validate Packets

src/emqttd_pubsub.erl

@@ -31,285 +31,171 @@
 -copy_mnesia({mnesia, [copy]}).
 
 %% API Exports
--export([start_link/4]).
+-export([start_link/3, create_topic/1, lookup_topic/1]).
 
--export([create/2, lookup/2, subscribe/1, subscribe/2,
+-export([subscribe/2, unsubscribe/2, publish/2, dispatch/2,
-         publish/1, unsubscribe/1, unsubscribe/2, delete/2]).
+         async_subscribe/2, async_unsubscribe/2]).
 
-%% Local node
+%% gen_server.
--export([match/1]).
-
-%% gen_server Function Exports
 -export([init/1, handle_call/3, handle_cast/2, handle_info/2,
          terminate/2, code_change/3]).
 
--record(state, {pool, id, statsfun}).
+-record(state, {pool, id, env}).
-
--define(ROUTER, emqttd_router).
 
 %%--------------------------------------------------------------------
 %% Mnesia callbacks
 %%--------------------------------------------------------------------
+
 mnesia(boot) ->
-    ok = create_table(topic, ram_copies),
+    ok = emqttd_mnesia:create_table(topic, [
-    if_subscription(fun(RamOrDisc) ->
+                {ram_copies, [node()]},
-                      ok = create_table(subscription, RamOrDisc)
+                {record_name, mqtt_topic},
-                    end);
+                {attributes, record_info(fields, mqtt_topic)}]);
 
 mnesia(copy) ->
-    ok = emqttd_mnesia:copy_table(topic),
+    ok = emqttd_mnesia:copy_table(topic).
-    %% Only one disc_copy???
-    if_subscription(fun(_RamOrDisc) ->
-                      ok = emqttd_mnesia:copy_table(subscription)
-                    end).
-
-%% Topic Table
-create_table(topic, RamOrDisc) ->
-    emqttd_mnesia:create_table(topic, [
-            {type, bag},
-            {RamOrDisc, [node()]},
-            {record_name, mqtt_topic},
-            {attributes, record_info(fields, mqtt_topic)}]);
-
-%% Subscription Table
-create_table(subscription, RamOrDisc) ->
-    emqttd_mnesia:create_table(subscription, [
-            {type, bag},
-            {RamOrDisc, [node()]},
-            {record_name, mqtt_subscription},
-            {attributes, record_info(fields, mqtt_subscription)},
-            {storage_properties, [{ets, [compressed]},
-                                  {dets, [{auto_save, 5000}]}]}]).
-
-if_subscription(Fun) ->
-    case env(subscription) of
-        disc      -> Fun(disc_copies);
-        ram       -> Fun(ram_copies);
-        false     -> ok;
-        undefined -> ok
-    end.
-
-env(Key) ->
-    case get({pubsub, Key}) of
-        undefined ->
-            cache_env(Key);
-        Val ->
-            Val
-    end.
-
-cache_env(Key) ->
-    Val = proplists:get_value(Key, emqttd_broker:env(pubsub)),
-    put({pubsub, Key}, Val),
-    Val.
 
 %%--------------------------------------------------------------------
-%% API
+%% Start PubSub
 %%--------------------------------------------------------------------
 
-%% @doc Start one pubsub server
+%% @doc Start one pubsub
--spec start_link(Pool, Id, StatsFun, Opts) -> {ok, pid()} | ignore | {error, any()} when
+-spec(start_link(Pool, Id, Env) -> {ok, pid()} | ignore | {error, any()} when
-    Pool     :: atom(),
+      Pool :: atom(),
-    Id       :: pos_integer(),
+      Id   :: pos_integer(),
-    StatsFun :: fun((atom()) -> any()),
+      Env  :: list(tuple())).
-    Opts     :: list(tuple()).
+start_link(Pool, Id, Env) ->
-start_link(Pool, Id, StatsFun, Opts) ->
+    gen_server2:start_link({local, ?PROC_NAME(?MODULE, Id)}, ?MODULE, [Pool, Id, Env], []).
-    gen_server2:start_link({local, ?PROC_NAME(?MODULE, Id)},
-                           ?MODULE, [Pool, Id, StatsFun, Opts], []).
 
-%% @doc Create Topic or Subscription.
+%% @doc Create a Topic.
--spec create(topic, emqttd_topic:topic()) -> ok | {error, any()};
+-spec(create_topic(binary()) -> ok | {error, any()}).
-            (subscription, {binary(), binary(), mqtt_qos()}) -> ok | {error, any()}.
+create_topic(Topic) when is_binary(Topic) ->
-create(topic, Topic) when is_binary(Topic) ->
+    case mnesia:transaction(fun add_topic_/2, [Topic, [static]]) of
-    Record = #mqtt_topic{topic = Topic, node = node()},
-    case mnesia:transaction(fun add_topic/1, [Record]) of
         {atomic, ok}     -> ok;
         {aborted, Error} -> {error, Error}
+    end.
+
+%% @doc Lookup a Topic.
+-spec(lookup_topic(binary()) -> list(mqtt_topic())).
+lookup_topic(Topic) when is_binary(Topic) ->
+    mnesia:dirty_read(topic, Topic).
+
+%%--------------------------------------------------------------------
+%% PubSub API
+%%--------------------------------------------------------------------
+
+%% @doc Subscribe a Topic
+-spec(subscribe(binary(), pid()) -> ok).
+subscribe(Topic, SubPid) when is_binary(Topic) ->
+    call(pick(Topic), {subscribe, Topic, SubPid}).
+
+%% @doc Asynchronous Subscribe
+-spec(async_subscribe(binary(), pid()) -> ok).
+async_subscribe(Topic, SubPid) when is_binary(Topic) ->
+    cast(pick(Topic), {subscribe, Topic, SubPid}).
+
+%% @doc Publish message to Topic.
+-spec(publish(binary(), any()) -> any()).
+publish(Topic, Msg) ->
+    lists:foreach(
+        fun(#mqtt_route{topic = To, node = Node}) when Node =:= node() ->
+            ?MODULE:dispatch(To, Msg);
+           (#mqtt_route{topic = To, node = Node}) ->
+            rpc:cast(Node, ?MODULE, dispatch, [To, Msg])
+        end, emqttd_router:lookup(Topic)).
+
+%% @doc Dispatch Message to Subscribers
+-spec(dispatch(binary(), mqtt_message()) -> ok).
+dispatch(Queue = <<"$queue/", _T>>, Msg) ->
+    case subscribers(Queue) of
+        [] ->
+            dropped(Queue);
+        [SubPid] ->
+            SubPid ! {dispatch, Queue, Msg};
+        SubPids ->
+            Idx = crypto:rand_uniform(1, length(SubPids) + 1),
+            SubPid = lists:nth(Idx, SubPids),
+            SubPid ! {dispatch, Queue, Msg}
     end;
 
-create(subscription, {SubId, Topic, Qos}) when is_binary(SubId) andalso is_binary(Topic) ->
+dispatch(Topic, Msg) ->
-    case mnesia:transaction(fun add_subscription/2, [SubId, {Topic, ?QOS_I(Qos)}]) of
+    case subscribers(Topic) of
-        {atomic, ok}     -> ok;
+        [] ->
-        {aborted, Error} -> {error, Error}
+            dropped(Topic);
+        [SubPid] ->
+            SubPid ! {dispatch, Topic, Msg};
+        SubPids ->
+            lists:foreach(fun(SubPid) ->
+                SubPid ! {dispatch, Topic, Msg}
+            end, SubPids)
     end.
 
-%% @doc Lookup Topic or Subscription.
+%% @private
--spec lookup(topic, emqttd_topic:topic()) -> list(mqtt_topic());
+%% @doc Find all subscribers
-            (subscription, binary())      -> list(mqtt_subscription()).
+subscribers(Topic) ->
-lookup(topic, Topic) when is_binary(Topic) ->
+    case ets:member(subscriber, Topic) of
-    mnesia:dirty_read(topic, Topic);
+        true -> %% faster then lookup?
+            try ets:lookup_element(subscriber, Topic, 2) catch error:badarg -> [] end;
+        false ->
+            []
+    end.
 
-lookup(subscription, SubId) when is_binary(SubId) ->
+%% @private
-    mnesia:dirty_read(subscription, SubId).
+%% @doc Ingore $SYS Messages.
+dropped(<<"$SYS/", _/binary>>) ->
+    ok;
+dropped(_Topic) ->
+    emqttd_metrics:inc('messages/dropped').
 
-%% @doc Delete Topic or Subscription.
+%% @doc Unsubscribe
--spec delete(topic, emqttd_topic:topic()) -> ok | {error, any()};
+-spec(unsubscribe(binary(), pid()) -> ok).
-            (subscription, binary() | {binary(), emqttd_topic:topic()}) -> ok.
+unsubscribe(Topic, SubPid) when is_binary(Topic) ->
-delete(topic, _Topic) ->
+    call(pick(Topic), {unsubscribe, Topic, SubPid}).
-    {error, unsupported};
 
-delete(subscription, SubId) when is_binary(SubId) ->
+%% @doc Asynchronous Unsubscribe
-    mnesia:dirty_delete({subscription, SubId});
+-spec(async_unsubscribe(binary(), pid()) -> ok).
+async_unsubscribe(Topic, SubPid)  when is_binary(Topic) ->
+    cast(pick(Topic), {unsubscribe, Topic, SubPid}).
 
-delete(subscription, {SubId, Topic}) when is_binary(SubId) andalso is_binary(Topic) ->
+call(PubSub, Req) when is_pid(PubSub) ->
-    mnesia:async_dirty(fun remove_subscriptions/2, [SubId, [Topic]]).
+    gen_server2:call(PubSub, Req, infinity).
 
-%% @doc Subscribe Topics
+cast(PubSub, Msg) when is_pid(PubSub) ->
--spec subscribe({Topic, Qos} | list({Topic, Qos})) ->
+    gen_server2:cast(PubSub, Msg).
-    {ok, Qos | list(Qos)} | {error, any()} when
-    Topic   :: binary(),
-    Qos     :: mqtt_qos() | mqtt_qos_name().
-subscribe({Topic, Qos}) ->
-    subscribe([{Topic, Qos}]);
-subscribe(TopicTable) when is_list(TopicTable) ->
-    call({subscribe, {undefined, self()}, fixqos(TopicTable)}).
 
--spec subscribe(ClientId, {Topic, Qos} | list({Topic, Qos})) ->
+pick(Topic) ->
-    {ok, Qos | list(Qos)} | {error, any()} when
+    gproc_pool:pick_worker(pubsub, Topic).
-    ClientId :: binary(),
-    Topic    :: binary(),
-    Qos      :: mqtt_qos() | mqtt_qos_name().
-subscribe(ClientId, {Topic, Qos}) when is_binary(ClientId) ->
-    subscribe(ClientId, [{Topic, Qos}]);
-subscribe(ClientId, TopicTable) when is_binary(ClientId) andalso is_list(TopicTable) ->
-    call({subscribe, {ClientId, self()}, fixqos(TopicTable)}).
-
-fixqos(TopicTable) ->
-    [{Topic, ?QOS_I(Qos)} || {Topic, Qos} <- TopicTable].
-
-%% @doc Unsubscribe Topic or Topics
--spec unsubscribe(emqttd_topic:topic() | list(emqttd_topic:topic())) -> ok.
-unsubscribe(Topic) when is_binary(Topic) ->
-    unsubscribe([Topic]);
-unsubscribe(Topics = [Topic|_]) when is_binary(Topic) ->
-    cast({unsubscribe, {undefined, self()}, Topics}).
-
--spec unsubscribe(binary(), emqttd_topic:topic() | list(emqttd_topic:topic())) -> ok.
-unsubscribe(ClientId, Topic) when is_binary(ClientId) andalso is_binary(Topic) ->
-    unsubscribe(ClientId, [Topic]);
-unsubscribe(ClientId, Topics = [Topic|_]) when is_binary(Topic) ->
-    cast({unsubscribe, {ClientId, self()}, Topics}).
-
-call(Request) ->
-    gen_server2:call(pick(self()), Request, infinity).
-
-cast(Msg) ->
-    gen_server2:cast(pick(self()), Msg).
-
-pick(Self) -> gproc_pool:pick_worker(pubsub, Self).
-
-%% @doc Publish to cluster nodes
--spec publish(Msg :: mqtt_message()) -> ok.
-publish(Msg = #mqtt_message{from = From}) ->
-    trace(publish, From, Msg),
-    Msg1 = #mqtt_message{topic = To}
-               = emqttd_broker:foldl_hooks('message.publish', [], Msg),
-
-    %% Retain message first. Don't create retained topic.
-    case emqttd_retainer:retain(Msg1) of
-        ok ->
-            %% TODO: why unset 'retain' flag?
-            publish(To, emqttd_message:unset_flag(Msg1));
-        ignore ->
-            publish(To, Msg1)
-     end.
-
-publish(To, Msg) ->
-    lists:foreach(fun(#mqtt_topic{topic = Topic, node = Node}) ->
-                    case Node =:= node() of
-                        true  -> ?ROUTER:route(Topic, Msg);
-                        false -> rpc:cast(Node, ?ROUTER, route, [Topic, Msg])
-                    end
-                  end, match(To)).
-
-%% @doc Match Topic Name with Topic Filters
--spec match(emqttd_topic:topic()) -> [mqtt_topic()].
-match(To) ->
-    Matched = mnesia:async_dirty(fun emqttd_trie:match/1, [To]),
-    %% ets:lookup for topic table will be replicated to all nodes.
-    lists:append([ets:lookup(topic, Topic) || Topic <- [To | Matched]]).
 
 %%--------------------------------------------------------------------
-%% gen_server callbacks
+%% gen_server Callbacks
 %%--------------------------------------------------------------------
 
-init([Pool, Id, StatsFun, _Opts]) ->
+init([Pool, Id, Env]) ->
     ?GPROC_POOL(join, Pool, Id),
-    {ok, #state{pool = Pool, id = Id, statsfun = StatsFun}}.
+    {ok, #state{pool = Pool, id = Id, env = Env}}.
 
-handle_call({subscribe, {SubId, SubPid}, TopicTable}, _From,
+handle_call({subscribe, Topic, SubPid}, _From, State) ->
-            State = #state{statsfun = StatsFun}) ->
+    add_subscriber_(Topic, SubPid),
+	{reply, ok, setstats(State)};
 
-    %% Monitor SubPid first
+handle_call({unsubscribe, Topic, SubPid}, _From, State) ->
-    try_monitor(SubPid),
+    del_subscriber_(Topic, SubPid),
-
+	{reply, ok, setstats(State)};
-    %% Topics
-    Topics = [Topic || {Topic, _Qos} <- TopicTable],
-
-    NewTopics = Topics -- reverse_routes(SubPid),
-
-    %% Add routes
-    ?ROUTER:add_routes(NewTopics, SubPid),
-
-    insert_reverse_routes(SubPid, NewTopics),
-
-    StatsFun(reverse_route),
-
-    %% Insert topic records to mnesia
-    Records = [#mqtt_topic{topic = Topic, node = node()} || Topic <- NewTopics],
-
-    case mnesia:transaction(fun add_topics/1, [Records]) of
-        {atomic, _} ->
-            StatsFun(topic),
-            if_subscription(
-                fun(_) ->
-                    %% Add subscriptions
-                    Args = [fun add_subscriptions/2, [SubId, TopicTable]],
-                    emqttd_pooler:async_submit({mnesia, async_dirty, Args}),
-                    StatsFun(subscription)
-                end),
-            %% Grant all qos...
-            {reply, {ok, [Qos || {_Topic, Qos} <- TopicTable]}, State};
-        {aborted, Error} ->
-            {reply, {error, Error}, State}
-    end;
 
 handle_call(Req, _From, State) ->
-   ?UNEXPECTED_REQ(Req, State).
+    ?UNEXPECTED_REQ(Req, State).
 
-handle_cast({unsubscribe, {SubId, SubPid}, Topics}, State = #state{statsfun = StatsFun}) ->
+handle_cast({subscribe, Topic, SubPid}, State) ->
+    add_subscriber_(Topic, SubPid),
+	{noreply, setstats(State)};
 
-    %% Delete routes first
+handle_cast({unsubscribe, Topic, SubPid}, State) ->
-    ?ROUTER:delete_routes(Topics, SubPid),
+    del_subscriber_(Topic, SubPid),
-
+	{noreply, setstats(State)};
-    delete_reverse_routes(SubPid, Topics),
-
-    StatsFun(reverse_route),
-
-    %% Remove subscriptions
-    if_subscription(
-        fun(_) ->
-            Args = [fun remove_subscriptions/2, [SubId, Topics]],
-            emqttd_pooler:async_submit({mnesia, async_dirty, Args}),
-            StatsFun(subscription)
-        end),
-
-    {noreply, State};
 
 handle_cast(Msg, State) ->
     ?UNEXPECTED_MSG(Msg, State).
 
-handle_info({'DOWN', _Mon, _Type, DownPid, _Info}, State = #state{statsfun = StatsFun}) ->
-
-    Topics = reverse_routes(DownPid),
-
-    ?ROUTER:delete_routes(Topics, DownPid),
-
-    delete_reverse_routes(DownPid),
-
-    StatsFun(reverse_route),
-
-    {noreply, State, hibernate};
-
 handle_info(Info, State) ->
     ?UNEXPECTED_INFO(Info, State).
 
@@ -317,98 +203,64 @@ terminate(_Reason, #state{pool = Pool, id = Id}) ->
     ?GPROC_POOL(leave, Pool, Id).
 
 code_change(_OldVsn, State, _Extra) ->
-    {ok, State}.
+	{ok, State}.
 
 %%--------------------------------------------------------------------
-%% Internal functions
+%% Internal Functions
 %%--------------------------------------------------------------------
 
-add_topics(Records) ->
+add_subscriber_(Topic, SubPid) ->
-    lists:foreach(fun add_topic/1, Records).
+    case ets:member(subscriber, Topic) of
-
+        false ->
-add_topic(TopicR = #mqtt_topic{topic = Topic}) ->
+            mnesia:transaction(fun add_topic_/1, [Topic]),
-    case mnesia:wread({topic, Topic}) of
+            emqttd_router:add_route(Topic, node()),
-        [] ->
+            setstats(topic);
-            case emqttd_topic:wildcard(Topic) of
-                true  -> emqttd_trie:insert(Topic);
-                false -> ok
-            end,
-            mnesia:write(topic, TopicR, write);
-        Records ->
-            case lists:member(TopicR, Records) of
-                true  -> ok;
-                false -> mnesia:write(topic, TopicR, write)
-            end
-    end.
-
-add_subscriptions(undefined, _TopicTable) ->
-    ok;
-add_subscriptions(SubId, TopicTable) ->
-    lists:foreach(fun({Topic, Qos}) ->
-                    add_subscription(SubId, {Topic, Qos})
-                  end,TopicTable).
-
-add_subscription(SubId, {Topic, Qos}) ->
-    Subscription = #mqtt_subscription{subid = SubId, topic = Topic, qos = Qos},
-    Pattern = #mqtt_subscription{subid = SubId, topic = Topic, qos = '_'},
-    Records = mnesia:match_object(subscription, Pattern, write),
-    case lists:member(Subscription, Records) of
         true ->
-            ok;
+            ok
-        false ->
+    end,
-            [delete_subscription(Record) || Record <- Records],
+    ets:insert(subscriber, {Topic, SubPid}).
-            insert_subscription(Subscription)
+
+del_subscriber_(Topic, SubPid) ->
+    ets:delete_object(subscriber, {Topic, SubPid}),
+    case ets:lookup(subscriber, Topic) of
+        [] ->
+            emqttd_router:del_route(Topic, node()),
+            mnesia:transaction(fun del_topic_/1, [Topic]),
+            setstats(topic);
+        [_|_] ->
+            ok
     end.
 
-insert_subscription(Record) ->
+add_topic_(Topic) ->
-    mnesia:write(subscription, Record, write).
+    add_topic_(Topic, []).
 
-remove_subscriptions(undefined, _Topics) ->
+add_topic_(Topic, Flags) ->
-    ok;
+    Record = #mqtt_topic{topic = Topic, flags = Flags},
-remove_subscriptions(SubId, Topics) ->
+    case mnesia:wread({topic, Topic}) of
-    lists:foreach(fun(Topic) ->
+        []  -> mnesia:write(topic, Record, write);
-         Pattern = #mqtt_subscription{subid = SubId, topic = Topic, qos = '_'},
+        [_] -> ok
-         Records = mnesia:match_object(subscription, Pattern, write),
-         lists:foreach(fun delete_subscription/1, Records)
-     end, Topics).
-
-delete_subscription(Record) ->
-    mnesia:delete_object(subscription, Record, write).
-
-reverse_routes(SubPid) ->
-    case ets:member(reverse_route, SubPid) of
-        true  ->
-            try ets:lookup_element(reverse_route, SubPid, 2) catch error:badarg -> [] end;
-        false ->
-            []
     end.
 
-insert_reverse_routes(SubPid, Topics) ->
+del_topic_(Topic) ->
-    ets:insert(reverse_route, [{SubPid, Topic} || Topic <- Topics]).
+    case emqttd_router:has_route(Topic) of
-
+        true  -> ok;
-delete_reverse_routes(SubPid, Topics) ->
+        false -> do_del_topic_(Topic)
-    lists:foreach(fun(Topic) ->
-                ets:delete_object(reverse_route, {SubPid, Topic})
-        end, Topics).
-
-delete_reverse_routes(SubPid) ->
-    ets:delete(reverse_route, SubPid).
-
-try_monitor(SubPid) ->
-    case ets:member(reverse_route, SubPid) of
-        true  -> ignore;
-        false -> erlang:monitor(process, SubPid)
     end.
 
-%%--------------------------------------------------------------------
+do_del_topic_(Topic) ->
-%% Trace Functions
+    case mnesia:wread({topic, Topic}) of
-%%--------------------------------------------------------------------
+        [#mqtt_topic{flags = []}] ->
+            mnesia:delete(topic, Topic, write);
+        _ ->
+            ok
+    end.
 
-trace(publish, From, _Msg) when is_atom(From) ->
+setstats(State) when is_record(State, state) ->
-    %% Dont' trace '$SYS' publish
+    setstats(subscriber), State;
-    ignore;
 
-trace(publish, From, #mqtt_message{topic = Topic, payload = Payload}) ->
+setstats(topic) ->
-    lager:info([{client, From}, {topic, Topic}],
+    emqttd_stats:setstats('topics/count', 'topics/max', mnesia:table_info(topic, size));
-               "~s PUBLISH to ~s: ~p", [From, Topic, Payload]).
+
+setstats(subscriber) ->
+    emqttd_stats:setstats('subscribers/count', 'subscribers/max', ets:info(subscriber, size)).
 

src/emqttd_pubsub_helper.erl

@@ -1,76 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2012-2016 Feng Lee <feng@emqtt.io>.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%%     http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
-
-%%% @doc PubSub Helper.
--module(emqttd_pubsub_helper).
-
--behaviour(gen_server).
-
--include("emqttd.hrl").
-
--include("emqttd_internal.hrl").
-
-%% API Function Exports
--export([start_link/1]).
-
-%% gen_server Function Exports
--export([init/1, handle_call/3, handle_cast/2, handle_info/2,
-         terminate/2, code_change/3]).
-
--record(state, {statsfun}).
-
--define(SERVER, ?MODULE).
-
-%% @doc Start PubSub Helper.
--spec start_link(fun()) -> {ok, pid()} | ignore | {error, any()}.
-start_link(StatsFun) ->
-    gen_server:start_link({local, ?SERVER}, ?MODULE, [StatsFun], []).
-
-%%--------------------------------------------------------------------
-%% gen_server callbacks
-%%--------------------------------------------------------------------
-
-init([StatsFun]) ->
-    mnesia:subscribe(system),
-    {ok, #state{statsfun = StatsFun}}.
-
-handle_call(Req, _From, State) ->
-    ?UNEXPECTED_REQ(Req, State).
-
-handle_cast(Msg, State) ->
-    ?UNEXPECTED_MSG(Msg, State).
-
-handle_info({mnesia_system_event, {mnesia_down, Node}}, State) ->
-    %% TODO: mnesia master?
-    Pattern = #mqtt_topic{_ = '_', node = Node},
-    F = fun() ->
-            [mnesia:delete_object(topic, R, write) ||
-                R <- mnesia:match_object(topic, Pattern, write)]
-        end,
-    mnesia:transaction(F), noreply(State);
-
-handle_info(Info, State) ->
-    ?UNEXPECTED_INFO(Info, State).
-
-terminate(_Reason, _State) ->
-    mnesia:unsubscribe(system).
-
-code_change(_OldVsn, State, _Extra) ->
-    {ok, State}.
-
-noreply(State = #state{statsfun = StatsFun}) ->
-    StatsFun(topic), {noreply, State}.
-

src/emqttd_pubsub_sup.erl

@@ -21,8 +21,6 @@
 
 -include("emqttd.hrl").
 
--define(HELPER, emqttd_pubsub_helper).
-
 -define(CONCURRENCY_OPTS, [{read_concurrency, true}, {write_concurrency, true}]).
 
 %% API
@@ -38,33 +36,37 @@ pubsub_pool() ->
     hd([Pid|| {pubsub_pool, Pid, _, _} <- supervisor:which_children(?MODULE)]).
 
 init([Env]) ->
-    %% Create tabs
-    create_tab(route), create_tab(reverse_route),
 
-    %% PubSub Helper
+    %% Create ETS Tabs
-    Helper = {helper, {?HELPER, start_link, [fun setstats/1]},
+    create_tab(subscriber), create_tab(subscribed),
-                permanent, infinity, worker, [?HELPER]},
 
-    %% Router Pool Sup
+    %% Router
-    RouterMFA = {emqttd_router, start_link, [fun setstats/1, Env]},
+    Router = {router, {emqttd_router, start_link, []},
-
+                permanent, 5000, worker, [emqttd_router]},
-    %% Pool_size / 2
-    RouterSup = emqttd_pool_sup:spec(router_pool, [router, hash, router_pool(Env), RouterMFA]),
 
     %% PubSub Pool Sup
-    PubSubMFA = {emqttd_pubsub, start_link, [fun setstats/1, Env]},
+    PubSubMFA = {emqttd_pubsub, start_link, [Env]},
-    PubSubSup = emqttd_pool_sup:spec(pubsub_pool, [pubsub, hash, pool_size(Env), PubSubMFA]),
+    PubSubPoolSup = emqttd_pool_sup:spec(pubsub_pool, [pubsub, hash, pool_size(Env), PubSubMFA]),
 
-    {ok, {{one_for_all, 10, 60}, [Helper, RouterSup, PubSubSup]}}.
+    %% Server Pool Sup
+    ServerMFA = {emqttd_server, start_link, [Env]},
+    ServerPoolSup = emqttd_pool_sup:spec(server_pool, [server, hash, pool_size(Env), ServerMFA]),
 
-create_tab(route) ->
+    {ok, {{one_for_all, 5, 60}, [Router, PubSubPoolSup, ServerPoolSup]}}.
-    %% Route Table: Topic -> Pid1, Pid2, ..., PidN
+
+pool_size(Env) ->
+    Schedulers = erlang:system_info(schedulers),
+    proplists:get_value(pool_size, Env, Schedulers).
+
+create_tab(subscriber) ->
+    %% subscriber: Topic -> Pid1, Pid2, ..., PidN
     %% duplicate_bag: o(1) insert
-    ensure_tab(route, [public, named_table, duplicate_bag | ?CONCURRENCY_OPTS]);
+    ensure_tab(subscriber, [public, named_table, duplicate_bag | ?CONCURRENCY_OPTS]);
 
-create_tab(reverse_route) ->
+create_tab(subscribed) ->
-    %% Reverse Route Table: Pid -> Topic1, Topic2, ..., TopicN
+    %% subscribed: Pid -> Topic1, Topic2, ..., TopicN
-    ensure_tab(reverse_route, [public, named_table, bag | ?CONCURRENCY_OPTS]).
+    %% bag: o(n) insert
+    ensure_tab(subscribed, [public, named_table, bag | ?CONCURRENCY_OPTS]).
 
 ensure_tab(Tab, Opts) ->
     case ets:info(Tab, name) of
@@ -72,26 +74,3 @@ ensure_tab(Tab, Opts) ->
         _ -> ok
     end.
 
-router_pool(Env) ->
-    case pool_size(Env) div 2 of
-        0 -> 1;
-        I -> I
-    end.
-
-pool_size(Env) ->
-    Schedulers = erlang:system_info(schedulers),
-    proplists:get_value(pool_size, Env, Schedulers).
-
-setstats(route) ->
-    emqttd_stats:setstat('routes/count', ets:info(route, size));
-
-setstats(reverse_route) ->
-    emqttd_stats:setstat('routes/reverse', ets:info(reverse_route, size));
-
-setstats(topic) ->
-    emqttd_stats:setstats('topics/count', 'topics/max', mnesia:table_info(topic, size));
-
-setstats(subscription) ->
-    emqttd_stats:setstats('subscriptions/count', 'subscriptions/max',
-                          mnesia:table_info(subscription, size)).
-

src/emqttd_retainer.erl

@@ -14,8 +14,7 @@
 %% limitations under the License.
 %%--------------------------------------------------------------------
 
-%% TODO: should match topic tree
+%% @doc MQTT retained message.
-%% @doc MQTT retained message storage.
 -module(emqttd_retainer).
 
 -behaviour(gen_server).
@@ -53,7 +52,7 @@
 mnesia(boot) ->
     ok = emqttd_mnesia:create_table(retained, [
                 {type, ordered_set},
-                {ram_copies, [node()]},
+                {disc_copies, [node()]},
                 {record_name, mqtt_retained},
                 {attributes, record_info(fields, mqtt_retained)}]);
 mnesia(copy) ->

src/emqttd_router.erl

@@ -14,268 +14,212 @@
 %% limitations under the License.
 %%--------------------------------------------------------------------
 
-%% @doc
-%% The Message Router on Local Node.
-%% @end
 -module(emqttd_router).
 
--behaviour(gen_server2).
+-behaviour(gen_server).
 
 -include("emqttd.hrl").
 
--include("emqttd_protocol.hrl").
+%% Mnesia Bootstrap
+-export([mnesia/1]).
 
--include("emqttd_internal.hrl").
+-boot_mnesia({mnesia, [boot]}).
+-copy_mnesia({mnesia, [copy]}).
 
--export([start_link/4]).
+-export([start_link/0, stop/0]).
 
-%% Route API
+-export([add_route/1, add_route/2, add_routes/1, lookup/1, print/1,
--export([route/2]).
+         del_route/1, del_route/2, del_routes/1, has_route/1]).
 
-%% Route Admin API
+%% gen_server Function Exports
--export([add_route/2, lookup_routes/1, has_route/1, delete_route/2]).
-
-%% Batch API
--export([add_routes/2, delete_routes/2]).
-
-%% For Test
--export([stop/1]).
-
-%% gen_server Callbacks
 -export([init/1, handle_call/3, handle_cast/2, handle_info/2,
          terminate/2, code_change/3]).
 
--record(aging, {topics, time, tref}).
+-record(state, {}).
 
--record(state, {pool, id, aging :: #aging{}, statsfun}).
+%%--------------------------------------------------------------------
+%% Mnesia Bootstrap
+%%--------------------------------------------------------------------
 
-%% @doc Start a router.
+mnesia(boot) ->
--spec start_link(atom(), pos_integer(), fun((atom()) -> ok), list()) -> {ok, pid()} | {error, any()}.
+    ok = emqttd_mnesia:create_table(route, [
-start_link(Pool, Id, StatsFun, Env) ->
+                {type, bag},
-    gen_server2:start_link({local, ?PROC_NAME(?MODULE, Id)},
+                {ram_copies, [node()]},
-                           ?MODULE, [Pool, Id, StatsFun, Env], []).
+                {record_name, mqtt_route},
+                {attributes, record_info(fields, mqtt_route)}]);
 
-%% @doc Route Message on this node.
+mnesia(copy) ->
--spec route(emqttd_topic:topic(), mqtt_message()) -> any().
+    ok = emqttd_mnesia:copy_table(route, ram_copies).
-route(Queue = <<"$Q/", _Q>>, Msg) ->
-    case lookup_routes(Queue) of
-        [] ->
-            dropped(Queue);
-        [SubPid] ->
-            SubPid ! {dispatch, Queue, Msg};
-        Routes ->
-            Idx = crypto:rand_uniform(1, length(Routes) + 1),
-            SubPid = lists:nth(Idx, Routes),
-            SubPid ! {dispatch, Queue, Msg}
-    end;
 
-route(Topic, Msg) ->
+%%--------------------------------------------------------------------
-    case lookup_routes(Topic) of
+%% Start the Router
-        [] ->
+%%--------------------------------------------------------------------
-            dropped(Topic);
+
-        [SubPid] ->
+start_link() ->
-            SubPid ! {dispatch, Topic, Msg};
+    gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
-        Routes ->
+
-            lists:foreach(fun(SubPid) ->
+%%--------------------------------------------------------------------
-                SubPid ! {dispatch, Topic, Msg}
+%% API
-            end, Routes)
+%%--------------------------------------------------------------------
+
+%% @doc Lookup Routes.
+-spec(lookup(Topic:: binary()) -> [mqtt_route()]).
+lookup(Topic) when is_binary(Topic) ->
+    Matched = mnesia:async_dirty(fun emqttd_trie:match/1, [Topic]),
+    %% Optimize: route table will be replicated to all nodes.
+    lists:append([ets:lookup(route, To) || To <- [Topic | Matched]]).
+
+%% @doc Print Routes.
+-spec(print(Topic :: binary()) -> [ok]).
+print(Topic) ->
+    [io:format("~s -> ~s~n", [To, Node]) ||
+        #mqtt_route{topic = To, node = Node} <- lookup(Topic)].
+
+%% @doc Add Route
+-spec(add_route(binary() | mqtt_route()) -> ok | {error, Reason :: any()}).
+add_route(Topic) when is_binary(Topic) ->
+    add_route(#mqtt_route{topic = Topic, node = node()});
+add_route(Route) when is_record(Route, mqtt_route) ->
+    add_routes([Route]).
+
+-spec(add_route(Topic :: binary(), Node :: node()) -> ok | {error, Reason :: any()}).
+add_route(Topic, Node) when is_binary(Topic), is_atom(Node) ->
+    add_route(#mqtt_route{topic = Topic, node = Node}).
+
+%% @doc Add Routes
+-spec(add_routes([mqtt_route()]) -> ok | {errory, Reason :: any()}).
+add_routes(Routes) ->
+    Add = fun() -> [add_route_(Route) || Route <- Routes] end,
+    case mnesia:transaction(Add) of
+        {atomic, _}      -> update_stats_(), ok;
+        {aborted, Error} -> {error, Error}
     end.
 
 %% @private
-%% @doc Ingore $SYS Messages.
+add_route_(Route = #mqtt_route{topic = Topic}) ->
-dropped(<<"$SYS/", _/binary>>) ->
+    case mnesia:wread({route, Topic}) of
-    ok;
+        [] ->
-dropped(_Topic) ->
+            case emqttd_topic:wildcard(Topic) of
-    emqttd_metrics:inc('messages/dropped').
+                true  -> emqttd_trie:insert(Topic);
-
+                false -> ok
-%% @doc Has Route?
+            end,
--spec has_route(emqttd_topic:topic()) -> boolean().
+            mnesia:write(route, Route, write);
-has_route(Topic) when is_binary(Topic) ->
+        Records ->
-    ets:member(route, Topic).
+            case lists:member(Route, Records) of
-
+                true  -> ok;
-%% @doc Lookup Routes
+                false -> mnesia:write(route, Route, write)
--spec lookup_routes(emqttd_topic:topic()) -> list(pid()).
+            end
-lookup_routes(Topic) when is_binary(Topic) ->
-    case ets:member(route, Topic) of
-        true  ->
-            try ets:lookup_element(route, Topic, 2) catch error:badarg -> [] end;
-        false ->
-            []
     end.
 
-%% @doc Add Route
--spec add_route(emqttd_topic:topic(), pid()) -> ok.
-add_route(Topic, Pid) when is_pid(Pid) ->
-    call(pick(Topic), {add_route, Topic, Pid}).
-
-%% @doc Add Routes
--spec add_routes(list(emqttd_topic:topic()), pid()) -> ok.
-add_routes([], _Pid) ->
-    ok;
-add_routes([Topic], Pid) ->
-    add_route(Topic, Pid);
-
-add_routes(Topics, Pid) ->
-    lists:foreach(fun({Router, Slice}) ->
-                call(Router, {add_routes, Slice, Pid})
-        end, slice(Topics)).
-
 %% @doc Delete Route
--spec delete_route(emqttd_topic:topic(), pid()) -> ok.
+-spec(del_route(binary() | mqtt_route()) -> ok | {error, Reason :: any()}).
-delete_route(Topic, Pid) ->
+del_route(Topic) when is_binary(Topic) ->
-    cast(pick(Topic), {delete_route, Topic, Pid}).
+    del_route(#mqtt_route{topic = Topic, node = node()});
+del_route(Route) when is_record(Route, mqtt_route) ->
+    del_routes([Route]).
+
+-spec(del_route(Topic :: binary(), Node :: node()) -> ok | {error, Reason :: any()}).
+del_route(Topic, Node) when is_binary(Topic), is_atom(Node) ->
+    del_route(#mqtt_route{topic = Topic, node = Node}).
 
 %% @doc Delete Routes
--spec delete_routes(list(emqttd_topic:topic()), pid()) -> ok.
+-spec(del_routes([mqtt_route()]) -> ok | {error, any()}).
-delete_routes([Topic], Pid) ->
+del_routes(Routes) ->
-    delete_route(Topic, Pid);
+    Del = fun() -> [del_route_(Route) || Route <- Routes] end,
+    case mnesia:transaction(Del) of
+        {atomic, _}      -> update_stats_(), ok;
+        {aborted, Error} -> {error, Error}
+    end.
 
-delete_routes(Topics, Pid) ->
+del_route_(Route = #mqtt_route{topic = Topic}) ->
-    lists:foreach(fun({Router, Slice}) ->
+    case mnesia:wread({route, Topic}) of
-                cast(Router, {delete_routes, Slice, Pid})
+        [] ->
-        end, slice(Topics)).
+            ok;
+        [Route] ->
+            %% Remove route and trie
+            mnesia:delete_object(route, Route, write),
+            case emqttd_topic:wildcard(Topic) of
+                true  -> emqttd_trie:delete(Topic);
+                false -> ok
+            end;
+        _More ->
+            %% Remove route only
+            mnesia:delete_object(route, Route, write)
+    end.
 
-%% @private Slice topics.
+%% @doc Has Route?
-slice(Topics) ->
+-spec has_route(binary()) -> boolean().
-    dict:to_list(lists:foldl(fun(Topic, Dict) ->
+has_route(Topic) ->
-                    dict:append(pick(Topic), Topic, Dict)
+    Routes = case mnesia:is_transaction() of
-            end, dict:new(), Topics)).
+                 true  -> mnesia:read(route, Topic);
+                 false -> mnesia:dirty_read(route, Topic)
+             end,
+    length(Routes) > 0.
 
-%% @private Pick a router.
+stop() -> gen_server:call(?MODULE, stop).
-pick(Topic) ->
-    gproc_pool:pick_worker(router, Topic).
 
-%% @doc For unit test.
+%%--------------------------------------------------------------------
-stop(Id) when is_integer(Id) ->
+%% gen_server Callbacks
-    gen_server2:call(?PROC_NAME(?MODULE, Id), stop);
+%%--------------------------------------------------------------------
-stop(Pid) when is_pid(Pid) ->
-    gen_server2:call(Pid, stop).
 
-call(Router, Request) ->
+init([]) ->
-    gen_server2:call(Router, Request, infinity).
+    mnesia:subscribe(system),
-
+    {ok, #state{}}.
-cast(Router, Msg) ->
-    gen_server2:cast(Router, Msg).
-
-init([Pool, Id, StatsFun, Opts]) ->
-
-    emqttd_time:seed(),
-
-    ?GPROC_POOL(join, Pool, Id),
-
-    Aging = init_aging(Opts),
-
-    {ok, #state{pool = Pool, id = Id, aging = Aging, statsfun = StatsFun}}.
-
-%% Init Aging
-init_aging(Opts) ->
-    AgingSecs = proplists:get_value(route_aging, Opts, 5),
-    {ok, AgingTref} = start_tick(AgingSecs + random:uniform(AgingSecs)),
-    #aging{topics = dict:new(), time = AgingSecs, tref = AgingTref}.
-
-start_tick(Secs) ->
-    timer:send_interval(timer:seconds(Secs), {clean, aged}).
 
 handle_call(stop, _From, State) ->
     {stop, normal, ok, State};
 
-handle_call({add_route, Topic, Pid}, _From, State) ->
+handle_call(_Req, _From, State) ->
-    ets:insert(route, {Topic, Pid}),
+    {reply, ignore, State}.
-    {reply, ok, setstats(State)};
 
-handle_call({add_routes, Topics, Pid}, _From, State) ->
+handle_cast(_Msg, State) ->
-    ets:insert(route, [{Topic, Pid} || Topic <- Topics]),
+    {noreply, State}.
-    {reply, ok, setstats(State)};
 
-handle_call(Req, _From, State) ->
+handle_info({mnesia_system_event, {mnesia_up, Node}}, State) ->
-   ?UNEXPECTED_REQ(Req, State).
+    lager:error("Mnesia up: ~p~n", [Node]),
+    {noreply, State};
 
-handle_cast({delete_route, Topic, Pid}, State = #state{aging = Aging}) ->
+handle_info({mnesia_system_event, {mnesia_down, Node}}, State) ->
-    Aging1 = delete_route_(Topic, Pid, Aging),
+    lager:error("Mnesia down: ~p~n", [Node]),
-    {noreply, setstats(State#state{aging = Aging1})};
+    clean_routes_(Node),
+    update_stats_(),
+    {noreply, State};
 
-handle_cast({delete_routes, Topics, Pid}, State) ->
+handle_info({mnesia_system_event, {inconsistent_database, Context, Node}}, State) ->
-    Aging1 =
+    %% 1. Backup and restart
-    lists:foldl(fun(Topic, Aging) ->
+    %% 2. Set master nodes
-                    delete_route_(Topic, Pid, Aging)
+    lager:critical("Mnesia inconsistent_database event: ~p, ~p~n", [Context, Node]),
-            end, State#state.aging, Topics),
+    {noreply, State};
-    {noreply, setstats(State#state{aging = Aging1})};
 
-handle_cast(Msg, State) ->
+handle_info({mnesia_system_event, {mnesia_overload, Details}}, State) ->
-    ?UNEXPECTED_MSG(Msg, State).
+    lager:critical("Mnesia overload: ~p~n", [Details]),
+    {noreply, State};
 
-handle_info({clean, aged}, State = #state{aging = Aging}) ->
+handle_info({mnesia_system_event, _Event}, State) ->
+    {noreply, State};
 
-    #aging{topics = Dict, time = Time} = Aging,
+handle_info(_Info, State) ->
+    {noreply, State}.
 
-    ByTime  = emqttd_time:now_to_secs() - Time,
+terminate(_Reason, _State) ->
-
+    mnesia:unsubscribe(system).
-    Dict1 = try_clean(ByTime, dict:to_list(Dict)),
-
-    Aging1 = Aging#aging{topics = dict:from_list(Dict1)},
-
-    {noreply, State#state{aging = Aging1}, hibernate};
-
-handle_info(Info, State) ->
-    ?UNEXPECTED_INFO(Info, State).
-
-terminate(_Reason, #state{pool = Pool, id = Id, aging = #aging{tref = TRef}}) ->
-    timer:cancel(TRef),
-    ?GPROC_POOL(leave, Pool, Id).
 
 code_change(_OldVsn, State, _Extra) ->
     {ok, State}.
 
-delete_route_(Topic, Pid, Aging) ->
+%%--------------------------------------------------------------------
-    ets:delete_object(route, {Topic, Pid}),
+%% Internal Functions
-    case has_route(Topic) of
+%%--------------------------------------------------------------------
-        false -> store_aged(Topic, Aging);
-        true  -> Aging
-    end.
 
-try_clean(ByTime, List) ->
+%% Clean Routes on Node
-    try_clean(ByTime, List, []).
+clean_routes_(Node) ->
+    Pattern = #mqtt_route{_ = '_', node = Node},
+    Clean = fun() ->
+                [mnesia:delete_object(route, R, write) ||
+                    R <- mnesia:match_object(route, Pattern, write)]
+            end,
+    mnesia:transaction(Clean).
 
-try_clean(_ByTime, [], Acc) ->
+update_stats_() ->
-    Acc;
+    emqttd_stats:setstats('routes/count', 'routes/max', mnesia:table_info(route, size)).
-
-try_clean(ByTime, [{Topic, TS} | Left], Acc) ->
-    case has_route(Topic) of
-        false ->
-            try_clean2(ByTime, {Topic, TS}, Left, Acc);
-        true  ->
-            try_clean(ByTime, Left, Acc)
-    end.
-
-try_clean2(ByTime, {Topic, TS}, Left, Acc) when TS > ByTime ->
-    try_clean(ByTime, Left, [{Topic, TS} | Acc]);
-
-try_clean2(ByTime, {Topic, _TS}, Left, Acc) ->
-    TopicR = #mqtt_topic{topic = Topic, node = node()},
-    case mnesia:transaction(fun try_remove_topic/1, [TopicR]) of
-        {atomic, _}      -> ok;
-        {aborted, Error} -> lager:error("Clean Topic '~s' Error: ~p", [Topic, Error])
-    end,
-    try_clean(ByTime, Left, Acc).
-
-try_remove_topic(TopicR = #mqtt_topic{topic = Topic}) ->
-    %% Lock topic first
-    case mnesia:wread({topic, Topic}) of
-        []       -> ok;
-        [TopicR] -> %% Remove topic and trie
-                    delete_topic(TopicR),
-                    emqttd_trie:delete(Topic);
-        _More    -> %% Remove topic only
-                    delete_topic(TopicR)
-    end.
-
-delete_topic(TopicR) ->
-    mnesia:delete_object(topic, TopicR, write).
-
-store_aged(Topic, Aging = #aging{topics = Dict}) ->
-    Now = emqttd_time:now_to_secs(),
-    Aging#aging{topics = dict:store(Topic, Now, Dict)}.
-
-setstats(State = #state{statsfun = StatsFun}) ->
-    StatsFun(route), State.
 

src/emqttd_server.erl

@@ -0,0 +1,263 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2012-2016 Feng Lee <feng@emqtt.io>.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqttd_server).
+
+-behaviour(gen_server2).
+
+-include("emqttd.hrl").
+
+-include("emqttd_protocol.hrl").
+
+-include("emqttd_internal.hrl").
+
+%% Mnesia Callbacks
+-export([mnesia/1]).
+
+-boot_mnesia({mnesia, [boot]}).
+-copy_mnesia({mnesia, [copy]}).
+
+%% API Exports
+-export([start_link/3]).
+
+%% PubSub API
+-export([subscribe/1, subscribe/3, publish/1, unsubscribe/1, unsubscribe/3,
+         lookup_subscription/1, update_subscription/4]).
+
+%% gen_server Function Exports
+-export([init/1, handle_call/3, handle_cast/2, handle_info/2,
+         terminate/2, code_change/3]).
+
+-record(state, {pool, id, env, monitors}).
+
+%%--------------------------------------------------------------------
+%% Mnesia callbacks
+%%--------------------------------------------------------------------
+
+mnesia(boot) ->
+    ok = emqttd_mnesia:create_table(subscription, [
+                {type, bag},
+                {ram_copies, [node()]},
+                {local_content, true}, %% subscription table is local
+                {record_name, mqtt_subscription},
+                {attributes, record_info(fields, mqtt_subscription)}]);
+
+mnesia(copy) ->
+    ok = emqttd_mnesia:copy_table(subscription).
+
+%%--------------------------------------------------------------------
+%% Start server
+%%--------------------------------------------------------------------
+
+%% @doc Start a Server
+-spec(start_link(Pool, Id, Env) -> {ok, pid()} | ignore | {error, any()} when
+      Pool :: atom(),
+      Id   :: pos_integer(),
+      Env  :: list(tuple())).
+start_link(Pool, Id, Env) ->
+    gen_server2:start_link({local, ?PROC_NAME(?MODULE, Id)}, ?MODULE, [Pool, Id, Env], []).
+
+%%--------------------------------------------------------------------
+%% PubSub API
+%%--------------------------------------------------------------------
+
+%% @doc Subscribe a Topic
+-spec(subscribe(binary()) -> ok).
+subscribe(Topic) when is_binary(Topic) ->
+    From = self(), call(server(From), {subscribe, From, Topic}).
+
+%% @doc Subscribe from a MQTT session.
+-spec(subscribe(binary(), binary(), mqtt_qos()) -> ok).
+subscribe(ClientId, Topic, Qos) ->
+    From = self(), call(server(From), {subscribe, From, ClientId, Topic, ?QOS_I(Qos)}).
+
+%% @doc Lookup subscriptions.
+-spec(lookup_subscription(binary()) -> [#mqtt_subscription{}]).
+lookup_subscription(ClientId) ->
+    mnesia:dirty_read(subscription, ClientId).
+
+%% @doc Update a subscription.
+-spec(update_subscription(binary(), binary(), mqtt_qos(), mqtt_qos()) -> ok).
+update_subscription(ClientId, Topic, OldQos, NewQos) ->
+    call(server(self()), {update_subscription, ClientId, Topic, ?QOS_I(OldQos), ?QOS_I(NewQos)}).
+
+%% @doc Publish a Message
+-spec(publish(Msg :: mqtt_message()) -> any()).
+publish(Msg = #mqtt_message{from = From}) ->
+    trace(publish, From, Msg),
+    case emqttd:run_hooks('message.publish', [], Msg) of
+        {ok, Msg1 = #mqtt_message{topic = Topic}} ->
+            %% Retain message first. Don't create retained topic.
+            Msg2 = case emqttd_retainer:retain(Msg1) of
+                       ok     -> emqttd_message:unset_flag(Msg1);
+                       ignore -> Msg1
+                   end,
+            emqttd_pubsub:publish(Topic, Msg2);
+        {stop, Msg1} ->
+            lager:warning("Stop publishing: ~s", [emqttd_message:format(Msg1)])
+    end.
+
+%% @doc Unsubscribe a Topic
+-spec(unsubscribe(binary()) -> ok).
+unsubscribe(Topic) when is_binary(Topic) ->
+    From = self(), call(server(From), {unsubscribe, From, Topic}).
+
+%% @doc Unsubscribe a Topic from a MQTT session
+-spec(unsubscribe(binary(), binary(), mqtt_qos()) -> ok).
+unsubscribe(ClientId, Topic, Qos) ->
+    From = self(), call(server(From), {unsubscribe, From, ClientId, Topic, Qos}).
+
+call(Server, Req) ->
+    gen_server2:call(Server, Req, infinity).
+
+server(From) ->
+    gproc_pool:pick_worker(server, From).
+
+%%--------------------------------------------------------------------
+%% gen_server Callbacks
+%%--------------------------------------------------------------------
+
+init([Pool, Id, Env]) ->
+    ?GPROC_POOL(join, Pool, Id),
+    {ok, #state{pool = Pool, id = Id, env = Env, monitors = dict:new()}}.
+
+handle_call({subscribe, SubPid, ClientId, Topic, Qos}, _From, State) ->
+    add_subscription_(ClientId, Topic, Qos),
+    set_subscription_stats(),
+    do_subscribe_(SubPid, Topic),
+    ok(monitor_subscriber_(ClientId, SubPid, State));
+
+handle_call({subscribe, SubPid, Topic}, _From, State) ->
+    do_subscribe_(SubPid, Topic),
+    ok(monitor_subscriber_(undefined, SubPid, State));
+
+handle_call({update_subscription, ClientId, Topic, OldQos, NewQos}, _From, State) ->
+    OldSub = #mqtt_subscription{subid = ClientId, topic = Topic, qos = OldQos},
+    NewSub = #mqtt_subscription{subid = ClientId, topic = Topic, qos = NewQos},
+    mnesia:transaction(fun update_subscription_/2, [OldSub, NewSub]),
+    set_subscription_stats(), ok(State);
+
+handle_call({unsubscribe, SubPid, ClientId, Topic, Qos}, From, State) ->
+    del_subscription_(ClientId, Topic, Qos),
+    set_subscription_stats(),
+    handle_call({unsubscribe, SubPid, Topic}, From, State);
+
+handle_call({unsubscribe, SubPid, Topic}, _From, State) ->
+    emqttd_pubsub:unsubscribe(Topic, SubPid),
+    ets:delete_object(subscribed, {SubPid, Topic}),
+    ok(State);
+
+handle_call(Req, _From, State) ->
+    ?UNEXPECTED_REQ(Req, State).
+
+handle_cast(Msg, State) ->
+    ?UNEXPECTED_MSG(Msg, State).
+
+handle_info({'DOWN', _MRef, process, DownPid, _Reason}, State = #state{monitors = Monitors}) ->
+    %% unsubscribe
+    lists:foreach(fun({_, Topic}) ->
+                emqttd_pubsub:async_unsubscribe(Topic, DownPid)
+        end, ets:lookup(subscribed, DownPid)),
+    ets:delete(subscribed, DownPid),
+
+    %% clean subscriptions
+    case dict:find(DownPid, Monitors) of
+        {ok, {undefined, _}} -> ok;
+        {ok, {ClientId,  _}} -> mnesia:dirty_delete(subscription, ClientId);
+        error                -> ok
+    end,
+    {noreply, State#state{monitors = dict:erase(DownPid, Monitors)}};
+
+handle_info(Info, State) ->
+    ?UNEXPECTED_INFO(Info, State).
+
+terminate(_Reason, #state{pool = Pool, id = Id}) ->
+    ?GPROC_POOL(leave, Pool, Id).
+
+code_change(_OldVsn, State, _Extra) ->
+    {ok, State}.
+
+%%--------------------------------------------------------------------
+%% Internal Functions
+%%--------------------------------------------------------------------
+
+%% @private
+%% @doc Add a subscription.
+-spec(add_subscription_(binary(), binary(), mqtt_qos()) -> ok).
+add_subscription_(ClientId, Topic, Qos) ->
+    add_subscription_(#mqtt_subscription{subid = ClientId, topic = Topic, qos = Qos}).
+
+-spec(add_subscription_(mqtt_subscription()) -> ok).
+add_subscription_(Subscription) when is_record(Subscription, mqtt_subscription) ->
+    mnesia:dirty_write(subscription, Subscription).
+
+update_subscription_(OldSub, NewSub) ->
+    mnesia:delete_object(subscription, OldSub, write),
+    mnesia:write(subscription, NewSub, write).
+
+%% @private
+%% @doc Delete a subscription
+-spec(del_subscription_(binary(), binary(), mqtt_qos()) -> ok).
+del_subscription_(ClientId, Topic, Qos) ->
+    del_subscription_(#mqtt_subscription{subid = ClientId, topic = Topic, qos = Qos}).
+
+del_subscription_(Subscription) when is_record(Subscription, mqtt_subscription) ->
+    mnesia:dirty_delete_object(subscription, Subscription).
+
+%% @private
+%% @doc Call pubsub to subscribe
+do_subscribe_(SubPid, Topic) ->
+    case ets:match(subscribed, {SubPid, Topic}) of
+        [] ->
+            emqttd_pubsub:subscribe(Topic, SubPid),
+            ets:insert(subscribed, {SubPid, Topic});
+        [_] ->
+            false
+    end.
+
+monitor_subscriber_(ClientId, SubPid, State = #state{monitors = Monitors}) ->
+    case dict:find(SubPid, Monitors) of
+        {ok, _} ->
+            State;
+        error ->
+            MRef = erlang:monitor(process, SubPid),
+            State#state{monitors = dict:store(SubPid, {ClientId, MRef}, Monitors)}
+    end.
+
+%%--------------------------------------------------------------------
+%% Trace Functions
+%%--------------------------------------------------------------------
+
+trace(publish, From, _Msg) when is_atom(From) ->
+    %% Dont' trace '$SYS' publish
+    ignore;
+
+trace(publish, From, #mqtt_message{topic = Topic, payload = Payload}) ->
+    lager:info([{client, From}, {topic, Topic}],
+               "~s PUBLISH to ~s: ~p", [From, Topic, Payload]).
+
+%%--------------------------------------------------------------------
+%% Subscription Statistics
+%%--------------------------------------------------------------------
+
+set_subscription_stats() ->
+    emqttd_stats:setstats('subscriptions/count', 'subscriptions/max',
+                          mnesia:table_info(subscription, size)).
+
+%%--------------------------------------------------------------------
+
+ok(State) -> {reply, ok, State}.
+

src/emqttd_session.erl

@@ -154,6 +154,10 @@ info(SessPid) ->
 destroy(SessPid, ClientId) ->
     gen_server2:cast(SessPid, {destroy, ClientId}).
 
+%%--------------------------------------------------------------------
+%% PubSub
+%%--------------------------------------------------------------------
+
 %% @doc Subscribe Topics
 -spec subscribe(pid(), [{binary(), mqtt_qos()}]) -> ok.
 subscribe(SessPid, TopicTable) ->
@@ -171,11 +175,11 @@ subscribe(SessPid, PacketId, TopicTable) ->
 -spec publish(pid(), mqtt_message()) -> ok | {error, any()}.
 publish(_SessPid, Msg = #mqtt_message{qos = ?QOS_0}) ->
     %% publish qos0 directly
-    emqttd_pubsub:publish(Msg);
+    emqttd:publish(Msg);
 
 publish(_SessPid, Msg = #mqtt_message{qos = ?QOS_1}) ->
     %% publish qos1 directly, and client will puback automatically
-    emqttd_pubsub:publish(Msg);
+    emqttd:publish(Msg);
 
 publish(SessPid, Msg = #mqtt_message{qos = ?QOS_2}) ->
     %% publish qos2 by session 
@@ -281,63 +285,63 @@ handle_call({publish, Msg = #mqtt_message{qos = ?QOS_2, pktid = PktId}},
 handle_call(Req, _From, State) ->
     ?UNEXPECTED_REQ(Req, State).
 
-handle_cast({subscribe, TopicTable0, AckFun}, Session = #session{client_id = ClientId,
+handle_cast({subscribe, TopicTable0, AckFun}, Session = #session{client_id     = ClientId,
                                                                  subscriptions = Subscriptions}) ->
 
-    TopicTable = emqttd_broker:foldl_hooks('client.subscribe', [ClientId], TopicTable0),
+    case emqttd:run_hooks('client.subscribe', [ClientId], TopicTable0) of
+        {ok, TopicTable} ->
+            ?LOG(info, "Subscribe ~p", [TopicTable], Session),
+            Subscriptions1 = lists:foldl(
+                fun({Topic, Qos}, SubDict) ->
+                    case dict:find(Topic, SubDict) of
+                        {ok, Qos} ->
+                            ?LOG(warning, "duplicated subscribe: ~s, qos = ~w", [Topic, Qos], Session),
+                            SubDict;
+                        {ok, OldQos} ->
+                            emqttd_server:update_subscription(ClientId, Topic, OldQos, Qos),
+                            ?LOG(warning, "duplicated subscribe ~s, old_qos=~w, new_qos=~w", [Topic, OldQos, Qos], Session),
+                            dict:store(Topic, Qos, SubDict);
+                        error ->
+                            emqttd:subscribe(ClientId, Topic, Qos),
+                            %%TODO: the design is ugly...
+                            %% <MQTT V3.1.1>: 3.8.4
+                            %% Where the Topic Filter is not identical to any existing Subscription’s filter,
+                            %% a new Subscription is created and all matching retained messages are sent.
+                            emqttd_retainer:dispatch(Topic, self()),
 
-    case TopicTable -- dict:to_list(Subscriptions) of
+                            dict:store(Topic, Qos, SubDict)
-        [] ->
+                    end
+                end, Subscriptions, TopicTable),
             AckFun([Qos || {_, Qos} <- TopicTable]),
-            hibernate(Session);
+            emqttd:run_hooks('client.subscribe.after', [ClientId], TopicTable),
-        _  ->
+            hibernate(Session#session{subscriptions = Subscriptions1});
-            %% subscribe first and don't care if the subscriptions have been existed
+        {stop, TopicTable} ->
-            {ok, GrantedQos} = emqttd_pubsub:subscribe(ClientId, TopicTable),
+            ?LOG(error, "Cannot subscribe: ~p", [TopicTable], Session),
-
+            hibernate(Session)
-            AckFun(GrantedQos),
-
-            emqttd_broker:foreach_hooks('client.subscribe.after', [ClientId, TopicTable]),
-
-            ?LOG(info, "Subscribe ~p, Granted QoS: ~p", [TopicTable, GrantedQos], Session),
-
-            Subscriptions1 =
-            lists:foldl(fun({Topic, Qos}, Dict) ->
-                            case dict:find(Topic, Dict) of
-                                {ok, Qos} ->
-                                    ?LOG(warning, "resubscribe ~s, qos = ~w", [Topic, Qos], Session),
-                                    Dict;
-                                {ok, OldQos} ->
-                                    ?LOG(warning, "resubscribe ~s, old qos=~w, new qos=~w", [Topic, OldQos, Qos], Session),
-                                    dict:store(Topic, Qos, Dict);
-                                error ->
-                                    %%TODO: the design is ugly, rewrite later...:(
-                                    %% <MQTT V3.1.1>: 3.8.4
-                                    %% Where the Topic Filter is not identical to any existing Subscription’s filter,
-                                    %% a new Subscription is created and all matching retained messages are sent.
-                                    emqttd_retainer:dispatch(Topic, self()),
-
-                                    dict:store(Topic, Qos, Dict)
-                            end
-                        end, Subscriptions, TopicTable),
-            hibernate(Session#session{subscriptions = Subscriptions1})
     end;
 
+
 handle_cast({unsubscribe, Topics0}, Session = #session{client_id     = ClientId,
                                                        subscriptions = Subscriptions}) ->
 
-    Topics = emqttd_broker:foldl_hooks('client.unsubscribe', [ClientId], Topics0),
+    case emqttd:run_hooks('client.unsubscribe', [ClientId], Topics0) of
-
+        {ok, Topics} ->
-    %% unsubscribe from topic tree
+            ?LOG(info, "unsubscribe ~p", [Topics], Session),
-    ok = emqttd_pubsub:unsubscribe(Topics),
+            Subscriptions1 = lists:foldl(
-
+                fun(Topic, SubDict) ->
-    ?LOG(info, "unsubscribe ~p", [Topics], Session),
+                    case dict:find(Topic, SubDict) of
-
+                        {ok, Qos} ->
-    Subscriptions1 =
+                            emqttd:unsubscribe(ClientId, Topic, Qos),
-    lists:foldl(fun(Topic, Dict) ->
+                            dict:erase(Topic, SubDict);
-                    dict:erase(Topic, Dict)
+                        error ->
+                            SubDict
+                    end
                 end, Subscriptions, Topics),
-
+            hibernate(Session#session{subscriptions = Subscriptions1});
-    hibernate(Session#session{subscriptions = Subscriptions1});
+        {stop, Topics} ->
+            ?LOG(info, "Cannot unsubscribe: ~p", [Topics], Session),
+            hibernate(Session)
+    end;
 
 handle_cast({destroy, ClientId}, Session = #session{client_id = ClientId}) ->
     ?LOG(warning, "destroyed", [], Session),
@@ -430,7 +434,7 @@ handle_cast({pubrel, PktId}, Session = #session{awaiting_rel = AwaitingRel}) ->
     case maps:find(PktId, AwaitingRel) of
         {ok, {Msg, TRef}} ->
             cancel_timer(TRef),
-            emqttd_pubsub:publish(Msg),
+            emqttd:publish(Msg),
             hibernate(Session#session{awaiting_rel = maps:remove(PktId, AwaitingRel)});
         error ->
             ?LOG(error, "Cannot find PUBREL: ~p", [PktId], Session),
@@ -651,7 +655,7 @@ acked(PktId, Session = #session{client_id      = ClientId,
                                 awaiting_ack   = Awaiting}) ->
     case lists:keyfind(PktId, 1, InflightQ) of
         {_, Msg} ->
-            emqttd_broker:foreach_hooks('message.acked', [ClientId, Msg]);
+            emqttd:run_hooks('message.acked', [ClientId], Msg);
         false ->
             ?LOG(error, "Cannot find acked pktid: ~p", [PktId], Session)
     end,

src/emqttd_stats.erl

@@ -52,14 +52,14 @@
 
 %% $SYS Topics for Subscribers
 -define(SYSTOP_PUBSUB, [
-    'routes/count',      % ...
+    'routes/count',        % ...
-    'routes/reverse',    % ...
+    'routes/max',          % ...
-    'topics/count',      % ...
+    'topics/count',        % ...
-    'topics/max',        % ...
+    'topics/max',          % ...
+    'subscribers/count',   % ...
+    'subscribers/max',     % ...
     'subscriptions/count', % ...
-    'subscriptions/max',   % ...
+    'subscriptions/max'    % ...
-    'queues/count',      % ...
-    'queues/max'         % ...
 ]).
 
 %% $SYS Topic for retained
@@ -122,7 +122,7 @@ init([]) ->
     Topics = ?SYSTOP_CLIENTS ++ ?SYSTOP_SESSIONS ++ ?SYSTOP_PUBSUB ++ ?SYSTOP_RETAINED,
     ets:insert(?STATS_TAB, [{Topic, 0} || Topic <- Topics]),
     % Create $SYS Topics
-    [ok = emqttd_pubsub:create(topic, stats_topic(Topic)) || Topic <- Topics],
+    [ok = emqttd:create(topic, stats_topic(Topic)) || Topic <- Topics],
     % Tick to publish stats
     {ok, #state{tick_tref = emqttd_broker:start_tick(tick)}, hibernate}.
 
@@ -165,8 +165,7 @@ code_change(_OldVsn, State, _Extra) ->
 %%--------------------------------------------------------------------
 
 publish(Stat, Val) ->
-    Msg = emqttd_message:make(stats, stats_topic(Stat), bin(Val)),
+    emqttd:publish(emqttd_message:make(stats, stats_topic(Stat), bin(Val))).
-    emqttd_pubsub:publish(Msg).
 
 stats_topic(Stat) ->
     emqttd_topic:systop(list_to_binary(lists:concat(['stats/', Stat]))).

src/emqttd_sysmon.erl

@@ -157,7 +157,7 @@ procinfo(Pid) ->
 
 publish(Sysmon, WarnMsg) ->
     Msg = emqttd_message:make(sysmon, topic(Sysmon), iolist_to_binary(WarnMsg)),
-    emqttd_pubsub:publish(emqttd_message:set_flag(sys, Msg)).
+    emqttd:publish(emqttd_message:set_flag(sys, Msg)).
 
 topic(Sysmon) ->
     emqttd_topic:systop(list_to_binary(lists:concat(['sysmon/', Sysmon]))).

src/emqttd_topic.erl

@@ -139,11 +139,9 @@ word(<<"+">>) -> '+';
 word(<<"#">>) -> '#';
 word(Bin)     -> Bin.
 
-%% @doc Queue is a special topic name that starts with "$Q/"
+%% @doc Queue is a special topic name that starts with "$queue/"
 -spec is_queue(topic()) -> boolean().
-is_queue(<<"$Q/", _Queue/binary>>) ->
+is_queue(<<"$queue/", _Queue/binary>>) ->
-    true;
-is_queue(<<"$q/", _Queue/binary>>) ->
     true;
 is_queue(_) ->
     false.

src/lager_emqtt_backend.erl

@@ -77,9 +77,7 @@ publish_log(Message, State = #state{formatter = Formatter,
                                     format_config = FormatConfig}) ->
     Severity = lager_msg:severity(Message),
     Payload = Formatter:format(Message, FormatConfig),
-    emqttd_pubsub:publish(
+    emqttd:publish(emqttd_message:make(log, topic(Severity), iolist_to_binary(Payload))),
-      emqttd_message:make(
-        log, topic(Severity), iolist_to_binary(Payload))),
     {ok, State}.
 
 topic(Severity) ->

test/emqttd_SUITE.erl

@@ -27,18 +27,21 @@ all() ->
      {group, retainer},
      {group, broker},
      {group, metrics},
-     {group, stats}].
+     {group, stats},
+     {group, hook},
+     {group, cli}].
 
 groups() ->
     [{pubsub, [sequence],
       [create_topic,
        create_subscription,
        subscribe_unsubscribe,
-       publish_message]},
+       publish, pubsub,
+       'pubsub#', 'pubsub+']},
      {router, [sequence],
-      [add_delete_routes,
+      [router_add_del,
-       add_delete_route,
+       router_print,
-       route_message]},
+       router_unused]},
      {session, [sequence],
       [start_session]},
      {retainer, [sequence],
@@ -48,7 +51,22 @@ groups() ->
      {metrics, [sequence],
       [inc_dec_metric]},
      {stats, [sequence],
-      [set_get_stat]}].
+      [set_get_stat]},
+     {hook, [sequence],
+      [add_delete_hook,
+       run_hooks]},
+     {cli, [sequence],
+      [ctl_register_cmd,
+       cli_status,
+       cli_broker,
+       cli_clients,
+       cli_sessions,
+       cli_routes,
+       cli_topics,
+       cli_subscriptions,
+       cli_bridges,
+       cli_plugins,
+       cli_listeners]}].
 
 init_per_suite(Config) ->
     application:start(lager),
@@ -62,98 +80,107 @@ end_per_suite(_Config) ->
     emqttd_mnesia:ensure_stopped().
 
 %%--------------------------------------------------------------------
-%% PubSub Group
+%% PubSub Test
 %%--------------------------------------------------------------------
 
 create_topic(_) ->
-    Node = node(),
+    ok = emqttd:create(topic, <<"topic/create">>),
-    ok = emqttd_pubsub:create(topic, <<"topic/create">>),
+    ok = emqttd:create(topic, <<"topic/create2">>),
-    ok = emqttd_pubsub:create(topic, <<"topic/create2">>),
+    [#mqtt_topic{topic = <<"topic/create">>, flags = [static]}]
-    [#mqtt_topic{topic = <<"topic/create">>, node  = Node}]
+        = emqttd:lookup(topic, <<"topic/create">>).
-        = emqttd_pubsub:lookup(topic, <<"topic/create">>).
 
 create_subscription(_) ->
-    ok = emqttd_pubsub:create(subscription, {<<"clientId">>, <<"topic/sub">>, qos2}),
+    ok = emqttd:create(subscription, {<<"clientId">>, <<"topic/sub">>, qos2}),
     [#mqtt_subscription{subid = <<"clientId">>, topic = <<"topic/sub">>, qos = 2}]
-        = emqttd_pubsub:lookup(subscription, <<"clientId">>),
+        = emqttd_backend:lookup_subscriptions(<<"clientId">>),
-    ok = emqttd_pubsub:delete(subscription, <<"clientId">>),
+    ok = emqttd_backend:del_subscriptions(<<"clientId">>),
-    [] = emqttd_pubsub:lookup(subscription, <<"clientId">>).
+    [] = emqttd_backend:lookup_subscriptions(<<"clientId">>).
 
 subscribe_unsubscribe(_) ->
-    {ok, [1]} = emqttd_pubsub:subscribe({<<"topic/subunsub">>, 1}),
+    ok = emqttd:subscribe(<<"topic/subunsub">>),
-    {ok, [1, 2]} = emqttd_pubsub:subscribe([{<<"topic/subunsub1">>, 1}, {<<"topic/subunsub2">>, 2}]),
+    ok = emqttd:subscribe(<<"clientId">>, <<"topic/subunsub1">>, 1),
-    ok = emqttd_pubsub:unsubscribe(<<"topic/subunsub">>),
+    ok = emqttd:subscribe(<<"clientId">>, <<"topic/subunsub2">>, 2),
-    ok = emqttd_pubsub:unsubscribe([<<"topic/subunsub1">>, <<"topic/subunsub2">>]),
+    ok = emqttd:unsubscribe(<<"topic/subunsub">>),
+    ok = emqttd:unsubscribe(<<"clientId">>, <<"topic/subunsub1">>, 1),
+    ok = emqttd:unsubscribe(<<"clientId">>, <<"topic/subunsub2">>, 2).
 
-    {ok, [1]} = emqttd_pubsub:subscribe(<<"client_subunsub">>, {<<"topic/subunsub">>, 1}),
+publish(_) ->
-    {ok, [1,2]} = emqttd_pubsub:subscribe(<<"client_subunsub">>, [{<<"topic/subunsub1">>, 1},
-                                                         {<<"topic/subunsub2">>, 2}]),
-    ok = emqttd_pubsub:unsubscribe(<<"client_subunsub">>, <<"topic/subunsub">>),
-    ok = emqttd_pubsub:unsubscribe(<<"client_subunsub">>, [<<"topic/subunsub1">>,
-                                                           <<"topic/subunsub2">>]).
-
-publish_message(_) ->
     Msg = emqttd_message:make(ct, <<"test/pubsub">>, <<"hello">>),
-    {ok, [1]} = emqttd_pubsub:subscribe({<<"test/+">>, qos1}),
+    ok = emqttd:subscribe(<<"test/+">>),
-    emqttd_pubsub:publish(Msg),
+    emqttd:publish(Msg),
     true = receive {dispatch, <<"test/+">>, Msg} -> true after 5 -> false end.
 
+pubsub(_) ->
+    Self = self(),
+    emqttd:subscribe({<<"clientId">>, <<"a/b/c">>, 1}),
+    emqttd:subscribe({<<"clientId">>, <<"a/b/c">>, 2}),
+    [{Self, <<"a/b/c">>}] = ets:lookup(subscribed, Self),
+    [{<<"a/b/c">>, Self}] = ets:lookup(subscriber, <<"a/b/c">>),
+    emqttd:publish(emqttd_message:make(ct, <<"a/b/c">>, <<"hello">>)),
+    true = receive {dispatch, <<"a/b/c">>, _} -> true after 2 -> false end,
+    spawn(fun() ->
+            emqttd:subscribe(<<"a/b/c">>),
+            emqttd:subscribe(<<"c/d/e">>),
+            timer:sleep(10),
+            emqttd:unsubscribe(<<"a/b/c">>)
+          end),
+    timer:sleep(20),
+    emqttd:unsubscribe(<<"a/b/c">>).
+
+'pubsub#'(_) ->
+    emqttd:subscribe(<<"a/#">>),
+    emqttd:publish(emqttd_message:make(ct, <<"a/b/c">>, <<"hello">>)),
+    true = receive {dispatch, <<"a/#">>, _} -> true after 2 -> false end,
+    emqttd:unsubscribe(<<"a/#">>).
+
+'pubsub+'(_) ->
+    emqttd:subscribe(<<"a/+/+">>),
+    emqttd:publish(emqttd_message:make(ct, <<"a/b/c">>, <<"hello">>)),
+    true = receive {dispatch, <<"a/+/+">>, _} -> true after 1 -> false end,
+    emqttd:unsubscribe(<<"a/+/+">>).
+
 %%--------------------------------------------------------------------
-%% Route Group
+%% Router Test
 %%--------------------------------------------------------------------
 
-add_delete_route(_) ->
+router_add_del(_) ->
-    Self = self(),
+    %% Add
-    emqttd_router:add_route(<<"topic1">>, Self),
+    emqttd_router:add_route(<<"#">>),
-    true = emqttd_router:has_route(<<"topic1">>),
+    emqttd_router:add_route(<<"a/b/c">>),
-    emqttd_router:add_route(<<"topic2">>, Self),
+    emqttd_router:add_route(<<"+/#">>, node()),
-    true = emqttd_router:has_route(<<"topic2">>),
+    Routes = [R1, R2 | _] = [
-    [Self] = emqttd_router:lookup_routes(<<"topic1">>),
+            #mqtt_route{topic = <<"#">>,     node = node()},
-    [Self] = emqttd_router:lookup_routes(<<"topic2">>),
+            #mqtt_route{topic = <<"+/#">>,   node = node()},
-    %% Del topic1
+            #mqtt_route{topic = <<"a/b/c">>, node = node()}],
-    emqttd_router:delete_route(<<"topic1">>, Self),
+    Routes = lists:sort(emqttd_router:lookup(<<"a/b/c">>)),
-    erlang:yield(),
-    timer:sleep(10),
-    false = emqttd_router:has_route(<<"topic1">>),
-    %% Del topic2
-    emqttd_router:delete_route(<<"topic2">>, Self),
-    erlang:yield(),
-    timer:sleep(10),
-    false = emqttd_router:has_route(<<"topic2">>).
 
-add_delete_routes(_) ->
+    %% Batch Add
-    Self = self(),
+    emqttd_router:add_routes(Routes),
-    emqttd_router:add_routes([], Self),
+    Routes = lists:sort(emqttd_router:lookup(<<"a/b/c">>)),
-    emqttd_router:add_routes([<<"t0">>], Self),
-    emqttd_router:add_routes([<<"t1">>,<<"t2">>,<<"t3">>], Self),
-    true = emqttd_router:has_route(<<"t1">>),
-    [Self] = emqttd_router:lookup_routes(<<"t1">>),
-    [Self] = emqttd_router:lookup_routes(<<"t2">>),
-    [Self] = emqttd_router:lookup_routes(<<"t3">>),
 
-    emqttd_router:delete_routes([<<"t3">>], Self),
+    %% Del
-    emqttd_router:delete_routes([<<"t0">>, <<"t1">>], Self),
+    emqttd_router:del_route(<<"a/b/c">>),
-    erlang:yield(),
+    [R1, R2] = lists:sort(emqttd_router:lookup(<<"a/b/c">>)),
-    timer:sleep(10),
+    {atomic, []} = mnesia:transaction(fun emqttd_trie:lookup/1, [<<"a/b/c">>]),
-    false = emqttd_router:has_route(<<"t0">>),
-    false = emqttd_router:has_route(<<"t1">>),
-    true = emqttd_router:has_route(<<"t2">>),
-    false = emqttd_router:has_route(<<"t3">>).
 
-route_message(_) ->
+    %% Batch Del
-    Self = self(),
+    R3 = #mqtt_route{topic = <<"#">>, node = 'a@127.0.0.1'},
-    Pid = spawn_link(fun() -> timer:sleep(1000) end),
+    emqttd_router:add_route(R3),
-    emqttd_router:add_routes([<<"$Q/1">>,<<"t/2">>,<<"t/3">>], Self),
+    emqttd_router:del_routes([R1, R2]),
-    emqttd_router:add_routes([<<"t/2">>], Pid),
+    emqttd_router:del_route(R3),
-    Msg1 = #mqtt_message{topic = <<"$Q/1">>, payload = <<"q">>},
+    [] = lists:sort(emqttd_router:lookup(<<"a/b/c">>)).
-    Msg2 = #mqtt_message{topic = <<"t/2">>, payload = <<"t2">>},
+
-    Msg3 = #mqtt_message{topic = <<"t/3">>, payload = <<"t3">>},
+router_print(_) ->
-    emqttd_router:route(<<"$Q/1">>, Msg1),
+    Routes = [#mqtt_route{topic = <<"a/b/c">>, node = node()},
-    emqttd_router:route(<<"t/2">>, Msg2),
+              #mqtt_route{topic = <<"#">>,     node = node()},
-    emqttd_router:route(<<"t/3">>, Msg3),
+              #mqtt_route{topic = <<"+/#">>,   node = node()}],
-    [Msg1, Msg2, Msg3] = recv_loop([]),
+    emqttd_router:add_routes(Routes),
-    emqttd_router:add_route(<<"$Q/1">>, Self),
+    emqttd_router:print(<<"a/b/c">>).
-    emqttd_router:route(<<"$Q/1">>, Msg1).
+
+router_unused(_) ->
+    gen_server:call(emqttd_router, bad_call),
+    gen_server:cast(emqttd_router, bad_msg),
+    emqttd_router ! bad_info.
 
 recv_loop(Msgs) ->
     receive
@@ -212,3 +239,113 @@ inc_dec_metric(_) ->
 set_get_stat(_) ->
     emqttd_stats:setstat('retained/max', 99),
     99 = emqttd_stats:getstat('retained/max').
+
+%%--------------------------------------------------------------------
+%% Hook Test
+%%--------------------------------------------------------------------
+
+add_delete_hook(_) ->
+    emqttd:hook(test_hook, fun ?MODULE:hook_fun1/1, []),
+    emqttd:hook(test_hook, fun ?MODULE:hook_fun2/1, []),
+    {error, already_hooked} = emqttd:hook(test_hook, fun ?MODULE:hook_fun2/1, []),
+    Callbacks = [{callback, fun ?MODULE:hook_fun1/1, [], 0},
+                 {callback, fun ?MODULE:hook_fun2/1, [], 0}],
+    Callbacks = emqttd_hook:lookup(test_hook),
+    emqttd:unhook(test_hook, fun ?MODULE:hook_fun1/1),
+    emqttd:unhook(test_hook, fun ?MODULE:hook_fun2/1),
+    ok = emqttd:unhook(test_hook, fun ?MODULE:hook_fun2/1),
+    {error, not_found} = emqttd:unhook(test_hook1, fun ?MODULE:hook_fun2/1),
+    [] = emqttd_hook:lookup(test_hook),
+
+    emqttd:hook(emqttd_hook, fun ?MODULE:hook_fun1/1, [], 9),
+    emqttd:hook(emqttd_hook, fun ?MODULE:hook_fun2/1, [], 8),
+    Callbacks2 = [{callback, fun ?MODULE:hook_fun2/1, [], 8},
+                  {callback, fun ?MODULE:hook_fun1/1, [], 9}],
+    Callbacks2 = emqttd_hook:lookup(emqttd_hook),
+    emqttd:unhook(emqttd_hook, fun ?MODULE:hook_fun1/1),
+    emqttd:unhook(emqttd_hook, fun ?MODULE:hook_fun2/1),
+    [] = emqttd_hook:lookup(emqttd_hook).
+
+run_hooks(_) ->
+    emqttd:hook(test_hook, fun ?MODULE:hook_fun3/4, [init]),
+    emqttd:hook(test_hook, fun ?MODULE:hook_fun4/4, [init]),
+    emqttd:hook(test_hook, fun ?MODULE:hook_fun5/4, [init]),
+    {stop, [r3, r2]} = emqttd:run_hooks(test_hook, [arg1, arg2], []),
+    {ok, []} = emqttd:run_hooks(unknown_hook, [], []).
+
+hook_fun1([]) -> ok.
+hook_fun2([]) -> {ok, []}.
+
+hook_fun3(arg1, arg2, _Acc, init) -> ok.
+hook_fun4(arg1, arg2, Acc, init)  -> {ok, [r2 | Acc]}.
+hook_fun5(arg1, arg2, Acc, init)  -> {stop, [r3 | Acc]}.
+
+%%--------------------------------------------------------------------
+%% CLI Group
+%%--------------------------------------------------------------------
+
+ctl_register_cmd(_) ->
+    emqttd_ctl:register_cmd(test_cmd, {?MODULE, test_cmd}),
+    erlang:yield(),
+    timer:sleep(5),
+    [{?MODULE, test_cmd}] = emqttd_ctl:lookup(test_cmd),
+    emqttd_ctl:run(["test_cmd", "arg1", "arg2"]),
+    emqttd_ctl:unregister_cmd(test_cmd).
+
+test_cmd(["arg1", "arg2"]) ->
+    ct:print("test_cmd is called");
+
+test_cmd([]) ->
+    io:format("test command").
+
+cli_status(_) ->
+    emqttd_cli:status([]).
+
+cli_broker(_) ->
+    emqttd_cli:broker([]),
+    emqttd_cli:broker(["stats"]),
+    emqttd_cli:broker(["metrics"]),
+    emqttd_cli:broker(["pubsub"]).
+
+cli_clients(_) ->
+    emqttd_cli:clients(["list"]),
+    emqttd_cli:clients(["show", "clientId"]),
+    emqttd_cli:clients(["kick", "clientId"]).
+
+cli_sessions(_) ->
+    emqttd_cli:sessions(["list"]),
+    emqttd_cli:sessions(["list", "persistent"]),
+    emqttd_cli:sessions(["list", "transient"]),
+    emqttd_cli:sessions(["show", "clientId"]).
+
+cli_routes(_) ->
+    emqttd:subscribe(<<"topic/route">>),
+    emqttd_cli:routes(["list"]),
+    emqttd_cli:routes(["show", "topic/route"]),
+    emqttd:unsubscribe(<<"topic/route">>).
+
+cli_topics(_) ->
+    emqttd:subscribe(<<"topic">>),
+    emqttd_cli:topics(["list"]),
+    emqttd_cli:topics(["show", "topic"]),
+    emqttd:unsubscribe(<<"topic">>).
+
+cli_subscriptions(_) ->
+    emqttd_cli:subscriptions(["list"]),
+    emqttd_cli:subscriptions(["show", "clientId"]),
+    emqttd_cli:subscriptions(["add", "clientId", "topic", "2"]),
+    emqttd_cli:subscriptions(["del", "clientId", "topic"]).
+
+cli_plugins(_) ->
+    emqttd_cli:plugins(["list"]),
+    emqttd_cli:plugins(["load", "emqttd_plugin_template"]),
+    emqttd_cli:plugins(["unload", "emqttd_plugin_template"]).
+
+cli_bridges(_) ->
+    emqttd_cli:bridges(["list"]),
+    emqttd_cli:bridges(["start", "a@127.0.0.1", "topic"]),
+    emqttd_cli:bridges(["stop", "a@127.0.0.1", "topic"]).
+
+cli_listeners(_) ->
+    emqttd_cli:listeners([]).
+

test/emqttd_access_SUITE.erl

@@ -74,7 +74,6 @@ end_per_testcase(_TestCase, _Config) ->
 %%--------------------------------------------------------------------
 
 reload_acl(_) ->
-    ct:print("~p~n", [whereis(?AC)]),
     [ok] = ?AC:reload_acl().
 
 register_mod(_) ->

test/emqttd_protocol_SUITE.erl

@@ -33,6 +33,7 @@ all() ->
 groups() ->
     [{parser, [],
       [parse_connect,
+       parse_bridge,
        parse_publish,
        parse_puback,
        parse_subscribe,
@@ -122,6 +123,26 @@ parse_connect(_) ->
                                                       password = <<"public">>}}, <<>>} = Parser(ConnBinWithWill),
     ok.
 
+parse_bridge(_) ->
+    Parser = emqttd_parser:new([]),
+    Data = <<16,86,0,6,77,81,73,115,100,112,131,44,0,60,0,19,67,95,48,48,58,48,67,58,50,57,58,50,66,58,55,55,58,53,50,
+             0,48,36,83,89,83,47,98,114,111,107,101,114,47,99,111,110,110,101,99,116,105,111,110,47,67,95,48,48,58,48,
+             67,58,50,57,58,50,66,58,55,55,58,53,50,47,115,116,97,116,101,0,1,48>>,
+
+    %% CONNECT(Q0, R0, D0, ClientId=C_00:0C:29:2B:77:52, ProtoName=MQIsdp, ProtoVsn=131, CleanSess=false, KeepAlive=60,
+    %% Username=undefined, Password=undefined, Will(Q1, R1, Topic=$SYS/broker/connection/C_00:0C:29:2B:77:52/state, Msg=0))
+    {ok, #mqtt_packet{variable = Variable}, <<>>} = Parser(Data),
+    #mqtt_packet_connect{client_id  = <<"C_00:0C:29:2B:77:52">>,
+                         proto_ver  = 16#03,
+                         proto_name = <<"MQIsdp">>,
+                         will_retain = true,
+                         will_qos   = 1,
+                         will_flag  = true,
+                         clean_sess = false,
+                         keep_alive = 60,
+                         will_topic = <<"$SYS/broker/connection/C_00:0C:29:2B:77:52/state">>,
+                         will_msg   = <<"0">>} = Variable.
+
 parse_publish(_) ->
     Parser = emqttd_parser:new([]),
     %%PUBLISH(Qos=1, Retain=false, Dup=false, TopicName=a/b/c, PacketId=1, Payload=<<"hahah">>)

test/emqttd_topic_SUITE.erl

@@ -156,8 +156,7 @@ t_join(_) ->
     <<"ab/+/#">> = join(words(<<"ab/+/#">>)).
 
 t_is_queue(_) ->
-    true  = is_queue(<<"$Q/queue">>),
+    true  = is_queue(<<"$queue/queue">>),
-    true  = is_queue(<<"$q/queue">>),
     false = is_queue(<<"xyz/queue">>).
 
 t_systop(_) ->
@@ -167,7 +166,7 @@ t_systop(_) ->
     SysTop2 = systop(<<"abc">>).
 
 t_feed_var(_) ->
-    <<"$Q/client/clientId">> = feed_var(<<"$c">>, <<"clientId">>, <<"$Q/client/$c">>),
+    <<"$queue/client/clientId">> = feed_var(<<"$c">>, <<"clientId">>, <<"$queue/client/$c">>),
     <<"username/test/client/x">> = feed_var(<<"%u">>, <<"test">>, <<"username/%u/client/x">>),
     <<"username/test/client/clientId">> = feed_var(<<"%c">>, <<"clientId">>, <<"username/test/client/%c">>).