chore: bump dashboard app version

fix: update mongodb driver to fix potential process leak

.ci/acl_migration_test/build.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -xe
+
+cd "$EMQX_PATH"
+
+rm -rf _build _upgrade_base
+
+mkdir _upgrade_base
+pushd _upgrade_base
+    wget "https://s3-us-west-2.amazonaws.com/packages.emqx/emqx-ce/v${EMQX_BASE}/emqx-ubuntu20.04-${EMQX_BASE}-amd64.zip"
+popd
+
+make emqx-zip

.ci/acl_migration_test/prepare.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -xe
+
+mkdir -p "$TEST_PATH"
+cd "$TEST_PATH"
+
+cp ../"$EMQX_PATH"/_upgrade_base/*.zip ./
+unzip ./*.zip
+
+cp ../"$EMQX_PATH"/_packages/emqx/*.zip ./emqx/releases/
+
+git clone --depth 1 https://github.com/terry-xiaoyu/one_more_emqx.git
+
+./one_more_emqx/one_more_emqx.sh emqx2

.ci/acl_migration_test/suite.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+set -xe
+
+export EMQX_PATH="$1"
+export EMQX_BASE="$2"
+
+export TEST_PATH="emqx_test"
+
+./build.sh
+
+VERSION=$("$EMQX_PATH"/pkg-vsn.sh)
+export VERSION
+
+./prepare.sh
+
+./test.sh

.ci/acl_migration_test/test.sh

@@ -0,0 +1,121 @@
+#!/bin/bash
+
+set -e
+
+EMQX_ENDPOINT="http://localhost:8081/api/v4/acl"
+EMQX2_ENDPOINT="http://localhost:8917/api/v4/acl"
+
+function run() {
+    emqx="$1"
+    shift
+
+    echo "[$emqx]" "$@"
+
+    pushd "$TEST_PATH/$emqx"
+        "$@"
+    popd
+}
+
+function post_rule() {
+    endpoint="$1"
+    rule="$2"
+    echo -n "->($endpoint) "
+    curl -s -u admin:public -X POST "$endpoint" -d "$rule"
+    echo
+}
+
+function verify_clientid_rule() {
+    endpoint="$1"
+    id="$2"
+    echo -n "<-($endpoint) "
+    curl -s -u admin:public "$endpoint/clientid/$id" | grep "$id" || (echo "verify rule for client $id failed" && return 1)
+}
+
+# Run nodes
+
+run emqx ./bin/emqx start
+run emqx2 ./bin/emqx start
+
+run emqx ./bin/emqx_ctl plugins load emqx_auth_mnesia
+run emqx2 ./bin/emqx_ctl plugins load emqx_auth_mnesia
+
+run emqx2 ./bin/emqx_ctl cluster join 'emqx@127.0.0.1'
+
+# Add ACL rule to unupgraded EMQX nodes
+
+post_rule "$EMQX_ENDPOINT" '{"clientid": "CLIENT1_A","topic": "t", "action": "pub", "access": "allow"}'
+post_rule "$EMQX2_ENDPOINT" '{"clientid": "CLIENT1_B","topic": "t", "action": "pub", "access": "allow"}'
+
+# Upgrade emqx2 node
+
+run emqx2 ./bin/emqx install "$VERSION"
+sleep 60
+
+# Verify upgrade blocked
+
+run emqx2 ./bin/emqx eval 'emqx_acl_mnesia_migrator:is_old_table_migrated().' | grep false || (echo "emqx2 shouldn't have migrated" && exit 1)
+
+# Verify old rules on both nodes
+
+verify_clientid_rule "$EMQX_ENDPOINT" 'CLIENT1_A'
+verify_clientid_rule "$EMQX2_ENDPOINT" 'CLIENT1_A'
+
+verify_clientid_rule "$EMQX_ENDPOINT" 'CLIENT1_B'
+verify_clientid_rule "$EMQX2_ENDPOINT" 'CLIENT1_B'
+
+# Add ACL on OLD and NEW node, verify on all nodes
+
+post_rule "$EMQX_ENDPOINT" '{"clientid": "CLIENT2_A","topic": "t", "action": "pub", "access": "allow"}'
+post_rule "$EMQX2_ENDPOINT" '{"clientid": "CLIENT2_B","topic": "t", "action": "pub", "access": "allow"}'
+
+verify_clientid_rule "$EMQX_ENDPOINT" 'CLIENT2_A'
+verify_clientid_rule "$EMQX2_ENDPOINT" 'CLIENT2_A'
+
+verify_clientid_rule "$EMQX_ENDPOINT" 'CLIENT2_B'
+verify_clientid_rule "$EMQX2_ENDPOINT" 'CLIENT2_B'
+
+# Upgrade emqx node
+
+run emqx ./bin/emqx install "$VERSION"
+
+# Wait for upgrade
+
+sleep 60
+
+# Verify if upgrade occured
+
+run emqx ./bin/emqx eval 'emqx_acl_mnesia_migrator:is_old_table_migrated().' | grep true || (echo "emqx should have migrated" && exit 1)
+run emqx2 ./bin/emqx eval 'emqx_acl_mnesia_migrator:is_old_table_migrated().' | grep true || (echo "emqx2 should have migrated" && exit 1)
+
+# Verify rules are kept
+
+verify_clientid_rule "$EMQX_ENDPOINT" 'CLIENT1_A'
+verify_clientid_rule "$EMQX2_ENDPOINT" 'CLIENT1_A'
+
+verify_clientid_rule "$EMQX_ENDPOINT" 'CLIENT1_B'
+verify_clientid_rule "$EMQX2_ENDPOINT" 'CLIENT1_B'
+
+verify_clientid_rule "$EMQX_ENDPOINT" 'CLIENT2_A'
+verify_clientid_rule "$EMQX2_ENDPOINT" 'CLIENT2_A'
+
+verify_clientid_rule "$EMQX_ENDPOINT" 'CLIENT2_B'
+verify_clientid_rule "$EMQX2_ENDPOINT" 'CLIENT2_B'
+
+# Add ACL on OLD and NEW node, verify on all nodes
+
+post_rule "$EMQX_ENDPOINT" '{"clientid": "CLIENT3_A","topic": "t", "action": "pub", "access": "allow"}'
+post_rule "$EMQX2_ENDPOINT" '{"clientid": "CLIENT3_B","topic": "t", "action": "pub", "access": "allow"}'
+
+verify_clientid_rule "$EMQX_ENDPOINT" 'CLIENT3_A'
+verify_clientid_rule "$EMQX2_ENDPOINT" 'CLIENT3_A'
+
+verify_clientid_rule "$EMQX_ENDPOINT" 'CLIENT3_B'
+verify_clientid_rule "$EMQX2_ENDPOINT" 'CLIENT3_B'
+
+# Stop nodes
+
+run emqx ./bin/emqx stop
+run emqx2 ./bin/emqx stop
+
+echo "Success!"
+

.ci/build_packages/Dockerfile

@@ -1,4 +1,4 @@
-ARG BUILD_FROM=emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04
+ARG BUILD_FROM=emqx/build-env:erl23.2.7.2-emqx-3-ubuntu20.04
 FROM ${BUILD_FROM}
 
 ARG EMQX_NAME=emqx

.ci/build_packages/tests.sh

@@ -88,6 +88,7 @@ emqx_test(){
             ;;
             "rpm")
                 packagename=$(basename "${PACKAGE_PATH}/${EMQX_NAME}"-*.rpm)
+
                 rpm -ivh "${PACKAGE_PATH}/${packagename}"
                 if ! rpm -q emqx | grep -q emqx; then
                     echo "package install error"
@@ -129,23 +130,6 @@ running_test(){
     pytest -v /paho-mqtt-testing/interoperability/test_client/V5/test_connect.py::test_basic
     # shellcheck disable=SC2009 # pgrep does not support Extended Regular Expressions
     emqx stop || kill "$(ps -ef | grep -E '\-progname\s.+emqx\s' |awk '{print $2}')"
-
-    if [ "$(sed -n '/^ID=/p' /etc/os-release | sed -r 's/ID=(.*)/\1/g' | sed 's/"//g')" = ubuntu ] \
-    || [ "$(sed -n '/^ID=/p' /etc/os-release | sed -r 's/ID=(.*)/\1/g' | sed 's/"//g')" = debian ] ;then
-        service emqx start || ( tail /var/log/emqx/emqx.log.1 && exit 1 )
-        IDLE_TIME=0
-        while ! emqx_ctl status | grep -E 'Node\s.*@.*\sis\sstarted'
-        do
-            if [ $IDLE_TIME -gt 10 ]
-            then
-                echo "emqx service error"
-                exit 1
-            fi
-            sleep 10
-            IDLE_TIME=$((IDLE_TIME+1))
-        done
-        service emqx stop
-    fi
 }
 
 relup_test(){
@@ -156,15 +140,21 @@ relup_test(){
         find . -maxdepth 1 -name "${EMQX_NAME}-*-${ARCH}.zip" |
             while read -r pkg; do
                 packagename=$(basename "${pkg}")
-                unzip "$packagename"
+                unzip -q "$packagename"
                 ./emqx/bin/emqx start || ( tail emqx/log/emqx.log.1 && exit 1 )
                 ./emqx/bin/emqx_ctl status
                 ./emqx/bin/emqx versions
                 cp "${PACKAGE_PATH}/${EMQX_NAME}"-*-"${TARGET_VERSION}-${ARCH}".zip ./emqx/releases
                 ./emqx/bin/emqx install "${TARGET_VERSION}"
                 [ "$(./emqx/bin/emqx versions |grep permanent | awk '{print $2}')" = "${TARGET_VERSION}" ] || exit 1
+                export EMQX_WAIT_FOR_STOP=300
                 ./emqx/bin/emqx_ctl status
-                ./emqx/bin/emqx stop
+                if ! ./emqx/bin/emqx stop; then
+                    cat emqx/log/erlang.log.1 || true
+                    cat emqx/log/emqx.log.1 || true
+                    echo "failed to stop emqx"
+                    exit 1
+                fi
                 rm -rf emqx
             done
    fi

.ci/docker-compose-file/docker-compose-emqx-broker-cluster.yaml

@@ -0,0 +1,99 @@
+version: '3.9'
+
+services:
+  haproxy:
+    container_name: haproxy
+    image: haproxy:2.3
+    depends_on:
+      - emqx1
+      - emqx2
+    volumes:
+      - ./haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg
+      - ../../etc/certs:/usr/local/etc/haproxy/certs
+    ports:
+      - "18083:18083"
+    #  - "1883:1883"
+    #  - "8883:8883"
+    #  - "8083:8083"
+    #  - "5683:5683/udp"
+    #  - "9999:9999"
+    #  - "8084:8084"
+    networks:
+      - emqx_bridge
+    working_dir: /usr/local/etc/haproxy
+    command:
+      - bash
+      - -c
+      - |
+        cat /usr/local/etc/haproxy/certs/cert.pem /usr/local/etc/haproxy/certs/key.pem > /usr/local/etc/haproxy/certs/emqx.pem
+        haproxy -f /usr/local/etc/haproxy/haproxy.cfg
+
+  emqx1:
+    restart: always
+    container_name: node1.emqx.io
+    image: $TARGET:$EMQX_TAG
+    env_file:
+      - conf.cluster.env
+    volumes:
+      - etc:/opt/emqx/etc
+    environment:
+      - "EMQX_HOST=node1.emqx.io"
+    ports:
+      - "11881:18083"
+#      - "1883:1883"
+    command:
+        - /bin/sh
+        - -c
+        - |
+          sed -i "s 127.0.0.1 $$(ip route show |grep "link" |awk '{print $$1}') g" /opt/emqx/etc/acl.conf
+          sed -i '/emqx_telemetry/d' /opt/emqx/data/loaded_plugins
+          /opt/emqx/bin/emqx foreground
+    healthcheck:
+      test: ["CMD", "/opt/emqx/bin/emqx_ctl", "status"]
+      interval: 5s
+      timeout: 25s
+      retries: 5
+    networks:
+      emqx_bridge:
+        aliases:
+        - node1.emqx.io
+
+  emqx2:
+    restart: always
+    container_name: node2.emqx.io
+    image: $TARGET:$EMQX_TAG
+    env_file:
+      - conf.cluster.env
+    volumes:
+      - etc:/opt/emqx/etc
+    environment:
+      - "EMQX_HOST=node2.emqx.io"
+    ports:
+      - "11882:18083"
+    command:
+      - /bin/sh
+      - -c
+      - |
+        sed -i "s 127.0.0.1 $$(ip route show |grep "link" |awk '{print $$1}') g" /opt/emqx/etc/acl.conf
+        sed -i '/emqx_telemetry/d' /opt/emqx/data/loaded_plugins
+        /opt/emqx/bin/emqx foreground
+    healthcheck:
+      test: ["CMD", "/opt/emqx/bin/emqx", "ping"]
+      interval: 5s
+      timeout: 25s
+      retries: 5
+    networks:
+      emqx_bridge:
+        aliases:
+        - node2.emqx.io
+volumes:
+  etc:
+networks:
+  emqx_bridge:
+    driver: bridge
+    name: emqx_bridge
+    ipam:
+      driver: default
+      config:
+        - subnet: 172.100.239.0/24
+          gateway: 172.100.239.1

.ci/docker-compose-file/docker-compose-emqx-cluster.yaml

@@ -27,6 +27,7 @@ services:
         haproxy -f /usr/local/etc/haproxy/haproxy.cfg
 
   emqx1:
+    restart: always
     container_name: node1.emqx.io
     image: $TARGET:$EMQX_TAG
     env_file:
@@ -51,6 +52,7 @@ services:
         - node1.emqx.io
 
   emqx2:
+    restart: always
     container_name: node2.emqx.io
     image: $TARGET:$EMQX_TAG
     env_file:

.ci/docker-compose-file/docker-compose-enterprise-tomcat-tcp.yaml

@@ -0,0 +1,10 @@
+version: '3.9'
+
+services:
+  web_server:
+    container_name: Tomcat
+    build:
+      context: ./http-service
+    image: web-server
+    networks:
+      - emqx_bridge

.ci/docker-compose-file/docker-compose.yaml

@@ -3,7 +3,7 @@ version: '3.9'
 services:
   erlang:
     container_name: erlang
-    image: emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04
+    image: emqx/build-env:erl23.2.7.2-emqx-3-ubuntu20.04
     env_file:
       - conf.env
     environment:

.ci/docker-compose-file/http-service/Dockerfile

@@ -0,0 +1,15 @@
+FROM tomcat:10.0.5
+
+RUN wget https://downloads.apache.org/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.zip \
+	&& unzip -q apache-maven-3.6.3-bin.zip \
+	&& mv apache-maven-3.6.3 /opt/apache-maven-3.6.3/ \
+	&& ln -s /opt/apache-maven-3.6.3/ /opt/maven
+ENV M2_HOME=/opt/maven
+ENV M2=$M2_HOME/bin
+ENV PATH=$M2:$PATH
+COPY ./web-server /code
+WORKDIR /code
+RUN mvn package -Dmaven.skip.test=true
+RUN mv ./target/emqx-web-0.0.1.war /usr/local/tomcat/webapps/emqx-web.war
+EXPOSE 8080
+CMD ["/usr/local/tomcat/bin/catalina.sh","run"]

.ci/docker-compose-file/http-service/web-server/pom.xml

@@ -0,0 +1,65 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <groupId>emqx-web</groupId>
+  <artifactId>emqx-web</artifactId>
+  <version>0.0.1</version>
+  <packaging>war</packaging>
+  <dependencies>
+	<dependency>
+    	<groupId>mysql</groupId>
+    	<artifactId>mysql-connector-java</artifactId>
+    	<version>8.0.16</version>
+    </dependency>
+    <dependency>
+    	<groupId>commons-dbutils</groupId>
+    	<artifactId>commons-dbutils</artifactId>
+    	<version>1.7</version>
+	</dependency>
+	<dependency>
+  	  <groupId>commons-logging</groupId>
+   	 <artifactId>commons-logging</artifactId>
+    	<version>1.2</version>
+	</dependency>
+	<dependency>
+    	<groupId>commons-dbcp</groupId>
+    	<artifactId>commons-dbcp</artifactId>
+   		<version>1.4</version>
+	</dependency>
+	<dependency>
+    	<groupId>commons-pool</groupId>
+    	<artifactId>commons-pool</artifactId>
+    	<version>1.6</version>
+	</dependency>
+	<dependency>
+     	<groupId>jakarta.servlet</groupId>
+     	<artifactId>jakarta.servlet-api</artifactId>
+     	<version>5.0.0</version>
+     	<scope>provided</scope>
+  	</dependency>
+  </dependencies>
+  <build>
+    <resources>
+      <resource>
+        <directory>src/main/reousrce</directory>
+        <excludes>
+          <exclude>**/*.java</exclude>
+        </excludes>
+      </resource>
+    </resources>
+    <plugins>
+      <plugin>
+        <artifactId>maven-compiler-plugin</artifactId>
+        <version>3.8.1</version>
+        <configuration>
+          <source>1.8</source>
+          <target>1.8</target>
+        </configuration>
+      </plugin>
+      <plugin>
+        <artifactId>maven-war-plugin</artifactId>
+        <version>3.2.3</version>
+      </plugin>
+    </plugins>
+  </build>
+
+</project>

.ci/docker-compose-file/http-service/web-server/src/main/java/com/emqx/dao/AuthDAO.java

@@ -0,0 +1,54 @@
+package com.emqx.dao;
+
+import java.io.IOException;
+import java.sql.SQLException;
+
+import org.apache.commons.dbutils.QueryRunner;
+import org.apache.commons.dbutils.handlers.ScalarHandler;
+
+import com.emqx.util.EmqxDatabaseUtil;
+
+public class AuthDAO {
+
+	public String getUserName(String userName) throws IOException, SQLException {
+		QueryRunner runner = new QueryRunner(EmqxDatabaseUtil.getDataSource());
+		String sql = "select password from http_user where username='"+userName+"'";
+		String password =runner.query(sql, new ScalarHandler<String>());
+		return password;
+	}
+	
+	public String getClient(String clientid) throws IOException, SQLException {
+		QueryRunner runner = new QueryRunner(EmqxDatabaseUtil.getDataSource());
+		String sql = "select password from http_user where clientid='"+clientid+"'";
+		String password =runner.query(sql, new ScalarHandler<String>());
+		return password;
+	}
+	
+	public String getUserAccess(String userName) throws IOException, SQLException {
+		QueryRunner runner = new QueryRunner(EmqxDatabaseUtil.getDataSource());
+		String sql = "select access from http_acl where username='"+userName+"'";
+		String access =runner.query(sql, new ScalarHandler<String>());
+		return access;
+	}
+	
+	public String getUserTopic(String userName) throws IOException, SQLException {
+		QueryRunner runner = new QueryRunner(EmqxDatabaseUtil.getDataSource());
+		String sql = "select topic from http_acl where username='"+userName+"'";
+		String topic =runner.query(sql, new ScalarHandler<String>());
+		return topic;
+	}
+	
+	public String getClientAccess(String clientid) throws IOException, SQLException {
+		QueryRunner runner = new QueryRunner(EmqxDatabaseUtil.getDataSource());
+		String sql = "select access from http_acl where clientid='"+clientid+"'";
+		String access =runner.query(sql, new ScalarHandler<String>());
+		return access;
+	}
+	
+	public String getClientTopic(String clientid) throws IOException, SQLException {
+		QueryRunner runner = new QueryRunner(EmqxDatabaseUtil.getDataSource());
+		String sql = "select topic from http_acl where clientid='"+clientid+"'";
+		String topic =runner.query(sql, new ScalarHandler<String>());
+		return topic;
+	}
+}

.ci/docker-compose-file/http-service/web-server/src/main/java/com/emqx/dao/DBUtilsTest.java

@@ -0,0 +1,45 @@
+package com.emqx.dao;
+
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.sql.SQLException;
+import java.util.Properties;
+
+import org.apache.commons.dbcp.BasicDataSource;
+import org.apache.commons.dbutils.QueryRunner;
+import org.apache.commons.dbutils.handlers.ColumnListHandler;
+import org.apache.commons.dbutils.handlers.ScalarHandler;
+import org.apache.commons.dbutils.handlers.columns.StringColumnHandler;
+
+
+public class DBUtilsTest {
+
+	public static void main(String args[]) throws FileNotFoundException, IOException, SQLException {
+		Properties property = new Properties();//流文件  
+		
+		property.load(DBUtilsTest.class.getClassLoader().getResourceAsStream("database.properties"));
+		
+		BasicDataSource dataSource = new BasicDataSource();
+		dataSource.setDriverClassName(property.getProperty("jdbc.driver"));
+		dataSource.setUrl(property.getProperty("jdbc.url"));
+		dataSource.setUsername(property.getProperty("jdbc.username"));
+		dataSource.setPassword(property.getProperty("jdbc.password"));
+
+		// 初始化连接数 if(initialSize!=null)
+		//dataSource.setInitialSize(Integer.parseInt(initialSize));
+
+		// 最小空闲连接 if(minIdle!=null)
+		//dataSource.setMinIdle(Integer.parseInt(minIdle));
+
+		// 最大空闲连接 if(maxIdle!=null)
+		//dataSource.setMaxIdle(Integer.parseInt(maxIdle));
+
+		QueryRunner runner = new QueryRunner(dataSource);
+		String sql="select username from mqtt_user where id=1";
+		String result = runner.query(sql, new ScalarHandler<String>());
+		
+		System.out.println(result);
+				
+	}
+}

.ci/docker-compose-file/http-service/web-server/src/main/java/com/emqx/servlet/AclServlet.java

@@ -0,0 +1,103 @@
+package com.emqx.servlet;
+
+import java.io.IOException;
+import java.sql.SQLException;
+
+import com.emqx.dao.AuthDAO;
+
+import jakarta.servlet.ServletException;
+import jakarta.servlet.http.HttpServlet;
+import jakarta.servlet.http.HttpServletRequest;
+import jakarta.servlet.http.HttpServletResponse;
+
+public class AclServlet extends HttpServlet {
+
+	@Override
+	protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException {
+		// TODO Auto-generated method stub
+		doPost(req, resp);
+	}
+	@Override
+	protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException {
+		String clientid = req.getParameter("clientid");
+		String username = req.getParameter("username");
+		String access = req.getParameter("access");
+		String topic = req.getParameter("topic");
+		//String password = req.getParameter("password");
+		
+		//step0: password is not null, or not pass.
+		
+		AuthDAO dao = new AuthDAO();
+		try {
+			//step1: check username access&topic
+			if(username != null) {
+				String access_1 = dao.getUserAccess(username);
+				String topic_1 = dao.getUserTopic(username);
+				
+				if(access.equals(access_1)) {
+					if(topic.equals(topic_1)) {
+						resp.setStatus(200);
+					}
+					else {
+						if(clientid != null){
+							String access_2 = dao.getClientAccess(clientid);
+							String topic_2 = dao.getClientTopic(clientid);
+								if(access.equals(access_2)) {
+									if(topic.equals(topic_2)) {
+										resp.setStatus(200);
+									}
+									else {
+										resp.setStatus(400);
+									}
+								}else {
+									resp.setStatus(400);
+								}
+						}else {
+							resp.setStatus(400);
+						}
+					}
+				}else {//step2.1: username password is not match, then check clientid password
+					if(clientid != null){
+						String access_3 = dao.getClientAccess(clientid);
+						String topic_3 = dao.getClientTopic(clientid);
+							if(access.equals(access_3)) {
+								if(topic.equals(topic_3)) {
+									resp.setStatus(200);
+								}
+								else {
+									resp.setStatus(400);
+								}
+							}else {
+								resp.setStatus(400);
+							}
+					}else {
+						resp.setStatus(400);
+					}
+				}
+			}else {//step2.2: username is null, then check clientid password
+				if(clientid != null){
+					String access_4 = dao.getClientAccess(clientid);
+					String topic_4 = dao.getClientTopic(clientid);
+						if(access.equals(access_4)) {
+							if(topic.equals(topic_4)) {
+								resp.setStatus(200);
+							}
+							else {
+								resp.setStatus(400);
+							}
+						}else {
+							resp.setStatus(400);
+						}
+				}else {
+					resp.setStatus(400);
+				}
+			}
+		} catch (IOException e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+		} catch (SQLException e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+		}
+	}
+}

.ci/docker-compose-file/http-service/web-server/src/main/java/com/emqx/servlet/AuthServlet.java

@@ -0,0 +1,72 @@
+package com.emqx.servlet;
+
+import java.io.IOException;
+import java.sql.SQLException;
+
+import com.emqx.dao.AuthDAO;
+
+import jakarta.servlet.ServletException;
+import jakarta.servlet.http.HttpServlet;
+import jakarta.servlet.http.HttpServletRequest;
+import jakarta.servlet.http.HttpServletResponse;
+
+public class AuthServlet extends HttpServlet {
+
+	@Override
+	protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException {
+		// TODO Auto-generated method stub
+		doPost(req, resp);
+	}
+	@Override
+	protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException {
+		String clientid = req.getParameter("clientid");
+		String username =req.getParameter("username");
+		String password = req.getParameter("password");
+		
+		//step0: password is not null, or not pass.
+		if(password == null) {
+			resp.setStatus(400);
+			return;
+		}
+		AuthDAO dao = new AuthDAO();
+		try {
+			//step1: check username password
+			if(username != null) {
+				String password_d = dao.getUserName(username);
+				
+				if(password.equals(password_d)) {
+					resp.setStatus(200);
+					//200
+				}else {//step2.1: username password is not match, then check clientid password
+					if(clientid != null){
+						String password_c = dao.getClient(clientid);
+							if(password.equals(password_c)) {
+								resp.setStatus(200);
+							}else {
+								resp.setStatus(400);
+							}
+					}else {
+						resp.setStatus(400);
+					}
+				}
+			}else {//step2.2: username is null, then check clientid password
+				if(clientid != null){
+					String password_c = dao.getClient(clientid);
+						if(password.equals(password_c)) {
+							resp.setStatus(200);
+						}else {
+							resp.setStatus(400);
+						}
+				}else {
+					resp.setStatus(400);
+				}
+			}
+		} catch (IOException e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+		} catch (SQLException e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+		}
+	}
+}

.ci/docker-compose-file/http-service/web-server/src/main/java/com/emqx/util/EmqxDatabaseUtil.java

@@ -0,0 +1,27 @@
+package com.emqx.util;
+
+import java.io.IOException;
+import java.util.Properties;
+
+import javax.sql.DataSource;
+
+import org.apache.commons.dbcp.BasicDataSource;
+
+import com.emqx.dao.DBUtilsTest;
+
+public class EmqxDatabaseUtil {
+
+	public static DataSource getDataSource() throws IOException {
+		Properties property = new Properties();// 流文件
+
+		property.load(EmqxDatabaseUtil.class.getClassLoader().getResourceAsStream("database.properties"));
+
+		BasicDataSource dataSource = new BasicDataSource();
+		dataSource.setDriverClassName(property.getProperty("jdbc.driver"));
+		dataSource.setUrl(property.getProperty("jdbc.url"));
+		dataSource.setUsername(property.getProperty("jdbc.username"));
+		dataSource.setPassword(property.getProperty("jdbc.password"));
+		
+		return dataSource;
+	}
+}

.ci/docker-compose-file/http-service/web-server/src/main/reousrce/database.properties

@@ -0,0 +1,4 @@
+jdbc.driver= com.mysql.jdbc.Driver
+jdbc.url= jdbc:mysql://mysql_server:3306/mqtt
+jdbc.username= root
+jdbc.password= public

.ci/docker-compose-file/http-service/web-server/src/main/webapp/META-INF/MANIFEST.MF

@@ -0,0 +1,3 @@
+Manifest-Version: 1.0
+Class-Path: 
+

.ci/docker-compose-file/http-service/web-server/src/main/webapp/WEB-INF/web.xml

@@ -0,0 +1,31 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<web-app xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xmlns="http://JAVA.sun.com/xml/ns/javaee"
+	xsi:schemaLocation="http://java.sun.com/xml/ns/javaee http://java.sun.com/xml/ns/javaee/web-app_2_5.xsd"
+	id="WebApp_ID" version="2.5">
+	<display-name>emqx-web</display-name>
+	<servlet>
+		<servlet-name>Auth</servlet-name>
+		<servlet-class>com.emqx.servlet.AuthServlet</servlet-class>
+	</servlet>
+	<servlet>
+		<servlet-name>Acl</servlet-name>
+		<servlet-class>com.emqx.servlet.AclServlet</servlet-class>
+	</servlet>
+	<servlet-mapping>
+		<servlet-name>Auth</servlet-name>
+		<url-pattern>/auth</url-pattern>
+	</servlet-mapping>
+	<servlet-mapping>
+		<servlet-name>Acl</servlet-name>
+		<url-pattern>/acl</url-pattern>
+	</servlet-mapping>
+	<welcome-file-list>
+		<welcome-file>index.html</welcome-file>
+		<welcome-file>index.htm</welcome-file>
+		<welcome-file>index.jsp</welcome-file>
+		<welcome-file>default.html</welcome-file>
+		<welcome-file>default.htm</welcome-file>
+		<welcome-file>default.jsp</welcome-file>
+	</welcome-file-list>
+</web-app>

.ci/docker-compose-file/http-service/web-server/src/main/webapp/index.html

@@ -0,0 +1,10 @@
+<!DOCTYPE html>
+<html>
+<head>
+<meta charset="UTF-8">
+<title>love</title>
+</head>
+<body>
+It's lucky, jiabanxiang.
+</body>
+</html>

.ci/docker-compose-file/python/pytest.sh

@@ -10,7 +10,7 @@ LB="haproxy"
 
 apk update && apk add git curl
 git clone -b develop-4.0 https://github.com/emqx/paho.mqtt.testing.git /paho.mqtt.testing
-pip install pytest
+pip install pytest==6.2.5
 
 pytest -v /paho.mqtt.testing/interoperability/test_client/V5/test_connect.py -k test_basic --host "$LB"
 RESULT=$?

.ci/fvt_tests/http_server/rebar.config

@@ -1,7 +1,7 @@
 {erl_opts, [debug_info]}.
 {deps, 
  [
-    {minirest, {git, "https://github.com/emqx/minirest.git", {tag, "0.3.5"}}}
+    {minirest, {git, "https://github.com/emqx/minirest.git", {tag, "0.3.6"}}}
  ]}.
 
 {shell, [

.ci/fvt_tests/local_relup_test_run.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+USAGE="$0 profile vsn old_vsn package_path"
+EXAMPLE="$0 emqx 4.3.8-b3bb6075 v4.3.2 /home/alice/relup_dubug/downloaded_packages"
+
+if [[ $# -ne 4 ]]; then
+    echo "$USAGE"
+    echo "$EXAMPLE"
+    exit 1
+fi
+
+set -ex
+
+PROFILE="$1"
+VSN="$2"
+OLD_VSN="$3"
+PACKAGE_PATH="$4"
+
+TEMPDIR=$(mktemp -d)
+trap '{ rm -rf -- "$TEMPDIR"; }' EXIT
+
+git clone --branch=master "https://github.com/terry-xiaoyu/one_more_emqx.git" "$TEMPDIR/one_more_emqx"
+cp -r "$PACKAGE_PATH" "$TEMPDIR/packages"
+cp relup.lux "$TEMPDIR/"
+cp -r http_server "$TEMPDIR/http_server"
+
+exec docker run \
+    -v "$TEMPDIR:/relup_test" \
+    -w "/relup_test" \
+    -e REBAR_COLOR=none \
+    -it emqx/relup-test-env:erl23.2.7.2-emqx-3-ubuntu20.04 \
+        lux \
+        --progress verbose \
+        --case_timeout infinity \
+        --var PROFILE="$PROFILE" \
+        --var PACKAGE_PATH="/relup_test/packages" \
+        --var ONE_MORE_EMQX_PATH="/relup_test/one_more_emqx" \
+        --var VSN="$VSN" \
+        --var OLD_VSN="$OLD_VSN" \
+        relup.lux

.ci/fvt_tests/relup.lux

@@ -1,15 +1,12 @@
 [config var=PROFILE]
 [config var=PACKAGE_PATH]
-[config var=BENCH_PATH]
 [config var=ONE_MORE_EMQX_PATH]
 [config var=VSN]
-[config var=OLD_VSNS]
+[config var=OLD_VSN]
 
 [config shell_cmd=/bin/bash]
 [config timeout=600000]
 
-[loop old_vsn $OLD_VSNS]
-
 [shell http_server]
     !cd http_server
     !rebar3 shell
@@ -21,29 +18,30 @@
     ?>
 
 [shell emqx]
+    !OLD_VSN=$(echo $OLD_VSN | sed  -r 's/[v|e]//g')
     !cd $PACKAGE_PATH
-    !unzip -q -o $PROFILE-ubuntu20.04-$(echo $old_vsn | sed  -r 's/[v|e]//g')-amd64.zip
+    !unzip -q -o $PROFILE-ubuntu20.04-$${OLD_VSN}-amd64.zip
     ?SH-PROMPT
 
     !cd emqx
-    !sed -i 's|listener.wss.external[ \t]*=.*|listener.wss.external = 8085|g' etc/emqx.conf
-    !sed -i '/emqx_telemetry/d' data/loaded_plugins
+    !export EMQX_LOG__LEVEL=debug
 
     !./bin/emqx start
     ?EMQ X .* is started successfully!
     ?SH-PROMPT
 
 [shell emqx2]
+    !OLD_VSN=$(echo $OLD_VSN | sed  -r 's/[v|e]//g')
     !cd $PACKAGE_PATH
     !cp -f $ONE_MORE_EMQX_PATH/one_more_$(echo $PROFILE | sed 's/-/_/g').sh .
     !./one_more_$(echo $PROFILE | sed 's/-/_/g').sh emqx2
     ?SH-PROMPT
     !cd emqx2
 
-    !sed -i '/emqx_telemetry/d' data/loaded_plugins
+    !export EMQX_LOG__LEVEL=debug
 
     !./bin/emqx start
-    ?EMQ X (.*) is started successfully!
+    ?EMQ X .* is started successfully!
     ?SH-PROMPT
 
     !./bin/emqx_ctl cluster join emqx@127.0.0.1
@@ -63,6 +61,8 @@
     !./bin/emqx_ctl rules create 'SELECT * FROM "t/#"' '[{"name":"data_to_webserver", "params": {"$$resource":  "resource:691c29ba"}}]'
     ?created
     ?SH-PROMPT
+    !sleep 5
+    ?SH-PROMPT
 
 [shell emqx]
     !./bin/emqx_ctl resources list
@@ -71,11 +71,11 @@
     !./bin/emqx_ctl rules list
     ?691c29ba
     ?SH-PROMPT
+    !./bin/emqx_ctl broker metrics | grep "messages.publish"
+    ???SH-PROMPT
 
 [shell bench]
-    !cd $BENCH_PATH
-
-    !./emqtt_bench pub -c 10 -I 1000 -t t/%i -s 64 -L 300
+    !emqtt_bench pub -c 10 -I 1000 -t t/%i -s 64 -L 300
     ???sent
 
 [shell emqx]
@@ -84,6 +84,27 @@
 
     !cp -f ../$PROFILE-ubuntu20.04-$VSN-amd64.zip releases/
 
+    ## upgrade to the new version
+    !./bin/emqx install $VSN
+    ?Made release permanent: "$VSN"
+    ?SH-PROMPT
+
+    !./bin/emqx versions |grep permanent
+    ?(.*)$VSN
+    ?SH-PROMPT
+
+    ## downgrade to the old version
+    !./bin/emqx install $${OLD_VSN}
+    ?Made release permanent:.*
+    ?SH-PROMPT
+
+    !./bin/emqx versions |grep permanent | grep -qs "$${OLD_VSN}"
+    ?SH-PROMPT:
+    !echo ==$$?==
+    ?^==0==
+    ?SH-PROMPT:
+
+    ## again, upgrade to the new version
     !./bin/emqx install $VSN
     ?Made release permanent: "$VSN"
     ?SH-PROMPT
@@ -99,12 +120,37 @@
     """
     ?SH-PROMPT
 
+    !./bin/emqx_ctl plugins list | grep --color=never emqx_management
+    ?Plugin\(emqx_management.*active=true\)
+    ?SH-PROMPT
+
 [shell emqx2]
     !echo "" > log/emqx.log.1
     ?SH-PROMPT
 
     !cp -f ../$PROFILE-ubuntu20.04-$VSN-amd64.zip releases/
 
+    ## upgrade to the new version
+    !./bin/emqx install $VSN
+    ?Made release permanent: "$VSN"
+    ?SH-PROMPT
+
+    !./bin/emqx versions |grep permanent
+    ?(.*)$VSN
+    ?SH-PROMPT
+
+    ## downgrade to the old version
+    !./bin/emqx install $${OLD_VSN}
+    ?Made release permanent:.*
+    ?SH-PROMPT
+
+    !./bin/emqx versions |grep permanent | grep -qs "$${OLD_VSN}"
+    ?SH-PROMPT:
+    !echo ==$$?==
+    ?^==0==
+    ?SH-PROMPT:
+
+    ## again, upgrade to the new version
     !./bin/emqx install $VSN
     ?Made release permanent: "$VSN"
     ?SH-PROMPT
@@ -120,11 +166,34 @@
     """
     ?SH-PROMPT
 
+    !./bin/emqx_ctl plugins list | grep --color=never emqx_management
+    ?Plugin\(emqx_management.*active=true\)
+    ?SH-PROMPT
+
 [shell bench]
     ???publish complete
     ??SH-PROMPT:
+    !sleep 30
+    ?SH-PROMPT
+
+[shell emqx]
+    !./bin/emqx_ctl broker metrics | grep "messages.publish"
+    ???SH-PROMPT
+
+## We don't guarantee not to lose a single message!
+## So even if we received 290~300 messages, we consider it as success
+[shell bench]
+    !curl --user admin:public --silent --show-error http://localhost:8081/api/v4/rules | jq -M --raw-output ".data[0].metrics[] | select(.node==\"emqx@127.0.0.1\").matched"
+    ?(29[0-9])|(300)
+    ?SH-PROMPT
+
+    !curl --user admin:public --silent --show-error http://localhost:8081/api/v4/rules | jq -M --raw-output ".data[0].actions[0].metrics[] | select(.node==\"emqx@127.0.0.1\").success"
+    ?(29[0-9])|(300)
+    ?SH-PROMPT
+
+    ## The /counter API is provided by .ci/fvt_test/http_server
     !curl http://127.0.0.1:8080/counter
-    ???{"data":300,"code":0}
+    ?\{"data":(29[0-9])|(300),"code":0\}
     ?SH-PROMPT
 
 [shell emqx2]
@@ -158,8 +227,6 @@
     !halt(3).
     ?SH-PROMPT:
 
-[endloop]
-
 [cleanup]
     !echo ==$$?==
     ?==0==

.github/ISSUE_TEMPLATE/bug-report.md

@@ -12,11 +12,11 @@ assignees: tigercl
 
 **Environment**:
 
-- EMQ X version (e.g. `emqx_ctl status`): 
+- EMQX version (e.g. `emqx_ctl status`):
 - Hardware configuration (e.g. `lscpu`):
 - OS (e.g. `cat /etc/os-release`):
 - Kernel (e.g. `uname -a`):
-- Erlang/OTP version (in case you build emqx from source code): 
+- Erlang/OTP version (in case you build emqx from source code):
 - Others:
 
 **What happened and what you expected to happen**:

.github/workflows/apps_version_check.yaml

@@ -6,7 +6,24 @@ jobs:
   check_apps_version:
     runs-on: ubuntu-20.04
 
+    strategy:
+      matrix:
+        erl_otp:
+        - erl23.2.7.2-emqx-3
+        os:
+        - ubuntu20.04
+
+    container: emqx/build-env:${{ matrix.erl_otp }}-${{ matrix.os }}
+
     steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0 # need full history
+      - name: Check relup (ce)
+        if: endsWith(github.repository, 'emqx')
+        run: ./scripts/update-appup.sh emqx --check
+      - name: Check relup (ee)
+        if: endsWith(github.repository, 'enterprise')
+        run: ./scripts/update-appup.sh emqx-ee --check
       - name: Check apps version
         run: ./scripts/apps-version-check.sh

.github/workflows/build_packages.yaml

@@ -1,21 +1,27 @@
 name: Cross build packages
 
+concurrency:
+  group: build-${{ github.event_name }}-${{ github.ref }}
+  cancel-in-progress: true
+
 on:
+  push:
+    branches:
+      - 'main-v4.**'
+    tags:
+      - v*
+      - e*
   schedule:
     - cron:  '0 */6 * * *'
-  release:
-    types:
-      - published
   workflow_dispatch:
 
 jobs:
   prepare:
     runs-on: ubuntu-20.04
-    container: emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04
+    container: emqx/build-env:erl23.2.7.2-emqx-3-ubuntu20.04
 
     outputs:
       profiles: ${{ steps.set_profile.outputs.profiles}}
-      old_vsns: ${{ steps.set_profile.outputs.old_vsns}}
 
     steps:
       - uses: actions/checkout@v2
@@ -27,15 +33,9 @@ jobs:
         shell: bash
         run: |
           cd source
-          vsn="$(./pkg-vsn.sh)"
-          pre_vsn="$(echo $vsn | grep -oE '^[0-9]+.[0-9]')"
           if make emqx-ee --dry-run > /dev/null 2>&1; then
-            old_vsns="$(git tag -l "e$pre_vsn.[0-9]" | xargs echo -n | sed "s/e$vsn//")"
-            echo "::set-output name=old_vsns::$old_vsns"
             echo "::set-output name=profiles::[\"emqx-ee\"]"
           else
-            old_vsns="$(git tag -l "v$pre_vsn.[0-9]" | xargs echo -n | sed "s/v$vsn//")"
-            echo "::set-output name=old_vsns::$old_vsns"
             echo "::set-output name=profiles::[\"emqx\", \"emqx-edge\"]"
           fi
       - name: get_all_deps
@@ -48,7 +48,6 @@ jobs:
         run: |
           echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials
           git config --global credential.helper store
-          echo "${{ secrets.CI_GIT_TOKEN }}" >> source/scripts/git-token
           make -C source deps-all
           zip -ryq source.zip source/* source/.[^.]*
       - uses: actions/upload-artifact@v2
@@ -58,13 +57,13 @@ jobs:
 
   windows:
     runs-on: windows-2019
-
     needs: prepare
     if: endsWith(github.repository, 'emqx')
-
     strategy:
       matrix:
         profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
+        otp:
+          - 23.3.4.13
         exclude:
           - profile: emqx-edge
 
@@ -76,16 +75,15 @@ jobs:
     - name: unzip source code
       run: Expand-Archive -Path source.zip -DestinationPath ./
     - uses: ilammy/msvc-dev-cmd@v1
-    - uses: gleam-lang/setup-erlang@v1.1.0
-      id: install_erlang
+    - uses: erlef/setup-beam@v1
       with:
-        otp-version: 23.2
+        otp-version: ${{ matrix.otp }}
     - name: build
       env:
         PYTHON: python
+        DIAGNOSTIC: 1
       run: |
-        $env:PATH = "${{ steps.install_erlang.outputs.erlpath }}\bin;$env:PATH"
-
+        erl -eval "erlang:display(crypto:info_lib())" -s init stop
         $version = $( "${{ github.ref }}" -replace "^(.*)/(.*)/" )
         if ($version -match "^v[0-9]+\.[0-9]+(\.[0-9]+)?") {
           $regex = "[0-9]+\.[0-9]+(-alpha|-beta|-rc)?\.[0-9]+"
@@ -101,14 +99,11 @@ jobs:
             Remove-Item -Force -Path rebar.lock
         }
         make ensure-rebar3
-        copy rebar3 "${{ steps.install_erlang.outputs.erlpath }}\bin"
-        ls "${{ steps.install_erlang.outputs.erlpath }}\bin"
-        rebar3 --help
         make ${{ matrix.profile }}
         mkdir -p _packages/${{ matrix.profile }}
         Compress-Archive -Path _build/${{ matrix.profile }}/rel/emqx -DestinationPath _build/${{ matrix.profile }}/rel/$pkg_name
         mv _build/${{ matrix.profile }}/rel/$pkg_name _packages/${{ matrix.profile }}
-        Get-FileHash -Path "_packages/${{ matrix.profile }}/$pkg_name" | Format-List | grep 'Hash' | awk '{print $3}'  > _packages/${{ matrix.profile }}/$pkg_name.sha256
+        sha256sum "_packages/${{ matrix.profile }}/$pkg_name" | head -c 64 > "_packages/${{ matrix.profile }}/${pkg_name}.sha256"
     - name: run emqx
       timeout-minutes: 1
       run: |
@@ -125,17 +120,18 @@ jobs:
         path: source/_packages/${{ matrix.profile }}/.
 
   mac:
-    runs-on: macos-10.15
-
     needs: prepare
 
     strategy:
       matrix:
         profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
         erl_otp:
-          - 23.2.7.2-emqx-2
+          - 23.2.7.2-emqx-3
         exclude:
           - profile: emqx-edge
+        macos:
+          - macos-10.15
+    runs-on: ${{ matrix.macos  }}
 
     steps:
     - uses: actions/download-artifact@v2
@@ -153,8 +149,8 @@ jobs:
     - uses: actions/cache@v2
       id: cache
       with:
-        path: ~/.kerl
-        key: erl${{ matrix.erl_otp }}-macos10.15
+        path: ~/.kerl/${{ matrix.erl_otp }}
+        key: otp-install-${{ matrix.erl_otp }}-${{ matrix.macos }}
     - name: build erlang
       if: steps.cache.outputs.cache-hit != 'true'
       timeout-minutes: 60
@@ -168,9 +164,11 @@ jobs:
     - name: build
       run: |
         . $HOME/.kerl/${{ matrix.erl_otp }}/activate
-        make -C source ensure-rebar3
-        sudo cp source/rebar3 /usr/local/bin/rebar3
-        make -C source ${{ matrix.profile }}-zip
+        cd source
+        make ensure-rebar3
+        sudo cp rebar3 /usr/local/bin/rebar3
+        rm -rf _build/${{ matrix.profile }}/lib
+        make ${{ matrix.profile }}-zip
     - name: test
       run: |
         cd source
@@ -192,8 +190,14 @@ jobs:
           exit 1
         fi
         ./emqx/bin/emqx_ctl status
-        ./emqx/bin/emqx stop
+        if ! ./emqx/bin/emqx stop; then
+          cat emqx/log/erlang.log.1 || true
+          cat emqx/log/emqx.log.1 || true
+          echo "failed to stop emqx"
+          exit 1
+        fi
         rm -rf emqx
+        #sha256sum ./_packages/${{ matrix.profile }}/$pkg_name | head -c64  > ./_packages/${{ matrix.profile }}/$pkg_name.sha256
         openssl dgst -sha256 ./_packages/${{ matrix.profile }}/$pkg_name | awk '{print $2}'  > ./_packages/${{ matrix.profile }}/$pkg_name.sha256
     - uses: actions/upload-artifact@v1
       if: startsWith(github.ref, 'refs/tags/')
@@ -207,6 +211,7 @@ jobs:
     needs: prepare
 
     strategy:
+      fail-fast: false
       matrix:
         profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
         arch:
@@ -260,35 +265,9 @@ jobs:
         path: .
     - name: unzip source code
       run: unzip -q source.zip
-    - name: downloads old emqx zip packages
-      env:
-        PROFILE: ${{ matrix.profile }}
-        ARCH: ${{ matrix.arch }}
-        SYSTEM: ${{ matrix.os }}
-        OLD_VSNS: ${{ needs.prepare.outputs.old_vsns }}
-      run: |
-        set -e -x -u
-        broker=$PROFILE
-        if [ $PROFILE = "emqx" ];then
-            broker="emqx-ce"
-        fi
-        if [ ! -z "$(echo $SYSTEM | grep -oE 'raspbian')" ]; then
-            export ARCH="arm"
-        fi
-
-        mkdir -p source/_upgrade_base
-        cd source/_upgrade_base
-        old_vsns=($(echo $OLD_VSNS | tr ' ' ' '))
-        for tag in ${old_vsns[@]}; do
-          if [ ! -z "$(echo $(curl -I -m 10 -o /dev/null -s -w %{http_code} https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip) | grep -oE "^[23]+")" ];then
-            wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip
-            wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip.sha256
-            echo "$(cat $PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip.sha256) $PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip" | sha256sum -c || exit 1
-          fi
-        done
     - name: build emqx packages
       env:
-        ERL_OTP: erl23.2.7.2-emqx-2
+        ERL_OTP: erl23.2.7.2-emqx-3
         PROFILE: ${{ matrix.profile }}
         ARCH: ${{ matrix.arch }}
         SYSTEM: ${{ matrix.os }}
@@ -333,19 +312,18 @@ jobs:
     needs: prepare
 
     strategy:
+      fail-fast: false
       matrix:
         profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
-        arch:
-          - [amd64, x86_64]
-          - [arm64v8, aarch64]
-          - [arm32v7, arm]
-          - [i386, i386]
-          - [s390x, s390x]
+        registry:
+          - 'docker.io'
+          - 'public.ecr.aws'
         exclude:
+          # we don't have an aws ecr repo for enterprise and edge yet
+          - profile: emqx-edge
+            registry: 'public.ecr.aws'
           - profile: emqx-ee
-            arch: [i386, i386]
-          - profile: emqx-ee
-            arch: [s390x, s390x]
+            registry: 'public.ecr.aws'
 
     steps:
     - uses: actions/download-artifact@v2
@@ -354,30 +332,70 @@ jobs:
         path: .
     - name: unzip source code
       run: unzip -q source.zip
-    - name: build emqx docker image
-      env:
-        PROFILE: ${{ matrix.profile }}
-        ARCH: ${{ matrix.arch[0] }}
-        QEMU_ARCH: ${{ matrix.arch[1] }}
-      run: |
-        sudo docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
-
-        cd source
-        sudo TARGET=emqx/$PROFILE ARCH=$ARCH QEMU_ARCH=$QEMU_ARCH  make docker
-        cd _packages/$PROFILE && for var in $(ls ${PROFILE}-docker-* ); do sudo bash -c "echo $(sha256sum $var | awk '{print $1}') > $var.sha256"; done && cd -
-    - uses: actions/upload-artifact@v1
-      if: startsWith(github.ref, 'refs/tags/')
+    - uses: docker/setup-buildx-action@v1
+    - uses: docker/setup-qemu-action@v1
       with:
-        name: ${{ matrix.profile }}
-        path: source/_packages/${{ matrix.profile }}/.
-
-  delete-artifact:
-    runs-on: ubuntu-20.04
-    needs: [prepare, mac, linux, docker]
-    steps:
-    - uses: geekyeggo/delete-artifact@v1
+        image: tonistiigi/binfmt:latest
+        platforms: all
+    - uses: aws-actions/configure-aws-credentials@v1
+      if: matrix.registry == 'public.ecr.aws'
       with:
-        name: source
+        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        aws-region: ${{ secrets.AWS_DEFAULT_REGION }}
+    - name: Docker login to aws ecr
+      if: matrix.registry == 'public.ecr.aws'
+      run: aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws
+    - uses: docker/login-action@v1
+      if: matrix.registry == 'docker.io'
+      with:
+        username: ${{ secrets.DOCKER_HUB_USER }}
+        password: ${{ secrets.DOCKER_HUB_TOKEN }}
+    - uses: docker/metadata-action@v3
+      id: meta
+      with:
+        images: ${{ matrix.registry }}/${{ github.repository_owner }}/${{ matrix.profile }}
+        ## only stable tag is latest
+        flavor: |
+          latest=${{ contains(github.ref, 'tags') && !contains(github.ref_name, 'rc') && !contains(github.ref_name, 'beta') && !contains(github.ref_name, 'alpha') }}
+        tags: |
+          type=ref,event=branch
+          type=ref,event=pr
+          type=match,pattern=[v|e](.*),group=1
+        labels:
+          org.opencontainers.image.otp.version=${{ matrix.otp }}
+    - uses: docker/build-push-action@v2
+      if: matrix.profile != 'emqx-ee'
+      with:
+        ## only push when stable tag and rc tag
+        push: ${{ contains(github.ref, 'tags') && !contains(github.ref_name, 'beta') && !contains(github.ref_name, 'alpha') }}
+        pull: true
+        no-cache: true
+        platforms: linux/amd64,linux/arm64
+        tags: ${{ steps.meta.outputs.tags }}
+        labels: ${{ steps.meta.outputs.labels }}
+        build-args: |
+          BUILD_FROM=emqx/build-env:erl23.2.7.2-emqx-3-alpine
+          RUN_FROM=alpine:3.12
+          EMQX_NAME=${{ matrix.profile }}
+        file: source/deploy/docker/Dockerfile
+        context: source
+    - uses: docker/build-push-action@v2
+      if: matrix.profile == 'emqx-ee'
+      with:
+        ## only push when stable tag and rc tag
+        push: ${{ contains(github.ref, 'tags') && !contains(github.ref_name, 'beta') && !contains(github.ref_name, 'alpha') }}
+        pull: true
+        no-cache: true
+        platforms: linux/amd64,linux/arm64
+        tags: ${{ steps.meta.outputs.tags }}
+        labels: ${{ steps.meta.outputs.labels }}
+        build-args: |
+          BUILD_FROM=emqx/build-env:erl23.2.7.2-emqx-3-alpine
+          RUN_FROM=alpine:3.12
+          EMQX_NAME=${{ matrix.profile }}
+        file: source/deploy/docker/Dockerfile.enterprise
+        context: source
 
   upload:
     runs-on: ubuntu-20.04
@@ -387,106 +405,37 @@ jobs:
     needs: [prepare, mac, linux, docker]
 
     strategy:
+      fail-fast: false
       matrix:
         profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
-
     steps:
-    - uses: actions/checkout@v2
-    - name: get_version
-      run: |
-        echo 'version<<EOF' >> $GITHUB_ENV
-        echo ${{ github.ref }} | sed -r  "s ^refs/heads/|^refs/tags/(.*) \1 g" >> $GITHUB_ENV
-        echo 'EOF' >> $GITHUB_ENV
     - uses: actions/download-artifact@v2
       with:
         name: ${{ matrix.profile }}
-        path: ./_packages/${{ matrix.profile }}
+        path: packages/${{ matrix.profile }}
     - name: install dos2unix
       run: sudo apt-get update && sudo apt install -y dos2unix
     - name: get packages
       run: |
         set -e -u
-        cd _packages/${{ matrix.profile }}
+        cd packages/${{ matrix.profile }}
         for var in $( ls |grep emqx |grep -v sha256); do
           dos2unix $var.sha256
           echo "$(cat $var.sha256) $var" | sha256sum -c || exit 1
         done
         cd -
+    - uses: aws-actions/configure-aws-credentials@v1
+      with:
+        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        aws-region: ${{ secrets.AWS_DEFAULT_REGION }}
     - name: upload aws s3
       run: |
         set -e -u
         if [ "${{ matrix.profile }}"  == "emqx" ];then
-            broker="emqx-ce"
+            s3dir="emqx-ce"
         else
-            broker=${{ matrix.profile }}
+            s3dir=${{ matrix.profile }}
         fi
-        aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID }}
-        aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-        aws configure set default.region ${{ secrets.AWS_DEFAULT_REGION }}
-
-        aws s3 cp --recursive _packages/${{ matrix.profile }} s3://${{ secrets.AWS_S3_BUCKET }}/$broker/${{ env.version }}
-        aws cloudfront create-invalidation --distribution-id ${{ secrets.AWS_CLOUDFRONT_ID }} --paths "/$broker/${{ env.version }}/*"
-    - uses: Rory-Z/upload-release-asset@v1
-      if: github.event_name == 'release' && matrix.profile != 'emqx-ee'
-      with:
-        repo: emqx
-        path: "_packages/${{ matrix.profile }}/emqx-*"
-        token: ${{ github.token }}
-    - uses: Rory-Z/upload-release-asset@v1
-      if: github.event_name == 'release' && matrix.profile == 'emqx-ee'
-      with:
-        repo: emqx-enterprise
-        path: "_packages/${{ matrix.profile }}/emqx-*"
-        token: ${{ github.token }}
-    - name: update to emqx.io
-      if: github.event_name == 'release'
-      run: |
-        set -e -x -u
-        curl -w %{http_code} \
-             --insecure \
-             -H "Content-Type: application/json" \
-             -H "token: ${{ secrets.EMQX_IO_TOKEN }}" \
-             -X POST \
-             -d "{\"repo\":\"emqx/emqx\", \"tag\": \"${{ env.version }}\" }" \
-             ${{ secrets.EMQX_IO_RELEASE_API }}
-    - name: push docker image to docker hub
-      if: github.event_name == 'release'
-      run: |
-        set -e -x -u
-        sudo make docker-prepare
-        cd _packages/${{ matrix.profile }} && for var in $(ls |grep docker |grep -v sha256); do unzip $var; sudo docker load < ${var%.*}; rm -f ${var%.*}; done && cd -
-        echo ${{ secrets.DOCKER_HUB_TOKEN }} |sudo docker login -u ${{ secrets.DOCKER_HUB_USER }} --password-stdin
-        sudo TARGET=emqx/${{ matrix.profile }} make docker-push
-        sudo TARGET=emqx/${{ matrix.profile }} make docker-manifest-list
-    - name: update repo.emqx.io
-      if: github.event_name == 'release' && endsWith(github.repository, 'enterprise') && matrix.profile == 'emqx-ee'
-      run: |
-        curl --silent --show-error \
-          -H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \
-          -H "Accept: application/vnd.github.v3+json" \
-          -X POST \
-          -d "{\"ref\":\"v1.0.1\",\"inputs\":{\"version\": \"${{ env.version }}\", \"emqx_ee\": \"true\"}}" \
-          "https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/update_emqx_repos.yaml/dispatches"
-    - name: update repo.emqx.io
-      if: github.event_name == 'release' && endsWith(github.repository, 'emqx') && matrix.profile == 'emqx'
-      run: |
-        curl --silent --show-error \
-          -H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \
-          -H "Accept: application/vnd.github.v3+json" \
-          -X POST \
-          -d "{\"ref\":\"v1.0.1\",\"inputs\":{\"version\": \"${{ env.version }}\", \"emqx_ce\": \"true\"}}" \
-          "https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/update_emqx_repos.yaml/dispatches"
-    - name: update homebrew packages
-      if: github.event_name == 'release' && endsWith(github.repository, 'emqx') && matrix.profile == 'emqx'
-      run: |
-        if [ -z $(echo $version | grep -oE "(alpha|beta|rc)\.[0-9]") ]; then
-            curl --silent --show-error \
-              -H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \
-              -H "Accept: application/vnd.github.v3+json" \
-              -X POST \
-              -d "{\"ref\":\"v1.0.1\",\"inputs\":{\"version\": \"${{ env.version }}\"}}" \
-              "https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/update_emqx_homebrew.yaml/dispatches"
-        fi
-    - uses: geekyeggo/delete-artifact@v1
-      with:
-        name: ${{ matrix.profile }}
+        aws s3 cp --recursive packages/${{ matrix.profile }} s3://${{ secrets.AWS_S3_BUCKET }}/${s3dir}/${{ github.ref_name }}
+        aws cloudfront create-invalidation --distribution-id ${{ secrets.AWS_CLOUDFRONT_ID }} --paths "/${s3dir}/${{ github.ref_name }}/*"

.github/workflows/build_slim_packages.yaml

@@ -1,10 +1,6 @@
 name: Build slim packages
 
 on:
-  push:
-    tags:
-      - v*
-      - e*
   pull_request:
   workflow_dispatch:
 
@@ -15,7 +11,7 @@ jobs:
     strategy:
       matrix:
         erl_otp:
-        - erl23.2.7.2-emqx-2
+        - erl23.2.7.2-emqx-3
         os:
         - ubuntu20.04
         - centos7
@@ -29,7 +25,6 @@ jobs:
         if make emqx-ee --dry-run > /dev/null 2>&1; then
           echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials
           git config --global credential.helper store
-          echo "${{ secrets.CI_GIT_TOKEN }}" >> ./scripts/git-token
           echo "EMQX_NAME=emqx-ee" >> $GITHUB_ENV
         else
           echo "EMQX_NAME=emqx" >> $GITHUB_ENV
@@ -38,6 +33,11 @@ jobs:
       run: make ${EMQX_NAME}-zip
     - name: build deb/rpm packages
       run: make ${EMQX_NAME}-pkg
+    - uses: actions/upload-artifact@v1
+      if: failure()
+      with:
+        name: rebar3.crashdump
+        path: ./rebar3.crashdump
     - name: pakcages test
       run: |
         export CODE_PATH=$GITHUB_WORKSPACE
@@ -48,13 +48,13 @@ jobs:
         path: _packages/**/*.zip
 
   mac:
-    runs-on: macos-10.15
-
     strategy:
       matrix:
         erl_otp:
-        - 23.2.7.2-emqx-2
-
+        - 23.2.7.2-emqx-3
+        macos:
+        - macos-11
+    runs-on: ${{ matrix.macos }}
     steps:
     - uses: actions/checkout@v1
     - name: prepare
@@ -62,7 +62,6 @@ jobs:
         if make emqx-ee --dry-run > /dev/null 2>&1; then
           echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials
           git config --global credential.helper store
-          echo "${{ secrets.CI_GIT_TOKEN }}" >> ./scripts/git-token
           echo "EMQX_NAME=emqx-ee" >> $GITHUB_ENV
         else
           echo "EMQX_NAME=emqx" >> $GITHUB_ENV
@@ -76,8 +75,8 @@ jobs:
     - uses: actions/cache@v2
       id: cache
       with:
-        path: ~/.kerl
-        key: erl${{ matrix.erl_otp }}-macos10.15
+        path: ~/.kerl/${{ matrix.erl_otp }}
+        key: otp-install-${{ matrix.erl_otp }}-${{ matrix.macos }}
     - name: build erlang
       if: steps.cache.outputs.cache-hit != 'true'
       timeout-minutes: 60
@@ -94,6 +93,11 @@ jobs:
         make ensure-rebar3
         sudo cp rebar3 /usr/local/bin/rebar3
         make ${EMQX_NAME}-zip
+    - uses: actions/upload-artifact@v1
+      if: failure()
+      with:
+        name: rebar3.crashdump
+        path: ./rebar3.crashdump
     - name: test
       run: |
         pkg_name=$(basename _packages/${EMQX_NAME}/emqx-*.zip)

.github/workflows/check_deps_integrity.yaml

@@ -5,7 +5,7 @@ on: [pull_request]
 jobs:
   check_deps_integrity:
     runs-on: ubuntu-20.04
-    container: emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04
+    container: emqx/build-env:erl23.2.7.2-emqx-3-ubuntu20.04
 
     steps:
       - uses: actions/checkout@v2

.github/workflows/elvis_lint.yaml

@@ -1,16 +0,0 @@
-name: Elvis Linter
-
-on: [pull_request]
-
-jobs:
-  build:
-    runs-on: ubuntu-20.04
-    steps:
-      - uses: actions/checkout@v2
-      - name: Set git token
-        if: endsWith(github.repository, 'enterprise')
-        run: |
-          echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials
-          git config --global credential.helper store
-      - run: |
-          ./scripts/elvis-check.sh $GITHUB_BASE_REF

.github/workflows/git_sync.yaml

@@ -3,7 +3,6 @@ name: Sync to enterprise
 on:
   push:
     branches:
-      - master
       - main-v*
 
 jobs:
@@ -23,11 +22,7 @@ jobs:
         id: create_pull_request
         run: |
           set -euo pipefail
-          if [ "$GITHUB_REF" = "refs/heads/master" ]; then
-            EE_REF="refs/heads/enterprise"
-          else
-            EE_REF="${GITHUB_REF}-enterprise"
-          fi
+          EE_REF="${GITHUB_REF}-enterprise"
           R=$(curl --silent --show-error \
             -H "Accept: application/vnd.github.v3+json" \
             -H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \

.github/workflows/release.yaml

@@ -0,0 +1,97 @@
+name: Upload release assets
+on:
+  release:
+    types:
+      - published
+
+jobs:
+  prepare:
+    runs-on: ubuntu-20.04
+    container: emqx/build-env:erl23.2.7.2-emqx-3-ubuntu20.04
+
+    outputs:
+      profiles: ${{ steps.set_profile.outputs.profiles}}
+      s3dir: ${{ steps.set_profile.outputs.s3dir}}
+
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          path: source
+          fetch-depth: 0
+      - name: set profile
+        id: set_profile
+        shell: bash
+        run: |
+          cd source
+          if make emqx-ee --dry-run > /dev/null 2>&1; then
+            echo "::set-output name=profiles::[\"emqx-ee\"]"
+          else
+            echo "::set-output name=profiles::[\"emqx\", \"emqx-edge\"]"
+          fi
+
+  upload:
+    runs-on: ubuntu-20.04
+    needs: prepare
+    strategy:
+      fail-fast: false
+      matrix:
+        profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
+
+    steps:
+      - uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ secrets.AWS_DEFAULT_REGION }}
+      - name: Get packages
+        run: |
+          if [ "${{ matrix.profile }}"  == "emqx" ];then
+              s3dir="emqx-ce"
+          else
+              s3dir=${{ matrix.profile }}
+          fi
+          aws s3 cp --recursive  s3://${{ secrets.AWS_S3_BUCKET }}/$s3dir/${{ github.ref_name }} packages
+      - uses: alexellis/upload-assets@0.2.2
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        with:
+          asset_paths: '["packages/*"]'
+      - name: update to emqx.io
+        run: |
+          set -e -x -u
+          curl -w %{http_code} \
+               --insecure \
+               -H "Content-Type: application/json" \
+               -H "token: ${{ secrets.EMQX_IO_TOKEN }}" \
+               -X POST \
+               -d "{\"repo\":\"emqx/emqx\", \"tag\": \"${{ github.ref_name }}\" }" \
+               ${{ secrets.EMQX_IO_RELEASE_API }}
+      - name: update repo.emqx.io
+        if: github.event_name == 'release' && endsWith(github.repository, 'enterprise') && matrix.profile == 'emqx-ee'
+        run: |
+          curl --silent --show-error \
+            -H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \
+            -H "Accept: application/vnd.github.v3+json" \
+            -X POST \
+            -d "{\"ref\":\"v1.0.3\",\"inputs\":{\"version\": \"${{ github.ref_name }}\", \"emqx_ee\": \"true\"}}" \
+            "https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/update_emqx_repos.yaml/dispatches"
+      - name: update repo.emqx.io
+        if: github.event_name == 'release' && endsWith(github.repository, 'emqx') && matrix.profile == 'emqx'
+        run: |
+          curl --silent --show-error \
+            -H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \
+            -H "Accept: application/vnd.github.v3+json" \
+            -X POST \
+            -d "{\"ref\":\"v1.0.3\",\"inputs\":{\"version\": \"${{ github.ref_name }}\", \"emqx_ce\": \"true\"}}" \
+            "https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/update_emqx_repos.yaml/dispatches"
+      - name: update homebrew packages
+        if: github.event_name == 'release' && endsWith(github.repository, 'emqx') && matrix.profile == 'emqx'
+        run: |
+          if [ -z $(echo $version | grep -oE "(alpha|beta|rc)\.[0-9]") ]; then
+              curl --silent --show-error \
+                -H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \
+                -H "Accept: application/vnd.github.v3+json" \
+                -X POST \
+                -d "{\"ref\":\"v1.0.3\",\"inputs\":{\"version\": \"${{ github.ref_name }}\"}}" \
+                "https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/update_emqx_homebrew.yaml/dispatches"
+          fi

.github/workflows/run_acl_migration_tests.yaml

@@ -0,0 +1,22 @@
+name: ACL fix & migration integration tests
+
+on: workflow_dispatch
+
+jobs:
+    test:
+        runs-on: ubuntu-20.04
+        container: emqx/build-env:erl23.2.7.2-emqx-3-ubuntu20.04
+        strategy:
+            fail-fast: true
+        env:
+            BASE_VERSION: "4.3.0"
+        steps:
+        - uses: actions/checkout@v2
+          with:
+            path: emqx
+        - name: Prepare scripts
+          run: |
+            cp ./emqx/.ci/acl_migration_test/*.sh ./
+        - name: Run tests
+          run: |
+            ./suite.sh emqx "$BASE_VERSION"

.github/workflows/run_automate_tests.yaml

@@ -0,0 +1,442 @@
+name: Integration Test Suites
+
+on:
+  push:
+    tags:
+      - "v4.*"
+  pull_request:
+    branches:
+      - "main-v4.*"
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    outputs:
+      imgname: ${{ steps.build_docker.outputs.imgname}}
+      version: ${{ steps.build_docker.outputs.version}}
+    steps:
+    - name: download jmeter
+      id: dload_jmeter
+      timeout-minutes: 1
+      env:
+        JMETER_VERSION: 5.4.3
+      run: |
+        wget --no-verbose --no-check-certificate -O /tmp/apache-jmeter.tgz https://downloads.apache.org/jmeter/binaries/apache-jmeter-$JMETER_VERSION.tgz
+    - uses: actions/upload-artifact@v2
+      with:
+        name: apache-jmeter.tgz
+        path: /tmp/apache-jmeter.tgz
+    - uses: actions/checkout@v2
+    - name: build docker
+      id: build_docker
+      run: |
+        if [ -f EMQX_ENTERPRISE ]; then
+          echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials
+          git config --global credential.helper store
+          make deps-emqx-ee
+          make clean
+          make emqx-ee-docker
+          echo "::set-output name=imgname::emqx-ee"
+          echo "::set-output name=version::$(./pkg-vsn.sh)"
+          docker save emqx/emqx-ee:$(./pkg-vsn.sh) -o emqx.tar
+        else
+          make emqx-docker
+          echo "::set-output name=imgname::emqx"
+          echo "::set-output name=version::$(./pkg-vsn.sh)"
+          docker save emqx/emqx:$(./pkg-vsn.sh) -o emqx.tar
+        fi
+    - uses: actions/upload-artifact@v2
+      with:
+        name: emqx-docker-image
+        path: emqx.tar
+
+  webhook:
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        webhook_type:
+        - webhook_data_bridge
+
+    needs: build
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/download-artifact@v2
+      with:
+        name: emqx-docker-image
+        path: /tmp
+    - name: load docker image
+      run: docker load < /tmp/emqx.tar
+    - name: docker compose up
+      timeout-minutes: 5
+      env:
+        TARGET: emqx/${{ needs.build.outputs.imgname }}
+        EMQX_TAG: ${{ needs.build.outputs.version }}
+      run: |
+        docker-compose \
+          -f .ci/docker-compose-file/docker-compose-emqx-cluster.yaml \
+          up -d --build
+    - uses: actions/checkout@v2
+      with:
+        repository: emqx/emqx-svt-web-server
+        ref: web-server-1.0
+        path: emqx-svt-web-server
+    - uses: actions/download-artifact@v2
+    - name: run webserver in docker
+      run: |
+        cd ./emqx-svt-web-server/svtserver
+        mvn clean package
+        cd target
+        docker run --name webserver --network emqx_bridge -d -v $(pwd)/svtserver-0.0.1.jar:/webserver/svtserver-0.0.1.jar --workdir /webserver openjdk:8-jdk bash \
+        -c "java -jar svtserver-0.0.1.jar"
+    - name: wait docker compose up
+      timeout-minutes: 5
+      run: |
+        while [ "$(docker inspect -f '{{ .State.Health.Status}}' node1.emqx.io)" != "healthy" ] || [ "$(docker inspect -f '{{ .State.Health.Status}}' node2.emqx.io)" != "healthy" ]; do
+          echo "['$(date -u +"%y-%m-%dt%h:%m:%sz")']:waiting emqx";
+          sleep 5;
+        done
+        docker ps -a
+        echo HAPROXY_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' haproxy) >> $GITHUB_ENV
+        echo WEB_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' webserver) >> $GITHUB_ENV
+    - uses: actions/checkout@v2
+      with:
+        repository: emqx/emqx-fvt
+        ref: v1.6.0
+        path: scripts
+    - uses: actions/setup-java@v1
+      with:
+        java-version: '8.0.282' # The JDK version to make available on the path.
+        java-package: jdk # (jre, jdk, or jdk+fx) - defaults to jdk
+        architecture: x64 # (x64 or x86) - defaults to x64
+    - uses: actions/download-artifact@v2
+      with:
+        name: apache-jmeter.tgz
+        path: /tmp
+    - name: install jmeter
+      timeout-minutes: 10
+      env:
+          JMETER_VERSION: 5.4.3
+      run: |
+        cd /tmp && tar -xvf apache-jmeter.tgz
+        echo "jmeter.save.saveservice.output_format=xml" >> /tmp/apache-jmeter-$JMETER_VERSION/user.properties
+        echo "jmeter.save.saveservice.response_data.on_error=true" >> /tmp/apache-jmeter-$JMETER_VERSION/user.properties
+        wget --no-verbose -O /tmp/apache-jmeter-$JMETER_VERSION/lib/ext/mqtt-xmeter-2.0.2-jar-with-dependencies.jar https://raw.githubusercontent.com/xmeter-net/mqtt-jmeter/master/Download/v2.0.2/mqtt-xmeter-2.0.2-jar-with-dependencies.jar
+        ln -s /tmp/apache-jmeter-$JMETER_VERSION /opt/jmeter
+    - name: run jmeter
+      run: |
+        /opt/jmeter/bin/jmeter.sh \
+          -Jjmeter.save.saveservice.output_format=xml -n \
+          -t scripts/automate-test-suite/${{ matrix.webhook_type }}.jmx \
+          -Demqx_ip=$HAPROXY_IP \
+          -Dweb_ip=$WEB_IP \
+          -l jmeter_logs/webhook_${{ matrix.webhook_type }}.jtl \
+          -j jmeter_logs/logs/webhook_${{ matrix.webhook_type }}.log
+    - name: check logs
+      run: |
+        if cat jmeter_logs/webhook_${{ matrix.webhook_type }}.jtl | grep -e '<failure>true</failure>' > /dev/null 2>&1; then
+          echo "check logs filed"
+          exit 1
+        fi
+    - uses: actions/upload-artifact@v1
+      if: always()
+      with:
+        name: jmeter_logs
+        path: ./jmeter_logs
+
+  mysql:
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        mysql_tag:
+        - 5.7
+        - 8
+        mysql_type:
+        - mysql_auth_acl
+
+    needs: build
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/download-artifact@v2
+      with:
+        name: emqx-docker-image
+        path: /tmp
+    - name: load docker image
+      run: docker load < /tmp/emqx.tar
+    - name: docker compose up
+      timeout-minutes: 5
+      env:
+        TARGET: emqx/${{ needs.build.outputs.imgname }}
+        EMQX_TAG: ${{ needs.build.outputs.version }}
+        MYSQL_TAG: ${{ matrix.mysql_tag }}
+      run: |
+        docker-compose \
+          -f .ci/docker-compose-file/docker-compose-emqx-cluster.yaml \
+          -f .ci/docker-compose-file/docker-compose-mysql-tls.yaml \
+          up -d --build
+    - name: wait docker compose up
+      timeout-minutes: 5
+      run: |
+        while [ "$(docker inspect -f '{{ .State.Health.Status}}' node1.emqx.io)" != "healthy" ] || [ "$(docker inspect -f '{{ .State.Health.Status}}' node2.emqx.io)" != "healthy" ]; do
+          echo "['$(date -u +"%y-%m-%dt%h:%m:%sz")']:waiting emqx";
+          sleep 5;
+        done
+        while [ $(docker ps -a --filter name=client --filter exited=0 | wc -l) \
+             != $(docker ps -a --filter name=client | wc -l) ]; do
+          sleep 1
+        done
+        docker ps -a
+        echo HAPROXY_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' haproxy) >> $GITHUB_ENV
+        echo MYSQL_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' mysql) >> $GITHUB_ENV
+    - uses: actions/checkout@v2
+      with:
+        repository: emqx/emqx-fvt
+        ref: v1.6.0
+        path: scripts
+    - uses: actions/setup-java@v1
+      with:
+        java-version: '8.0.282' # The JDK version to make available on the path.
+        java-package: jdk # (jre, jdk, or jdk+fx) - defaults to jdk
+        architecture: x64 # (x64 or x86) - defaults to x64
+    - uses: actions/download-artifact@v2
+      with:
+        name: apache-jmeter.tgz
+        path: /tmp
+    - name: install jmeter
+      timeout-minutes: 10
+      env:
+          JMETER_VERSION: 5.4.3
+      run: |
+        cd /tmp && tar -xvf apache-jmeter.tgz
+        echo "jmeter.save.saveservice.output_format=xml" >> /tmp/apache-jmeter-$JMETER_VERSION/user.properties
+        echo "jmeter.save.saveservice.response_data.on_error=true" >> /tmp/apache-jmeter-$JMETER_VERSION/user.properties
+        wget --no-verbose -O /tmp/apache-jmeter-$JMETER_VERSION/lib/ext/mqtt-xmeter-2.0.2-jar-with-dependencies.jar https://raw.githubusercontent.com/xmeter-net/mqtt-jmeter/master/Download/v2.0.2/mqtt-xmeter-2.0.2-jar-with-dependencies.jar
+        ln -s /tmp/apache-jmeter-$JMETER_VERSION /opt/jmeter
+    - name: install jmeter plugin
+      run: |
+        wget --no-verbose -O "/opt/jmeter/lib/mysql-connector-java-8.0.16.jar" https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar
+    - name: run jmeter
+      run: |
+        /opt/jmeter/bin/jmeter.sh \
+          -Jjmeter.save.saveservice.output_format=xml -n \
+          -t scripts/automate-test-suite/${{ matrix.mysql_type }}.jmx \
+          -Droute="apps/emqx_auth_mysql/test/emqx_auth_mysql_SUITE_data" \
+          -Dmysql_ip=$MYSQL_IP \
+          -Demqx_ip=$HAPROXY_IP \
+          -Ddbname="mqtt" \
+          -Dmysql_user="ssluser" \
+          -Ddb_user="root" \
+          -Dmysql_pwd="public" \
+          -Dconfig_path="/tmp/etc" \
+          -Ddocker_path=".ci/docker-compose-file" \
+          -l jmeter_logs/${{ matrix.mysql_type }}_${{ matrix.mysql_tag }}.jtl \
+          -j jmeter_logs/logs/${{ matrix.mysql_type }}_${{ matrix.mysql_tag }}.log
+    - name: check logs
+      run: |
+        if cat jmeter_logs/${{ matrix.mysql_type }}_${{ matrix.mysql_tag }}.jtl | grep -e '<failure>true</failure>' > /dev/null 2>&1; then
+          echo "check logs filed"
+          exit 1
+        fi
+    - uses: actions/upload-artifact@v1
+      if: always()
+      with:
+        name: jmeter_logs
+        path: ./jmeter_logs
+
+
+  postgresql:
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        pgsql_type:
+        - pgsql_auth_acl
+        pgsql_tag:
+        - 9
+        - 10
+        - 11
+        - 12
+        - 13
+
+    needs: build
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/download-artifact@v2
+      with:
+        name: emqx-docker-image
+        path: /tmp
+    - name: load docker image
+      run: docker load < /tmp/emqx.tar
+    - name: docker compose up
+      timeout-minutes: 5
+      env:
+        TARGET: emqx/${{ needs.build.outputs.imgname }}
+        EMQX_TAG: ${{ needs.build.outputs.version }}
+        PGSQL_TAG: ${{ matrix.pgsql_tag }}
+      run: |
+        docker-compose \
+          -f .ci/docker-compose-file/docker-compose-emqx-broker-cluster.yaml \
+          -f .ci/docker-compose-file/docker-compose-pgsql-tls.yaml \
+          up -d --build
+    - name: wait docker compose up
+      timeout-minutes: 5
+      run: |
+        while [ "$(docker inspect -f '{{ .State.Health.Status}}' node1.emqx.io)" != "healthy" ] || [ "$(docker inspect -f '{{ .State.Health.Status}}' node2.emqx.io)" != "healthy" ]; do
+          echo "['$(date -u +"%y-%m-%dt%h:%m:%sz")']:waiting emqx";
+          sleep 5;
+        done
+        docker ps -a
+        echo HAPROXY_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' haproxy) >> $GITHUB_ENV
+        echo PGSQL_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' pgsql) >> $GITHUB_ENV
+        echo CONFIG_PATH=$(docker inspect -f '{{ range .Mounts }}{{ if eq .Name "docker-compose-file_etc" }}{{ .Source }}{{ end }}{{ end }}' node1.emqx.io) >> $GITHUB_ENV
+    - uses: actions/checkout@v2
+      with:
+        repository: emqx/emqx-fvt
+        ref: v1.6.0
+        path: scripts
+    - uses: actions/setup-java@v1
+      with:
+        java-version: '8.0.282' # The JDK version to make available on the path.
+        java-package: jdk # (jre, jdk, or jdk+fx) - defaults to jdk
+        architecture: x64 # (x64 or x86) - defaults to x64
+    - uses: actions/download-artifact@v2
+      with:
+        name: apache-jmeter.tgz
+        path: /tmp
+    - name: install jmeter
+      timeout-minutes: 10
+      env:
+          JMETER_VERSION: 5.4.3
+      run: |
+        cd /tmp && tar -xvf apache-jmeter.tgz
+        echo "jmeter.save.saveservice.output_format=xml" >> /tmp/apache-jmeter-$JMETER_VERSION/user.properties
+        echo "jmeter.save.saveservice.response_data.on_error=true" >> /tmp/apache-jmeter-$JMETER_VERSION/user.properties
+        wget --no-verbose -O /tmp/apache-jmeter-$JMETER_VERSION/lib/ext/mqtt-xmeter-2.0.2-jar-with-dependencies.jar https://raw.githubusercontent.com/xmeter-net/mqtt-jmeter/master/Download/v2.0.2/mqtt-xmeter-2.0.2-jar-with-dependencies.jar
+        ln -s /tmp/apache-jmeter-$JMETER_VERSION /opt/jmeter
+    - name: install jmeter plugin
+      run: |
+        wget --no-verbose -O "/opt/jmeter/lib/postgresql-42.2.18.jar" https://repo1.maven.org/maven2/org/postgresql/postgresql/42.2.18/postgresql-42.2.18.jar
+    - name: run jmeter
+      run: |
+        sudo /opt/jmeter/bin/jmeter.sh \
+          -Jjmeter.save.saveservice.output_format=xml -n \
+          -t scripts/automate-test-suite/${{ matrix.pgsql_type }}.jmx \
+          -Droute="apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data" \
+          -Dca_name="ca.pem" \
+          -Dkey_name="client-key.pem" \
+          -Dcert_name="client-cert.pem" \
+          -Ddb_ip=$PGSQL_IP \
+          -Dpgsql_ip=$PGSQL_IP \
+          -Demqx_ip=$HAPROXY_IP \
+          -Dpgsql_user="root" \
+          -Dpgsql_pwd="public" \
+          -Ddbname="mqtt" \
+          -Dpgsql_db="mqtt" \
+          -Dport="5432" \
+          -Dconfig_path=$CONFIG_PATH \
+          -Ddocker_path=".ci/docker-compose-file" \
+          -l jmeter_logs/${{ matrix.pgsql_type }}_${{ matrix.pgsql_tag }}.jtl \
+          -j jmeter_logs/logs/${{ matrix.pgsql_type }}_${{ matrix.pgsql_tag }}.log
+    - name: check logs
+      run: |
+        if cat jmeter_logs/${{ matrix.pgsql_type }}_${{ matrix.pgsql_tag }}.jtl | grep -e '<failure>true</failure>' > /dev/null 2>&1; then
+          echo "check logs filed"
+          exit 1
+        fi
+    - uses: actions/upload-artifact@v1
+      if: always()
+      with:
+        name: jmeter_logs
+        path: ./jmeter_logs
+
+  http:
+    runs-on: ubuntu-latest
+
+    needs: build
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/download-artifact@v2
+      with:
+        name: emqx-docker-image
+        path: /tmp
+    - name: load docker image
+      run: docker load < /tmp/emqx.tar
+    - name: docker compose up
+      timeout-minutes: 5
+      env:
+        TARGET: emqx/${{ needs.build.outputs.imgname }}
+        EMQX_TAG: ${{ needs.build.outputs.version }}
+        MYSQL_TAG: 8
+      run: |
+        docker-compose \
+          -f .ci/docker-compose-file/docker-compose-emqx-broker-cluster.yaml \
+          -f .ci/docker-compose-file/docker-compose-mysql-tcp.yaml \
+          -f .ci/docker-compose-file/docker-compose-enterprise-tomcat-tcp.yaml \
+          up -d --build
+    - name: wait docker compose up
+      timeout-minutes: 5
+      run: |
+        while [ "$(docker inspect -f '{{ .State.Health.Status}}' node1.emqx.io)" != "healthy" ] || [ "$(docker inspect -f '{{ .State.Health.Status}}' node2.emqx.io)" != "healthy" ]; do
+          echo "['$(date -u +"%y-%m-%dt%h:%m:%sz")']:waiting emqx";
+          sleep 5;
+        done
+        docker ps -a
+        echo HAPROXY_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' haproxy) >> $GITHUB_ENV
+        echo HTTP_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' Tomcat) >> $GITHUB_ENV
+        echo MYSQL_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' mysql) >> $GITHUB_ENV
+        echo CONFIG_PATH=$(docker inspect -f '{{ range .Mounts }}{{ if eq .Name "docker-compose-file_etc" }}{{ .Source }}{{ end }}{{ end }}' node1.emqx.io) >> $GITHUB_ENV
+    - uses: actions/checkout@v2
+      with:
+        repository: emqx/emqx-fvt
+        ref: v1.6.0
+        path: scripts
+    - uses: actions/setup-java@v1
+      with:
+        java-version: '8.0.282' # The JDK version to make available on the path.
+        java-package: jdk # (jre, jdk, or jdk+fx) - defaults to jdk
+        architecture: x64 # (x64 or x86) - defaults to x64
+    - uses: actions/download-artifact@v2
+      with:
+        name: apache-jmeter.tgz
+        path: /tmp
+    - name: install jmeter
+      timeout-minutes: 10
+      env:
+          JMETER_VERSION: 5.4.3
+      run: |
+        cd /tmp && tar -xvf apache-jmeter.tgz
+        echo "jmeter.save.saveservice.output_format=xml" >> /tmp/apache-jmeter-$JMETER_VERSION/user.properties
+        echo "jmeter.save.saveservice.response_data.on_error=true" >> /tmp/apache-jmeter-$JMETER_VERSION/user.properties
+        wget --no-verbose -O /tmp/apache-jmeter-$JMETER_VERSION/lib/ext/mqtt-xmeter-2.0.2-jar-with-dependencies.jar https://raw.githubusercontent.com/xmeter-net/mqtt-jmeter/master/Download/v2.0.2/mqtt-xmeter-2.0.2-jar-with-dependencies.jar
+        ln -s /tmp/apache-jmeter-$JMETER_VERSION /opt/jmeter
+    - name: install jmeter plugin
+      run: |
+        wget --no-verbose -O "/opt/jmeter/lib/mysql-connector-java-8.0.16.jar" https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar
+    - name: run jmeter
+      run: |
+        sudo /opt/jmeter/bin/jmeter.sh \
+          -Jjmeter.save.saveservice.output_format=xml -n \
+          -t scripts/automate-test-suite/http_auth_acl.jmx \
+          -Dmysql_ip=$MYSQL_IP \
+          -Demqx_ip=$HAPROXY_IP \
+          -Dweb_server_ip=$HTTP_IP \
+          -Dconfig_path=$CONFIG_PATH \
+          -Ddocker_path=".ci/docker-compose-file" \
+          -l jmeter_logs/http_auth_acl.jtl \
+          -j jmeter_logs/logs/http_auth_acl.log
+    - name: check logs
+      run: |
+        if cat jmeter_logs/http_auth_acl.jtl | grep -e '<failure>true</failure>' > /dev/null 2>&1; then
+          echo "check logs filed"
+          sudo cat /var/lib/docker/volumes/docker-compose-file_etc/_data/emqx.conf
+          exit 1
+        fi
+    - uses: actions/upload-artifact@v1
+      if: always()
+      with:
+        name: jmeter_logs
+        path: ./jmeter_logs

.github/workflows/run_cts_tests.yaml

@@ -1,11 +1,13 @@
 name: Compatibility Test Suite
 
 on:
+  schedule:
+    - cron:  '0 */6 * * *'
+  pull_request:
   push:
     tags:
       - v*
       - e*
-  pull_request:
 
 jobs:
   ldap:
@@ -45,11 +47,12 @@ jobs:
           fi
       - name: run test cases
         run: |
-          export CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_
-          printenv > .env
           docker exec -i erlang sh -c "make ensure-rebar3"
-          docker exec -i erlang sh -c "./rebar3 eunit --dir apps/emqx_auth_ldap"
-          docker exec --env-file .env -i erlang sh -c "./rebar3 ct --dir apps/emqx_auth_ldap"
+          printenv | grep "^EMQX_" > .env
+          docker exec -i \
+            -e "CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_" \
+            --env-file .env \
+            erlang sh -c "make apps/emqx_auth_ldap-ct"
       - uses: actions/upload-artifact@v1
         if: failure()
         with:
@@ -114,11 +117,11 @@ jobs:
           fi
       - name: run test cases
         run: |
-          export CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_
-          printenv > .env
-          docker exec -i erlang sh -c "make ensure-rebar3"
-          docker exec -i erlang sh -c "./rebar3 eunit --dir apps/emqx_auth_mongo"
-          docker exec --env-file .env -i erlang sh -c "./rebar3 ct --dir apps/emqx_auth_mongo"
+          printenv | grep "^EMQX_" > .env
+          docker exec -i \
+            -e "CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_" \
+            --env-file .env \
+            erlang sh -c "make apps/emqx_auth_mongo-ct"
       - uses: actions/upload-artifact@v1
         if: failure()
         with:
@@ -196,11 +199,11 @@ jobs:
           fi
       - name: run test cases
         run: |
-          export CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_
-          printenv > .env
-          docker exec -i erlang sh -c "make ensure-rebar3"
-          docker exec -i erlang sh -c "./rebar3 eunit --dir apps/emqx_auth_mysql"
-          docker exec --env-file .env -i erlang sh -c "./rebar3 ct --dir apps/emqx_auth_mysql"
+          printenv | grep "^EMQX_" > .env
+          docker exec -i \
+            -e "CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_" \
+            --env-file .env \
+            erlang sh -c "make apps/emqx_auth_mysql-ct"
       - uses: actions/upload-artifact@v1
         if: failure()
         with:
@@ -269,12 +272,12 @@ jobs:
         run: |
           export EMQX_AUTH__PGSQL__USERNAME=root \
                  EMQX_AUTH__PGSQL__PASSWORD=public \
-                 EMQX_AUTH__PGSQL__DATABASE=mqtt \
-                 CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_
-          printenv > .env
-          docker exec -i erlang sh -c "make ensure-rebar3"
-          docker exec -i erlang sh -c "./rebar3 eunit --dir apps/emqx_auth_pgsql"
-          docker exec --env-file .env -i erlang sh -c "./rebar3 ct --dir apps/emqx_auth_pgsql"
+                 EMQX_AUTH__PGSQL__DATABASE=mqtt
+          printenv | grep "^EMQX_" > .env
+          docker exec -i \
+            -e "CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_" \
+            --env-file .env \
+            erlang sh -c "make apps/emqx_auth_pgsql-ct"
       - uses: actions/upload-artifact@v1
         if: failure()
         with:
@@ -390,12 +393,12 @@ jobs:
           fi
       - name: run test cases
         run: |
-          export CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_
           export EMQX_AUTH__REIDS__PASSWORD=public
-          printenv > .env
-          docker exec -i erlang sh -c "make ensure-rebar3"
-          docker exec -i erlang sh -c "./rebar3 eunit --dir apps/emqx_auth_redis"
-          docker exec --env-file .env -i erlang sh -c "./rebar3 ct --dir apps/emqx_auth_redis"
+          printenv | grep "^EMQX_" > .env
+          docker exec -i \
+            -e "CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_" \
+            --env-file .env \
+            erlang sh -c "make apps/emqx_auth_redis-ct"
       - uses: actions/upload-artifact@v1
         if: failure()
         with:

.github/workflows/run_fvt_tests.yaml

@@ -17,21 +17,20 @@ jobs:
           id: install_erlang
           with:
             otp-version: 23.2
-        - name: prepare
+        - name: make docker
           run: |
             if make emqx-ee --dry-run > /dev/null 2>&1; then
               echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials
               git config --global credential.helper store
-              echo "${{ secrets.CI_GIT_TOKEN }}" >> scripts/git-token
               make deps-emqx-ee
               echo "TARGET=emqx/emqx-ee" >> $GITHUB_ENV
               echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV
+              make emqx-ee-docker
             else
               echo "TARGET=emqx/emqx" >> $GITHUB_ENV
               echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV
+              make emqx-docker
             fi
-        - name: make emqx image
-          run: make docker
         - name: run emqx
           timeout-minutes: 5
           run: |
@@ -76,14 +75,13 @@ jobs:
             if make emqx-ee --dry-run > /dev/null 2>&1; then
               echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials
               git config --global credential.helper store
-              echo "${{ secrets.CI_GIT_TOKEN }}" >> scripts/git-token
               make deps-emqx-ee
               echo "TARGET=emqx/emqx-ee" >> $GITHUB_ENV
+              make emqx-ee-docker
             else
               echo "TARGET=emqx/emqx" >> $GITHUB_ENV
+              make emqx-docker
             fi
-        - name: make emqx image
-          run: make docker
         - name: install k3s
           env:
             KUBECONFIG: "/etc/rancher/k3s/k3s.yaml"
@@ -130,11 +128,27 @@ jobs:
               echo "waiting emqx started";
               sleep 10;
             done
-        - name: get pods log
+        - name: get emqx-0 pods log
           if: failure()
           env:
             KUBECONFIG: "/etc/rancher/k3s/k3s.yaml"
-          run: kubectl describe pods emqx-0
+          run: |
+            kubectl describe pods emqx-0
+            kubectl logs emqx-0
+        - name: get emqx-1 pods log
+          if: failure()
+          env:
+            KUBECONFIG: "/etc/rancher/k3s/k3s.yaml"
+          run: |
+            kubectl describe pods emqx-1
+            kubectl logs emqx-1
+        - name: get emqx-2 pods log
+          if: failure()
+          env:
+            KUBECONFIG: "/etc/rancher/k3s/k3s.yaml"
+          run: |
+            kubectl describe pods emqx-2
+            kubectl logs emqx-2
         - uses: actions/checkout@v2
           with:
             repository: emqx/paho.mqtt.testing
@@ -162,117 +176,140 @@ jobs:
             fi
             exit $RESULT
 
-    relup_test:
+    relup_test_plan:
         runs-on: ubuntu-20.04
-        container: emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04
+        container: emqx/build-env:erl23.2.7.2-emqx-3-ubuntu20.04
+        outputs:
+          profile: ${{ steps.profile-and-versions.outputs.profile }}
+          vsn: ${{ steps.profile-and-versions.outputs.vsn }}
+          old_vsns: ${{ steps.profile-and-versions.outputs.old_vsns }}
+          broker: ${{ steps.profile-and-versions.outputs.broker }}
+          matrix: ${{ steps.generate-matrix.outputs.matrix }}
         defaults:
           run:
             shell: bash
         steps:
-        - uses: actions/setup-python@v2
-          with:
-            python-version: '3.8'
-            architecture: 'x64'
         - uses: actions/checkout@v2
+          name: Checkout
           with:
-            repository: emqx/paho.mqtt.testing
-            ref: develop-4.0
-            path: paho.mqtt.testing
+            path: emqx
+            fetch-depth: 0
+        - name: Get profile and version list
+          id: profile-and-versions
+          run: |
+            cd emqx
+            vsn="$(./pkg-vsn.sh)"
+            if make emqx-ee --dry-run > /dev/null 2>&1; then
+              profile="emqx-ee"
+              old_vsns="$(./scripts/relup-base-vsns.sh enterprise | xargs)"
+              broker="emqx-ee"
+            else
+              profile="emqx"
+              old_vsns="$(./scripts/relup-base-vsns.sh community | xargs)"
+              broker="emqx-ce"
+            fi
+
+            echo "OLD_VSNS=$old_vsns" >> $GITHUB_ENV
+
+            echo "::set-output name=vsn::$vsn"
+            echo "::set-output name=profile::$profile"
+            echo "::set-output name=broker::$broker"
+            echo "::set-output name=old_vsns::$old_vsns"
+        - name: Generate matrix
+          id: generate-matrix
+          run: |
+            matrix=$(echo -n "$OLD_VSNS" | sed 's/ $//g' | jq -R -s -c 'split(" ")')
+            echo "::set-output name=matrix::$matrix"
+
+    relup_test_build:
+        needs: relup_test_plan
+        runs-on: ubuntu-20.04
+        container: emqx/build-env:erl23.2.7.2-emqx-3-ubuntu20.04
+        defaults:
+          run:
+            shell: bash
+        env:
+          OLD_VSNS: "${{ needs.relup_test_plan.outputs.old_vsns }}"
+          PROFILE: "${{ needs.relup_test_plan.outputs.profile }}"
+          BROKER: "${{ needs.relup_test_plan.outputs.broker }}"
+        steps:
         - uses: actions/checkout@v2
+          name: Checkout
+          with:
+            path: emqx
+            fetch-depth: 0
+        - name: Prepare credentials
+          run: |
+            if [ "$PROFILE" = "emqx-ee" ]; then
+              echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials
+              git config --global credential.helper store
+            fi
+        - name: Build emqx
+          run: make -C emqx ${PROFILE}-zip
+        - uses: actions/upload-artifact@v2
+          name: Upload built emqx and test scenario
+          with:
+            name: emqx_built
+            path: |
+              emqx/_packages/*/*.zip
+              emqx/.ci/fvt_tests
+
+    relup_test_run:
+        needs:
+          - relup_test_plan
+          - relup_test_build
+        runs-on: ubuntu-20.04
+        container: emqx/relup-test-env:erl23.2.7.2-emqx-3-ubuntu20.04
+        strategy:
+          fail-fast: false
+          matrix:
+            old_vsn: ${{ fromJson(needs.relup_test_plan.outputs.matrix) }}
+        env:
+          OLD_VSN: "${{ matrix.old_vsn }}"
+          PROFILE: "${{ needs.relup_test_plan.outputs.profile }}"
+          VSN: "${{ needs.relup_test_plan.outputs.vsn }}"
+          BROKER: "${{ needs.relup_test_plan.outputs.broker }}"
+        defaults:
+          run:
+            shell: bash
+        steps:
+        - uses: actions/download-artifact@v2
+          name: Download built emqx and test scenario
+          with:
+            name: emqx_built
+            path: emqx_built
+        - uses: actions/checkout@v2
+          name: Checkout one_more_emqx
           with:
             repository: terry-xiaoyu/one_more_emqx
             ref: master
             path: one_more_emqx
-        - uses: actions/checkout@v2
-          with:
-            repository: emqx/emqtt-bench
-            ref: master
-            path: emqtt-bench
-        - uses: actions/checkout@v2
-          with:
-            repository: hawk/lux
-            ref: lux-2.6
-            path: lux
-        - uses: actions/checkout@v2
-          with:
-            repository: ${{ github.repository }}
-            path: emqx
-            fetch-depth: 0
-        - name: prepare
-          run: |
-            if make -C emqx emqx-ee --dry-run > /dev/null 2>&1; then
-              echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials
-              git config --global credential.helper store
-              echo "${{ secrets.CI_GIT_TOKEN }}" >> emqx/scripts/git-token
-              echo "PROFILE=emqx-ee" >> $GITHUB_ENV
-            else
-              echo "PROFILE=emqx" >> $GITHUB_ENV
-            fi
-        - name: get version
+        - name: Prepare packages
           run: |
             set -e -x -u
-            cd emqx
-            if [ $PROFILE = "emqx" ];then
-                broker="emqx-ce"
-                edition='opensource'
-            else
-                broker="emqx-ee"
-                edition='enterprise'
-            fi
-            echo "BROKER=$broker" >> $GITHUB_ENV
-
-            vsn="$(./pkg-vsn.sh)"
-            echo "VSN=$vsn" >> $GITHUB_ENV
-
-            pre_vsn="$(echo $vsn | grep -oE '^[0-9]+.[0-9]')"
-            if [ $PROFILE = "emqx" ]; then
-                old_vsns="$(git tag -l "v$pre_vsn.[0-9]" | xargs echo -n | sed "s/v$vsn//")"
-            else
-                old_vsns="$(git tag -l "e$pre_vsn.[0-9]" | xargs echo -n | sed "s/e$vsn//")"
-            fi
-            echo "OLD_VSNS=$old_vsns" >> $GITHUB_ENV
-        - name: download emqx
+            mkdir -p packages
+            cp emqx_built/_packages/*/*.zip packages
+            cd packages
+            wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$BROKER/$OLD_VSN/$PROFILE-ubuntu20.04-${OLD_VSN#[e|v]}-amd64.zip
+        - name: Run relup test scenario
+          timeout-minutes: 5
           run: |
-            set -e -x -u
-            mkdir -p emqx/_upgrade_base
-            cd emqx/_upgrade_base
-            old_vsns=($(echo $OLD_VSNS | tr ' ' ' '))
-            for old_vsn in ${old_vsns[@]}; do
-              wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$BROKER/$old_vsn/$PROFILE-ubuntu20.04-${old_vsn#[e|v]}-amd64.zip
-            done
-        - name: build emqx
-          run: make -C emqx ${PROFILE}-zip
-        - name: build emqtt-bench
-          run: make -C emqtt-bench
-        - name: build lux
-          run: |
-            set -e -u -x
-            cd lux
-            autoconf
-            ./configure
-            make
-            make install
-        - name: run relup test
-          timeout-minutes: 20
-          run: |
-            set -e -x -u
-            if [ -n "$OLD_VSNS" ]; then
-                mkdir -p packages
-                cp emqx/_packages/${PROFILE}/*.zip packages
-                cp emqx/_upgrade_base/*.zip packages
-                lux \
-                --case_timeout infinity \
-                --var PROFILE=$PROFILE \
-                --var PACKAGE_PATH=$(pwd)/packages \
-                --var BENCH_PATH=$(pwd)/emqtt-bench \
-                --var ONE_MORE_EMQX_PATH=$(pwd)/one_more_emqx \
-                --var VSN="$VSN" \
-                --var OLD_VSNS="$OLD_VSNS" \
-                emqx/.ci/fvt_tests/relup.lux
-            fi
-        - uses: actions/upload-artifact@v1
+            lux \
+            --progress verbose \
+            --case_timeout infinity \
+            --var PROFILE=$PROFILE \
+            --var PACKAGE_PATH=$(pwd)/packages \
+            --var ONE_MORE_EMQX_PATH=$(pwd)/one_more_emqx \
+            --var VSN="$VSN" \
+            --var OLD_VSN="$OLD_VSN" \
+            emqx_built/.ci/fvt_tests/relup.lux
+        - uses: actions/upload-artifact@v2
+          name: Save debug data
           if: failure()
           with:
-            name: lux_logs
-            path: lux_logs
-
+            name: debug_data
+            path: |
+              packages/emqx/log/emqx.log.1
+              packages/emqx2/log/emqx.log.1
+              packages/*.zip
+              lux_logs

.github/workflows/run_test_cases.yaml

@@ -8,26 +8,9 @@ on:
   pull_request:
 
 jobs:
-    run_static_analysis:
-        runs-on: ubuntu-20.04
-        container: emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04
-
-        steps:
-        - uses: actions/checkout@v2
-        - name: set git credentials
-          run: |
-            if make emqx-ee --dry-run > /dev/null 2>&1; then
-              echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials
-              git config --global credential.helper store
-            fi
-        - name: xref
-          run: make xref
-        - name: dialyzer
-          run: make dialyzer
-
     run_proper_test:
         runs-on: ubuntu-20.04
-        container: emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04
+        container: emqx/build-env:erl23.2.7.2-emqx-3-ubuntu20.04
 
         steps:
         - uses: actions/checkout@v2

.gitignore

@@ -47,3 +47,16 @@ dist.zip
 scripts/git-token
 etc/*.seg
 _upgrade_base/
+erlang_ls.config
+.els_cache/
+# VSCode files
+.vs/
+.vscode/
+# Emacs Backup files
+*~
+# Emacs temporary files
+.#*
+*#
+# For direnv
+.envrc
+mix.lock

.tool-versions

@@ -1 +1 @@
-erlang 24.0.1-emqx-1
+erlang 23.2.7.2-emqx-3

CHANGES-4.3.md

@@ -0,0 +1,177 @@
+# EMQX 4.3 Changes
+
+Started tracking changes in CHANGE.md since EMQX v4.3.11
+
+NOTE: Keep prepending to the head of the file instead of the tail
+
+File format:
+
+- Use weight-2 heading for releases
+- One list item per change topic
+  Change log ends with a list of github PRs
+
+## v4.3.14
+
+### Enhancements
+
+* In order to fix the execution order of exhook, e.g. before/after other plugins/modules,
+  ExHook now supports user customizing emqx_hook execute priority.
+* add api: PUT /rules/{id}/reset_metrics.
+  This api reset the metrics of the rule engine of a rule, and reset the metrics of the action related to this rule. [#7474]
+* Enhanced rule engine error handling when json parsing error.
+
+### Bug fixes
+
+* Prohibit empty topics in strict mode
+* Make sure ehttpc delete useless pool always succeed.
+* Update mongodb driver to fix potential process leak.
+
+## v4.3.13
+
+### Important changes
+
+* For docker image, /opt/emqx/etc has been removed from the VOLUME list,
+  this made it easier for the users to rebuild image on top with changed configs.
+* CentOS 7 Erlang runtime is rebuilt on OpenSSL-1.1.1n (previously on 1.0),
+  Prior to v4.3.13, EMQX pick certain cipher suites proposed by the clients,
+  but then fail to handshake resulting in a `malformed_handshake_data` exception.
+* CentOS 8 Erlang runtime is rebuilt on RockyLinux 8.
+  'centos8' will remain in the package name to keep it backward compatible.
+
+### Enhancements
+
+* CLI `emqx_ctl pem_cache clean` to force purge x509 certificate cache,
+  to force an immediate reload of all certificates after the files are updated on disk.
+* Refactor the ExProto so that anonymous clients can also be displayed on the dashboard [#6983]
+* Force shutdown of processes that cannot answer takeover event [#7026]
+* `topic` parameter in bridge configuration can have `${node}` substitution (just like in `clientid` parameter)
+* Add UTF-8 string validity check in `strict_mode` for MQTT packet.
+  When set to true, invalid UTF-8 strings will cause the client to be disconnected. i.e. client ID, topic name. [#7261]
+* Changed systemd service restart delay from 10 seconds to 60 seconds.
+* MQTT-SN gateway supports initiative to synchronize registered topics after session resumed. [#7300]
+* Add load control app for future development.
+* Change the precision of float to 17 digits after the decimal point when formatting a
+  float using payload templates of rule actions. The old precision is 10 digits before
+  this change. [#7336]
+* Return the cached resource status when querying a resource using HTTP APIs.
+  This is to avoid blocking the HTTP request if the resource is unavailable. [#7374]
+
+### Bug fixes
+
+* Fix the `{error,eexist}` error when do release upgrade again if last run failed. [#7121]
+* Fix case where publishing to a non-existent topic alias would crash the connection [#6979]
+* Fix HTTP-API 500 error on querying the lwm2m client list on the another node [#7009]
+* Fix the ExProto connection registry is not released after the client process abnormally exits [#6983]
+* Fix Server-KeepAlive wrongly applied on MQTT v3.0/v3.1 [#7085]
+* Fix Stomp client can not trigger `$event/client_connection` message [#7096]
+* Fix system memory false alarm at boot
+* Fix the MQTT-SN message replay when the topic is not registered to the client [#6970]
+* Fix rpc get node info maybe crash when other nodes is not ready.
+* Fix false alert level log “cannot_find_plugins” caused by duplicate plugin names in `loaded_plugins` files.
+* Prompt user how to change the dashboard's initial default password when emqx start.
+* Fix errno=13 'Permission denied' Cannot create FIFO boot error in Amazon Linux 2022 (el8 package)
+* Fix user or appid created, name only allow `^[A-Za-z]+[A-Za-z0-9-_]*$`
+* Fix subscribe http api crash by bad_qos `/mqtt/subscribe`,`/mqtt/subscribe_batch`.
+* Send DISCONNECT packet with reason code 0x98 if connection has been kicked [#7309]
+* Auto subscribe to an empty topic will be simply ignored now
+
+## v4.3.12
+### Important changes
+
+### Minor changes
+* Fix updating `emqx_auth_mnesia.conf` password and restarting the new password does not take effect [#6717]
+* Fix import data crash when emqx_auth_mnesia's record is not empty [#6717]
+* Fix `os_mon.sysmem_high_watermark` may not alert after reboot.
+* Enhancement: Log client status before killing it for holding the lock for too long.
+  [emqx-6959](https://github.com/emqx/emqx/pull/6959)
+  [ekka-144](https://github.com/emqx/ekka/pull/144)
+  [ekka-146](https://github.com/emqx/ekka/pull/146)
+
+## v4.3.11
+
+Important notes:
+
+- For Debian/Ubuntu users
+
+  We changed the package installed service from init.d to systemd.
+  The upgrade from init.d to systemd is verified, however it is
+  recommended to verify it before rolling out to production.
+  At least to ensure systemd is available in your system.
+
+- For Centos Users
+
+  RPM package now depends on `openssl11` which is NOT available
+  in certain centos distributions.
+  Please make sure the yum repo [epel-release](https://docs.fedoraproject.org/en-US/epel) is installed.
+
+### Important changes
+
+* Debian/Ubuntu package (deb) installed EMQX now runs on systemd [#6389]<br>
+  This is to take advantage of systemd's supervision functionality to ensure
+  EMQX service is restarted after crashes.
+
+### Minor changes
+
+* Clustering malfunction fixes [#6221, #6381]
+  Mostly changes made in [ekka](https://github.com/emqx/ekka/pull/134)<br>
+  From 0.8.1.4 to 0.8.1.6, fixes included intra-cluster RPC call timeouts,<br>
+  also fixed `ekka_locker` process crashed after killing a hanged lock owner.
+
+* Improved log message when TCP proxy is in use but proxy_protocol configuration is not turned on [#6416]<br>
+  "please check proxy_protocol config for specific listeners and zones" to hint a misconfiguration
+
+* Helm chart supports networking.k8s.io/v1 [#6368]
+
+* Fix session takeover race condition which may lead to message loss [#6396]
+
+* EMQX docker images are pushed to aws public ecr in an automated CI job [#6271]<br>
+  `docker pull public.ecr.aws/emqx/emqx:4.3.10`
+
+* Fix webhook URL path to allow rule-engine variable substitution [#6399]
+
+* Corrected RAM usage display [#6379]
+
+* Changed emqx_sn_registry table creation to runtime [#6357]<br>
+  This was a bug introduced in 4.3.3, in which the table is changed from ets to mnesia<br>
+  this will cause upgrade to fail when a later version node joins a 4.3.0-2 cluster<br>
+
+* Log level for normal termination changed from info to debug [#6358]
+
+* Added config `retainer.stop_publish_clear_msg` to enable/disable empty message retained message publish [#6343]<br>
+  In MQTT 3.1.1, it is unclear if a MQTT broker should publish the 'clear' (no payload) message<br>
+  to the subscribers, or just delete the retained message. So we have made it configurable
+
+* Fix mqtt bridge malfunction when remote host is unreachable (hangs the connection) [#6286, #6323]
+
+* System monitor now inspects `current_stacktrace` of suspicious process [#6290]<br>
+  `current_function` was not quite helpful
+
+* Changed default `max_topc_levels` config value to 128 [#6294, #6420]<br>
+  previously it has no limit (config value = 0), which can be a potential DoS threat
+
+* Collect only libcrypto and libtinfo so files for zip package [#6259]<br>
+  in 4.3.10 we tried to collect all so files, however glibc is not quite portable
+
+* Added openssl-1.1 to RPM dependency [#6239]
+
+* Http client duplicated header fix [#6195]
+
+* Fix `node_dump` issues when working with deb or rpm installation [#6209]
+
+* Pin Erlang/OTP 23.2.7.2-emqx-3 [#6246]<br>
+  4.3.10 is on 23.2.7.2-emqx-2, this bump is to fix an ECC signature name typo:
+  ecdsa_secp512r1_sha512 -> ecdsa_secp521r1_sha512
+
+* HTTP client performance improvement [#6474, #6414]<br>
+  The changes are mostly done in the dependency [repo](https://github.com/emqx/ehttpc).
+
+* For messages from gateways add message properties as MQTT message headers [#6142]<br>
+  e.g. messages from CoAP, LwM2M, Stomp, ExProto, when translated into MQTT message<br>
+  properties such as protocol name, protocol version, username (if any) peer-host<br>
+  etc. are filled as MQTT message headers.
+
+* Format the message id to hex strings in the log message [#6961]
+
+## v4.3.0~10
+
+Older version changes are not tracked here.

Makefile

@@ -3,11 +3,16 @@ REBAR_VERSION = 3.14.3-emqx-8
 REBAR = $(CURDIR)/rebar3
 BUILD = $(CURDIR)/build
 SCRIPTS = $(CURDIR)/scripts
+export EMQX_RELUP ?= true
+export EMQX_DEFAULT_BUILDER = emqx/build-env:erl23.2.7.2-emqx-3-alpine
+export EMQX_DEFAULT_RUNNER = alpine:3.12
 export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh)
-export EMQX_DESC ?= EMQ X
-export EMQX_CE_DASHBOARD_VERSION ?= v4.3.1
+export DOCKERFILE := deploy/docker/Dockerfile
 ifeq ($(OS),Windows_NT)
 	export REBAR_COLOR=none
+	FIND=/usr/bin/find
+else
+	FIND=find
 endif
 
 PROFILE ?= emqx
@@ -51,7 +56,7 @@ APPS=$(shell $(CURDIR)/scripts/find-apps.sh)
 ## app/name-ct targets are intended for local tests hence cover is not enabled
 .PHONY: $(APPS:%=%-ct)
 define gen-app-ct-target
-$1-ct:
+$1-ct: $(REBAR)
 	$(REBAR) ct --name 'test@127.0.0.1' -v --suite $(shell $(CURDIR)/scripts/find-suites.sh $1)
 endef
 $(foreach app,$(APPS),$(eval $(call gen-app-ct-target,$(app))))
@@ -85,16 +90,20 @@ $(REL_PROFILES:%=%): $(REBAR) get-dashboard
 clean: $(PROFILES:%=clean-%)
 $(PROFILES:%=clean-%):
 	@if [ -d _build/$(@:clean-%=%) ]; then \
+		rm rebar.lock \
 		rm -rf _build/$(@:clean-%=%)/rel; \
-		find _build/$(@:clean-%=%) -name '*.beam' -o -name '*.so' -o -name '*.app' -o -name '*.appup' -o -name '*.o' -o -name '*.d' -type f | xargs rm -f; \
+		$(FIND) _build/$(@:clean-%=%) -name '*.beam' -o -name '*.so' -o -name '*.app' -o -name '*.appup' -o -name '*.o' -o -name '*.d' -type f | xargs rm -f; \
+		$(FIND) _build/$(@:clean-%=%) -type l -delete; \
 	fi
 
 .PHONY: clean-all
 clean-all:
 	@rm -rf _build
+	@rm rebar.lock
 
 .PHONY: deps-all
 deps-all: $(REBAR) $(PROFILES:%=deps-%)
+	@make clean # ensure clean at the end
 
 ## deps-<profile> is used in CI scripts to download deps and the
 ## share downloads between CI steps and/or copied into containers
@@ -102,6 +111,7 @@ deps-all: $(REBAR) $(PROFILES:%=deps-%)
 .PHONY: $(PROFILES:%=deps-%)
 $(PROFILES:%=deps-%): $(REBAR) get-dashboard
 	@$(REBAR) as $(@:deps-%=%) get-deps
+	@rm -f rebar.lock
 
 .PHONY: xref
 xref: $(REBAR)
@@ -118,10 +128,19 @@ COMMON_DEPS := $(REBAR) get-dashboard $(CONF_SEGS)
 $(REL_PROFILES:%=%-rel) $(PKG_PROFILES:%=%-rel): $(COMMON_DEPS)
 	@$(BUILD) $(subst -rel,,$(@)) rel
 
+## download relup base packages
+.PHONY: $(REL_PROFILES:%=%-relup-downloads)
+define download-relup-packages
+$1-relup-downloads:
+	@if [ "$${EMQX_RELUP}" = "true" ]; then $(CURDIR)/scripts/relup-base-packages.sh $1; fi
+endef
+ALL_ZIPS = $(REL_PROFILES)
+$(foreach zt,$(ALL_ZIPS),$(eval $(call download-relup-packages,$(zt))))
+
 ## relup target is to create relup instructions
 .PHONY: $(REL_PROFILES:%=%-relup)
 define gen-relup-target
-$1-relup: $(COMMON_DEPS)
+$1-relup: $1-relup-downloads $(COMMON_DEPS)
 	@$(BUILD) $1 relup
 endef
 ALL_ZIPS = $(REL_PROFILES)
@@ -144,11 +163,18 @@ $1: $1-rel
 endef
 $(foreach pt,$(PKG_PROFILES),$(eval $(call gen-pkg-target,$(pt))))
 
+## docker target is to create docker instructions
+.PHONY: $(REL_PROFILES:%=%-docker)
+define gen-docker-target
+$1-docker: $(COMMON_DEPS)
+	@$(BUILD) $1 docker
+endef
+ALL_ZIPS = $(REL_PROFILES)
+$(foreach zt,$(ALL_ZIPS),$(eval $(call gen-docker-target,$(zt))))
+
 .PHONY: run
 run: $(PROFILE) quickrun
 
 .PHONY: quickrun
 quickrun:
 	./_build/$(PROFILE)/rel/emqx/bin/emqx console
-
-include docker.mk

NOTICE

@@ -0,0 +1,5 @@
+EMQX, a highly scalable, highly available distributed MQTT messaging broker for IoT.
+Copyright (c) 2017-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+
+This product contains code developed at EMQ Technologies Co., Ltd.
+Visit https://www.emqx.come to learn more.

README-CN.md

@@ -1,4 +1,4 @@
-# EMQ X Broker
+# EMQX Broker
 
 [![GitHub Release](https://img.shields.io/github/release/emqx/emqx?color=brightgreen)](https://github.com/emqx/emqx/releases)
 [![Build Status](https://travis-ci.org/emqx/emqx.svg)](https://travis-ci.org/emqx/emqx)
@@ -13,20 +13,20 @@
 
 [English](./README.md) | 简体中文 | [日本語](./README-JP.md) | [русский](./README-RU.md)
 
-*EMQ X* 是一款完全开源，高度可伸缩，高可用的分布式 MQTT 消息服务器，适用于 IoT、M2M 和移动应用程序，可处理千万级别的并发客户端。
+*EMQX* 是一款完全开源，高度可伸缩，高可用的分布式 MQTT 消息服务器，适用于 IoT、M2M 和移动应用程序，可处理千万级别的并发客户端。
 
-从 3.0 版本开始，*EMQ X* 完整支持 MQTT V5.0 协议规范，向下兼容 MQTT V3.1 和 V3.1.1，并支持 MQTT-SN、CoAP、LwM2M、WebSocket 和 STOMP 等通信协议。EMQ X 3.0 单集群可支持千万级别的 MQTT 并发连接。
+从 3.0 版本开始，*EMQX* 完整支持 MQTT V5.0 协议规范，向下兼容 MQTT V3.1 和 V3.1.1，并支持 MQTT-SN、CoAP、LwM2M、WebSocket 和 STOMP 等通信协议。EMQX 3.0 单集群可支持千万级别的 MQTT 并发连接。
 
-- 新功能的完整列表，请参阅 [EMQ X Release Notes](https://github.com/emqx/emqx/releases)。
-- 获取更多信息，请访问 [EMQ X 官网](https://www.emqx.cn/)。
+- 新功能的完整列表，请参阅 [EMQX Release Notes](https://github.com/emqx/emqx/releases)。
+- 获取更多信息，请访问 [EMQX 官网](https://www.emqx.cn/)。
 
 ## 安装
 
-*EMQ X* 是跨平台的，支持 Linux、Unix、macOS 以及 Windows。这意味着 *EMQ X* 可以部署在 x86_64 架构的服务器上，也可以部署在 Raspberry Pi 这样的 ARM 设备上。
+*EMQX* 是跨平台的，支持 Linux、Unix、macOS 以及 Windows。这意味着 *EMQX* 可以部署在 x86_64 架构的服务器上，也可以部署在 Raspberry Pi 这样的 ARM 设备上。
 
-Windows 上编译和运行 *EMQ X* 的详情参考：[Windows.md](./Windows.md)
+Windows 上编译和运行 *EMQX* 的详情参考：[Windows.md](./Windows.md)
 
-#### EMQ X Docker 镜像安装
+#### EMQX Docker 镜像安装
 
 ```
 docker run -d --name emqx -p 1883:1883 -p 8081:8081 -p 8083:8083 -p 8883:8883 -p 8084:8084 -p 18083:18083 emqx/emqx
@@ -34,14 +34,14 @@ docker run -d --name emqx -p 1883:1883 -p 8081:8081 -p 8083:8083 -p 8883:8883 -p
 
 #### 二进制软件包安装
 
-需从 [EMQ X 下载](https://www.emqx.cn/downloads) 页面获取相应操作系统的二进制软件包。
+需从 [EMQX 下载](https://www.emqx.cn/downloads) 页面获取相应操作系统的二进制软件包。
 
 - [单节点安装文档](https://docs.emqx.cn/broker/latest/getting-started/install.html)
 - [集群配置文档](https://docs.emqx.cn/broker/latest/advanced/cluster.html)
 
 ## 从源码构建
 
-3.0 版本开始，构建 *EMQ X* 需要 Erlang/OTP R21+。
+3.0 版本开始，构建 *EMQX* 需要 Erlang/OTP R21+。
 
 4.3 及以后的版本：
 
@@ -77,7 +77,7 @@ _build/emqx/rel/emqx/bin/emqx console
 ./bin/emqx stop
 ```
 
-*EMQ X* 启动，可以使用浏览器访问 http://localhost:18083 来查看 Dashboard。
+*EMQX* 启动，可以使用浏览器访问 http://localhost:18083 来查看 Dashboard。
 
 ## 测试
 
@@ -108,7 +108,7 @@ DIALYZER_ANALYSE_APP=emqx_lwm2m,emqx_auth_jwt,emqx_auth_ldap make dialyzer
 
 ### FAQ
 
-访问 [EMQ X FAQ](https://docs.emqx.cn/broker/latest/faq/faq.html) 以获取常见问题的帮助。
+访问 [EMQX FAQ](https://docs.emqx.cn/broker/latest/faq/faq.html) 以获取常见问题的帮助。
 
 ### 问答
 
@@ -117,7 +117,7 @@ DIALYZER_ANALYSE_APP=emqx_lwm2m,emqx_auth_jwt,emqx_auth_ldap make dialyzer
 
 ### 参与设计
 
-如果对 EMQ X 有改进建议，可以向[EIP](https://github.com/emqx/eip) 提交 PR 和 ISSUE
+如果对 EMQX 有改进建议，可以向[EIP](https://github.com/emqx/eip) 提交 PR 和 ISSUE
 
 ### 插件开发
 

README-JP.md

@@ -1,4 +1,4 @@
-# EMQ X Broker
+# EMQX Broker
 
 [![GitHub Release](https://img.shields.io/github/release/emqx/emqx?color=brightgreen)](https://github.com/emqx/emqx/releases)
 [![Build Status](https://travis-ci.org/emqx/emqx.svg)](https://travis-ci.org/emqx/emqx)
@@ -12,22 +12,22 @@
 
 [English](./README.md) | [简体中文](./README-CN.md) | 日本語 | [русский](./README-RU.md)
 
-*EMQ X* は、高い拡張性と可用性をもつ、分散型のMQTTブローカーです。数千万のクライアントを同時に処理するIoT、M2M、モバイルアプリケーション向けです。
+*EMQX* は、高い拡張性と可用性をもつ、分散型のMQTTブローカーです。数千万のクライアントを同時に処理するIoT、M2M、モバイルアプリケーション向けです。
 
-version 3.0 以降、*EMQ X* は MQTT V5.0 の仕様を完全にサポートしており、MQTT V3.1およびV3.1.1とも下位互換性があります。
+version 3.0 以降、*EMQX* は MQTT V5.0 の仕様を完全にサポートしており、MQTT V3.1およびV3.1.1とも下位互換性があります。
 MQTT-SN、CoAP、LwM2M、WebSocket、STOMPなどの通信プロトコルをサポートしています。 MQTTの同時接続数は1つのクラスター上で1,000万以上にまでスケールできます。
 
-- 新機能の一覧については、[EMQ Xリリースノート](https://github.com/emqx/emqx/releases)を参照してください。
-- 詳細はこちら[EMQ X公式ウェブサイト](https://www.emqx.io/)をご覧ください。
+- 新機能の一覧については、[EMQXリリースノート](https://github.com/emqx/emqx/releases)を参照してください。
+- 詳細はこちら[EMQX公式ウェブサイト](https://www.emqx.io/)をご覧ください。
 
 ## インストール
 
-*EMQ X* はクロスプラットフォームで、Linux、Unix、macOS、Windowsをサポートしています。
-そのため、x86_64アーキテクチャサーバー、またはRaspberryPiなどのARMデバイスに *EMQ X* をデプロイすることもできます。
+*EMQX* はクロスプラットフォームで、Linux、Unix、macOS、Windowsをサポートしています。
+そのため、x86_64アーキテクチャサーバー、またはRaspberryPiなどのARMデバイスに *EMQX* をデプロイすることもできます。
 
-Windows上における *EMQ X* のビルドと実行については、[Windows.md](./Windows.md)をご参照ください。
+Windows上における *EMQX* のビルドと実行については、[Windows.md](./Windows.md)をご参照ください。
 
-#### Docker イメージによる EMQ X のインストール
+#### Docker イメージによる EMQX のインストール
 
 ```
 docker run -d --name emqx -p 1883:1883 -p 8083:8083 -p 8883:8883 -p 8084:8084 -p 18083:18083 emqx/emqx
@@ -35,14 +35,14 @@ docker run -d --name emqx -p 1883:1883 -p 8083:8083 -p 8883:8883 -p 8084:8084 -p
 
 #### バイナリパッケージによるインストール
 
-それぞれのOSに対応したバイナリソフトウェアパッケージは、[EMQ Xのダウンロード](https://www.emqx.io/downloads)ページから取得できます。
+それぞれのOSに対応したバイナリソフトウェアパッケージは、[EMQXのダウンロード](https://www.emqx.io/downloads)ページから取得できます。
 
 - [シングルノードインストール](https://docs.emqx.io/broker/latest/en/getting-started/installation.html)
 - [マルチノードインストール](https://docs.emqx.io/broker/latest/en/advanced/cluster.html)
 
 ## ソースからビルド
 
-version 3.0 以降の *EMQ X* をビルドするには Erlang/OTP R21+ が必要です。
+version 3.0 以降の *EMQX* をビルドするには Erlang/OTP R21+ が必要です。
 
 version 4.3 以降の場合：
 
@@ -71,7 +71,7 @@ emqx をソースコードからビルドした場合は、
 ./bin/emqx stop
 ```
 
-*EMQ X* の起動後、ブラウザで http://localhost:18083 にアクセスするとダッシュボードが表示されます。
+*EMQX* の起動後、ブラウザで http://localhost:18083 にアクセスするとダッシュボードが表示されます。
 
 ## テスト
 
@@ -102,7 +102,7 @@ DIALYZER_ANALYSE_APP=emqx_lwm2m,emqx_auth_jwt,emqx_auth_ldap make dialyzer
 
 ### FAQ
 
-よくある質問については、[EMQ X FAQ](https://docs.emqx.io/broker/latest/en/faq/faq.html)をご確認ください。
+よくある質問については、[EMQX FAQ](https://docs.emqx.io/broker/latest/en/faq/faq.html)をご確認ください。
 
 ### 質問する
 

README-RU.md

@@ -1,4 +1,4 @@
-# Брокер EMQ X
+# Брокер EMQX
 
 [![GitHub Release](https://img.shields.io/github/release/emqx/emqx?color=brightgreen)](https://github.com/emqx/emqx/releases)
 [![Build Status](https://travis-ci.org/emqx/emqx.svg)](https://travis-ci.org/emqx/emqx)
@@ -13,20 +13,20 @@
 
 [English](./README.md) | [简体中文](./README-CN.md) | [日本語](./README-JP.md) | русский
 
-*EMQ X* — это масштабируемый, высоко доступный, распределённый MQTT брокер с полностью открытым кодом для интернета вещей, межмашинного взаимодействия и мобильных приложений, который поддерживает миллионы одновременных подключений.
+*EMQX* — это масштабируемый, высоко доступный, распределённый MQTT брокер с полностью открытым кодом для интернета вещей, межмашинного взаимодействия и мобильных приложений, который поддерживает миллионы одновременных подключений.
 
-Начиная с релиза 3.0, брокер *EMQ X* полностью поддерживает протокол MQTT версии 5.0, и обратно совместим с версиями 3.1 и 3.1.1, а также протоколами MQTT-SN, CoAP, LwM2M, WebSocket и STOMP. Начиная с релиза 3.0, брокер *EMQ X* может масштабироваться до более чем 10 миллионов одновременных MQTT соединений на один кластер.
+Начиная с релиза 3.0, брокер *EMQX* полностью поддерживает протокол MQTT версии 5.0, и обратно совместим с версиями 3.1 и 3.1.1, а также протоколами MQTT-SN, CoAP, LwM2M, WebSocket и STOMP. Начиная с релиза 3.0, брокер *EMQX* может масштабироваться до более чем 10 миллионов одновременных MQTT соединений на один кластер.
 
-- Полный список возможностей доступен по ссылке: [EMQ X Release Notes](https://github.com/emqx/emqx/releases).
-- Более подробная информация доступна на нашем сайте: [EMQ X homepage](https://www.emqx.io).
+- Полный список возможностей доступен по ссылке: [EMQX Release Notes](https://github.com/emqx/emqx/releases).
+- Более подробная информация доступна на нашем сайте: [EMQX homepage](https://www.emqx.io).
 
 ## Установка
 
-Брокер *EMQ X* кросплатформенный, и поддерживает Linux, Unix, macOS и Windows. Он может работать на серверах с архитектурой x86_64 и устройствах на архитектуре ARM, таких как Raspberry Pi.
+Брокер *EMQX* кросплатформенный, и поддерживает Linux, Unix, macOS и Windows. Он может работать на серверах с архитектурой x86_64 и устройствах на архитектуре ARM, таких как Raspberry Pi.
 
 Более подробная информация о запуске на Windows по ссылке: [Windows.md](./Windows.md)
 
-#### Установка EMQ X с помощью Docker-образа
+#### Установка EMQX с помощью Docker-образа
 
 ```
 docker run -d --name emqx -p 1883:1883 -p 8081:8081 -p 8083:8083 -p 8883:8883 -p 8084:8084 -p 18083:18083 emqx/emqx
@@ -34,7 +34,7 @@ docker run -d --name emqx -p 1883:1883 -p 8081:8081 -p 8083:8083 -p 8883:8883 -p
 
 #### Установка бинарного пакета
 
-Сборки для различных операционных систем: [Загрузить EMQ X](https://www.emqx.io/downloads).
+Сборки для различных операционных систем: [Загрузить EMQX](https://www.emqx.io/downloads).
 
 - [Установка на одном сервере](https://docs.emqx.io/en/broker/latest/getting-started/install.html)
 - [Установка на кластере](https://docs.emqx.io/en/broker/latest/advanced/cluster.html)
@@ -111,7 +111,7 @@ DIALYZER_ANALYSE_APP=emqx_lwm2m,emqx_auth_jwt,emqx_auth_ldap make dialyzer
 
 ### FAQ
 
-Наиболее частые проблемы разобраны в [EMQ X FAQ](https://docs.emqx.io/en/broker/latest/faq/faq.html).
+Наиболее частые проблемы разобраны в [EMQX FAQ](https://docs.emqx.io/en/broker/latest/faq/faq.html).
 
 
 ### Вопросы

README.md

@@ -1,4 +1,4 @@
-# EMQ X Broker
+# EMQX Broker
 
 [![GitHub Release](https://img.shields.io/github/release/emqx/emqx?color=brightgreen)](https://github.com/emqx/emqx/releases)
 [![Build Status](https://travis-ci.org/emqx/emqx.svg)](https://travis-ci.org/emqx/emqx)
@@ -12,20 +12,20 @@
 
 English | [简体中文](./README-CN.md) | [日本語](./README-JP.md) | [русский](./README-RU.md)
 
-*EMQ X* broker is a fully open source, highly scalable, highly available distributed MQTT messaging broker for IoT, M2M and Mobile applications that can handle tens of millions of concurrent clients.
+*EMQX* broker is a fully open source, highly scalable, highly available distributed MQTT messaging broker for IoT, M2M and Mobile applications that can handle tens of millions of concurrent clients.
 
-Starting from 3.0 release, *EMQ X* broker fully supports MQTT V5.0 protocol specifications and backward compatible with MQTT V3.1 and V3.1.1,  as well as other communication protocols such as MQTT-SN, CoAP, LwM2M, WebSocket and STOMP. The 3.0 release of the *EMQ X* broker can scaled to 10+ million concurrent MQTT connections on one cluster.
+Starting from 3.0 release, *EMQX* broker fully supports MQTT V5.0 protocol specifications and backward compatible with MQTT V3.1 and V3.1.1,  as well as other communication protocols such as MQTT-SN, CoAP, LwM2M, WebSocket and STOMP. The 3.0 release of the *EMQX* broker can scaled to 10+ million concurrent MQTT connections on one cluster.
 
-- For full list of new features, please read [EMQ X Release Notes](https://github.com/emqx/emqx/releases).
-- For more information, please visit [EMQ X homepage](https://www.emqx.io).
+- For full list of new features, please read [EMQX Release Notes](https://github.com/emqx/emqx/releases).
+- For more information, please visit [EMQX homepage](https://www.emqx.io).
 
 ## Installation
 
-The *EMQ X* broker is cross-platform, which supports Linux, Unix, macOS and Windows. It means *EMQ X* can be deployed on x86_64 architecture servers and ARM devices like Raspberry Pi.
+The *EMQX* broker is cross-platform, which supports Linux, Unix, macOS and Windows. It means *EMQX* can be deployed on x86_64 architecture servers and ARM devices like Raspberry Pi.
 
-See more details for building and running *EMQ X* on Windows in [Windows.md](./Windows.md)
+See more details for building and running *EMQX* on Windows in [Windows.md](./Windows.md)
 
-#### Installing via EMQ X Docker Image
+#### Installing via EMQX Docker Image
 
 ```
 docker run -d --name emqx -p 1883:1883 -p 8081:8081 -p 8083:8083 -p 8883:8883 -p 8084:8084 -p 18083:18083 emqx/emqx
@@ -33,7 +33,7 @@ docker run -d --name emqx -p 1883:1883 -p 8081:8081 -p 8083:8083 -p 8883:8883 -p
 
 #### Installing via Binary Package
 
-Get the binary package of the corresponding OS from [EMQ X Download](https://www.emqx.io/downloads) page.
+Get the binary package of the corresponding OS from [EMQX Download](https://www.emqx.io/downloads) page.
 
 - [Single Node Install](https://docs.emqx.io/en/broker/latest/getting-started/install.html)
 - [Multi Node Install](https://docs.emqx.io/en/broker/latest/advanced/cluster.html)
@@ -41,7 +41,7 @@ Get the binary package of the corresponding OS from [EMQ X Download](https://www
 
 ## Build From Source
 
-The *EMQ X* broker requires Erlang/OTP R21+ to build since 3.0 release.
+The *EMQX* broker requires Erlang/OTP R21+ to build since 3.0 release.
 
 For 4.3 and later versions.
 
@@ -110,7 +110,7 @@ DIALYZER_ANALYSE_APP=emqx_lwm2m,emqx_auth_jwt,emqx_auth_ldap make dialyzer
 
 ### FAQ
 
-Visiting [EMQ X FAQ](https://docs.emqx.io/en/broker/latest/faq/faq.html) to get help of common problems.
+Visiting [EMQX FAQ](https://docs.emqx.io/en/broker/latest/faq/faq.html) to get help of common problems.
 
 
 ### Questions

Windows.md

@@ -1,4 +1,4 @@
-# Build and run EMQ X on Windows
+# Build and run EMQX on Windows
 
 NOTE: The instructions and examples are based on Windows 10.
 
@@ -6,7 +6,7 @@ NOTE: The instructions and examples are based on Windows 10.
 
 ### Visual studio for C/C++ compile and link
 
-EMQ X includes Erlang NIF (Native Implmented Function) components, implemented
+EMQX includes Erlang NIF (Native Implmented Function) components, implemented
 in C/C++. To compile and link C/C++ libraries, the easiest way is perhaps to
 install Visual Studio.
 
@@ -25,17 +25,17 @@ C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\Build
 
 Depending on your visual studio version and OS, the paths may differ.
 The first path is for rebar3 port compiler to find `cl.exe` and `link.exe`
-The second path is for Powershell or CMD to setup environment variables.
+The second path is for CMD to setup environment variables.
 
 ### Erlang/OTP
 
 Install Erlang/OTP 23.2 from https://www.erlang.org/downloads
 You may need to edit the `Path` environment variable to allow running
-Erlang commands such as `erl` from powershell.
+Erlang commands such as `erl` from CMD.
 
-To validate Erlang installation in CMD or powershell:
+To validate Erlang installation in CMD :
 
-* Start (or restart) CMD or powershell
+* Start (or restart) CMD
 
 * Execute `erl` command to enter Erlang shell
 
@@ -51,7 +51,7 @@ Eshell V11.1.4  (abort with ^G)
 
 ### bash
 
-All EMQ X build/run scripts are either in `bash` or `escript`.
+All EMQX build/run scripts are either in `bash` or `escript`.
 `escript` is installed as a part of Erlang. To install a `bash`
 environment in Windows, there are quite a few options.
 
@@ -63,12 +63,12 @@ Cygwin is what we tested with.
   to `Path` list.
 
 * Validate installation.
-  Start (restart) CMD or powershell console and execute `which bash`, it should
+  Start (restart) CMD console and execute `which bash`, it should
   print out `/usr/bin/bash`
 
 ### Other tools
 
-Some of the unix world tools are required to build EMQ X.  Including:
+Some of the unix world tools are required to build EMQX.  Including:
 
 * git
 * curl
@@ -84,11 +84,11 @@ When using scoop:
 scoop install git curl make jq zip unzip
 ```
 
-## Build EMQ X source code
+## Build EMQX source code
 
 * Clone the repo: `git clone https://github.com/emqx/emqx.git`
 
-* Start CMD or Powershell
+* Start CMD
 
 * Execute `vcvarsall.bat x86_amd64` to load environment variables
 
@@ -112,11 +112,11 @@ scoop install git curl make jq zip unzip
   To fix it, Visual Studio's bin paths should be ordered prior to Cygwin's (or similar installation's)
   bin paths in `Path` environment variable.
 
-## Run EMQ X
+## Run EMQX
 
-To start EMQ X broker.
+To start EMQX broker.
 
-Execute `_build\emqx\rel\emqx>.\bin\emqx console` or `_build\emqx\rel\emqx>.\bin\emqx start` to start EMQ X.
+Execute `_build\emqx\rel\emqx>.\bin\emqx console` or `_build\emqx\rel\emqx>.\bin\emqx start` to start EMQX.
 
 Then execute `_build\emqx\rel\emqx>.\bin\emqx_ctl status` to check status.
 If everything works fine, it should print out

apps/emqx_auth_http/README.md

@@ -1,7 +1,7 @@
 emqx_auth_http
 ==============
 
-EMQ X HTTP Auth/ACL Plugin
+EMQX HTTP Auth/ACL Plugin
 
 Build
 -----
@@ -96,5 +96,5 @@ Apache License Version 2.0
 Author
 ------
 
-EMQ X Team.
+EMQX Team.
 

apps/emqx_auth_http/etc/emqx_auth_http.conf

@@ -42,18 +42,18 @@ auth.http.auth_req.params = clientid=%c,username=%u,password=%P
 ## Value: URL
 ##
 ## Examples: http://127.0.0.1:80/mqtt/superuser, https://[::1]:80/mqtt/superuser
-auth.http.super_req.url = http://127.0.0.1:80/mqtt/superuser
+# auth.http.super_req.url = http://127.0.0.1:80/mqtt/superuser
 
 ## HTTP Request Method for SuperUser Request
 ##
 ## Value: post | get
-auth.http.super_req.method = post
+# auth.http.super_req.method = post
 
 ## HTTP Request Headers for SuperUser Request, Content-Type header is configured by default.
 ## The possible values of the Content-Type header: application/x-www-form-urlencoded, application/json
 ##
 ## Examples: auth.http.super_req.headers.accept = */*
-auth.http.super_req.headers.content-type = application/x-www-form-urlencoded
+# auth.http.super_req.headers.content-type = application/x-www-form-urlencoded
 
 ## Parameters used to construct the request body or query string parameters
 ## When the request method is GET, these parameters will be converted into query string parameters
@@ -70,7 +70,7 @@ auth.http.super_req.headers.content-type = application/x-www-form-urlencoded
 ##  - %d: subject of client TLS cert
 ##
 ## Value: <K1>=<V1>,<K2>=<V2>,...
-auth.http.super_req.params = clientid=%c,username=%u
+# auth.http.super_req.params = clientid=%c,username=%u
 
 ## HTTP URL API path for ACL Request
 ## Comment out this config to disable ACL checks
@@ -136,6 +136,11 @@ auth.http.connect_timeout = 5s
 ## Value: Number
 auth.http.pool_size = 32
 
+## Whether to enable HTTP Pipelining
+##
+## See: https://en.wikipedia.org/wiki/HTTP_pipelining
+auth.http.enable_pipelining = true
+
 ##------------------------------------------------------------------------------
 ## SSL options
 
@@ -163,7 +168,7 @@ auth.http.pool_size = 32
 
 ## If not specified, the server's names returned in server's certificate is validated against
 ## what's provided `auth.http.auth_req.url` config's host part.
-## Setting to 'disable' will make EMQ X ignore unmatched server names.
+## Setting to 'disable' will make EMQX ignore unmatched server names.
 ## If set with a host name, the server's names returned in server's certificate is validated
 ## against this value.
 ##

apps/emqx_auth_http/include/emqx_auth_http.hrl

@@ -7,17 +7,8 @@
         ignore = 'client.auth.ignore'
     }).
 
--record(acl_metrics, {
-        allow = 'client.acl.allow',
-        deny = 'client.acl.deny',
-        ignore = 'client.acl.ignore'
-    }).
-
 -define(METRICS(Type), tl(tuple_to_list(#Type{}))).
 -define(METRICS(Type, K), #Type{}#Type.K).
 
 -define(AUTH_METRICS, ?METRICS(auth_metrics)).
 -define(AUTH_METRICS(K), ?METRICS(auth_metrics, K)).
-
--define(ACL_METRICS, ?METRICS(acl_metrics)).
--define(ACL_METRICS(K), ?METRICS(acl_metrics, K)).

apps/emqx_auth_http/priv/emqx_auth_http.schema

@@ -109,6 +109,11 @@ end}.
   {datatype, integer}
 ]}.
 
+{mapping, "auth.http.enable_pipelining", "emqx_auth_http.enable_pipelining", [
+  {default, true},
+  {datatype, {enum, [true, false]}}
+]}.
+
 {mapping, "auth.http.ssl.cacertfile", "emqx_auth_http.cacertfile", [
   {datatype, string}
 ]}.

apps/emqx_auth_http/src/emqx_acl_http.erl

@@ -29,26 +29,17 @@
         ]).
 
 %% ACL callbacks
--export([ register_metrics/0
-        , check_acl/5
+-export([ check_acl/5
         , description/0
         ]).
 
--spec(register_metrics() -> ok).
-register_metrics() ->
-    lists:foreach(fun emqx_metrics:ensure/1, ?ACL_METRICS).
-
 %%--------------------------------------------------------------------
 %% ACL callbacks
 %%--------------------------------------------------------------------
 
-check_acl(ClientInfo, PubSub, Topic, AclResult, Params) ->
-    return_with(fun inc_metrics/1,
-                do_check_acl(ClientInfo, PubSub, Topic, AclResult, Params)).
-
-do_check_acl(#{username := <<$$, _/binary>>}, _PubSub, _Topic, _AclResult, _Params) ->
+check_acl(#{username := <<$$, _/binary>>}, _PubSub, _Topic, _AclResult, _Params) ->
     ok;
-do_check_acl(ClientInfo, PubSub, Topic, _AclResult, #{acl := ACLParams = #{path := Path}}) ->
+check_acl(ClientInfo, PubSub, Topic, _AclResult, #{acl := ACLParams = #{path := Path}}) ->
     ClientInfo1 = ClientInfo#{access => access(PubSub), topic => Topic},
     case check_acl_request(ACLParams, ClientInfo1) of
         {ok, 200, <<"ignore">>} -> ok;
@@ -65,16 +56,6 @@ description() -> "ACL with HTTP API".
 %% Internal functions
 %%--------------------------------------------------------------------
 
-inc_metrics(ok) ->
-    emqx_metrics:inc(?ACL_METRICS(ignore));
-inc_metrics({stop, allow}) ->
-    emqx_metrics:inc(?ACL_METRICS(allow));
-inc_metrics({stop, deny}) ->
-    emqx_metrics:inc(?ACL_METRICS(deny)).
-
-return_with(Fun, Result) ->
-    Fun(Result), Result.
-
 check_acl_request(#{pool_name := PoolName,
                     path := Path,
                     method := Method,

apps/emqx_auth_http/src/emqx_auth_http.app.src

@@ -1,6 +1,6 @@
 {application, emqx_auth_http,
  [{description, "EMQ X Authentication/ACL with HTTP API"},
-  {vsn, "4.3.0"}, % strict semver, bump manually!
+  {vsn, "4.3.4"}, % strict semver, bump manually!
   {modules, []},
   {registered, [emqx_auth_http_sup]},
   {applications, [kernel,stdlib,ehttpc]},

apps/emqx_auth_http/src/emqx_auth_http.appup.src

@@ -0,0 +1,24 @@
+%% -*- mode: erlang -*-
+{VSN,
+  [{"4.3.3",
+    [{load_module,emqx_auth_http_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_http,brutal_purge,soft_purge,[]}]},
+   {"4.3.2",
+    [{apply,{application,stop,[emqx_auth_http]}},
+     {load_module,emqx_auth_http_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_http,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_http_cli,brutal_purge,soft_purge,[]}]},
+   {<<"4.3.[0-1]">>,
+    [{restart_application,emqx_auth_http}]},
+   {<<".*">>,[]}],
+  [{"4.3.3",
+    [{load_module,emqx_auth_http_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_http,brutal_purge,soft_purge,[]}]},
+   {"4.3.2",
+    [{apply,{application,stop,[emqx_auth_http]}},
+     {load_module,emqx_auth_http_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_http,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_http_cli,brutal_purge,soft_purge,[]}]},
+   {<<"4.3.[0-1]">>,
+    [{restart_application,emqx_auth_http}]},
+   {<<".*">>,[]}]}.

apps/emqx_auth_http/src/emqx_auth_http_app.erl

@@ -50,14 +50,14 @@ translate_env(EnvName) ->
     case application:get_env(?APP, EnvName) of
         undefined -> ok;
         {ok, Req} ->
+            {ok, EnablePipelining} = application:get_env(?APP, enable_pipelining),
             {ok, PoolSize} = application:get_env(?APP, pool_size),
             {ok, ConnectTimeout} = application:get_env(?APP, connect_timeout),
             URL = proplists:get_value(url, Req),
             {ok, #{host := Host,
-                   path := Path0,
                    port := Port,
-                   scheme := Scheme}} = emqx_http_lib:uri_parse(URL),
-            Path = path(Path0),
+                   scheme := Scheme} = URIMap} = emqx_http_lib:uri_parse(URL),
+            Path = path(URIMap),
             MoreOpts = case Scheme of
                         http ->
                             [{transport_opts, emqx_misc:ipv6_probe([])}];
@@ -89,6 +89,7 @@ translate_env(EnvName) ->
                         end,
             PoolOpts = [{host, Host},
                         {port, Port},
+                        {enable_pipelining, EnablePipelining},
                         {pool_size, PoolSize},
                         {pool_type, random},
                         {connect_timeout, ConnectTimeout},
@@ -129,7 +130,6 @@ load_hooks() ->
     case application:get_env(?APP, acl_req) of
         undefined -> ok;
         {ok, ACLReq} ->
-            ok = emqx_acl_http:register_metrics(),
             PoolOpts2 = proplists:get_value(pool_opts, ACLReq),
             PoolName2 = proplists:get_value(pool_name, ACLReq),
             {ok, _} = ehttpc_sup:start_pool(PoolName2, PoolOpts2),
@@ -149,10 +149,14 @@ ensure_content_type_header(Method, Headers)
   when Method =:= post orelse Method =:= put ->
     Headers;
 ensure_content_type_header(_Method, Headers) ->
-    lists:keydelete("content-type", 1, Headers).
+    lists:keydelete(<<"content-type">>, 1, Headers).
 
-path("") ->
+path(#{path := "", 'query' := Query}) ->
+    "?" ++ Query;
+path(#{path := Path, 'query' := Query}) ->
+    Path ++ "?" ++ Query;
+path(#{path := ""}) ->
     "/";
-path(Path) ->
+path(#{path := Path}) ->
     Path.
 

apps/emqx_auth_http/src/emqx_auth_http_cli.erl

@@ -29,16 +29,16 @@
 
 request(PoolName, get, Path, Headers, Params, Timeout) ->
     NewPath = Path ++ "?" ++ binary_to_list(cow_qs:qs(bin_kw(Params))),
-    reply(ehttpc:request(ehttpc_pool:pick_worker(PoolName), get, {NewPath, Headers}, Timeout));
+    reply(ehttpc:request(PoolName, get, {NewPath, Headers}, Timeout));
 
 request(PoolName, post, Path, Headers, Params, Timeout) ->
-    Body = case proplists:get_value("content-type", Headers) of
+    Body = case proplists:get_value(<<"content-type">>, Headers) of
                "application/x-www-form-urlencoded" ->
                    cow_qs:qs(bin_kw(Params));
                "application/json" -> 
                    emqx_json:encode(bin_kw(Params))
            end,
-    reply(ehttpc:request(ehttpc_pool:pick_worker(PoolName), post, {Path, Headers, Body}, Timeout)).
+    reply(ehttpc:request(PoolName, post, {Path, Headers, Body}, Timeout)).
 
 reply({ok, StatusCode, _Headers}) ->
     {ok, StatusCode, <<>>};

apps/emqx_auth_jwt/README.md

@@ -1,7 +1,7 @@
 
 # emqx-auth-jwt
 
-EMQ X JWT Authentication Plugin
+EMQX JWT Authentication Plugin
 
 Build
 -----
@@ -87,4 +87,4 @@ Apache License Version 2.0
 Author
 ------
 
-EMQ X Team.
+EMQX Team.

apps/emqx_auth_jwt/src/emqx_auth_jwt.app.src

@@ -1,6 +1,6 @@
 {application, emqx_auth_jwt,
  [{description, "EMQ X Authentication with JWT"},
-  {vsn, "4.3.1"}, % strict semver, bump manually!
+  {vsn, "4.3.2"}, % strict semver, bump manually!
   {modules, []},
   {registered, [emqx_auth_jwt_sup]},
   {applications, [kernel,stdlib,jose]},

apps/emqx_auth_jwt/src/emqx_auth_jwt.appup.src

@@ -1,13 +1,13 @@
 %% -*-: erlang -*-
 {VSN,
  [
-    {"4.3.0", [
+    {<<"4\\.3\\.[0-1]">>, [
       {load_module, emqx_auth_jwt_svr, brutal_purge, soft_purge, []}
     ]},
     {<<".*">>, []}
  ],
  [
-    {"4.3.0", [
+    {<<"4\\.3\\.[0-1]">>, [
       {load_module, emqx_auth_jwt_svr, brutal_purge, soft_purge, []}
     ]},
     {<<".*">>, []}

apps/emqx_auth_jwt/src/emqx_auth_jwt_svr.erl

@@ -91,7 +91,7 @@ do_init_jwks(Options) ->
                                   [K, V, Reason]),
                              undefined;
                          J -> J
-                     catch T:R:_ ->
+                     catch T:R ->
                          ?LOG(warning, "Build ~p JWK ~p failed: {~p, ~p}~n",
                               [K, V, T, R]),
                          undefined

apps/emqx_auth_ldap/README.md

@@ -1,7 +1,7 @@
 emqx_auth_ldap
 ==============
 
-EMQ X LDAP Authentication Plugin
+EMQX LDAP Authentication Plugin
 
 Build
 -----
@@ -92,5 +92,5 @@ Apache License Version 2.0
 Author
 ------
 
-EMQ X Team.
+EMQX Team.
 

apps/emqx_auth_ldap/include/emqx_auth_ldap.hrl

@@ -7,17 +7,8 @@
         ignore = 'client.auth.ignore'
     }).
 
--record(acl_metrics, {
-        allow = 'client.acl.allow',
-        deny = 'client.acl.deny',
-        ignore = 'client.acl.ignore'
-    }).
-
 -define(METRICS(Type), tl(tuple_to_list(#Type{}))).
 -define(METRICS(Type, K), #Type{}#Type.K).
 
 -define(AUTH_METRICS, ?METRICS(auth_metrics)).
 -define(AUTH_METRICS(K), ?METRICS(auth_metrics, K)).
-
--define(ACL_METRICS, ?METRICS(acl_metrics)).
--define(ACL_METRICS(K), ?METRICS(acl_metrics, K)).

apps/emqx_auth_ldap/src/emqx_acl_ldap.erl

@@ -22,24 +22,15 @@
 -include_lib("eldap/include/eldap.hrl").
 -include_lib("emqx/include/logger.hrl").
 
--export([ register_metrics/0
-        , check_acl/5
+-export([ check_acl/5
         , description/0
         ]).
 
--import(proplists, [get_value/2]).
-
--import(emqx_auth_ldap_cli, [search/4]).
-
--spec(register_metrics() -> ok).
-register_metrics() ->
-    lists:foreach(fun emqx_metrics:ensure/1, ?ACL_METRICS).
-
 check_acl(ClientInfo, PubSub, Topic, NoMatchAction, State) ->
     case do_check_acl(ClientInfo, PubSub, Topic, NoMatchAction, State) of
-        ok -> emqx_metrics:inc(?ACL_METRICS(ignore)), ok;
-        {stop, allow} -> emqx_metrics:inc(?ACL_METRICS(allow)), {stop, allow};
-        {stop, deny} -> emqx_metrics:inc(?ACL_METRICS(deny)), {stop, deny}
+        ok -> ok;
+        {stop, allow} -> {stop, allow};
+        {stop, deny} -> {stop, deny}
     end.
 
 do_check_acl(#{username := <<$$, _/binary>>}, _PubSub, _Topic, _NoMatchAction, _State) ->
@@ -70,14 +61,14 @@ do_check_acl(#{username := Username}, PubSub, Topic, _NoMatchAction,
 
     BaseDN = emqx_auth_ldap:replace_vars(CustomBaseDN, ReplaceRules),
 
-    case search(Pool, BaseDN, Filter, [Attribute, Attribute1]) of
+    case emqx_auth_ldap_cli:search(Pool, BaseDN, Filter, [Attribute, Attribute1]) of
         {error, noSuchObject} ->
             ok;
         {ok, #eldap_search_result{entries = []}} ->
             ok;
         {ok, #eldap_search_result{entries = [Entry]}} ->
-            Topics = get_value(Attribute, Entry#eldap_entry.attributes)
-                ++ get_value(Attribute1, Entry#eldap_entry.attributes),
+            Topics = proplists:get_value(Attribute, Entry#eldap_entry.attributes, [])
+                ++ proplists:get_value(Attribute1, Entry#eldap_entry.attributes, []),
             match(Topic, Topics);
         Error ->
             ?LOG(error, "[LDAP] search error:~p", [Error]),
@@ -95,4 +86,3 @@ match(Topic, [Filter | Topics]) ->
 
 description() ->
     "ACL with LDAP".
-

apps/emqx_auth_ldap/src/emqx_auth_ldap.app.src

@@ -1,6 +1,6 @@
 {application, emqx_auth_ldap,
  [{description, "EMQ X Authentication/ACL with LDAP"},
-  {vsn, "4.3.0"}, % strict semver, bump manually!
+  {vsn, "4.3.3"}, % strict semver, bump manually!
   {modules, []},
   {registered, [emqx_auth_ldap_sup]},
   {applications, [kernel,stdlib,eldap2,ecpool]},

apps/emqx_auth_ldap/src/emqx_auth_ldap.appup.src

@@ -0,0 +1,36 @@
+%% -*-: erlang -*-
+{VSN,
+ [ {"4.3.0",
+    [ {load_module, emqx_acl_ldap, brutal_purge, soft_purge, []}
+    , {load_module, emqx_auth_ldap_cli, brutal_purge, soft_purge, []}
+    , {load_module, emqx_auth_ldap_app, brutal_purge, soft_purge, []}
+    ]},
+   {"4.3.1",
+    [ {load_module, emqx_auth_ldap_cli, brutal_purge, soft_purge, []}
+    , {load_module, emqx_acl_ldap, brutal_purge, soft_purge, []}
+    , {load_module, emqx_auth_ldap_app, brutal_purge, soft_purge, []}
+    ]},
+   {"4.3.2",
+    [ {load_module, emqx_acl_ldap, brutal_purge, soft_purge, []}
+    , {load_module, emqx_auth_ldap_app, brutal_purge, soft_purge, []}
+    ]},
+   {<<".*">>, []}
+ ],
+ [
+   {"4.3.0",
+    [ {load_module, emqx_acl_ldap, brutal_purge, soft_purge, []}
+    , {load_module, emqx_auth_ldap_cli, brutal_purge, soft_purge, []}
+    , {load_module, emqx_auth_ldap_app, brutal_purge, soft_purge, []}
+    ]},
+   {"4.3.1",
+    [ {load_module, emqx_auth_ldap_cli, brutal_purge, soft_purge, []}
+    , {load_module, emqx_acl_ldap, brutal_purge, soft_purge, []}
+    , {load_module, emqx_auth_ldap_app, brutal_purge, soft_purge, []}
+    ]},
+   {"4.3.2",
+    [ {load_module, emqx_acl_ldap, brutal_purge, soft_purge, []}
+    , {load_module, emqx_auth_ldap_app, brutal_purge, soft_purge, []}
+    ]},
+   {<<".*">>, []}
+ ]
+}.

apps/emqx_auth_ldap/src/emqx_auth_ldap_app.erl

@@ -54,7 +54,6 @@ load_auth_hook(DeviceDn) ->
     emqx:hook('client.authenticate', fun emqx_auth_ldap:check/3, [Params#{pool => ?APP}]).
 
 load_acl_hook(DeviceDn) ->
-    ok = emqx_acl_ldap:register_metrics(),
     Params = maps:from_list(DeviceDn),
     emqx:hook('client.check_acl', fun emqx_acl_ldap:check_acl/5 , [Params#{pool => ?APP}]).
 

apps/emqx_auth_ldap/src/emqx_auth_ldap_cli.erl

@@ -76,8 +76,8 @@ connect(Opts) ->
 search(Pool, Base, Filter) ->
     ecpool:with_client(Pool,
         fun(C) ->
-                case application:get_env(?APP, bind_as_user) of
-                    {ok, true} ->
+                case application:get_env(?APP, bind_as_user, false) of
+                    true ->
                         {ok, Opts} = application:get_env(?APP, ldap),
                         BindDn       = get_value(bind_dn, Opts),
                         BindPassword = get_value(bind_password, Opts),
@@ -91,7 +91,7 @@ search(Pool, Base, Filter) ->
                         catch
                             error:Reason -> {error, Reason}
                         end;
-                    {ok, false} ->
+                    false ->
                         eldap2:search(C, [{base, Base},
                                           {filter, Filter},
                                           {deref, eldap2:derefFindingBaseObj()}])
@@ -101,8 +101,8 @@ search(Pool, Base, Filter) ->
 search(Pool, Base, Filter, Attributes) ->
     ecpool:with_client(Pool,
         fun(C) ->
-                case application:get_env(?APP, bind_as_user) of
-                    {ok, true} ->
+                case application:get_env(?APP, bind_as_user, false) of
+                    true ->
                         {ok, Opts} = application:get_env(?APP, ldap),
                         BindDn       = get_value(bind_dn, Opts),
                         BindPassword = get_value(bind_password, Opts),
@@ -117,7 +117,7 @@ search(Pool, Base, Filter, Attributes) ->
                         catch
                             error:Reason -> {error, Reason}
                         end;
-                    {ok, false} ->
+                    false ->
                         eldap2:search(C, [{base, Base},
                                           {filter, Filter},
                                           {attributes, Attributes},

apps/emqx_auth_mnesia/include/emqx_auth_mnesia.hrl

@@ -1,38 +1,55 @@
 -define(APP, emqx_auth_mnesia).
 
--type(login():: {clientid, binary()}
+-type(login() :: {clientid, binary()}
               | {username, binary()}).
 
+-type(acl_target() :: login() | all).
+
+-type(acl_target_type() :: clientid | username | all).
+
+-type(access():: allow | deny).
+-type(action():: pub | sub).
+-type(legacy_action():: action() | pubsub).
+-type(created_at():: integer()).
+
 -record(emqx_user, {
           login :: login(),
           password :: binary(),
-          created_at :: integer()
+          created_at :: created_at()
         }).
 
--record(emqx_acl, {
-          filter:: {login() | all, emqx_topic:topic()},
-          action :: pub | sub | pubsub,
-          access :: allow | deny,
-          created_at :: integer()
+-define(ACL_TABLE, emqx_acl).
+
+-define(MIGRATION_MARK_KEY, emqx_acl2_migration_started).
+
+-record(?ACL_TABLE, {
+          filter :: {acl_target(), emqx_topic:topic()} | ?MIGRATION_MARK_KEY,
+          action :: legacy_action(),
+          access :: access(),
+          created_at :: created_at()
          }).
 
+-define(MIGRATION_MARK_RECORD, #?ACL_TABLE{filter = ?MIGRATION_MARK_KEY, action = pub, access = deny, created_at = 0}).
+
+-type(rule() :: {access(), action(), emqx_topic:topic(), created_at()}).
+
+-define(ACL_TABLE2, emqx_acl2).
+
+-record(?ACL_TABLE2, {
+          who :: acl_target(),
+          rules :: [ rule() ]
+         }).
+
+-type(acl_record() :: {acl_target(), emqx_topic:topic(), action(), access(), created_at()}).
+
 -record(auth_metrics, {
         success = 'client.auth.success',
         failure = 'client.auth.failure',
         ignore = 'client.auth.ignore'
     }).
 
--record(acl_metrics, {
-        allow = 'client.acl.allow',
-        deny = 'client.acl.deny',
-        ignore = 'client.acl.ignore'
-    }).
-
 -define(METRICS(Type), tl(tuple_to_list(#Type{}))).
 -define(METRICS(Type, K), #Type{}#Type.K).
 
 -define(AUTH_METRICS, ?METRICS(auth_metrics)).
 -define(AUTH_METRICS(K), ?METRICS(auth_metrics, K)).
-
--define(ACL_METRICS, ?METRICS(acl_metrics)).
--define(ACL_METRICS(K), ?METRICS(acl_metrics, K)).

apps/emqx_auth_mnesia/src/emqx_acl_mnesia.erl

@@ -18,51 +18,35 @@
 
 -include("emqx_auth_mnesia.hrl").
 
--include_lib("stdlib/include/ms_transform.hrl").
-
--define(TABLE, emqx_acl).
-
 %% ACL Callbacks
 -export([ init/0
-        , register_metrics/0
         , check_acl/5
         , description/0
-        ]).
+       ]).
 
 init() ->
-    ok = ekka_mnesia:create_table(emqx_acl, [
-            {type, bag},
-            {disc_copies, [node()]},
-            {attributes, record_info(fields, emqx_acl)},
-            {storage_properties, [{ets, [{read_concurrency, true}]}]}]),
-    ok = ekka_mnesia:copy_table(emqx_acl, disc_copies).
-
--spec(register_metrics() -> ok).
-register_metrics() ->
-    lists:foreach(fun emqx_metrics:ensure/1, ?ACL_METRICS).
+    ok = emqx_acl_mnesia_db:create_table(),
+    ok = emqx_acl_mnesia_db:create_table2().
 
 check_acl(ClientInfo = #{ clientid := Clientid }, PubSub, Topic, _NoMatchAction, _Params) ->
     Username = maps:get(username, ClientInfo, undefined),
 
     Acls = case Username of
                undefined ->
-                   emqx_acl_mnesia_cli:lookup_acl({clientid, Clientid}) ++
-                   emqx_acl_mnesia_cli:lookup_acl(all);
+                   emqx_acl_mnesia_db:lookup_acl({clientid, Clientid}) ++
+                   emqx_acl_mnesia_db:lookup_acl(all);
                _ ->
-                   emqx_acl_mnesia_cli:lookup_acl({clientid, Clientid}) ++
-                   emqx_acl_mnesia_cli:lookup_acl({username, Username}) ++
-                   emqx_acl_mnesia_cli:lookup_acl(all)
+                   emqx_acl_mnesia_db:lookup_acl({clientid, Clientid}) ++
+                   emqx_acl_mnesia_db:lookup_acl({username, Username}) ++
+                   emqx_acl_mnesia_db:lookup_acl(all)
            end,
 
     case match(ClientInfo, PubSub, Topic, Acls) of
         allow ->
-            emqx_metrics:inc(?ACL_METRICS(allow)),
             {stop, allow};
         deny ->
-            emqx_metrics:inc(?ACL_METRICS(deny)),
             {stop, deny};
         _ ->
-            emqx_metrics:inc(?ACL_METRICS(ignore)),
             ok
     end.
 
@@ -83,7 +67,6 @@ match(ClientInfo, PubSub, Topic, [ {_, ACLTopic, Action, Access, _} | Acls]) ->
 match_topic(ClientInfo, Topic, ACLTopic) when is_binary(Topic) ->
     emqx_topic:match(Topic, feed_var(ClientInfo, ACLTopic)).
 
-match_actions(_, pubsub) -> true;
 match_actions(subscribe, sub) -> true;
 match_actions(publish, pub) -> true;
 match_actions(_, _) -> false.

apps/emqx_auth_mnesia/src/emqx_acl_mnesia_api.erl

@@ -16,8 +16,6 @@
 
 -module(emqx_acl_mnesia_api).
 
--include("emqx_auth_mnesia.hrl").
-
 -include_lib("stdlib/include/ms_transform.hrl").
 
 -import(proplists, [ get_value/2
@@ -99,26 +97,22 @@
         ]).
 
 list_clientid(_Bindings, Params) ->
-    MatchSpec = ets:fun2ms(
-                  fun({emqx_acl, {{clientid, Clientid}, Topic}, Action, Access, CreatedAt}) -> {{clientid,Clientid}, Topic, Action,Access, CreatedAt} end),
-    return({ok, emqx_auth_mnesia_api:paginate(emqx_acl, MatchSpec, Params, fun emqx_acl_mnesia_cli:comparing/2, fun format/1)}).
+    Table = emqx_acl_mnesia_db:login_acl_table(clientid),
+    return({ok, emqx_auth_mnesia_api:paginate_qh(Table, count(Table), Params, fun emqx_acl_mnesia_db:comparing/2, fun format/1)}).
 
 list_username(_Bindings, Params) ->
-    MatchSpec = ets:fun2ms(
-                  fun({emqx_acl, {{username, Username}, Topic}, Action, Access, CreatedAt}) -> {{username, Username}, Topic, Action,Access, CreatedAt} end),
-    return({ok, emqx_auth_mnesia_api:paginate(emqx_acl, MatchSpec, Params, fun emqx_acl_mnesia_cli:comparing/2, fun format/1)}).
+    Table = emqx_acl_mnesia_db:login_acl_table(username),
+    return({ok, emqx_auth_mnesia_api:paginate_qh(Table, count(Table), Params, fun emqx_acl_mnesia_db:comparing/2, fun format/1)}).
 
 list_all(_Bindings, Params) ->
-    MatchSpec = ets:fun2ms(
-                  fun({emqx_acl, {all, Topic}, Action, Access, CreatedAt}) -> {all, Topic, Action,Access, CreatedAt}end
-                 ),
-    return({ok, emqx_auth_mnesia_api:paginate(emqx_acl, MatchSpec, Params, fun emqx_acl_mnesia_cli:comparing/2, fun format/1)}).
+    Table = emqx_acl_mnesia_db:login_acl_table(all),
+    return({ok, emqx_auth_mnesia_api:paginate_qh(Table, count(Table), Params, fun emqx_acl_mnesia_db:comparing/2, fun format/1)}).
 
 
 lookup(#{clientid := Clientid}, _Params) ->
-    return({ok, format(emqx_acl_mnesia_cli:lookup_acl({clientid, urldecode(Clientid)}))});
+    return({ok, format(emqx_acl_mnesia_db:lookup_acl({clientid, urldecode(Clientid)}))});
 lookup(#{username := Username}, _Params) ->
-    return({ok, format(emqx_acl_mnesia_cli:lookup_acl({username, urldecode(Username)}))}).
+    return({ok, format(emqx_acl_mnesia_db:lookup_acl({username, urldecode(Username)}))}).
 
 add(_Bindings, Params) ->
     [ P | _] = Params,
@@ -144,15 +138,15 @@ do_add(Params) ->
     Username = get_value(<<"username">>, Params, undefined),
     Login = case {Clientid, Username} of
                 {undefined, undefined} -> all;
-                {_, undefined} -> {clientid, urldecode(Clientid)};
-                {undefined, _} -> {username, urldecode(Username)}
+                {_, undefined} -> {clientid, Clientid};
+                {undefined, _} -> {username, Username}
             end,
-    Topic = urldecode(get_value(<<"topic">>, Params)),
-    Action = urldecode(get_value(<<"action">>, Params)),
-    Access = urldecode(get_value(<<"access">>, Params)),
+    Topic = get_value(<<"topic">>, Params),
+    Action = get_value(<<"action">>, Params),
+    Access = get_value(<<"access">>, Params),
     Re = case validate([login, topic, action, access], [Login, Topic, Action, Access]) of
         ok ->
-            emqx_acl_mnesia_cli:add_acl(Login, Topic, erlang:binary_to_atom(Action, utf8), erlang:binary_to_atom(Access, utf8));
+            emqx_acl_mnesia_db:add_acl(Login, Topic, erlang:binary_to_atom(Action, utf8), erlang:binary_to_atom(Access, utf8));
         Err -> Err
     end,
     maps:merge(#{topic => Topic,
@@ -165,15 +159,19 @@ do_add(Params) ->
                    end).
 
 delete(#{clientid := Clientid, topic := Topic}, _) ->
-    return(emqx_acl_mnesia_cli:remove_acl({clientid, urldecode(Clientid)}, urldecode(Topic)));
+    return(emqx_acl_mnesia_db:remove_acl({clientid, urldecode(Clientid)}, urldecode(Topic)));
 delete(#{username := Username, topic := Topic}, _) ->
-    return(emqx_acl_mnesia_cli:remove_acl({username, urldecode(Username)}, urldecode(Topic)));
+    return(emqx_acl_mnesia_db:remove_acl({username, urldecode(Username)}, urldecode(Topic)));
 delete(#{topic := Topic}, _) ->
-    return(emqx_acl_mnesia_cli:remove_acl(all, urldecode(Topic))).
+    return(emqx_acl_mnesia_db:remove_acl(all, urldecode(Topic))).
 
 %%------------------------------------------------------------------------------
 %% Interval Funcs
 %%------------------------------------------------------------------------------
+
+count(QH) ->
+    qlc:fold(fun(_, Count) -> Count + 1 end, 0, QH).
+
 format({{clientid, Clientid}, Topic, Action, Access, _CreatedAt}) ->
     #{clientid => Clientid, topic => Topic, action => Action, access => Access};
 format({{username, Username}, Topic, Action, Access, _CreatedAt}) ->

apps/emqx_auth_mnesia/src/emqx_acl_mnesia_cli.erl

@@ -16,110 +16,28 @@
 
 -module(emqx_acl_mnesia_cli).
 
--include("emqx_auth_mnesia.hrl").
--include_lib("emqx/include/logger.hrl").
--include_lib("stdlib/include/ms_transform.hrl").
--define(TABLE, emqx_acl).
-
-%% Acl APIs
--export([ add_acl/4
-        , lookup_acl/1
-        , all_acls/0
-        , all_acls/1
-        , remove_acl/2
-        ]).
-
 -export([cli/1]).
--export([comparing/2]).
-%%--------------------------------------------------------------------
-%% Acl API
-%%--------------------------------------------------------------------
-
-%% @doc Add Acls
--spec(add_acl(login() | all, emqx_topic:topic(), pub | sub | pubsub, allow | deny) ->
-        ok | {error, any()}).
-add_acl(Login, Topic, Action, Access) ->
-    Filter = {Login, Topic},
-    Acl = #?TABLE{
-             filter = Filter,
-             action = Action,
-             access = Access,
-             created_at = erlang:system_time(millisecond)
-            },
-    ret(mnesia:transaction(
-          fun() ->
-                  OldRecords = mnesia:wread({?TABLE, Filter}),
-                  case Action of
-                      pubsub ->
-                          update_permission(pub, Acl, OldRecords),
-                          update_permission(sub, Acl, OldRecords);
-                      _ ->
-                          update_permission(Action, Acl, OldRecords)
-                  end
-          end)).
-
-%% @doc Lookup acl by login
--spec(lookup_acl(login() | all) -> list()).
-lookup_acl(undefined) -> [];
-lookup_acl(Login) ->
-    MatchSpec = ets:fun2ms(fun({?TABLE, {Filter, ACLTopic}, Action, Access, CreatedAt})
-                                 when Filter =:= Login ->
-                                   {Filter, ACLTopic, Action, Access, CreatedAt}
-                           end),
-    lists:sort(fun comparing/2, ets:select(?TABLE, MatchSpec)).
-
-%% @doc Remove acl
--spec(remove_acl(login() | all, emqx_topic:topic()) -> ok | {error, any()}).
-remove_acl(Login, Topic) ->
-    ret(mnesia:transaction(fun mnesia:delete/1, [{?TABLE, {Login, Topic}}])).
-
-%% @doc All logins
--spec(all_acls() -> list()).
-all_acls() ->
-    all_acls(clientid) ++
-    all_acls(username) ++
-    all_acls(all).
-
-all_acls(clientid) ->
-    MatchSpec = ets:fun2ms(
-                  fun({?TABLE, {{clientid, Clientid}, Topic}, Action, Access, CreatedAt}) ->
-                          {{clientid, Clientid}, Topic, Action, Access, CreatedAt}
-                  end),
-    lists:sort(fun comparing/2, ets:select(?TABLE, MatchSpec));
-all_acls(username) ->
-    MatchSpec = ets:fun2ms(
-                  fun({?TABLE, {{username, Username}, Topic}, Action, Access, CreatedAt}) ->
-                          {{username, Username}, Topic, Action, Access, CreatedAt}
-                  end),
-    lists:sort(fun comparing/2, ets:select(?TABLE, MatchSpec));
-all_acls(all) ->
-    MatchSpec = ets:fun2ms(
-                  fun({?TABLE, {all, Topic}, Action, Access, CreatedAt}) ->
-                          {all, Topic, Action, Access, CreatedAt}
-                  end
-                 ),
-    lists:sort(fun comparing/2, ets:select(?TABLE, MatchSpec)).
 
 %%--------------------------------------------------------------------
 %% ACL Cli
 %%--------------------------------------------------------------------
 
 cli(["list"]) ->
-    [print_acl(Acl) || Acl <- all_acls()];
+    [print_acl(Acl) || Acl <- emqx_acl_mnesia_db:all_acls()];
 
 cli(["list", "clientid"]) ->
-    [print_acl(Acl) || Acl <- all_acls(clientid)];
+    [print_acl(Acl) || Acl <- emqx_acl_mnesia_db:all_acls(clientid)];
 
 cli(["list", "username"]) ->
-    [print_acl(Acl) || Acl <- all_acls(username)];
+    [print_acl(Acl) || Acl <- emqx_acl_mnesia_db:all_acls(username)];
 
 cli(["list", "_all"]) ->
-    [print_acl(Acl) || Acl <- all_acls(all)];
+    [print_acl(Acl) || Acl <- emqx_acl_mnesia_db:all_acls(all)];
 
 cli(["add", "clientid", Clientid, Topic, Action, Access]) ->
     case validate(action, Action) andalso validate(access, Access) of
         true ->
-            case add_acl(
+            case emqx_acl_mnesia_db:add_acl(
                    {clientid, iolist_to_binary(Clientid)},
                    iolist_to_binary(Topic),
                    list_to_existing_atom(Action),
@@ -135,7 +53,7 @@ cli(["add", "clientid", Clientid, Topic, Action, Access]) ->
 cli(["add", "username", Username, Topic, Action, Access]) ->
     case validate(action, Action) andalso validate(access, Access) of
         true ->
-            case add_acl(
+            case emqx_acl_mnesia_db:add_acl(
                    {username, iolist_to_binary(Username)},
                    iolist_to_binary(Topic),
                    list_to_existing_atom(Action),
@@ -151,7 +69,7 @@ cli(["add", "username", Username, Topic, Action, Access]) ->
 cli(["add", "_all", Topic, Action, Access]) ->
     case validate(action, Action) andalso validate(access, Access) of
         true ->
-            case add_acl(
+            case emqx_acl_mnesia_db:add_acl(
                    all,
                    iolist_to_binary(Topic),
                    list_to_existing_atom(Action),
@@ -165,16 +83,16 @@ cli(["add", "_all", Topic, Action, Access]) ->
     end;
 
 cli(["show", "clientid", Clientid]) ->
-    [print_acl(Acl) || Acl <- lookup_acl({clientid, iolist_to_binary(Clientid)})];
+    [print_acl(Acl) || Acl <- emqx_acl_mnesia_db:lookup_acl({clientid, iolist_to_binary(Clientid)})];
 
 cli(["show", "username", Username]) ->
-    [print_acl(Acl) || Acl <- lookup_acl({username, iolist_to_binary(Username)})];
+    [print_acl(Acl) || Acl <- emqx_acl_mnesia_db:lookup_acl({username, iolist_to_binary(Username)})];
 
 cli(["del", "clientid", Clientid, Topic])->
     cli(["delete", "clientid", Clientid, Topic]);
 
 cli(["delete", "clientid", Clientid, Topic])->
-    case remove_acl({clientid, iolist_to_binary(Clientid)}, iolist_to_binary(Topic)) of
+    case emqx_acl_mnesia_db:remove_acl({clientid, iolist_to_binary(Clientid)}, iolist_to_binary(Topic)) of
          ok -> emqx_ctl:print("ok~n");
         {error, Reason} -> emqx_ctl:print("Error: ~p~n", [Reason])
     end;
@@ -183,7 +101,7 @@ cli(["del", "username", Username, Topic])->
     cli(["delete", "username", Username, Topic]);
 
 cli(["delete", "username", Username, Topic])->
-    case remove_acl({username, iolist_to_binary(Username)}, iolist_to_binary(Topic)) of
+    case emqx_acl_mnesia_db:remove_acl({username, iolist_to_binary(Username)}, iolist_to_binary(Topic)) of
          ok -> emqx_ctl:print("ok~n");
         {error, Reason} -> emqx_ctl:print("Error: ~p~n", [Reason])
     end;
@@ -192,7 +110,7 @@ cli(["del", "_all", Topic])->
     cli(["delete", "_all", Topic]);
 
 cli(["delete", "_all", Topic])->
-    case remove_acl(all, iolist_to_binary(Topic)) of
+    case emqx_acl_mnesia_db:remove_acl(all, iolist_to_binary(Topic)) of
          ok -> emqx_ctl:print("ok~n");
         {error, Reason} -> emqx_ctl:print("Error: ~p~n", [Reason])
     end;
@@ -215,13 +133,6 @@ cli(_) ->
 %% Internal functions
 %%--------------------------------------------------------------------
 
-comparing({_, _, _, _, CreatedAt1},
-          {_, _, _, _, CreatedAt2}) ->
-    CreatedAt1 >= CreatedAt2.
-
-ret({atomic, ok})     -> ok;
-ret({aborted, Error}) -> {error, Error}.
-
 validate(action, "pub") -> true;
 validate(action, "sub") -> true;
 validate(action, "pubsub") -> true;
@@ -244,27 +155,3 @@ print_acl({all, Topic, Action, Access, _}) ->
         "Acl($all topic = ~p action = ~p access = ~p)~n",
         [Topic, Action, Access]
      ).
-
-update_permission(Action, Acl0, OldRecords) ->
-    Acl = Acl0 #?TABLE{action = Action},
-    maybe_delete_shadowed_records(Action, OldRecords),
-    mnesia:write(Acl).
-
-maybe_delete_shadowed_records(_, []) ->
-    ok;
-maybe_delete_shadowed_records(Action1, [Rec = #emqx_acl{action = Action2} | Rest]) ->
-    if Action1 =:= Action2 ->
-            ok = mnesia:delete_object(Rec);
-       Action2 =:= pubsub ->
-            %% Perform migration from the old data format on the
-            %% fly. This is needed only for the enterprise version,
-            %% delete this branch on 5.0
-            mnesia:delete_object(Rec),
-            mnesia:write(Rec#?TABLE{action = other_action(Action1)});
-       true ->
-            ok
-    end,
-    maybe_delete_shadowed_records(Action1, Rest).
-
-other_action(pub) -> sub;
-other_action(sub) -> pub.

apps/emqx_auth_mnesia/src/emqx_acl_mnesia_db.erl

@@ -0,0 +1,339 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_acl_mnesia_db).
+
+-include("emqx_auth_mnesia.hrl").
+-include_lib("stdlib/include/ms_transform.hrl").
+-include_lib("stdlib/include/qlc.hrl").
+
+%% ACL APIs
+-export([ create_table/0
+        , create_table2/0
+        ]).
+
+-export([ add_acl/4
+        , lookup_acl/1
+        , all_acls_export/0
+        , all_acls/0
+        , all_acls/1
+        , remove_acl/2
+        , merge_acl_records/3
+        , login_acl_table/1
+        , is_migration_started/0
+        ]).
+
+-export([comparing/2]).
+
+%%--------------------------------------------------------------------
+%% ACL API
+%%--------------------------------------------------------------------
+
+%% @doc Create table `emqx_acl` of old format rules
+-spec(create_table() -> ok).
+create_table() ->
+    ok = ekka_mnesia:create_table(?ACL_TABLE, [
+        {type, bag},
+        {disc_copies, [node()]},
+        {attributes, record_info(fields, ?ACL_TABLE)},
+        {storage_properties, [{ets, [{read_concurrency, true}]}]}]),
+    ok = ekka_mnesia:copy_table(?ACL_TABLE, disc_copies).
+
+%% @doc Create table `emqx_acl2` of new format rules
+-spec(create_table2() -> ok).
+create_table2() ->
+    ok = ekka_mnesia:create_table(?ACL_TABLE2, [
+            {type, ordered_set},
+            {disc_copies, [node()]},
+            {attributes, record_info(fields, ?ACL_TABLE2)},
+            {storage_properties, [{ets, [{read_concurrency, true}]}]}]),
+    ok = ekka_mnesia:copy_table(?ACL_TABLE2, disc_copies).
+
+%% @doc Add Acls
+-spec(add_acl(acl_target(), emqx_topic:topic(), legacy_action(), access()) ->
+        ok | {error, any()}).
+add_acl(Login, Topic, Action, Access) ->
+    ret(mnesia:transaction(fun() ->
+                              case is_migration_started() of
+                                  true -> add_acl_new(Login, Topic, Action, Access);
+                                  false -> add_acl_old(Login, Topic, Action, Access)
+                              end
+                           end)).
+
+%% @doc Lookup acl by login
+-spec(lookup_acl(acl_target()) -> list(acl_record())).
+lookup_acl(undefined) -> [];
+lookup_acl(Login) ->
+    % After migration to ?ACL_TABLE2, ?ACL_TABLE never has any rules. This lookup should be removed later.
+    MatchSpec = ets:fun2ms(fun(#?ACL_TABLE{filter = {Filter, _}} = Rec)
+                                 when Filter =:= Login -> Rec
+                           end),
+    OldRecs = ets:select(?ACL_TABLE, MatchSpec),
+
+    NewAcls = ets:lookup(?ACL_TABLE2, Login),
+    MergedAcl = merge_acl_records(Login, OldRecs, NewAcls),
+    lists:sort(fun comparing/2, acl_to_list(MergedAcl)).
+
+%% @doc Remove ACL
+-spec remove_acl(acl_target(), emqx_topic:topic()) -> ok | {error, any()}.
+remove_acl(Login, Topic) ->
+    ret(mnesia:transaction(fun() ->
+                              mnesia:delete({?ACL_TABLE, {Login, Topic}}),
+                              case mnesia:wread({?ACL_TABLE2, Login}) of
+                                  [] -> ok;
+                                  [#?ACL_TABLE2{rules = Rules} = Acl] ->
+                                      case delete_topic_rules(Topic, Rules) of
+                                          [] -> mnesia:delete({?ACL_TABLE2, Login});
+                                          [_ | _] = RemainingRules ->
+                                              mnesia:write(Acl#?ACL_TABLE2{rules = RemainingRules})
+                                      end
+                              end
+                           end)).
+
+%% @doc All ACL rules
+-spec(all_acls() -> list(acl_record())).
+all_acls() ->
+    all_acls(username) ++
+    all_acls(clientid) ++
+    all_acls(all).
+
+%% @doc All ACL rules of specified type
+-spec(all_acls(acl_target_type()) -> list(acl_record())).
+all_acls(AclTargetType) ->
+    lists:sort(fun comparing/2, qlc:eval(login_acl_table(AclTargetType))).
+
+%% @doc All ACL rules fetched transactionally
+-spec(all_acls_export() -> list(acl_record())).
+all_acls_export() ->
+    AclTargetTypes = [username, clientid, all],
+    MatchSpecNew = lists:flatmap(fun login_match_spec_new/1, AclTargetTypes),
+    MatchSpecOld = lists:flatmap(fun login_match_spec_old/1, AclTargetTypes),
+
+    {atomic, Records} = mnesia:transaction(
+        fun() ->
+            QH = acl_table(MatchSpecNew, MatchSpecOld, fun mnesia:table/2, fun lookup_mnesia/2),
+            qlc:eval(QH)
+        end),
+    Records.
+
+%% @doc QLC table of logins matching spec
+-spec(login_acl_table(acl_target_type()) -> qlc:query_handle()).
+login_acl_table(AclTargetType) ->
+    MatchSpecNew = login_match_spec_new(AclTargetType),
+    MatchSpecOld = login_match_spec_old(AclTargetType),
+    acl_table(MatchSpecNew, MatchSpecOld, fun ets:table/2, fun lookup_ets/2).
+
+%% @doc Combine old `emqx_acl` ACL records with a new `emqx_acl2` ACL record for a given login
+-spec(merge_acl_records(acl_target(), [#?ACL_TABLE{}], [#?ACL_TABLE2{}]) -> #?ACL_TABLE2{}).
+merge_acl_records(Login, OldRecs, Acls) ->
+    OldRules = old_recs_to_rules(OldRecs),
+    NewRules = case Acls of
+        [] -> [];
+        [#?ACL_TABLE2{rules = Rules}] -> Rules
+    end,
+    #?ACL_TABLE2{who = Login, rules = merge_rules(NewRules, OldRules)}.
+
+%% @doc Checks if background migration of ACL rules from `emqx_acl` to `emqx_acl2` format started.
+%% Should be run in transaction
+-spec(is_migration_started() -> boolean()).
+is_migration_started() ->
+    case mnesia:read({?ACL_TABLE, ?MIGRATION_MARK_KEY}) of
+        [?MIGRATION_MARK_RECORD | _] -> true;
+        [] -> false
+    end.
+
+%%--------------------------------------------------------------------
+%% Internal functions
+%%--------------------------------------------------------------------
+
+add_acl_new(Login, Topic, Action, Access) ->
+    Rule = {Access, Action, Topic, erlang:system_time(millisecond)},
+    Rules = normalize_rule(Rule),
+    OldAcl = mnesia:wread({?ACL_TABLE2, Login}),
+    NewAcl = case OldAcl of
+        [#?ACL_TABLE2{rules = OldRules} = Acl] ->
+            Acl#?ACL_TABLE2{rules = merge_rules(Rules, OldRules)};
+        [] ->
+            #?ACL_TABLE2{who = Login, rules = Rules}
+    end,
+    mnesia:write(NewAcl).
+
+add_acl_old(Login, Topic, Action, Access) ->
+    Filter = {Login, Topic},
+    Acl = #?ACL_TABLE{
+             filter = Filter,
+             action = Action,
+             access = Access,
+             created_at = erlang:system_time(millisecond)
+            },
+    OldRecords = mnesia:wread({?ACL_TABLE, Filter}),
+    case Action of
+        pubsub ->
+            update_permission(pub, Acl, OldRecords),
+            update_permission(sub, Acl, OldRecords);
+        _ ->
+            update_permission(Action, Acl, OldRecords)
+    end.
+
+old_recs_to_rules(OldRecs) ->
+    lists:flatmap(fun old_rec_to_rules/1, OldRecs).
+
+old_rec_to_rules(#?ACL_TABLE{filter = {_, Topic}, action = Action, access = Access, created_at = CreatedAt}) ->
+    normalize_rule({Access, Action, Topic, CreatedAt}).
+
+normalize_rule({Access, pubsub, Topic, CreatedAt}) ->
+    [{Access, pub, Topic, CreatedAt}, {Access, sub, Topic, CreatedAt}];
+normalize_rule({Access, Action, Topic, CreatedAt}) ->
+    [{Access, Action, Topic, CreatedAt}].
+
+merge_rules([], OldRules) -> OldRules;
+merge_rules([NewRule | RestNewRules], OldRules) ->
+    merge_rules(RestNewRules, merge_rule(NewRule, OldRules)).
+
+merge_rule({_, Action, Topic, _ } = NewRule, OldRules) ->
+    [NewRule | lists:filter(
+        fun({_, OldAction, OldTopic, _}) ->
+            {Action, Topic} =/= {OldAction, OldTopic}
+        end, OldRules)].
+
+acl_to_list(#?ACL_TABLE2{who = Login, rules = Rules}) ->
+    [{Login, Topic, Action, Access, CreatedAt} || {Access, Action, Topic, CreatedAt} <- Rules].
+
+delete_topic_rules(Topic, Rules) ->
+    [Rule || {_, _, T, _} = Rule <- Rules, T =/= Topic].
+
+comparing({_, _, _, _, CreatedAt} = Rec1,
+          {_, _, _, _, CreatedAt} = Rec2) ->
+        Rec1 >= Rec2;
+
+comparing({_, _, _, _, CreatedAt1},
+          {_, _, _, _, CreatedAt2}) ->
+    CreatedAt1 >= CreatedAt2.
+
+login_match_spec_old(all) ->
+    ets:fun2ms(fun(#?ACL_TABLE{filter = {all, _}} = Record) ->
+                Record
+               end);
+
+login_match_spec_old(Type) when (Type =:= username) or (Type =:= clientid) ->
+    ets:fun2ms(fun(#?ACL_TABLE{filter = {{RecordType, _}, _}} = Record)
+                    when RecordType =:= Type -> Record
+               end).
+
+login_match_spec_new(all) ->
+    ets:fun2ms(fun(#?ACL_TABLE2{who = all} = Record) ->
+                Record
+               end);
+
+login_match_spec_new(Type) when (Type =:= username) or (Type =:= clientid) ->
+    ets:fun2ms(fun(#?ACL_TABLE2{who = {RecordType, _}} = Record)
+                    when RecordType =:= Type -> Record
+               end).
+
+acl_table(MatchSpecNew, MatchSpecOld, TableFun, LookupFun) ->
+    TraverseFun =
+        fun() ->
+           CursorNew =
+               qlc:cursor(
+                   TableFun(?ACL_TABLE2, [{traverse, {select, MatchSpecNew}}])),
+           CursorOld =
+               qlc:cursor(
+                   TableFun(?ACL_TABLE, [{traverse, {select, MatchSpecOld}}])),
+           traverse_new(CursorNew, CursorOld, #{}, LookupFun)
+        end,
+
+    qlc:table(TraverseFun, []).
+
+
+% These are traverse funs for qlc table created by `acl_table/4`.
+% Traversing consumes memory: it collects logins present in `?ACL_TABLE` and
+% at the same time having rules in `?ACL_TABLE2`.
+% Such records appear if ACLs are inserted before migration started.
+% After migration, number of such logins is zero, so traversing starts working in
+% constant memory.
+
+traverse_new(CursorNew, CursorOld, FoundKeys, LookupFun) ->
+    Acls = qlc:next_answers(CursorNew, 1),
+    case Acls of
+        [] ->
+            qlc:delete_cursor(CursorNew),
+            traverse_old(CursorOld, FoundKeys);
+        [#?ACL_TABLE2{who = Login, rules = Rules} = Acl] ->
+            Keys = lists:usort([{Login, Topic} || {_, _, Topic, _} <- Rules]),
+            OldRecs = lists:flatmap(fun(Key) -> LookupFun(?ACL_TABLE, Key) end, Keys),
+            MergedAcl = merge_acl_records(Login, OldRecs, [Acl]),
+            NewFoundKeys =
+                lists:foldl(fun(#?ACL_TABLE{filter = Key}, Found) -> maps:put(Key, true, Found) end,
+                            FoundKeys,
+                            OldRecs),
+            case acl_to_list(MergedAcl) of
+                [] ->
+                    traverse_new(CursorNew, CursorOld, NewFoundKeys, LookupFun);
+                List ->
+                    List ++ fun() -> traverse_new(CursorNew, CursorOld, NewFoundKeys, LookupFun) end
+            end
+    end.
+
+traverse_old(CursorOld, FoundKeys) ->
+    OldAcls = qlc:next_answers(CursorOld),
+    case OldAcls of
+        [] ->
+            qlc:delete_cursor(CursorOld),
+            [];
+        _ ->
+            Records = [ {Login, Topic, Action, Access, CreatedAt}
+                || #?ACL_TABLE{filter = {Login, Topic}, action = LegacyAction, access = Access, created_at = CreatedAt} <- OldAcls,
+                {_, Action, _, _} <- normalize_rule({Access, LegacyAction, Topic, CreatedAt}),
+                not maps:is_key({Login, Topic}, FoundKeys)
+            ],
+            case Records of
+                [] -> traverse_old(CursorOld, FoundKeys);
+                List -> List ++ fun() -> traverse_old(CursorOld, FoundKeys) end
+            end
+    end.
+
+lookup_mnesia(Tab, Key) ->
+    mnesia:read({Tab, Key}).
+
+lookup_ets(Tab, Key) ->
+    ets:lookup(Tab, Key).
+
+update_permission(Action, Acl0, OldRecords) ->
+    Acl = Acl0 #?ACL_TABLE{action = Action},
+    maybe_delete_shadowed_records(Action, OldRecords),
+    mnesia:write(Acl).
+
+maybe_delete_shadowed_records(_, []) ->
+    ok;
+maybe_delete_shadowed_records(Action1, [Rec = #emqx_acl{action = Action2} | Rest]) ->
+    if Action1 =:= Action2 ->
+            ok = mnesia:delete_object(Rec);
+       Action2 =:= pubsub ->
+            %% Perform migration from the old data format on the
+            %% fly. This is needed only for the enterprise version,
+            %% delete this branch on 5.0
+            mnesia:delete_object(Rec),
+            mnesia:write(Rec#?ACL_TABLE{action = other_action(Action1)});
+       true ->
+            ok
+    end,
+    maybe_delete_shadowed_records(Action1, Rest).
+
+other_action(pub) -> sub;
+other_action(sub) -> pub.
+
+ret({atomic, ok})     -> ok;
+ret({aborted, Error}) -> {error, Error}.

apps/emqx_auth_mnesia/src/emqx_acl_mnesia_migrator.erl

@@ -0,0 +1,215 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_acl_mnesia_migrator).
+
+-include("emqx_auth_mnesia.hrl").
+-include_lib("emqx/include/logger.hrl").
+-include_lib("snabbkaffe/include/snabbkaffe.hrl").
+
+-behaviour(gen_statem).
+
+-define(CHECK_ALL_NODES_INTERVAL, 60000).
+
+-type(migration_delay_reason() :: old_nodes | bad_nodes).
+
+-export([
+    callback_mode/0,
+    init/1
+]).
+
+-export([
+    waiting_all_nodes/3,
+    checking_old_table/3,
+    migrating/3
+]).
+
+-export([
+    start_link/0,
+    start_link/1,
+    start_supervised/0,
+    stop_supervised/0,
+    migrate_records/0,
+    is_migrating_on_node/1,
+    is_old_table_migrated/0
+]).
+
+%%--------------------------------------------------------------------
+%% External interface
+%%--------------------------------------------------------------------
+
+start_link() ->
+    start_link(?MODULE).
+
+start_link(Name) when is_atom(Name) ->
+    start_link(#{
+        name => Name
+    });
+
+start_link(#{name := Name} = Opts) ->
+    gen_statem:start_link({local, Name}, ?MODULE, Opts, []).
+
+start_supervised() ->
+    try
+        {ok, _} = supervisor:restart_child(emqx_auth_mnesia_sup, ?MODULE),
+        ok
+    catch
+        exit:{noproc, _} -> ok
+    end.
+
+stop_supervised() ->
+    try
+        ok = supervisor:terminate_child(emqx_auth_mnesia_sup, ?MODULE),
+        ok = supervisor:delete_child(emqx_auth_mnesia_sup, ?MODULE)
+    catch
+        exit:{noproc, _} -> ok
+    end.
+
+%%--------------------------------------------------------------------
+%% gen_statem callbacks
+%%--------------------------------------------------------------------
+
+callback_mode() -> state_functions.
+
+init(Opts) ->
+    ok = emqx_acl_mnesia_db:create_table(),
+    ok = emqx_acl_mnesia_db:create_table2(),
+    Name = maps:get(name, Opts, ?MODULE),
+    CheckNodesInterval = maps:get(check_nodes_interval, Opts, ?CHECK_ALL_NODES_INTERVAL),
+    GetNodes = maps:get(get_nodes, Opts, fun all_nodes/0),
+    Data =
+        #{name => Name,
+          check_nodes_interval => CheckNodesInterval,
+          get_nodes => GetNodes},
+    {ok, waiting_all_nodes, Data, [{state_timeout, 0, check_nodes}]}.
+
+%%--------------------------------------------------------------------
+%% state callbacks
+%%--------------------------------------------------------------------
+
+waiting_all_nodes(state_timeout, check_nodes, Data) ->
+    #{name := Name, check_nodes_interval := CheckNodesInterval, get_nodes := GetNodes} = Data,
+    case is_all_nodes_migrating(Name, GetNodes()) of
+        true ->
+            ?tp(info, emqx_acl_mnesia_migrator_check_old_table, #{}),
+            {next_state, checking_old_table, Data, [{next_event, internal, check_old_table}]};
+        {false, Reason, Nodes} ->
+            ?tp(info,
+                emqx_acl_mnesia_migrator_bad_nodes_delay,
+                #{delay => CheckNodesInterval,
+                  reason => Reason,
+                  name => Name,
+                  nodes => Nodes}),
+            {keep_state_and_data, [{state_timeout, CheckNodesInterval, check_nodes}]}
+    end.
+
+checking_old_table(internal, check_old_table, Data) ->
+    case is_old_table_migrated() of
+        true ->
+            ?tp(info, emqx_acl_mnesia_migrator_finish, #{}),
+            {next_state, finished, Data, [{hibernate, true}]};
+        false ->
+            ?tp(info, emqx_acl_mnesia_migrator_start_migration, #{}),
+            {next_state, migrating, Data, [{next_event, internal, start_migration}]}
+    end.
+
+migrating(internal, start_migration, Data) ->
+    ok = migrate_records(),
+    {next_state, checking_old_table, Data, [{next_event, internal, check_old_table}]}.
+
+%% @doc Returns `true` if migration is started in the local node, otherwise crash.
+-spec(is_migrating_on_node(atom()) -> true).
+is_migrating_on_node(Name) ->
+    true = is_pid(erlang:whereis(Name)).
+
+%% @doc Run migration of records
+-spec(migrate_records() -> ok).
+migrate_records() ->
+    ok = add_migration_mark(),
+    Key = peek_record(),
+    do_migrate_records(Key).
+
+%% @doc Run migration of records
+-spec(is_all_nodes_migrating(atom(), list(node())) -> true | {false, migration_delay_reason(), list(node())}).
+is_all_nodes_migrating(Name, Nodes) ->
+    case rpc:multicall(Nodes, ?MODULE, is_migrating_on_node, [Name]) of
+        {Results, []} ->
+            OldNodes = [ Node || {Node, Result} <- lists:zip(Nodes, Results), Result =/= true ],
+            case OldNodes of
+                [] -> true;
+                _ -> {false, old_nodes, OldNodes}
+            end;
+        {_, [_BadNode | _] = BadNodes} ->
+            {false, bad_nodes, BadNodes}
+    end.
+
+%%--------------------------------------------------------------------
+%% Internal functions
+%%--------------------------------------------------------------------
+
+all_nodes() ->
+    ekka_mnesia:cluster_nodes(all).
+
+is_old_table_migrated() ->
+    Result =
+        mnesia:transaction(fun() ->
+                              case mnesia:first(?ACL_TABLE) of
+                                  ?MIGRATION_MARK_KEY ->
+                                      case mnesia:next(?ACL_TABLE, ?MIGRATION_MARK_KEY) of
+                                          '$end_of_table' -> true;
+                                          _OtherKey -> false
+                                      end;
+                                  '$end_of_table' -> false;
+                                  _OtherKey -> false
+                              end
+                           end),
+    case Result of
+        {atomic, true} ->
+            true;
+        _ ->
+            false
+    end.
+
+add_migration_mark() ->
+    {atomic, ok} = mnesia:transaction(fun() -> mnesia:write(?MIGRATION_MARK_RECORD) end),
+    ok.
+
+peek_record() ->
+    Key = mnesia:dirty_first(?ACL_TABLE),
+    case Key of
+        ?MIGRATION_MARK_KEY ->
+            mnesia:dirty_next(?ACL_TABLE, Key);
+        _ -> Key
+    end.
+
+do_migrate_records('$end_of_table') -> ok;
+do_migrate_records({_Login, _Topic} = Key) ->
+    ?tp(emqx_acl_mnesia_migrator_record_selected, #{key => Key}),
+    _ = mnesia:transaction(fun migrate_one_record/1, [Key]),
+    do_migrate_records(peek_record()).
+
+migrate_one_record({Login, _Topic} = Key) ->
+    case mnesia:wread({?ACL_TABLE, Key}) of
+        [] ->
+            ?tp(emqx_acl_mnesia_migrator_record_missed, #{key => Key}),
+            record_missing;
+        OldRecs ->
+            Acls = mnesia:wread({?ACL_TABLE2, Login}),
+            UpdatedAcl = emqx_acl_mnesia_db:merge_acl_records(Login, OldRecs, Acls),
+            ok = mnesia:write(UpdatedAcl),
+            ok = mnesia:delete({?ACL_TABLE, Key}),
+            ?tp(emqx_acl_mnesia_migrator_record_migrated, #{key => Key})
+    end.

apps/emqx_auth_mnesia/src/emqx_auth_mnesia.app.src

@@ -1,6 +1,6 @@
 {application, emqx_auth_mnesia,
  [{description, "EMQ X Authentication with Mnesia"},
-  {vsn, "4.3.0"}, % strict semver, bump manually
+  {vsn, "4.3.6"}, % strict semver, bump manually
   {modules, []},
   {registered, []},
   {applications, [kernel,stdlib,mnesia]},

apps/emqx_auth_mnesia/src/emqx_auth_mnesia.appup.src

@@ -0,0 +1,47 @@
+%% -*- mode: erlang -*-
+%% Unless you know what you are doing, DO NOT edit manually!!
+{VSN,
+  [{"4.3.5",
+    [{load_module,emqx_auth_mnesia_api,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_mnesia,brutal_purge,soft_purge,[]}]},
+   {<<"4\\.3\\.[0-3]">>,
+    [{load_module,emqx_auth_mnesia_cli,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_mnesia,brutal_purge,soft_purge,[]},
+     {add_module,emqx_acl_mnesia_db},
+     {add_module,emqx_acl_mnesia_migrator,[emqx_acl_mnesia_db]},
+     {update,emqx_auth_mnesia_sup,supervisor},
+     {apply,{emqx_acl_mnesia_migrator,start_supervised,[]}},
+     {load_module,emqx_auth_mnesia_api,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_mnesia,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_mnesia_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_mnesia_api,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_mnesia_cli,brutal_purge,soft_purge,[]}]},
+   {"4.3.4",
+    [{load_module,emqx_auth_mnesia_api,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_mnesia,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_mnesia_cli,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_mnesia,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_mnesia_app,brutal_purge,soft_purge,[]}]},
+   {<<".*">>,[]}],
+  [{"4.3.5",
+    [{load_module,emqx_auth_mnesia_api,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_mnesia,brutal_purge,soft_purge,[]}]},
+   {<<"4\\.3\\.[0-3]">>,
+    [{load_module,emqx_auth_mnesia_cli,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_mnesia,brutal_purge,soft_purge,[]},
+     {apply,{emqx_acl_mnesia_migrator,stop_supervised,[]}},
+     {update,emqx_auth_mnesia_sup,supervisor},
+     {load_module,emqx_acl_mnesia_cli,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_mnesia_api,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_mnesia_api,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_mnesia,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_mnesia_app,brutal_purge,soft_purge,[]},
+     {delete_module,emqx_acl_mnesia_migrator},
+     {delete_module,emqx_acl_mnesia_db}]},
+   {"4.3.4",
+    [{load_module,emqx_auth_mnesia_api,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_mnesia,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_mnesia_cli,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_mnesia,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_mnesia_app,brutal_purge,soft_purge,[]}]},
+   {<<".*">>,[]}]}.

apps/emqx_auth_mnesia/src/emqx_auth_mnesia.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -32,25 +32,32 @@
         , description/0
         ]).
 
+-export([ match_password/3
+        , hash_type/0
+        ]).
+
 init(#{clientid_list := ClientidList, username_list := UsernameList}) ->
     ok = ekka_mnesia:create_table(?TABLE, [
             {disc_copies, [node()]},
             {attributes, record_info(fields, emqx_user)},
             {storage_properties, [{ets, [{read_concurrency, true}]}]}]),
-    _ = [ add_default_user({{clientid, iolist_to_binary(Clientid)}, iolist_to_binary(Password)})
-      || {Clientid, Password} <- ClientidList],
-    _ = [ add_default_user({{username, iolist_to_binary(Username)}, iolist_to_binary(Password)})
-      || {Username, Password} <- UsernameList],
-    ok = ekka_mnesia:copy_table(?TABLE, disc_copies).
+    lists:foreach(fun({Clientid, Password}) ->
+        emqx_auth_mnesia_cli:add_default_user(clientid, iolist_to_binary(Clientid), iolist_to_binary(Password))
+    end, ClientidList),
 
-%% @private
-add_default_user({Login, Password}) when is_tuple(Login) ->
-    emqx_auth_mnesia_cli:add_user(Login, Password).
+    lists:foreach(fun({Username, Password}) ->
+        emqx_auth_mnesia_cli:add_default_user(username, iolist_to_binary(Username), iolist_to_binary(Password))
+    end, UsernameList),
+
+    ok = ekka_mnesia:copy_table(?TABLE, disc_copies).
 
 -spec(register_metrics() -> ok).
 register_metrics() ->
     lists:foreach(fun emqx_metrics:ensure/1, ?AUTH_METRICS).
 
+hash_type() ->
+    application:get_env(emqx_auth_mnesia, password_hash, sha256).
+
 check(ClientInfo = #{ clientid := Clientid
                     , password := NPassword
                     }, AuthResult, #{hash_type := HashType}) ->
@@ -65,7 +72,8 @@ check(ClientInfo = #{ clientid := Clientid
         List ->
             case match_password(NPassword, HashType, List)  of
                 false ->
-                    ?LOG(error, "[Mnesia] Auth from mnesia failed: ~p", [ClientInfo]),
+                    Info = maps:without([password], ClientInfo),
+                    ?LOG(info, "[Mnesia] Auth from mnesia failed: ~p", [Info]),
                     emqx_metrics:inc(?AUTH_METRICS(failure)),
                     {stop, AuthResult#{anonymous => false, auth_result => password_error}};
                 _ ->

apps/emqx_auth_mnesia/src/emqx_auth_mnesia_api.erl

@@ -23,7 +23,7 @@
 
 -import(proplists, [get_value/2]).
 -import(minirest,  [return/1]).
--export([paginate/5]).
+-export([paginate_qh/5]).
 
 -export([ list_clientid/2
         , lookup_clientid/2
@@ -133,15 +133,15 @@ add_clientid(_Bindings, Params) ->
     end.
 
 do_add_clientid([ Params | ParamsN ], ReList ) ->
-    Clientid = urldecode(get_value(<<"clientid">>, Params)),
+    Clientid = get_value(<<"clientid">>, Params),
     do_add_clientid(ParamsN, [{Clientid, format_msg(do_add_clientid(Params))} | ReList]);
 
 do_add_clientid([], ReList) ->
     {ok, ReList}.
 
 do_add_clientid(Params) ->
-    Clientid = urldecode(get_value(<<"clientid">>, Params)),
-    Password = urldecode(get_value(<<"password">>, Params)),
+    Clientid = get_value(<<"clientid">>, Params),
+    Password = get_value(<<"password">>, Params),
     Login = {clientid, Clientid},
     case validate([login, password], [Login, Password]) of
         ok ->
@@ -152,7 +152,7 @@ do_add_clientid(Params) ->
 update_clientid(#{clientid := Clientid}, Params) ->
     Password = get_value(<<"password">>, Params),
     case validate([password], [Password]) of
-        ok -> return(emqx_auth_mnesia_cli:update_user({clientid, urldecode(Clientid)}, urldecode(Password)));
+        ok -> return(emqx_auth_mnesia_cli:update_user({clientid, urldecode(Clientid)}, Password));
         Err -> return(Err)
     end.
 
@@ -182,15 +182,15 @@ add_username(_Bindings, Params) ->
     end.
 
 do_add_username([ Params | ParamsN ], ReList ) ->
-    Username = urldecode(get_value(<<"username">>, Params)),
+    Username = get_value(<<"username">>, Params),
     do_add_username(ParamsN, [{Username, format_msg(do_add_username(Params))} | ReList]);
 
 do_add_username([], ReList) ->
     {ok, ReList}.
 
 do_add_username(Params) ->
-    Username = urldecode(get_value(<<"username">>, Params)),
-    Password = urldecode(get_value(<<"password">>, Params)),
+    Username = get_value(<<"username">>, Params),
+    Password = get_value(<<"password">>, Params),
     Login = {username, Username},
     case validate([login, password], [Login, Password]) of
         ok ->
@@ -201,7 +201,7 @@ do_add_username(Params) ->
 update_username(#{username := Username}, Params) ->
     Password = get_value(<<"password">>, Params),
     case validate([password], [Password]) of
-        ok -> return(emqx_auth_mnesia_cli:update_user({username, urldecode(Username)}, urldecode(Password)));
+        ok -> return(emqx_auth_mnesia_cli:update_user({username, urldecode(Username)}, Password));
         Err -> return(Err)
     end.
 
@@ -212,9 +212,12 @@ delete_username(#{username := Username}, _) ->
 %% Paging Query
 %%------------------------------------------------------------------------------
 
-paginate(Tables, MatchSpec, Params, ComparingFun, RowFun) ->
-    Qh = query_handle(Tables, MatchSpec),
-    Count = count(Tables, MatchSpec),
+paginate(Table, MatchSpec, Params, ComparingFun, RowFun) ->
+    Qh = query_handle(Table, MatchSpec),
+    Count = count(Table, MatchSpec),
+    paginate_qh(Qh, Count, Params, ComparingFun, RowFun).
+
+paginate_qh(Qh, Count, Params, ComparingFun, RowFun) ->
     Page = page(Params),
     Limit = limit(Params),
     Cursor = qlc:cursor(Qh),
@@ -231,24 +234,12 @@ paginate(Tables, MatchSpec, Params, ComparingFun, RowFun) ->
 
 query_handle(Table, MatchSpec) when is_atom(Table) ->
     Options = {traverse, {select, MatchSpec}},
-    qlc:q([R|| R <- ets:table(Table, Options)]);
-query_handle([Table], MatchSpec) when is_atom(Table) ->
-    Options = {traverse, {select, MatchSpec}},
-    qlc:q([R|| R <- ets:table(Table, Options)]);
-query_handle(Tables, MatchSpec) ->
-    Options = {traverse, {select, MatchSpec}},
-    qlc:append([qlc:q([E || E <- ets:table(T, Options)]) || T <- Tables]).
+    qlc:q([R || R <- ets:table(Table, Options)]).
 
 count(Table, MatchSpec) when is_atom(Table) ->
     [{MatchPattern, Where, _Re}] = MatchSpec,
     NMatchSpec = [{MatchPattern, Where, [true]}],
-    ets:select_count(Table, NMatchSpec);
-count([Table], MatchSpec) when is_atom(Table) ->
-    [{MatchPattern, Where, _Re}] = MatchSpec,
-    NMatchSpec = [{MatchPattern, Where, [true]}],
-    ets:select_count(Table, NMatchSpec);
-count(Tables, MatchSpec) ->
-    lists:sum([count(T, MatchSpec) || T <- Tables]).
+    ets:select_count(Table, NMatchSpec).
 
 page(Params) ->
     binary_to_integer(proplists:get_value(<<"_page">>, Params, <<"1">>)).
@@ -263,13 +254,11 @@ limit(Params) ->
 %% Interval Funcs
 %%------------------------------------------------------------------------------
 
-format([{?TABLE, {clientid, ClientId}, Password, _InterTime}]) ->
-    #{clientid => ClientId,
-      password => Password};
+format([{?TABLE, {clientid, ClientId}, _Password, _InterTime}]) ->
+    #{clientid => ClientId};
 
-format([{?TABLE, {username, Username}, Password, _InterTime}]) ->
-    #{username => Username,
-      password => Password};
+format([{?TABLE, {username, Username}, _Password, _InterTime}]) ->
+    #{username => Username};
 
 format([]) ->
     #{}.

apps/emqx_auth_mnesia/src/emqx_auth_mnesia_app.erl

@@ -57,12 +57,9 @@ load_auth_hook() ->
     UsernameList = application:get_env(?APP, username_list, []),
     ok = emqx_auth_mnesia:init(#{clientid_list => ClientidList, username_list => UsernameList}),
     ok = emqx_auth_mnesia:register_metrics(),
-    Params = #{
-            hash_type => application:get_env(emqx_auth_mnesia, password_hash, sha256)
-            },
+    Params = #{hash_type => emqx_auth_mnesia:hash_type()},
     emqx:hook('client.authenticate', fun emqx_auth_mnesia:check/3, [Params]).
 
 load_acl_hook() ->
     ok = emqx_acl_mnesia:init(),
-    ok = emqx_acl_mnesia:register_metrics(),
     emqx:hook('client.check_acl', fun emqx_acl_mnesia:check_acl/5, [#{}]).

apps/emqx_auth_mnesia/src/emqx_auth_mnesia_cli.erl

@@ -22,6 +22,8 @@
 -define(TABLE, emqx_user).
 %% Auth APIs
 -export([ add_user/2
+        , force_add_user/2
+        , add_default_user/3
         , update_user/2
         , remove_user/1
         , lookup_user/1
@@ -56,6 +58,65 @@ insert_user(User = #emqx_user{login = Login}) ->
         [_|_] -> mnesia:abort(existed)
     end.
 
+-spec(add_default_user(clientid | username, tuple(), binary()) -> ok | {error, any()}).
+add_default_user(Type, Key, Password) ->
+    Login = {Type, Key},
+    case add_user(Login, Password) of
+        ok -> ok;
+        {error, existed} ->
+            NewPwd = encrypted_data(Password),
+            [#emqx_user{password = OldPwd}] = emqx_auth_mnesia_cli:lookup_user(Login),
+            HashType = emqx_auth_mnesia:hash_type(),
+            case emqx_auth_mnesia:match_password(NewPwd, HashType, [OldPwd])  of
+                true -> ok;
+                false ->
+                    %% We can't force add default,
+                    %% otherwise passwords that have been updated via HTTP API will be reset after reboot.
+                    TypeCtl =
+                        case Type of
+                            clientid -> clientid;
+                            username -> user
+                        end,
+                    ?LOG(warning,
+                        "[Auth Mnesia] auth.client.x.~p=~s password in the emqx_auth_mnesia.conf\n"
+                        "does not match the password in the database(mnesia).\n"
+                        "1. If you have already changed the password via the HTTP API, this warning has no effect.\n"
+                        "You can remove the `auth.client.x.~p=~s` from emqx_auth_mnesia.conf to resolve this warning.\n"
+                        "2. If you just want to update the password by manually changing the configuration file,\n"
+                        "you need to delete the old user and password using `emqx_ctl ~p delete ~s` first\n"
+                        "the new password in emqx_auth_mnesia.conf can take effect after reboot.",
+                        [Type, Key, Type, Key, TypeCtl, Key]),
+                    ok
+            end;
+        Error -> Error
+    end.
+
+force_add_user(Login, Password) ->
+    User = #emqx_user{
+        login = Login,
+        password = encrypted_data(Password),
+        created_at = erlang:system_time(millisecond)
+    },
+    case ret(mnesia:transaction(fun insert_or_update_user/2, [Password, User])) of
+        {ok, override} ->
+            ?LOG(warning, "[Mnesia] (~p)'s password has be updated.", [Login]),
+            ok;
+        Other -> Other
+    end.
+
+insert_or_update_user(NewPwd, User = #emqx_user{login = Login}) ->
+    case mnesia:read(?TABLE, Login) of
+        []    -> mnesia:write(User);
+        [#emqx_user{password = Pwd}] ->
+            case emqx_auth_mnesia:match_password(NewPwd, emqx_auth_mnesia:hash_type(), [Pwd])  of
+                true -> ok;
+                false ->
+                    ok = mnesia:write(User),
+                    {ok, override}
+            end
+    end.
+
+
 %% @doc Update User
 -spec(update_user(tuple(), binary()) -> ok | {error, any()}).
 update_user(Login, NewPassword) ->
@@ -105,11 +166,11 @@ comparing({?TABLE, _, _, CreatedAt1},
           {?TABLE, _, _, CreatedAt2}) ->
     CreatedAt1 >= CreatedAt2.
 
-ret({atomic, ok})     -> ok;
+ret({atomic, Res}) -> Res;
 ret({aborted, Error}) -> {error, Error}.
 
 encrypted_data(Password) ->
-    HashType = application:get_env(emqx_auth_mnesia, password_hash, sha256),
+    HashType = emqx_auth_mnesia:hash_type(),
     SaltBin = salt(),
     <<SaltBin/binary, (hash(Password, SaltBin, HashType))/binary>>.
 

apps/emqx_auth_mnesia/src/emqx_auth_mnesia_sup.erl

@@ -33,4 +33,16 @@ start_link() ->
 %%--------------------------------------------------------------------
 
 init([]) ->
-    {ok, {{one_for_one, 10, 100}, []}}.
+    {ok, {{one_for_one, 10, 100}, [
+        child_spec(emqx_acl_mnesia_migrator, worker, [])
+    ]}}.
+
+child_spec(M, worker, Args) ->
+    #{id       => M,
+      start    => {M, start_link, Args},
+      restart  => permanent,
+      shutdown => 5000,
+      type     => worker,
+      modules  => [M]
+     }.
+

apps/emqx_auth_mnesia/test/emqx_acl_mnesia_SUITE.erl

@@ -22,6 +22,7 @@
 -include("emqx_auth_mnesia.hrl").
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("common_test/include/ct.hrl").
+-include_lib("snabbkaffe/include/snabbkaffe.hrl").
 
 -import(emqx_ct_http, [ request_api/3
                       , request_api/5
@@ -39,10 +40,17 @@ all() ->
     emqx_ct:all(?MODULE).
 
 groups() ->
-    [].
+    [{async_migration_tests, [sequence], [
+        t_old_and_new_acl_migration_by_migrator,
+        t_old_and_new_acl_migration_repeated_by_migrator,
+        t_migration_concurrency
+    ]}].
 
 init_per_suite(Config) ->
-    emqx_ct_helpers:start_apps([emqx_modules, emqx_management, emqx_auth_mnesia], fun set_special_configs/1),
+    emqx_ct_helpers:start_apps( [emqx_modules, emqx_management, emqx_auth_mnesia]
+                              , fun set_special_configs/1
+                              ),
+    supervisor:terminate_child(emqx_auth_mnesia_sup, emqx_acl_mnesia_migrator),
     create_default_app(),
     Config.
 
@@ -50,14 +58,32 @@ end_per_suite(_Config) ->
     delete_default_app(),
     emqx_ct_helpers:stop_apps([emqx_modules, emqx_management, emqx_auth_mnesia]).
 
-init_per_testcase(t_check_acl_as_clientid, Config) ->
+init_per_testcase_clean(_, Config) ->
+    mnesia:clear_table(?ACL_TABLE),
+    mnesia:clear_table(?ACL_TABLE2),
+    Config.
+
+init_per_testcase_emqx_hook(t_check_acl_as_clientid, Config) ->
     emqx:hook('client.check_acl', fun emqx_acl_mnesia:check_acl/5, [#{key_as => clientid}]),
     Config;
-
-init_per_testcase(_, Config) ->
+init_per_testcase_emqx_hook(_, Config) ->
     emqx:hook('client.check_acl', fun emqx_acl_mnesia:check_acl/5, [#{key_as => username}]),
     Config.
 
+init_per_testcase_migration(t_management_before_migration, Config) ->
+    Config;
+init_per_testcase_migration(_, Config) ->
+    emqx_acl_mnesia_migrator:migrate_records(),
+    Config.
+
+init_per_testcase(Case, Config) ->
+    PerTestInitializers = [
+        fun init_per_testcase_clean/2,
+        fun init_per_testcase_migration/2,
+        fun init_per_testcase_emqx_hook/2
+    ],
+    lists:foldl(fun(Init, Conf) -> Init(Case, Conf) end, Config, PerTestInitializers).
+
 end_per_testcase(_, Config) ->
     emqx:unhook('client.check_acl', fun emqx_acl_mnesia:check_acl/5),
     Config.
@@ -76,25 +102,34 @@ set_special_configs(_App) ->
 %% Testcases
 %%------------------------------------------------------------------------------
 
-t_management(_Config) ->
-    clean_all_acls(),
-    ?assertEqual("Acl with Mnesia", emqx_acl_mnesia:description()),
-    ?assertEqual([], emqx_acl_mnesia_cli:all_acls()),
+t_management_before_migration(_Config) ->
+    {atomic, IsStarted} = mnesia:transaction(fun emqx_acl_mnesia_db:is_migration_started/0),
+    ?assertNot(IsStarted),
+    run_acl_tests().
 
-    ok = emqx_acl_mnesia_cli:add_acl({clientid, <<"test_clientid">>}, <<"topic/%c">>, sub, allow),
-    ok = emqx_acl_mnesia_cli:add_acl({clientid, <<"test_clientid">>}, <<"topic/+">>, pub, deny),
-    ok = emqx_acl_mnesia_cli:add_acl({username, <<"test_username">>}, <<"topic/%u">>, sub, deny),
-    ok = emqx_acl_mnesia_cli:add_acl({username, <<"test_username">>}, <<"topic/+">>, pub, allow),
-    ok = emqx_acl_mnesia_cli:add_acl(all, <<"#">>, pubsub, deny),
+t_management_after_migration(_Config) ->
+    {atomic, IsStarted} = mnesia:transaction(fun emqx_acl_mnesia_db:is_migration_started/0),
+    ?assert(IsStarted),
+    run_acl_tests().
+
+run_acl_tests() ->
+    ?assertEqual("Acl with Mnesia", emqx_acl_mnesia:description()),
+    ?assertEqual([], emqx_acl_mnesia_db:all_acls()),
+
+    ok = emqx_acl_mnesia_db:add_acl({clientid, <<"test_clientid">>}, <<"topic/%c">>, sub, allow),
+    ok = emqx_acl_mnesia_db:add_acl({clientid, <<"test_clientid">>}, <<"topic/+">>, pub, deny),
+    ok = emqx_acl_mnesia_db:add_acl({username, <<"test_username">>}, <<"topic/%u">>, sub, deny),
+    ok = emqx_acl_mnesia_db:add_acl({username, <<"test_username">>}, <<"topic/+">>, pub, allow),
+    ok = emqx_acl_mnesia_db:add_acl(all, <<"#">>, pubsub, deny),
     %% Sleeps below are needed to hide the race condition between
     %% mnesia and ets dirty select in check_acl, that make this test
     %% flaky
     timer:sleep(100),
 
-    ?assertEqual(2, length(emqx_acl_mnesia_cli:lookup_acl({clientid, <<"test_clientid">>}))),
-    ?assertEqual(2, length(emqx_acl_mnesia_cli:lookup_acl({username, <<"test_username">>}))),
-    ?assertEqual(2, length(emqx_acl_mnesia_cli:lookup_acl(all))),
-    ?assertEqual(6, length(emqx_acl_mnesia_cli:all_acls())),
+    ?assertEqual(2, length(emqx_acl_mnesia_db:lookup_acl({clientid, <<"test_clientid">>}))),
+    ?assertEqual(2, length(emqx_acl_mnesia_db:lookup_acl({username, <<"test_username">>}))),
+    ?assertEqual(2, length(emqx_acl_mnesia_db:lookup_acl(all))),
+    ?assertEqual(6, length(emqx_acl_mnesia_db:all_acls())),
 
     User1 = #{zone => external, clientid => <<"test_clientid">>},
     User2 = #{zone => external, clientid => <<"no_exist">>, username => <<"test_username">>},
@@ -110,30 +145,32 @@ t_management(_Config) ->
     deny  = emqx_access_control:check_acl(User3, publish,   <<"topic/A/B">>),
 
     %% Test merging of pubsub capability:
-    ok = emqx_acl_mnesia_cli:add_acl({clientid, <<"test_clientid">>}, <<"topic/mix">>, pubsub, deny),
+    ok = emqx_acl_mnesia_db:add_acl({clientid, <<"test_clientid">>}, <<"topic/mix">>, pubsub, deny),
     timer:sleep(100),
     deny  = emqx_access_control:check_acl(User1, subscribe,   <<"topic/mix">>),
     deny  = emqx_access_control:check_acl(User1, publish,     <<"topic/mix">>),
-    ok = emqx_acl_mnesia_cli:add_acl({clientid, <<"test_clientid">>}, <<"topic/mix">>, pub, allow),
+    ok = emqx_acl_mnesia_db:add_acl({clientid, <<"test_clientid">>}, <<"topic/mix">>, pub, allow),
     timer:sleep(100),
     deny  = emqx_access_control:check_acl(User1, subscribe,   <<"topic/mix">>),
     allow = emqx_access_control:check_acl(User1, publish,     <<"topic/mix">>),
-    ok = emqx_acl_mnesia_cli:add_acl({clientid, <<"test_clientid">>}, <<"topic/mix">>, pubsub, allow),
+    ok = emqx_acl_mnesia_db:add_acl( {clientid, <<"test_clientid">>}
+                                   , <<"topic/mix">>, pubsub, allow
+                                   ),
     timer:sleep(100),
     allow = emqx_access_control:check_acl(User1, subscribe,   <<"topic/mix">>),
     allow = emqx_access_control:check_acl(User1, publish,     <<"topic/mix">>),
-    ok = emqx_acl_mnesia_cli:add_acl({clientid, <<"test_clientid">>}, <<"topic/mix">>, sub, deny),
+    ok = emqx_acl_mnesia_db:add_acl({clientid, <<"test_clientid">>}, <<"topic/mix">>, sub, deny),
     timer:sleep(100),
     deny  = emqx_access_control:check_acl(User1, subscribe,   <<"topic/mix">>),
     allow = emqx_access_control:check_acl(User1, publish,     <<"topic/mix">>),
-    ok = emqx_acl_mnesia_cli:add_acl({clientid, <<"test_clientid">>}, <<"topic/mix">>, pub, deny),
+    ok = emqx_acl_mnesia_db:add_acl({clientid, <<"test_clientid">>}, <<"topic/mix">>, pub, deny),
     timer:sleep(100),
     deny  = emqx_access_control:check_acl(User1, subscribe,   <<"topic/mix">>),
     deny  = emqx_access_control:check_acl(User1, publish,     <<"topic/mix">>),
 
     %% Test implicit migration of pubsub to pub and sub:
-    ok = emqx_acl_mnesia_cli:remove_acl({clientid, <<"test_clientid">>}, <<"topic/mix">>),
-    ok = mnesia:dirty_write(#emqx_acl{
+    ok = emqx_acl_mnesia_db:remove_acl({clientid, <<"test_clientid">>}, <<"topic/mix">>),
+    ok = mnesia:dirty_write(#?ACL_TABLE{
                                filter = {{clientid, <<"test_clientid">>}, <<"topic/mix">>},
                                action = pubsub,
                                access = allow,
@@ -142,34 +179,138 @@ t_management(_Config) ->
     timer:sleep(100),
     allow = emqx_access_control:check_acl(User1, subscribe,   <<"topic/mix">>),
     allow = emqx_access_control:check_acl(User1, publish,     <<"topic/mix">>),
-    ok = emqx_acl_mnesia_cli:add_acl({clientid, <<"test_clientid">>}, <<"topic/mix">>, pub, deny),
+    ok = emqx_acl_mnesia_db:add_acl({clientid, <<"test_clientid">>}, <<"topic/mix">>, pub, deny),
     timer:sleep(100),
     allow = emqx_access_control:check_acl(User1, subscribe,   <<"topic/mix">>),
     deny  = emqx_access_control:check_acl(User1, publish,     <<"topic/mix">>),
-    ok = emqx_acl_mnesia_cli:add_acl({clientid, <<"test_clientid">>}, <<"topic/mix">>, sub, deny),
+    ok = emqx_acl_mnesia_db:add_acl({clientid, <<"test_clientid">>}, <<"topic/mix">>, sub, deny),
     timer:sleep(100),
     deny  = emqx_access_control:check_acl(User1, subscribe,   <<"topic/mix">>),
     deny  = emqx_access_control:check_acl(User1, publish,     <<"topic/mix">>),
 
-    ok = emqx_acl_mnesia_cli:remove_acl({clientid, <<"test_clientid">>}, <<"topic/%c">>),
-    ok = emqx_acl_mnesia_cli:remove_acl({clientid, <<"test_clientid">>}, <<"topic/+">>),
-    ok = emqx_acl_mnesia_cli:remove_acl({clientid, <<"test_clientid">>}, <<"topic/mix">>),
-    ok = emqx_acl_mnesia_cli:remove_acl({username, <<"test_username">>}, <<"topic/%u">>),
-    ok = emqx_acl_mnesia_cli:remove_acl({username, <<"test_username">>}, <<"topic/+">>),
-    ok = emqx_acl_mnesia_cli:remove_acl(all, <<"#">>),
+    ok = emqx_acl_mnesia_db:remove_acl({clientid, <<"test_clientid">>}, <<"topic/%c">>),
+    ok = emqx_acl_mnesia_db:remove_acl({clientid, <<"test_clientid">>}, <<"topic/+">>),
+    ok = emqx_acl_mnesia_db:remove_acl({clientid, <<"test_clientid">>}, <<"topic/mix">>),
+    ok = emqx_acl_mnesia_db:remove_acl({username, <<"test_username">>}, <<"topic/%u">>),
+    ok = emqx_acl_mnesia_db:remove_acl({username, <<"test_username">>}, <<"topic/+">>),
+    ok = emqx_acl_mnesia_db:remove_acl(all, <<"#">>),
     timer:sleep(100),
 
-    ?assertEqual([], emqx_acl_mnesia_cli:all_acls()).
+    ?assertEqual([], emqx_acl_mnesia_db:all_acls()).
+
+t_old_and_new_acl_combination(_Config) ->
+    create_conflicting_records(),
+
+    ?assertEqual(combined_conflicting_records(), emqx_acl_mnesia_db:all_acls()),
+    ?assertEqual(
+        lists:usort(combined_conflicting_records()),
+        lists:usort(emqx_acl_mnesia_db:all_acls_export())).
+
+t_old_and_new_acl_migration(_Config) ->
+    create_conflicting_records(),
+    emqx_acl_mnesia_migrator:migrate_records(),
+
+    ?assertEqual(combined_conflicting_records(), emqx_acl_mnesia_db:all_acls()),
+    ?assertEqual(
+        lists:usort(combined_conflicting_records()),
+        lists:usort(emqx_acl_mnesia_db:all_acls_export())),
+
+    % check that old table is not popoulated anymore
+    ok = emqx_acl_mnesia_db:add_acl({clientid, <<"test_clientid">>}, <<"topic/%c">>, sub, allow),
+    ?assert(emqx_acl_mnesia_migrator:is_old_table_migrated()).
+
+
+t_migration_concurrency(_Config) ->
+    Key = {{clientid,<<"client6">>}, <<"t">>},
+    Record = #?ACL_TABLE{filter = Key, action = pubsub, access = deny, created_at = 0},
+    {atomic, ok} = mnesia:transaction(fun mnesia:write/1, [Record]),
+
+    LockWaitAndDelete =
+        fun() ->
+           [_Rec] = mnesia:wread({?ACL_TABLE, Key}),
+           {{Pid, Ref}, _} =
+               ?wait_async_action(spawn_monitor(fun emqx_acl_mnesia_migrator:migrate_records/0),
+                                  #{?snk_kind := emqx_acl_mnesia_migrator_record_selected},
+                                  1000),
+           mnesia:delete({?ACL_TABLE, Key}),
+           {Pid, Ref}
+        end,
+
+    ?check_trace(
+        begin
+            {atomic, {Pid, Ref}} = mnesia:transaction(LockWaitAndDelete),
+            receive {'DOWN', Ref, process, Pid, _} -> ok end
+        end,
+        fun(_, Trace) ->
+            ?assertMatch([_], ?of_kind(emqx_acl_mnesia_migrator_record_missed, Trace))
+        end),
+
+    ?assert(emqx_acl_mnesia_migrator:is_old_table_migrated()),
+    ?assertEqual([], emqx_acl_mnesia_db:all_acls()).
+
+
+t_old_and_new_acl_migration_by_migrator(_Config) ->
+    create_conflicting_records(),
+
+    meck:new(fake_nodes, [non_strict]),
+    meck:expect(fake_nodes, all, fun() -> [node(), 'somebadnode@127.0.0.1'] end),
+
+    ?check_trace(
+        begin
+            % check all nodes every 30 ms
+            {ok, _} = emqx_acl_mnesia_migrator:start_link(#{
+                name => ct_migrator,
+                check_nodes_interval => 30,
+                get_nodes => fun fake_nodes:all/0
+            }),
+            timer:sleep(100)
+        end,
+        fun(_, Trace) ->
+            ?assertEqual([], ?of_kind(emqx_acl_mnesia_migrator_start_migration, Trace))
+        end),
+
+    ?check_trace(
+        begin
+            meck:expect(fake_nodes, all, fun() -> [node()] end),
+            timer:sleep(100)
+        end,
+        fun(_, Trace) ->
+            ?assertMatch([_], ?of_kind(emqx_acl_mnesia_migrator_finish, Trace))
+        end),
+
+    meck:unload(fake_nodes),
+
+    ?assertEqual(combined_conflicting_records(), emqx_acl_mnesia_db:all_acls()),
+    ?assert(emqx_acl_mnesia_migrator:is_old_table_migrated()).
+
+t_old_and_new_acl_migration_repeated_by_migrator(_Config) ->
+    create_conflicting_records(),
+    emqx_acl_mnesia_migrator:migrate_records(),
+
+    ?check_trace(
+        begin
+            {ok, _} = emqx_acl_mnesia_migrator:start_link(ct_migrator),
+            timer:sleep(100)
+        end,
+        fun(_, Trace) ->
+            ?assertEqual([], ?of_kind(emqx_acl_mnesia_migrator_start_migration, Trace)),
+            ?assertMatch([_], ?of_kind(emqx_acl_mnesia_migrator_finish, Trace))
+        end).
+
+t_start_stop_supervised(_Config) ->
+    ?assertEqual(undefined, whereis(emqx_acl_mnesia_migrator)),
+    ok = emqx_acl_mnesia_migrator:start_supervised(),
+    ?assert(is_pid(whereis(emqx_acl_mnesia_migrator))),
+    ok = emqx_acl_mnesia_migrator:stop_supervised(),
+    ?assertEqual(undefined, whereis(emqx_acl_mnesia_migrator)).
 
 t_acl_cli(_Config) ->
     meck:new(emqx_ctl, [non_strict, passthrough]),
-    meck:expect(emqx_ctl, print, fun(Arg) -> emqx_ctl:format(Arg) end),
+    meck:expect(emqx_ctl, print, fun(Arg) -> emqx_ctl:format(Arg, []) end),
     meck:expect(emqx_ctl, print, fun(Msg, Arg) -> emqx_ctl:format(Msg, Arg) end),
     meck:expect(emqx_ctl, usage, fun(Usages) -> emqx_ctl:format_usage(Usages) end),
     meck:expect(emqx_ctl, usage, fun(Cmd, Descr) -> emqx_ctl:format_usage(Cmd, Descr) end),
 
-    clean_all_acls(),
-
     ?assertEqual(0, length(emqx_acl_mnesia_cli:cli(["list"]))),
 
     emqx_acl_mnesia_cli:cli(["add", "clientid", "test_clientid", "topic/A", "pub", "deny"]),
@@ -202,8 +343,6 @@ t_acl_cli(_Config) ->
     meck:unload(emqx_ctl).
 
 t_rest_api(_Config) ->
-    clean_all_acls(),
-
     Params1 = [#{<<"clientid">> => <<"test_clientid">>,
                  <<"topic">> => <<"topic/A">>,
                  <<"action">> => <<"pub">>,
@@ -273,13 +412,28 @@ t_rest_api(_Config) ->
     {ok, Res3} = request_http_rest_list(["$all"]),
     ?assertMatch([], get_http_data(Res3)).
 
-%%------------------------------------------------------------------------------
-%% Helpers
-%%------------------------------------------------------------------------------
 
-clean_all_acls() ->
-    [ mnesia:dirty_delete({emqx_acl, Login})
-      || Login <- mnesia:dirty_all_keys(emqx_acl)].
+create_conflicting_records() ->
+    Records = [
+        #?ACL_TABLE{ filter = {{clientid,<<"client6">>}, <<"t">>}
+                   , action = pubsub, access = deny, created_at = 0
+                   },
+        #?ACL_TABLE{ filter = {{clientid,<<"client5">>}, <<"t">>}
+                   , action = pubsub, access = deny, created_at = 1
+                   },
+        #?ACL_TABLE2{who = {clientid,<<"client5">>}, rules = [{allow, sub, <<"t">>, 2}]}
+    ],
+    mnesia:transaction(fun() -> lists:foreach(fun mnesia:write/1, Records) end).
+
+
+combined_conflicting_records() ->
+    % pubsub's are split, ACL_TABLE2 rules shadow ACL_TABLE rules
+    [
+        {{clientid,<<"client5">>},<<"t">>,sub,allow,2},
+        {{clientid,<<"client5">>},<<"t">>,pub,deny,1},
+        {{clientid,<<"client6">>},<<"t">>,sub,deny,0},
+        {{clientid,<<"client6">>},<<"t">>,pub,deny,0}
+    ].
 
 %%--------------------------------------------------------------------
 %% HTTP Request

apps/emqx_auth_mnesia/test/emqx_auth_mnesia_SUITE.erl

@@ -46,11 +46,15 @@ all() ->
 groups() ->
     [].
 
+init_per_suite(t_boot) ->
+    ok;
 init_per_suite(Config) ->
     ok = emqx_ct_helpers:start_apps([emqx_management, emqx_auth_mnesia], fun set_special_configs/1),
     create_default_app(),
     Config.
 
+end_per_suite(t_boot) ->
+    ok;
 end_per_suite(_Config) ->
     delete_default_app(),
     emqx_ct_helpers:stop_apps([emqx_management, emqx_auth_mnesia]).
@@ -65,10 +69,85 @@ set_special_configs(emqx) ->
 set_special_configs(_App) ->
     ok.
 
+set_default(ClientId, UserName, Pwd, HashType) ->
+    application:set_env(emqx_auth_mnesia, clientid_list, [{ClientId, Pwd}]),
+    application:set_env(emqx_auth_mnesia, username_list, [{UserName, Pwd}]),
+    application:set_env(emqx_auth_mnesia, password_hash, HashType),
+    ok.
 %%------------------------------------------------------------------------------
 %% Testcases
 %%------------------------------------------------------------------------------
 
+t_boot(_Config) ->
+    clean_all_users(),
+    emqx_ct_helpers:stop_apps([emqx_auth_mnesia]),
+    ClientId = <<"clientid-test">>,
+    UserName = <<"username-test">>,
+    Pwd = <<"emqx123456">>,
+    ok = emqx_ct_helpers:start_apps([emqx_auth_mnesia],
+        fun(_) -> set_default(ClientId, UserName, Pwd, sha256) end),
+    Ok = {stop, #{anonymous => false, auth_result => success}},
+    Failed = {stop, #{anonymous => false, auth_result => password_error}},
+    ?assertEqual(Ok,
+        emqx_auth_mnesia:check(#{clientid => ClientId, password => Pwd}, #{}, #{hash_type => sha256})),
+    ?assertEqual(Ok,
+        emqx_auth_mnesia:check(#{clientid => <<"NotExited">>, username => UserName, password => Pwd},
+            #{}, #{hash_type => sha256})),
+    ?assertEqual(Failed,
+        emqx_auth_mnesia:check(#{clientid => ClientId, password => <<Pwd/binary, "bad">>},
+            #{}, #{hash_type => sha256})),
+    ?assertEqual(Failed,
+        emqx_auth_mnesia:check(#{clientid => ClientId, username => UserName, password => <<Pwd/binary, "bad">>},
+            #{}, #{hash_type => sha256})),
+    emqx_ct_helpers:stop_apps([emqx_auth_mnesia]),
+
+    %% change default pwd
+    NewPwd = <<"emqx654321">>,
+    ok = emqx_ct_helpers:start_apps([emqx_auth_mnesia],
+        fun(_) -> set_default(ClientId, UserName, NewPwd, sha256) end),
+    ?assertEqual(Failed,
+        emqx_auth_mnesia:check(#{clientid => ClientId, password => NewPwd},
+            #{}, #{hash_type => sha256})),
+    ?assertEqual(Failed,
+        emqx_auth_mnesia:check(#{clientid => <<"NotExited">>, username => UserName, password => NewPwd},
+            #{}, #{hash_type => sha256})),
+    clean_all_users(),
+    emqx_ct_helpers:stop_apps([emqx_auth_mnesia]),
+
+    ok = emqx_ct_helpers:start_apps([emqx_auth_mnesia],
+        fun(_) -> set_default(ClientId, UserName, NewPwd, sha256) end),
+    ?assertEqual(Ok,
+        emqx_auth_mnesia:check(#{clientid => ClientId, password => NewPwd},
+            #{}, #{hash_type => sha256})),
+    ?assertEqual(Ok,
+        emqx_auth_mnesia:check(#{clientid => <<"NotExited">>, username => UserName, password => NewPwd},
+            #{}, #{hash_type => sha256})),
+    emqx_ct_helpers:stop_apps([emqx_auth_mnesia]),
+
+    %% change hash_type
+    NewPwd2 = <<"emqx6543210">>,
+    ok = emqx_ct_helpers:start_apps([emqx_auth_mnesia],
+        fun(_) -> set_default(ClientId, UserName, NewPwd2, plain) end),
+    ?assertEqual(Failed,
+        emqx_auth_mnesia:check(#{clientid => ClientId, password => NewPwd2},
+            #{}, #{hash_type => plain})),
+    ?assertEqual(Failed,
+        emqx_auth_mnesia:check(#{clientid => <<"NotExited">>, username => UserName, password => NewPwd2},
+            #{}, #{hash_type => plain})),
+    clean_all_users(),
+    emqx_ct_helpers:stop_apps([emqx_auth_mnesia]),
+
+    ok = emqx_ct_helpers:start_apps([emqx_auth_mnesia],
+        fun(_) -> set_default(ClientId, UserName, NewPwd2, plain) end),
+    ?assertEqual(Ok,
+        emqx_auth_mnesia:check(#{clientid => ClientId, password => NewPwd2},
+            #{}, #{hash_type => plain})),
+    ?assertEqual(Ok,
+        emqx_auth_mnesia:check(#{clientid => <<"NotExited">>, username => UserName, password => NewPwd2},
+            #{}, #{hash_type => plain})),
+    clean_all_users(),
+    ok.
+
 t_management(_Config) ->
     clean_all_users(),
 

apps/emqx_auth_mongo/README.md

@@ -1,7 +1,7 @@
 emqx_auth_mongo
 ===============
 
-EMQ X Authentication/ACL with MongoDB
+EMQX Authentication/ACL with MongoDB
 
 Build the Plugin
 ----------------
@@ -188,5 +188,5 @@ Apache License Version 2.0
 Author
 ------
 
-EMQ X Team.
+EMQX Team.
 

apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf

@@ -78,7 +78,7 @@ auth.mongo.database = mqtt
 
 ## If not specified, the server's names returned in server's certificate is validated against
 ## what's provided `auth.mongo.server` config's host part.
-## Setting to 'disable' will make EMQ X ignore unmatched server names.
+## Setting to 'disable' will make EMQX ignore unmatched server names.
 ## If set with a host name, the server's names returned in server's certificate is validated
 ## against this value.
 ##

apps/emqx_auth_mongo/include/emqx_auth_mongo.hrl

@@ -21,17 +21,8 @@
         ignore  = 'client.auth.ignore'
     }).
 
--record(acl_metrics, {
-        allow = 'client.acl.allow',
-        deny = 'client.acl.deny',
-        ignore = 'client.acl.ignore'
-    }).
-
 -define(METRICS(Type), tl(tuple_to_list(#Type{}))).
 -define(METRICS(Type, K), #Type{}#Type.K).
 
 -define(AUTH_METRICS, ?METRICS(auth_metrics)).
 -define(AUTH_METRICS(K), ?METRICS(auth_metrics, K)).
-
--define(ACL_METRICS, ?METRICS(acl_metrics)).
--define(ACL_METRICS(K), ?METRICS(acl_metrics, K)).

apps/emqx_auth_mongo/rebar.config

@@ -1,7 +1,6 @@
 {deps,
   %% NOTE: mind poolboy version when updating mongodb-erlang version
- [{mongodb, {git,"https://github.com/emqx/mongodb-erlang", {tag, "v3.0.7"}}},
-  %% mongodb-erlang uses a special fork https://github.com/comtihon/poolboy.git
+ [%% mongodb-erlang uses a special fork https://github.com/comtihon/poolboy.git
   %% (which has overflow_ttl feature added).
   %% However, it references `{branch, "master}` (commit 9c06a9a on 2021-04-07).
   %% By accident, We have always been using the upstream fork due to
@@ -29,4 +28,3 @@
 {cover_enabled, true}.
 {cover_opts, [verbose]}.
 {cover_export_enabled, true}.
-

apps/emqx_auth_mongo/src/emqx_acl_mongo.erl

@@ -21,17 +21,12 @@
 -include_lib("emqx/include/logger.hrl").
 
 %% ACL callbacks
--export([ register_metrics/0
-        , check_acl/5
+-export([ check_acl/5
         , description/0
         ]).
--spec(register_metrics() -> ok).
-register_metrics() ->
-    lists:foreach(fun emqx_metrics:ensure/1, ?ACL_METRICS).
 
 check_acl(#{username := <<$$, _/binary>>}, _PubSub, _Topic, _AclResult, _State) ->
     ok;
-
 check_acl(ClientInfo, PubSub, Topic, _AclResult, Env = #{aclquery := AclQuery}) ->
     #aclquery{collection = Coll, selector = SelectorList} = AclQuery,
     Pool = maps:get(pool, Env, ?APP),
@@ -43,20 +38,16 @@ check_acl(ClientInfo, PubSub, Topic, _AclResult, Env = #{aclquery := AclQuery})
         [] -> ok;
         Rows ->
             try match(ClientInfo, Topic, topics(PubSub, Rows)) of
-                matched -> emqx_metrics:inc(?ACL_METRICS(allow)),
-                           {stop, allow};
-                nomatch -> emqx_metrics:inc(?ACL_METRICS(deny)),
-                           {stop, deny}
+                matched -> {stop, allow};
+                nomatch -> {stop, deny}
             catch
                 _Err:Reason->
                     ?LOG(error, "[MongoDB] Check mongo ~p ACL failed, got ACL config: ~p, error: :~p",
                                 [PubSub, Rows, Reason]),
-                    emqx_metrics:inc(?ACL_METRICS(ignore)),
                     ignore
             end
     end.
 
-
 match(_ClientInfo, _Topic, []) ->
     nomatch;
 match(ClientInfo, Topic, [TopicFilter|More]) ->

apps/emqx_auth_mongo/src/emqx_auth_mongo.app.src

@@ -1,6 +1,6 @@
 {application, emqx_auth_mongo,
  [{description, "EMQ X Authentication/ACL with MongoDB"},
-  {vsn, "4.3.0"}, % strict semver, bump manually!
+  {vsn, "4.3.2"}, % strict semver, bump manually!
   {modules, []},
   {registered, [emqx_auth_mongo_sup]},
   {applications, [kernel,stdlib,mongodb,ecpool]},

apps/emqx_auth_mongo/src/emqx_auth_mongo.appup.src

@@ -0,0 +1,17 @@
+%% -*- mode: erlang -*-
+{VSN,
+  [{"4.3.1",
+    [{load_module,emqx_auth_mongo,brutal_purge,soft_purge,[]}]},
+   {"4.3.0",
+    [{load_module,emqx_auth_mongo_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_mongo,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_mongo,brutal_purge,soft_purge,[]}]},
+   {<<".*">>,[]}],
+  [{"4.3.1",
+    [{load_module,emqx_auth_mongo,brutal_purge,soft_purge,[]}]},
+   {"4.3.0",
+    [{load_module,emqx_auth_mongo_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_mongo,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_mongo,brutal_purge,soft_purge,[]}]},
+   {<<".*">>,[]}]
+}.

apps/emqx_auth_mongo/src/emqx_auth_mongo.erl

@@ -129,6 +129,9 @@ query_multi(Pool, Collection, SelectorList) ->
     lists:reverse(lists:flatten(lists:foldl(fun(Selector, Acc1) ->
         Batch = ecpool:with_client(Pool, fun(Conn) ->
                   case mongo_api:find(Conn, Collection, Selector, #{}) of
+                      {error, Reason} ->
+                          ?LOG(error, "[MongoDB] query_multi failed, got error: ~p", [Reason]),
+                          [];
                       [] -> [];
                       {ok, Cursor} ->
                           mc_cursor:foldl(fun(O, Acc2) -> [O|Acc2] end, [], Cursor, 1000)

apps/emqx_auth_mongo/src/emqx_auth_mongo_app.erl

@@ -55,7 +55,6 @@ reg_authmod(AuthQuery) ->
                    [#{authquery => AuthQuery, superquery => SuperQuery, pool => ?APP}]).
 
 reg_aclmod(AclQuery) ->
-    emqx_acl_mongo:register_metrics(),
     ok = emqx:hook('client.check_acl', fun emqx_acl_mongo:check_acl/5, [#{aclquery => AclQuery, pool => ?APP}]).
 
 %%--------------------------------------------------------------------

apps/emqx_auth_mysql/README.md

@@ -164,4 +164,4 @@ Apache License Version 2.0
 Author
 ------
 
-EMQ X Team.
+EMQX Team.

apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf

@@ -123,7 +123,7 @@ auth.mysql.acl_query = select allow, ipaddr, username, clientid, access, topic f
 
 ## If not specified, the server's names returned in server's certificate is validated against
 ## what's provided `auth.mysql.server` config's host part.
-## Setting to 'disable' will make EMQ X ignore unmatched server names.
+## Setting to 'disable' will make EMQX ignore unmatched server names.
 ## If set with a host name, the server's names returned in server's certificate is validated
 ## against this value.
 ##

apps/emqx_auth_mysql/include/emqx_auth_mysql.hrl

@@ -7,17 +7,8 @@
         ignore = 'client.auth.ignore'
     }).
 
--record(acl_metrics, {
-        allow = 'client.acl.allow',
-        deny = 'client.acl.deny',
-        ignore = 'client.acl.ignore'
-    }).
-
 -define(METRICS(Type), tl(tuple_to_list(#Type{}))).
 -define(METRICS(Type, K), #Type{}#Type.K).
 
 -define(AUTH_METRICS, ?METRICS(auth_metrics)).
 -define(AUTH_METRICS(K), ?METRICS(auth_metrics, K)).
-
--define(ACL_METRICS, ?METRICS(acl_metrics)).
--define(ACL_METRICS(K), ?METRICS(acl_metrics, K)).

apps/emqx_auth_mysql/src/emqx_acl_mysql.erl

@@ -22,20 +22,15 @@
 -include_lib("emqx/include/logger.hrl").
 
 %% ACL Callbacks
--export([ register_metrics/0
-        , check_acl/5
+-export([ check_acl/5
         , description/0
         ]).
 
--spec(register_metrics() -> ok).
-register_metrics() ->
-    lists:foreach(fun emqx_metrics:ensure/1, ?ACL_METRICS).
-
 check_acl(ClientInfo, PubSub, Topic, NoMatchAction, #{pool := Pool} = State) ->
     case do_check_acl(Pool, ClientInfo, PubSub, Topic, NoMatchAction, State) of
-        ok -> emqx_metrics:inc(?ACL_METRICS(ignore)), ok;
-        {stop, allow} -> emqx_metrics:inc(?ACL_METRICS(allow)), {stop, allow};
-        {stop, deny} -> emqx_metrics:inc(?ACL_METRICS(deny)), {stop, deny}
+        ok -> ok;
+        {stop, allow} -> {stop, allow};
+        {stop, deny} -> {stop, deny}
     end.
 
 do_check_acl(_Pool, #{username := <<$$, _/binary>>}, _PubSub, _Topic, _NoMatchAction, _State) ->

apps/emqx_auth_mysql/src/emqx_auth_mysql.app.src

@@ -1,6 +1,6 @@
 {application, emqx_auth_mysql,
  [{description, "EMQ X Authentication/ACL with MySQL"},
-  {vsn, "4.3.0"}, % strict semver, bump manually!
+  {vsn, "4.3.1"}, % strict semver, bump manually!
   {modules, []},
   {registered, [emqx_auth_mysql_sup]},
   {applications, [kernel,stdlib,mysql,ecpool]},

apps/emqx_auth_mysql/src/emqx_auth_mysql.appup.src

@@ -0,0 +1,11 @@
+%% -*- mode: erlang -*-
+{VSN,
+  [{"4.3.0",
+    [{load_module,emqx_auth_mysql_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_mysql,brutal_purge,soft_purge,[]}]},
+   {<<".*">>,[]}],
+  [{"4.3.0",
+    [{load_module,emqx_auth_mysql_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_mysql,brutal_purge,soft_purge,[]}]},
+   {<<".*">>,[]}]
+}.

apps/emqx_auth_mysql/src/emqx_auth_mysql_app.erl

@@ -60,7 +60,6 @@ load_auth_hook(AuthQuery) ->
     emqx:hook('client.authenticate', fun emqx_auth_mysql:check/3, [Params]).
 
 load_acl_hook(AclQuery) ->
-    ok = emqx_acl_mysql:register_metrics(),
     emqx:hook('client.check_acl', fun emqx_acl_mysql:check_acl/5, [#{acl_query => AclQuery, pool =>?APP}]).
 
 %%--------------------------------------------------------------------

apps/emqx_auth_pgsql/README.md

@@ -179,5 +179,5 @@ Apache License Version 2.0
 Author
 ------
 
-EMQ X Team.
+EMQX Team.