test: add test case to verify QoS 0 message is never redispatched

test: allow starting ct-slave without join cluster

.ci/acl_migration_test/build.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -xe
+
+cd "$EMQX_PATH"
+
+rm -rf _build _upgrade_base
+
+mkdir _upgrade_base
+pushd _upgrade_base
+    wget "https://s3-us-west-2.amazonaws.com/packages.emqx/emqx-ce/v${EMQX_BASE}/emqx-ubuntu20.04-${EMQX_BASE}-amd64.zip"
+popd
+
+make emqx-zip

.ci/acl_migration_test/prepare.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -xe
+
+mkdir -p "$TEST_PATH"
+cd "$TEST_PATH"
+
+cp ../"$EMQX_PATH"/_upgrade_base/*.zip ./
+unzip ./*.zip
+
+cp ../"$EMQX_PATH"/_packages/emqx/*.zip ./emqx/releases/
+
+git clone --depth 1 https://github.com/terry-xiaoyu/one_more_emqx.git
+
+./one_more_emqx/one_more_emqx.sh emqx2

.ci/acl_migration_test/suite.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+set -xe
+
+export EMQX_PATH="$1"
+export EMQX_BASE="$2"
+
+export TEST_PATH="emqx_test"
+
+./build.sh
+
+VERSION=$("$EMQX_PATH"/pkg-vsn.sh)
+export VERSION
+
+./prepare.sh
+
+./test.sh

.ci/acl_migration_test/test.sh

@@ -0,0 +1,121 @@
+#!/bin/bash
+
+set -e
+
+EMQX_ENDPOINT="http://localhost:8081/api/v4/acl"
+EMQX2_ENDPOINT="http://localhost:8917/api/v4/acl"
+
+function run() {
+    emqx="$1"
+    shift
+
+    echo "[$emqx]" "$@"
+
+    pushd "$TEST_PATH/$emqx"
+        "$@"
+    popd
+}
+
+function post_rule() {
+    endpoint="$1"
+    rule="$2"
+    echo -n "->($endpoint) "
+    curl -s -u admin:public -X POST "$endpoint" -d "$rule"
+    echo
+}
+
+function verify_clientid_rule() {
+    endpoint="$1"
+    id="$2"
+    echo -n "<-($endpoint) "
+    curl -s -u admin:public "$endpoint/clientid/$id" | grep "$id" || (echo "verify rule for client $id failed" && return 1)
+}
+
+# Run nodes
+
+run emqx ./bin/emqx start
+run emqx2 ./bin/emqx start
+
+run emqx ./bin/emqx_ctl plugins load emqx_auth_mnesia
+run emqx2 ./bin/emqx_ctl plugins load emqx_auth_mnesia
+
+run emqx2 ./bin/emqx_ctl cluster join 'emqx@127.0.0.1'
+
+# Add ACL rule to unupgraded EMQX nodes
+
+post_rule "$EMQX_ENDPOINT" '{"clientid": "CLIENT1_A","topic": "t", "action": "pub", "access": "allow"}'
+post_rule "$EMQX2_ENDPOINT" '{"clientid": "CLIENT1_B","topic": "t", "action": "pub", "access": "allow"}'
+
+# Upgrade emqx2 node
+
+run emqx2 ./bin/emqx install "$VERSION"
+sleep 60
+
+# Verify upgrade blocked
+
+run emqx2 ./bin/emqx eval 'emqx_acl_mnesia_migrator:is_old_table_migrated().' | grep false || (echo "emqx2 shouldn't have migrated" && exit 1)
+
+# Verify old rules on both nodes
+
+verify_clientid_rule "$EMQX_ENDPOINT" 'CLIENT1_A'
+verify_clientid_rule "$EMQX2_ENDPOINT" 'CLIENT1_A'
+
+verify_clientid_rule "$EMQX_ENDPOINT" 'CLIENT1_B'
+verify_clientid_rule "$EMQX2_ENDPOINT" 'CLIENT1_B'
+
+# Add ACL on OLD and NEW node, verify on all nodes
+
+post_rule "$EMQX_ENDPOINT" '{"clientid": "CLIENT2_A","topic": "t", "action": "pub", "access": "allow"}'
+post_rule "$EMQX2_ENDPOINT" '{"clientid": "CLIENT2_B","topic": "t", "action": "pub", "access": "allow"}'
+
+verify_clientid_rule "$EMQX_ENDPOINT" 'CLIENT2_A'
+verify_clientid_rule "$EMQX2_ENDPOINT" 'CLIENT2_A'
+
+verify_clientid_rule "$EMQX_ENDPOINT" 'CLIENT2_B'
+verify_clientid_rule "$EMQX2_ENDPOINT" 'CLIENT2_B'
+
+# Upgrade emqx node
+
+run emqx ./bin/emqx install "$VERSION"
+
+# Wait for upgrade
+
+sleep 60
+
+# Verify if upgrade occured
+
+run emqx ./bin/emqx eval 'emqx_acl_mnesia_migrator:is_old_table_migrated().' | grep true || (echo "emqx should have migrated" && exit 1)
+run emqx2 ./bin/emqx eval 'emqx_acl_mnesia_migrator:is_old_table_migrated().' | grep true || (echo "emqx2 should have migrated" && exit 1)
+
+# Verify rules are kept
+
+verify_clientid_rule "$EMQX_ENDPOINT" 'CLIENT1_A'
+verify_clientid_rule "$EMQX2_ENDPOINT" 'CLIENT1_A'
+
+verify_clientid_rule "$EMQX_ENDPOINT" 'CLIENT1_B'
+verify_clientid_rule "$EMQX2_ENDPOINT" 'CLIENT1_B'
+
+verify_clientid_rule "$EMQX_ENDPOINT" 'CLIENT2_A'
+verify_clientid_rule "$EMQX2_ENDPOINT" 'CLIENT2_A'
+
+verify_clientid_rule "$EMQX_ENDPOINT" 'CLIENT2_B'
+verify_clientid_rule "$EMQX2_ENDPOINT" 'CLIENT2_B'
+
+# Add ACL on OLD and NEW node, verify on all nodes
+
+post_rule "$EMQX_ENDPOINT" '{"clientid": "CLIENT3_A","topic": "t", "action": "pub", "access": "allow"}'
+post_rule "$EMQX2_ENDPOINT" '{"clientid": "CLIENT3_B","topic": "t", "action": "pub", "access": "allow"}'
+
+verify_clientid_rule "$EMQX_ENDPOINT" 'CLIENT3_A'
+verify_clientid_rule "$EMQX2_ENDPOINT" 'CLIENT3_A'
+
+verify_clientid_rule "$EMQX_ENDPOINT" 'CLIENT3_B'
+verify_clientid_rule "$EMQX2_ENDPOINT" 'CLIENT3_B'
+
+# Stop nodes
+
+run emqx ./bin/emqx stop
+run emqx2 ./bin/emqx stop
+
+echo "Success!"
+

.ci/build_packages/Dockerfile

@@ -1,4 +1,4 @@
-ARG BUILD_FROM=emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04
+ARG BUILD_FROM=emqx/build-env:erl23.3.4.9-3-ubuntu20.04
 FROM ${BUILD_FROM}
 
 ARG EMQX_NAME=emqx

.ci/build_packages/tests.sh

@@ -88,6 +88,7 @@ emqx_test(){
             ;;
             "rpm")
                 packagename=$(basename "${PACKAGE_PATH}/${EMQX_NAME}"-*.rpm)
+
                 rpm -ivh "${PACKAGE_PATH}/${packagename}"
                 if ! rpm -q emqx | grep -q emqx; then
                     echo "package install error"
@@ -129,23 +130,6 @@ running_test(){
     pytest -v /paho-mqtt-testing/interoperability/test_client/V5/test_connect.py::test_basic
     # shellcheck disable=SC2009 # pgrep does not support Extended Regular Expressions
     emqx stop || kill "$(ps -ef | grep -E '\-progname\s.+emqx\s' |awk '{print $2}')"
-
-    if [ "$(sed -n '/^ID=/p' /etc/os-release | sed -r 's/ID=(.*)/\1/g' | sed 's/"//g')" = ubuntu ] \
-    || [ "$(sed -n '/^ID=/p' /etc/os-release | sed -r 's/ID=(.*)/\1/g' | sed 's/"//g')" = debian ] ;then
-        service emqx start || ( tail /var/log/emqx/emqx.log.1 && exit 1 )
-        IDLE_TIME=0
-        while ! emqx_ctl status | grep -E 'Node\s.*@.*\sis\sstarted'
-        do
-            if [ $IDLE_TIME -gt 10 ]
-            then
-                echo "emqx service error"
-                exit 1
-            fi
-            sleep 10
-            IDLE_TIME=$((IDLE_TIME+1))
-        done
-        service emqx stop
-    fi
 }
 
 relup_test(){
@@ -155,16 +139,37 @@ relup_test(){
 
         find . -maxdepth 1 -name "${EMQX_NAME}-*-${ARCH}.zip" |
             while read -r pkg; do
+                if [[ "${pkg}" == *4.3.13* ]]; then
+                    echo "skipping upgrade test from 4.3.13 because this release had crypto linked with openssl 1.1.1n, it was in later version rolled back (to default 1.1.1k)."
+                    continue
+                fi
                 packagename=$(basename "${pkg}")
-                unzip "$packagename"
+                unzip -q "$packagename"
                 ./emqx/bin/emqx start || ( tail emqx/log/emqx.log.1 && exit 1 )
                 ./emqx/bin/emqx_ctl status
                 ./emqx/bin/emqx versions
+                OldVsn="$(./emqx/bin/emqx eval 'Versions=[{S, V} ||  {_,V,_, S} <- release_handler:which_releases()],
+                            Current = proplists:get_value(current, Versions, proplists:get_value(permanent, Versions)),
+                            io:format("~s", [Current])')"
                 cp "${PACKAGE_PATH}/${EMQX_NAME}"-*-"${TARGET_VERSION}-${ARCH}".zip ./emqx/releases
                 ./emqx/bin/emqx install "${TARGET_VERSION}"
                 [ "$(./emqx/bin/emqx versions |grep permanent | awk '{print $2}')" = "${TARGET_VERSION}" ] || exit 1
+                export EMQX_WAIT_FOR_STOP=300
                 ./emqx/bin/emqx_ctl status
-                ./emqx/bin/emqx stop
+
+                # also test remove old rel
+                ./emqx/bin/emqx uninstall "$OldVsn"
+
+                # check emqx still runs
+                ./emqx/bin/emqx ping
+
+                if ! ./emqx/bin/emqx stop; then
+                    cat emqx/log/erlang.log.1 || true
+                    cat emqx/log/emqx.log.1 || true
+                    echo "failed to stop emqx"
+                    exit 1
+                fi
+
                 rm -rf emqx
             done
    fi

.ci/docker-compose-file/docker-compose-emqx-broker-cluster.yaml

@@ -0,0 +1,99 @@
+version: '3.9'
+
+services:
+  haproxy:
+    container_name: haproxy
+    image: haproxy:2.3
+    depends_on:
+      - emqx1
+      - emqx2
+    volumes:
+      - ./haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg
+      - ../../etc/certs:/usr/local/etc/haproxy/certs
+    ports:
+      - "18083:18083"
+    #  - "1883:1883"
+    #  - "8883:8883"
+    #  - "8083:8083"
+    #  - "5683:5683/udp"
+    #  - "9999:9999"
+    #  - "8084:8084"
+    networks:
+      - emqx_bridge
+    working_dir: /usr/local/etc/haproxy
+    command:
+      - bash
+      - -c
+      - |
+        cat /usr/local/etc/haproxy/certs/cert.pem /usr/local/etc/haproxy/certs/key.pem > /usr/local/etc/haproxy/certs/emqx.pem
+        haproxy -f /usr/local/etc/haproxy/haproxy.cfg
+
+  emqx1:
+    restart: always
+    container_name: node1.emqx.io
+    image: $TARGET:$EMQX_TAG
+    env_file:
+      - conf.cluster.env
+    volumes:
+      - etc:/opt/emqx/etc
+    environment:
+      - "EMQX_HOST=node1.emqx.io"
+    ports:
+      - "11881:18083"
+#      - "1883:1883"
+    command:
+        - /bin/sh
+        - -c
+        - |
+          sed -i "s 127.0.0.1 $$(ip route show |grep "link" |awk '{print $$1}') g" /opt/emqx/etc/acl.conf
+          sed -i '/emqx_telemetry/d' /opt/emqx/data/loaded_plugins
+          /opt/emqx/bin/emqx foreground
+    healthcheck:
+      test: ["CMD", "/opt/emqx/bin/emqx_ctl", "status"]
+      interval: 5s
+      timeout: 25s
+      retries: 5
+    networks:
+      emqx_bridge:
+        aliases:
+        - node1.emqx.io
+
+  emqx2:
+    restart: always
+    container_name: node2.emqx.io
+    image: $TARGET:$EMQX_TAG
+    env_file:
+      - conf.cluster.env
+    volumes:
+      - etc:/opt/emqx/etc
+    environment:
+      - "EMQX_HOST=node2.emqx.io"
+    ports:
+      - "11882:18083"
+    command:
+      - /bin/sh
+      - -c
+      - |
+        sed -i "s 127.0.0.1 $$(ip route show |grep "link" |awk '{print $$1}') g" /opt/emqx/etc/acl.conf
+        sed -i '/emqx_telemetry/d' /opt/emqx/data/loaded_plugins
+        /opt/emqx/bin/emqx foreground
+    healthcheck:
+      test: ["CMD", "/opt/emqx/bin/emqx", "ping"]
+      interval: 5s
+      timeout: 25s
+      retries: 5
+    networks:
+      emqx_bridge:
+        aliases:
+        - node2.emqx.io
+volumes:
+  etc:
+networks:
+  emqx_bridge:
+    driver: bridge
+    name: emqx_bridge
+    ipam:
+      driver: default
+      config:
+        - subnet: 172.100.239.0/24
+          gateway: 172.100.239.1

.ci/docker-compose-file/docker-compose-emqx-cluster.yaml

@@ -27,6 +27,7 @@ services:
         haproxy -f /usr/local/etc/haproxy/haproxy.cfg
 
   emqx1:
+    restart: always
     container_name: node1.emqx.io
     image: $TARGET:$EMQX_TAG
     env_file:
@@ -51,6 +52,7 @@ services:
         - node1.emqx.io
 
   emqx2:
+    restart: always
     container_name: node2.emqx.io
     image: $TARGET:$EMQX_TAG
     env_file:

.ci/docker-compose-file/docker-compose-enterprise-tomcat-tcp.yaml

@@ -0,0 +1,10 @@
+version: '3.9'
+
+services:
+  web_server:
+    container_name: Tomcat
+    build:
+      context: ./http-service
+    image: web-server
+    networks:
+      - emqx_bridge

.ci/docker-compose-file/docker-compose.yaml

@@ -3,7 +3,7 @@ version: '3.9'
 services:
   erlang:
     container_name: erlang
-    image: emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04
+    image: emqx/build-env:erl23.3.4.9-3-ubuntu20.04
     env_file:
       - conf.env
     environment:

.ci/docker-compose-file/http-service/Dockerfile

@@ -0,0 +1,15 @@
+FROM tomcat:10.0.5
+
+RUN wget https://downloads.apache.org/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.zip \
+	&& unzip -q apache-maven-3.6.3-bin.zip \
+	&& mv apache-maven-3.6.3 /opt/apache-maven-3.6.3/ \
+	&& ln -s /opt/apache-maven-3.6.3/ /opt/maven
+ENV M2_HOME=/opt/maven
+ENV M2=$M2_HOME/bin
+ENV PATH=$M2:$PATH
+COPY ./web-server /code
+WORKDIR /code
+RUN mvn package -Dmaven.skip.test=true
+RUN mv ./target/emqx-web-0.0.1.war /usr/local/tomcat/webapps/emqx-web.war
+EXPOSE 8080
+CMD ["/usr/local/tomcat/bin/catalina.sh","run"]

.ci/docker-compose-file/http-service/web-server/pom.xml

@@ -0,0 +1,65 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <groupId>emqx-web</groupId>
+  <artifactId>emqx-web</artifactId>
+  <version>0.0.1</version>
+  <packaging>war</packaging>
+  <dependencies>
+	<dependency>
+    	<groupId>mysql</groupId>
+    	<artifactId>mysql-connector-java</artifactId>
+    	<version>8.0.16</version>
+    </dependency>
+    <dependency>
+    	<groupId>commons-dbutils</groupId>
+    	<artifactId>commons-dbutils</artifactId>
+    	<version>1.7</version>
+	</dependency>
+	<dependency>
+  	  <groupId>commons-logging</groupId>
+   	 <artifactId>commons-logging</artifactId>
+    	<version>1.2</version>
+	</dependency>
+	<dependency>
+    	<groupId>commons-dbcp</groupId>
+    	<artifactId>commons-dbcp</artifactId>
+   		<version>1.4</version>
+	</dependency>
+	<dependency>
+    	<groupId>commons-pool</groupId>
+    	<artifactId>commons-pool</artifactId>
+    	<version>1.6</version>
+	</dependency>
+	<dependency>
+     	<groupId>jakarta.servlet</groupId>
+     	<artifactId>jakarta.servlet-api</artifactId>
+     	<version>5.0.0</version>
+     	<scope>provided</scope>
+  	</dependency>
+  </dependencies>
+  <build>
+    <resources>
+      <resource>
+        <directory>src/main/reousrce</directory>
+        <excludes>
+          <exclude>**/*.java</exclude>
+        </excludes>
+      </resource>
+    </resources>
+    <plugins>
+      <plugin>
+        <artifactId>maven-compiler-plugin</artifactId>
+        <version>3.8.1</version>
+        <configuration>
+          <source>1.8</source>
+          <target>1.8</target>
+        </configuration>
+      </plugin>
+      <plugin>
+        <artifactId>maven-war-plugin</artifactId>
+        <version>3.2.3</version>
+      </plugin>
+    </plugins>
+  </build>
+
+</project>

.ci/docker-compose-file/http-service/web-server/src/main/java/com/emqx/dao/AuthDAO.java

@@ -0,0 +1,54 @@
+package com.emqx.dao;
+
+import java.io.IOException;
+import java.sql.SQLException;
+
+import org.apache.commons.dbutils.QueryRunner;
+import org.apache.commons.dbutils.handlers.ScalarHandler;
+
+import com.emqx.util.EmqxDatabaseUtil;
+
+public class AuthDAO {
+
+	public String getUserName(String userName) throws IOException, SQLException {
+		QueryRunner runner = new QueryRunner(EmqxDatabaseUtil.getDataSource());
+		String sql = "select password from http_user where username='"+userName+"'";
+		String password =runner.query(sql, new ScalarHandler<String>());
+		return password;
+	}
+	
+	public String getClient(String clientid) throws IOException, SQLException {
+		QueryRunner runner = new QueryRunner(EmqxDatabaseUtil.getDataSource());
+		String sql = "select password from http_user where clientid='"+clientid+"'";
+		String password =runner.query(sql, new ScalarHandler<String>());
+		return password;
+	}
+	
+	public String getUserAccess(String userName) throws IOException, SQLException {
+		QueryRunner runner = new QueryRunner(EmqxDatabaseUtil.getDataSource());
+		String sql = "select access from http_acl where username='"+userName+"'";
+		String access =runner.query(sql, new ScalarHandler<String>());
+		return access;
+	}
+	
+	public String getUserTopic(String userName) throws IOException, SQLException {
+		QueryRunner runner = new QueryRunner(EmqxDatabaseUtil.getDataSource());
+		String sql = "select topic from http_acl where username='"+userName+"'";
+		String topic =runner.query(sql, new ScalarHandler<String>());
+		return topic;
+	}
+	
+	public String getClientAccess(String clientid) throws IOException, SQLException {
+		QueryRunner runner = new QueryRunner(EmqxDatabaseUtil.getDataSource());
+		String sql = "select access from http_acl where clientid='"+clientid+"'";
+		String access =runner.query(sql, new ScalarHandler<String>());
+		return access;
+	}
+	
+	public String getClientTopic(String clientid) throws IOException, SQLException {
+		QueryRunner runner = new QueryRunner(EmqxDatabaseUtil.getDataSource());
+		String sql = "select topic from http_acl where clientid='"+clientid+"'";
+		String topic =runner.query(sql, new ScalarHandler<String>());
+		return topic;
+	}
+}

.ci/docker-compose-file/http-service/web-server/src/main/java/com/emqx/dao/DBUtilsTest.java

@@ -0,0 +1,45 @@
+package com.emqx.dao;
+
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.sql.SQLException;
+import java.util.Properties;
+
+import org.apache.commons.dbcp.BasicDataSource;
+import org.apache.commons.dbutils.QueryRunner;
+import org.apache.commons.dbutils.handlers.ColumnListHandler;
+import org.apache.commons.dbutils.handlers.ScalarHandler;
+import org.apache.commons.dbutils.handlers.columns.StringColumnHandler;
+
+
+public class DBUtilsTest {
+
+	public static void main(String args[]) throws FileNotFoundException, IOException, SQLException {
+		Properties property = new Properties();//流文件  
+		
+		property.load(DBUtilsTest.class.getClassLoader().getResourceAsStream("database.properties"));
+		
+		BasicDataSource dataSource = new BasicDataSource();
+		dataSource.setDriverClassName(property.getProperty("jdbc.driver"));
+		dataSource.setUrl(property.getProperty("jdbc.url"));
+		dataSource.setUsername(property.getProperty("jdbc.username"));
+		dataSource.setPassword(property.getProperty("jdbc.password"));
+
+		// 初始化连接数 if(initialSize!=null)
+		//dataSource.setInitialSize(Integer.parseInt(initialSize));
+
+		// 最小空闲连接 if(minIdle!=null)
+		//dataSource.setMinIdle(Integer.parseInt(minIdle));
+
+		// 最大空闲连接 if(maxIdle!=null)
+		//dataSource.setMaxIdle(Integer.parseInt(maxIdle));
+
+		QueryRunner runner = new QueryRunner(dataSource);
+		String sql="select username from mqtt_user where id=1";
+		String result = runner.query(sql, new ScalarHandler<String>());
+		
+		System.out.println(result);
+				
+	}
+}

.ci/docker-compose-file/http-service/web-server/src/main/java/com/emqx/servlet/AclServlet.java

@@ -0,0 +1,103 @@
+package com.emqx.servlet;
+
+import java.io.IOException;
+import java.sql.SQLException;
+
+import com.emqx.dao.AuthDAO;
+
+import jakarta.servlet.ServletException;
+import jakarta.servlet.http.HttpServlet;
+import jakarta.servlet.http.HttpServletRequest;
+import jakarta.servlet.http.HttpServletResponse;
+
+public class AclServlet extends HttpServlet {
+
+	@Override
+	protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException {
+		// TODO Auto-generated method stub
+		doPost(req, resp);
+	}
+	@Override
+	protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException {
+		String clientid = req.getParameter("clientid");
+		String username = req.getParameter("username");
+		String access = req.getParameter("access");
+		String topic = req.getParameter("topic");
+		//String password = req.getParameter("password");
+		
+		//step0: password is not null, or not pass.
+		
+		AuthDAO dao = new AuthDAO();
+		try {
+			//step1: check username access&topic
+			if(username != null) {
+				String access_1 = dao.getUserAccess(username);
+				String topic_1 = dao.getUserTopic(username);
+				
+				if(access.equals(access_1)) {
+					if(topic.equals(topic_1)) {
+						resp.setStatus(200);
+					}
+					else {
+						if(clientid != null){
+							String access_2 = dao.getClientAccess(clientid);
+							String topic_2 = dao.getClientTopic(clientid);
+								if(access.equals(access_2)) {
+									if(topic.equals(topic_2)) {
+										resp.setStatus(200);
+									}
+									else {
+										resp.setStatus(400);
+									}
+								}else {
+									resp.setStatus(400);
+								}
+						}else {
+							resp.setStatus(400);
+						}
+					}
+				}else {//step2.1: username password is not match, then check clientid password
+					if(clientid != null){
+						String access_3 = dao.getClientAccess(clientid);
+						String topic_3 = dao.getClientTopic(clientid);
+							if(access.equals(access_3)) {
+								if(topic.equals(topic_3)) {
+									resp.setStatus(200);
+								}
+								else {
+									resp.setStatus(400);
+								}
+							}else {
+								resp.setStatus(400);
+							}
+					}else {
+						resp.setStatus(400);
+					}
+				}
+			}else {//step2.2: username is null, then check clientid password
+				if(clientid != null){
+					String access_4 = dao.getClientAccess(clientid);
+					String topic_4 = dao.getClientTopic(clientid);
+						if(access.equals(access_4)) {
+							if(topic.equals(topic_4)) {
+								resp.setStatus(200);
+							}
+							else {
+								resp.setStatus(400);
+							}
+						}else {
+							resp.setStatus(400);
+						}
+				}else {
+					resp.setStatus(400);
+				}
+			}
+		} catch (IOException e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+		} catch (SQLException e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+		}
+	}
+}

.ci/docker-compose-file/http-service/web-server/src/main/java/com/emqx/servlet/AuthServlet.java

@@ -0,0 +1,72 @@
+package com.emqx.servlet;
+
+import java.io.IOException;
+import java.sql.SQLException;
+
+import com.emqx.dao.AuthDAO;
+
+import jakarta.servlet.ServletException;
+import jakarta.servlet.http.HttpServlet;
+import jakarta.servlet.http.HttpServletRequest;
+import jakarta.servlet.http.HttpServletResponse;
+
+public class AuthServlet extends HttpServlet {
+
+	@Override
+	protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException {
+		// TODO Auto-generated method stub
+		doPost(req, resp);
+	}
+	@Override
+	protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException {
+		String clientid = req.getParameter("clientid");
+		String username =req.getParameter("username");
+		String password = req.getParameter("password");
+		
+		//step0: password is not null, or not pass.
+		if(password == null) {
+			resp.setStatus(400);
+			return;
+		}
+		AuthDAO dao = new AuthDAO();
+		try {
+			//step1: check username password
+			if(username != null) {
+				String password_d = dao.getUserName(username);
+				
+				if(password.equals(password_d)) {
+					resp.setStatus(200);
+					//200
+				}else {//step2.1: username password is not match, then check clientid password
+					if(clientid != null){
+						String password_c = dao.getClient(clientid);
+							if(password.equals(password_c)) {
+								resp.setStatus(200);
+							}else {
+								resp.setStatus(400);
+							}
+					}else {
+						resp.setStatus(400);
+					}
+				}
+			}else {//step2.2: username is null, then check clientid password
+				if(clientid != null){
+					String password_c = dao.getClient(clientid);
+						if(password.equals(password_c)) {
+							resp.setStatus(200);
+						}else {
+							resp.setStatus(400);
+						}
+				}else {
+					resp.setStatus(400);
+				}
+			}
+		} catch (IOException e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+		} catch (SQLException e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+		}
+	}
+}

.ci/docker-compose-file/http-service/web-server/src/main/java/com/emqx/util/EmqxDatabaseUtil.java

@@ -0,0 +1,27 @@
+package com.emqx.util;
+
+import java.io.IOException;
+import java.util.Properties;
+
+import javax.sql.DataSource;
+
+import org.apache.commons.dbcp.BasicDataSource;
+
+import com.emqx.dao.DBUtilsTest;
+
+public class EmqxDatabaseUtil {
+
+	public static DataSource getDataSource() throws IOException {
+		Properties property = new Properties();// 流文件
+
+		property.load(EmqxDatabaseUtil.class.getClassLoader().getResourceAsStream("database.properties"));
+
+		BasicDataSource dataSource = new BasicDataSource();
+		dataSource.setDriverClassName(property.getProperty("jdbc.driver"));
+		dataSource.setUrl(property.getProperty("jdbc.url"));
+		dataSource.setUsername(property.getProperty("jdbc.username"));
+		dataSource.setPassword(property.getProperty("jdbc.password"));
+		
+		return dataSource;
+	}
+}

.ci/docker-compose-file/http-service/web-server/src/main/reousrce/database.properties

@@ -0,0 +1,4 @@
+jdbc.driver= com.mysql.jdbc.Driver
+jdbc.url= jdbc:mysql://mysql_server:3306/mqtt
+jdbc.username= root
+jdbc.password= public

.ci/docker-compose-file/http-service/web-server/src/main/webapp/META-INF/MANIFEST.MF

@@ -0,0 +1,3 @@
+Manifest-Version: 1.0
+Class-Path: 
+

.ci/docker-compose-file/http-service/web-server/src/main/webapp/WEB-INF/web.xml

@@ -0,0 +1,31 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<web-app xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xmlns="http://JAVA.sun.com/xml/ns/javaee"
+	xsi:schemaLocation="http://java.sun.com/xml/ns/javaee http://java.sun.com/xml/ns/javaee/web-app_2_5.xsd"
+	id="WebApp_ID" version="2.5">
+	<display-name>emqx-web</display-name>
+	<servlet>
+		<servlet-name>Auth</servlet-name>
+		<servlet-class>com.emqx.servlet.AuthServlet</servlet-class>
+	</servlet>
+	<servlet>
+		<servlet-name>Acl</servlet-name>
+		<servlet-class>com.emqx.servlet.AclServlet</servlet-class>
+	</servlet>
+	<servlet-mapping>
+		<servlet-name>Auth</servlet-name>
+		<url-pattern>/auth</url-pattern>
+	</servlet-mapping>
+	<servlet-mapping>
+		<servlet-name>Acl</servlet-name>
+		<url-pattern>/acl</url-pattern>
+	</servlet-mapping>
+	<welcome-file-list>
+		<welcome-file>index.html</welcome-file>
+		<welcome-file>index.htm</welcome-file>
+		<welcome-file>index.jsp</welcome-file>
+		<welcome-file>default.html</welcome-file>
+		<welcome-file>default.htm</welcome-file>
+		<welcome-file>default.jsp</welcome-file>
+	</welcome-file-list>
+</web-app>

.ci/docker-compose-file/http-service/web-server/src/main/webapp/index.html

@@ -0,0 +1,10 @@
+<!DOCTYPE html>
+<html>
+<head>
+<meta charset="UTF-8">
+<title>love</title>
+</head>
+<body>
+It's lucky, jiabanxiang.
+</body>
+</html>

.ci/docker-compose-file/python/pytest.sh

@@ -10,7 +10,7 @@ LB="haproxy"
 
 apk update && apk add git curl
 git clone -b develop-4.0 https://github.com/emqx/paho.mqtt.testing.git /paho.mqtt.testing
-pip install pytest
+pip install pytest==6.2.5
 
 pytest -v /paho.mqtt.testing/interoperability/test_client/V5/test_connect.py -k test_basic --host "$LB"
 RESULT=$?

.ci/fvt_tests/http_server/rebar.config

@@ -1,7 +1,7 @@
 {erl_opts, [debug_info]}.
 {deps, 
  [
-    {minirest, {git, "https://github.com/emqx/minirest.git", {tag, "0.3.5"}}}
+    {minirest, {git, "https://github.com/emqx/minirest.git", {tag, "0.3.6"}}}
  ]}.
 
 {shell, [

.ci/fvt_tests/local_relup_test_run.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+USAGE="$0 profile vsn old_vsn package_path"
+EXAMPLE="$0 emqx 4.3.8-b3bb6075 v4.3.2 /home/alice/relup_dubug/downloaded_packages"
+
+if [[ $# -ne 4 ]]; then
+    echo "$USAGE"
+    echo "$EXAMPLE"
+    exit 1
+fi
+
+set -ex
+
+PROFILE="$1"
+VSN="$2"
+OLD_VSN="$3"
+PACKAGE_PATH="$4"
+
+TEMPDIR=$(mktemp -d)
+trap '{ rm -rf -- "$TEMPDIR"; }' EXIT
+
+git clone --branch=master "https://github.com/terry-xiaoyu/one_more_emqx.git" "$TEMPDIR/one_more_emqx"
+cp -r "$PACKAGE_PATH" "$TEMPDIR/packages"
+cp relup.lux "$TEMPDIR/"
+cp -r http_server "$TEMPDIR/http_server"
+
+exec docker run \
+    -v "$TEMPDIR:/relup_test" \
+    -w "/relup_test" \
+    -e REBAR_COLOR=none \
+    -it emqx/relup-test-env:erl23.2.7.2-emqx-3-ubuntu20.04 \
+        lux \
+        --progress verbose \
+        --case_timeout infinity \
+        --var PROFILE="$PROFILE" \
+        --var PACKAGE_PATH="/relup_test/packages" \
+        --var ONE_MORE_EMQX_PATH="/relup_test/one_more_emqx" \
+        --var VSN="$VSN" \
+        --var OLD_VSN="$OLD_VSN" \
+        relup.lux

.ci/fvt_tests/relup.lux

@@ -1,15 +1,12 @@
 [config var=PROFILE]
 [config var=PACKAGE_PATH]
-[config var=BENCH_PATH]
 [config var=ONE_MORE_EMQX_PATH]
 [config var=VSN]
-[config var=OLD_VSNS]
+[config var=OLD_VSN]
 
 [config shell_cmd=/bin/bash]
 [config timeout=600000]
 
-[loop old_vsn $OLD_VSNS]
-
 [shell http_server]
     !cd http_server
     !rebar3 shell
@@ -21,29 +18,30 @@
     ?>
 
 [shell emqx]
+    !OLD_VSN=$(echo $OLD_VSN | sed  -r 's/[v|e]//g')
     !cd $PACKAGE_PATH
-    !unzip -q -o $PROFILE-ubuntu20.04-$(echo $old_vsn | sed  -r 's/[v|e]//g')-amd64.zip
+    !unzip -q -o $PROFILE-ubuntu20.04-$${OLD_VSN}-amd64.zip
     ?SH-PROMPT
 
     !cd emqx
-    !sed -i 's|listener.wss.external[ \t]*=.*|listener.wss.external = 8085|g' etc/emqx.conf
+    !export EMQX_LOG__LEVEL=debug
-    !sed -i '/emqx_telemetry/d' data/loaded_plugins
 
     !./bin/emqx start
     ?EMQ X .* is started successfully!
     ?SH-PROMPT
 
 [shell emqx2]
+    !OLD_VSN=$(echo $OLD_VSN | sed  -r 's/[v|e]//g')
     !cd $PACKAGE_PATH
     !cp -f $ONE_MORE_EMQX_PATH/one_more_$(echo $PROFILE | sed 's/-/_/g').sh .
     !./one_more_$(echo $PROFILE | sed 's/-/_/g').sh emqx2
     ?SH-PROMPT
     !cd emqx2
 
-    !sed -i '/emqx_telemetry/d' data/loaded_plugins
+    !export EMQX_LOG__LEVEL=debug
 
     !./bin/emqx start
-    ?EMQ X (.*) is started successfully!
+    ?EMQ X .* is started successfully!
     ?SH-PROMPT
 
     !./bin/emqx_ctl cluster join emqx@127.0.0.1
@@ -63,6 +61,8 @@
     !./bin/emqx_ctl rules create 'SELECT * FROM "t/#"' '[{"name":"data_to_webserver", "params": {"$$resource":  "resource:691c29ba"}}]'
     ?created
     ?SH-PROMPT
+    !sleep 5
+    ?SH-PROMPT
 
 [shell emqx]
     !./bin/emqx_ctl resources list
@@ -71,11 +71,11 @@
     !./bin/emqx_ctl rules list
     ?691c29ba
     ?SH-PROMPT
+    !./bin/emqx_ctl broker metrics | grep "messages.publish"
+    ???SH-PROMPT
 
 [shell bench]
-    !cd $BENCH_PATH
+    !emqtt_bench pub -c 10 -I 1000 -t t/%i -s 64 -L 300
-
-    !./emqtt_bench pub -c 10 -I 1000 -t t/%i -s 64 -L 300
     ???sent
 
 [shell emqx]
@@ -84,6 +84,27 @@
 
     !cp -f ../$PROFILE-ubuntu20.04-$VSN-amd64.zip releases/
 
+    ## upgrade to the new version
+    !./bin/emqx install $VSN
+    ?Made release permanent: "$VSN"
+    ?SH-PROMPT
+
+    !./bin/emqx versions |grep permanent
+    ?(.*)$VSN
+    ?SH-PROMPT
+
+    ## downgrade to the old version
+    !./bin/emqx install $${OLD_VSN}
+    ?Made release permanent:.*
+    ?SH-PROMPT
+
+    !./bin/emqx versions |grep permanent | grep -qs "$${OLD_VSN}"
+    ?SH-PROMPT:
+    !echo ==$$?==
+    ?^==0==
+    ?SH-PROMPT:
+
+    ## again, upgrade to the new version
     !./bin/emqx install $VSN
     ?Made release permanent: "$VSN"
     ?SH-PROMPT
@@ -99,12 +120,37 @@
     """
     ?SH-PROMPT
 
+    !./bin/emqx_ctl plugins list | grep --color=never emqx_management
+    ?Plugin\(emqx_management.*active=true\)
+    ?SH-PROMPT
+
 [shell emqx2]
     !echo "" > log/emqx.log.1
     ?SH-PROMPT
 
     !cp -f ../$PROFILE-ubuntu20.04-$VSN-amd64.zip releases/
 
+    ## upgrade to the new version
+    !./bin/emqx install $VSN
+    ?Made release permanent: "$VSN"
+    ?SH-PROMPT
+
+    !./bin/emqx versions |grep permanent
+    ?(.*)$VSN
+    ?SH-PROMPT
+
+    ## downgrade to the old version
+    !./bin/emqx install $${OLD_VSN}
+    ?Made release permanent:.*
+    ?SH-PROMPT
+
+    !./bin/emqx versions |grep permanent | grep -qs "$${OLD_VSN}"
+    ?SH-PROMPT:
+    !echo ==$$?==
+    ?^==0==
+    ?SH-PROMPT:
+
+    ## again, upgrade to the new version
     !./bin/emqx install $VSN
     ?Made release permanent: "$VSN"
     ?SH-PROMPT
@@ -120,11 +166,34 @@
     """
     ?SH-PROMPT
 
+    !./bin/emqx_ctl plugins list | grep --color=never emqx_management
+    ?Plugin\(emqx_management.*active=true\)
+    ?SH-PROMPT
+
 [shell bench]
     ???publish complete
     ??SH-PROMPT:
+    !sleep 30
+    ?SH-PROMPT
+
+[shell emqx]
+    !./bin/emqx_ctl broker metrics | grep "messages.publish"
+    ???SH-PROMPT
+
+## We don't guarantee not to lose a single message!
+## So even if we received 290~300 messages, we consider it as success
+[shell bench]
+    !curl --user admin:public --silent --show-error http://localhost:8081/api/v4/rules | jq -M --raw-output ".data[0].metrics[] | select(.node==\"emqx@127.0.0.1\").matched"
+    ?(29[0-9])|(300)
+    ?SH-PROMPT
+
+    !curl --user admin:public --silent --show-error http://localhost:8081/api/v4/rules | jq -M --raw-output ".data[0].actions[0].metrics[] | select(.node==\"emqx@127.0.0.1\").success"
+    ?(29[0-9])|(300)
+    ?SH-PROMPT
+
+    ## The /counter API is provided by .ci/fvt_test/http_server
     !curl http://127.0.0.1:8080/counter
-    ???{"data":300,"code":0}
+    ?\{"data":(29[0-9])|(300),"code":0\}
     ?SH-PROMPT
 
 [shell emqx2]
@@ -158,8 +227,6 @@
     !halt(3).
     ?SH-PROMPT:
 
-[endloop]
-
 [cleanup]
     !echo ==$$?==
     ?==0==

.github/workflows/apps_version_check.yaml

@@ -3,10 +3,41 @@ name: Check Apps Version
 on: [pull_request]
 
 jobs:
-  check_apps_version:
+  check_apps_version_4_x:
     runs-on: ubuntu-20.04
 
+    strategy:
+      matrix:
+        erl_otp:
+        - erl23.3.4.9-3
+        os:
+        - ubuntu20.04
+
+    container: emqx/build-env:${{ matrix.erl_otp }}-${{ matrix.os }}
+
     steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0 # need full history
+      - name: fix-git-unsafe-repository
+        run: git config --global --add safe.directory /__w/emqx/emqx
+      - name: Check relup (ce)
+        if: endsWith(github.repository, 'emqx')
+        run: ./scripts/update-appup.sh emqx --check
+      - name: Check relup (ee)
+        if: endsWith(github.repository, 'enterprise')
+        run: ./scripts/update-appup.sh emqx-ee --check
       - name: Check apps version
-        run: ./scripts/apps-version-check.sh
+        run: ./scripts/check-apps-vsn.sh
+      - name: Check chart versions
+        run: ./scripts/check-chart-vsn.sh
+      - uses: actions/upload-artifact@v3.1.0
+        if: failure()
+        with:
+          name: expected_appup_files
+          path: |
+            apps/*/src/*.appup.src
+            src/*.appup.src
+            lib-ce/*/src/*.appup.src
+            lib-ee/*/src/*.appup.src
+          retention-days: 1

.github/workflows/build_packages.yaml

@@ -1,21 +1,29 @@
 name: Cross build packages
 
+concurrency:
+  group: build-${{ github.event_name }}-${{ github.ref }}
+  cancel-in-progress: true
+
 on:
+  push:
+    branches:
+      - 'main-v4.**'
+    tags:
+      - v*
+      - e*
   schedule:
     - cron:  '0 */6 * * *'
-  release:
-    types:
-      - published
   workflow_dispatch:
 
 jobs:
   prepare:
+    # avoid building when syncing to ee repo
+    if: endsWith(github.repository, 'emqx')
     runs-on: ubuntu-20.04
-    container: emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04
+    container: emqx/build-env:erl23.3.4.9-3-ubuntu20.04
 
     outputs:
       profiles: ${{ steps.set_profile.outputs.profiles}}
-      old_vsns: ${{ steps.set_profile.outputs.old_vsns}}
 
     steps:
       - uses: actions/checkout@v2
@@ -27,15 +35,9 @@ jobs:
         shell: bash
         run: |
           cd source
-          vsn="$(./pkg-vsn.sh)"
-          pre_vsn="$(echo $vsn | grep -oE '^[0-9]+.[0-9]')"
           if make emqx-ee --dry-run > /dev/null 2>&1; then
-            old_vsns="$(git tag -l "e$pre_vsn.[0-9]" | xargs echo -n | sed "s/e$vsn//")"
-            echo "::set-output name=old_vsns::$old_vsns"
             echo "::set-output name=profiles::[\"emqx-ee\"]"
           else
-            old_vsns="$(git tag -l "v$pre_vsn.[0-9]" | xargs echo -n | sed "s/v$vsn//")"
-            echo "::set-output name=old_vsns::$old_vsns"
             echo "::set-output name=profiles::[\"emqx\", \"emqx-edge\"]"
           fi
       - name: get_all_deps
@@ -48,7 +50,6 @@ jobs:
         run: |
           echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials
           git config --global credential.helper store
-          echo "${{ secrets.CI_GIT_TOKEN }}" >> source/scripts/git-token
           make -C source deps-all
           zip -ryq source.zip source/* source/.[^.]*
       - uses: actions/upload-artifact@v2
@@ -58,13 +59,13 @@ jobs:
 
   windows:
     runs-on: windows-2019
-
     needs: prepare
     if: endsWith(github.repository, 'emqx')
-
     strategy:
       matrix:
         profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
+        otp:
+          - 23.3.4.13
         exclude:
           - profile: emqx-edge
 
@@ -76,16 +77,15 @@ jobs:
     - name: unzip source code
       run: Expand-Archive -Path source.zip -DestinationPath ./
     - uses: ilammy/msvc-dev-cmd@v1
-    - uses: gleam-lang/setup-erlang@v1.1.0
+    - uses: erlef/setup-beam@v1
-      id: install_erlang
       with:
-        otp-version: 23.2
+        otp-version: ${{ matrix.otp }}
     - name: build
       env:
         PYTHON: python
+        DIAGNOSTIC: 1
       run: |
-        $env:PATH = "${{ steps.install_erlang.outputs.erlpath }}\bin;$env:PATH"
+        erl -eval "erlang:display(crypto:info_lib())" -s init stop
-
         $version = $( "${{ github.ref }}" -replace "^(.*)/(.*)/" )
         if ($version -match "^v[0-9]+\.[0-9]+(\.[0-9]+)?") {
           $regex = "[0-9]+\.[0-9]+(-alpha|-beta|-rc)?\.[0-9]+"
@@ -101,14 +101,11 @@ jobs:
             Remove-Item -Force -Path rebar.lock
         }
         make ensure-rebar3
-        copy rebar3 "${{ steps.install_erlang.outputs.erlpath }}\bin"
-        ls "${{ steps.install_erlang.outputs.erlpath }}\bin"
-        rebar3 --help
         make ${{ matrix.profile }}
         mkdir -p _packages/${{ matrix.profile }}
         Compress-Archive -Path _build/${{ matrix.profile }}/rel/emqx -DestinationPath _build/${{ matrix.profile }}/rel/$pkg_name
         mv _build/${{ matrix.profile }}/rel/$pkg_name _packages/${{ matrix.profile }}
-        Get-FileHash -Path "_packages/${{ matrix.profile }}/$pkg_name" | Format-List | grep 'Hash' | awk '{print $3}'  > _packages/${{ matrix.profile }}/$pkg_name.sha256
+        sha256sum "_packages/${{ matrix.profile }}/$pkg_name" | head -c 64 > "_packages/${{ matrix.profile }}/${pkg_name}.sha256"
     - name: run emqx
       timeout-minutes: 1
       run: |
@@ -119,23 +116,23 @@ jobs:
         ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx install
         ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx uninstall
     - uses: actions/upload-artifact@v1
-      if: startsWith(github.ref, 'refs/tags/')
       with:
         name: ${{ matrix.profile }}
         path: source/_packages/${{ matrix.profile }}/.
 
   mac:
-    runs-on: macos-10.15
-
     needs: prepare
 
     strategy:
       matrix:
         profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
         erl_otp:
-          - 23.2.7.2-emqx-2
+          - 23.3.4.9-3
         exclude:
           - profile: emqx-edge
+        macos:
+          - macos-10.15
+    runs-on: ${{ matrix.macos  }}
 
     steps:
     - uses: actions/download-artifact@v2
@@ -153,8 +150,8 @@ jobs:
     - uses: actions/cache@v2
       id: cache
       with:
-        path: ~/.kerl
+        path: ~/.kerl/${{ matrix.erl_otp }}
-        key: erl${{ matrix.erl_otp }}-macos10.15
+        key: otp-install-${{ matrix.erl_otp }}-${{ matrix.macos }}
     - name: build erlang
       if: steps.cache.outputs.cache-hit != 'true'
       timeout-minutes: 60
@@ -168,9 +165,11 @@ jobs:
     - name: build
       run: |
         . $HOME/.kerl/${{ matrix.erl_otp }}/activate
-        make -C source ensure-rebar3
+        cd source
-        sudo cp source/rebar3 /usr/local/bin/rebar3
+        make ensure-rebar3
-        make -C source ${{ matrix.profile }}-zip
+        sudo cp rebar3 /usr/local/bin/rebar3
+        rm -rf _build/${{ matrix.profile }}/lib
+        make ${{ matrix.profile }}-zip
     - name: test
       run: |
         cd source
@@ -192,11 +191,16 @@ jobs:
           exit 1
         fi
         ./emqx/bin/emqx_ctl status
-        ./emqx/bin/emqx stop
+        if ! ./emqx/bin/emqx stop; then
+          cat emqx/log/erlang.log.1 || true
+          cat emqx/log/emqx.log.1 || true
+          echo "failed to stop emqx"
+          exit 1
+        fi
         rm -rf emqx
+        #sha256sum ./_packages/${{ matrix.profile }}/$pkg_name | head -c64  > ./_packages/${{ matrix.profile }}/$pkg_name.sha256
         openssl dgst -sha256 ./_packages/${{ matrix.profile }}/$pkg_name | awk '{print $2}'  > ./_packages/${{ matrix.profile }}/$pkg_name.sha256
     - uses: actions/upload-artifact@v1
-      if: startsWith(github.ref, 'refs/tags/')
       with:
         name: ${{ matrix.profile }}
         path: source/_packages/${{ matrix.profile }}/.
@@ -207,6 +211,7 @@ jobs:
     needs: prepare
 
     strategy:
+      fail-fast: false
       matrix:
         profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
         arch:
@@ -222,8 +227,11 @@ jobs:
           - centos8
           - centos7
           - centos6
-          - raspbian10
+          # - raspbian10 #armv6l is too slow to emulate
           # - raspbian9
+        otp_version:
+          #- 23.2.7.2-emqx-3
+          - 23.3.4.9-3
         exclude:
         - os: centos6
           arch: arm64
@@ -260,35 +268,9 @@ jobs:
         path: .
     - name: unzip source code
       run: unzip -q source.zip
-    - name: downloads old emqx zip packages
-      env:
-        PROFILE: ${{ matrix.profile }}
-        ARCH: ${{ matrix.arch }}
-        SYSTEM: ${{ matrix.os }}
-        OLD_VSNS: ${{ needs.prepare.outputs.old_vsns }}
-      run: |
-        set -e -x -u
-        broker=$PROFILE
-        if [ $PROFILE = "emqx" ];then
-            broker="emqx-ce"
-        fi
-        if [ ! -z "$(echo $SYSTEM | grep -oE 'raspbian')" ]; then
-            export ARCH="arm"
-        fi
-
-        mkdir -p source/_upgrade_base
-        cd source/_upgrade_base
-        old_vsns=($(echo $OLD_VSNS | tr ' ' ' '))
-        for tag in ${old_vsns[@]}; do
-          if [ ! -z "$(echo $(curl -I -m 10 -o /dev/null -s -w %{http_code} https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip) | grep -oE "^[23]+")" ];then
-            wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip
-            wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip.sha256
-            echo "$(cat $PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip.sha256) $PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip" | sha256sum -c || exit 1
-          fi
-        done
     - name: build emqx packages
       env:
-        ERL_OTP: erl23.2.7.2-emqx-2
+        ERL_OTP: erl${{ matrix.otp_version }}
         PROFILE: ${{ matrix.profile }}
         ARCH: ${{ matrix.arch }}
         SYSTEM: ${{ matrix.os }}
@@ -313,16 +295,25 @@ jobs:
     - name: create sha256
       env:
         PROFILE: ${{ matrix.profile }}
+        ERL_OTP: erl${{ matrix.otp_version }}
+        ARCH: ${{ matrix.arch }}
       run: |
         if [ -d /tmp/packages/$PROFILE ]; then
           cd /tmp/packages/$PROFILE
             for var in $(ls emqx-* ); do
+              if [[ $ERL_OTP == erl23.2* ]]; then
+                # Keep package with new OTP as default
+                # But move package with old otp to track 2
+                echo "rename track 2 package"
+                oldfile="$var"
+                var="${var/${ARCH}/otp23.2-${ARCH}}"
+                mv "$oldfile" "$var"
+              fi
               bash -c "echo $(sha256sum $var | awk '{print $1}') > $var.sha256"
             done
           cd -
         fi
     - uses: actions/upload-artifact@v1
-      if: startsWith(github.ref, 'refs/tags/')
       with:
         name: ${{ matrix.profile }}
         path: /tmp/packages/${{ matrix.profile }}/.
@@ -333,19 +324,18 @@ jobs:
     needs: prepare
 
     strategy:
+      fail-fast: false
       matrix:
         profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
-        arch:
+        registry:
-          - [amd64, x86_64]
+          - 'docker.io'
-          - [arm64v8, aarch64]
+          - 'public.ecr.aws'
-          - [arm32v7, arm]
-          - [i386, i386]
-          - [s390x, s390x]
         exclude:
+          # we don't have an aws ecr repo for enterprise and edge yet
+          - profile: emqx-edge
+            registry: 'public.ecr.aws'
           - profile: emqx-ee
-            arch: [i386, i386]
+            registry: 'public.ecr.aws'
-          - profile: emqx-ee
-            arch: [s390x, s390x]
 
     steps:
     - uses: actions/download-artifact@v2
@@ -354,32 +344,72 @@ jobs:
         path: .
     - name: unzip source code
       run: unzip -q source.zip
-    - name: build emqx docker image
+    - uses: docker/setup-buildx-action@v1
-      env:
+    - uses: docker/setup-qemu-action@v1
-        PROFILE: ${{ matrix.profile }}
-        ARCH: ${{ matrix.arch[0] }}
-        QEMU_ARCH: ${{ matrix.arch[1] }}
-      run: |
-        sudo docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
-
-        cd source
-        sudo TARGET=emqx/$PROFILE ARCH=$ARCH QEMU_ARCH=$QEMU_ARCH  make docker
-        cd _packages/$PROFILE && for var in $(ls ${PROFILE}-docker-* ); do sudo bash -c "echo $(sha256sum $var | awk '{print $1}') > $var.sha256"; done && cd -
-    - uses: actions/upload-artifact@v1
-      if: startsWith(github.ref, 'refs/tags/')
       with:
-        name: ${{ matrix.profile }}
+        image: tonistiigi/binfmt:latest
-        path: source/_packages/${{ matrix.profile }}/.
+        platforms: all
-
+    - uses: aws-actions/configure-aws-credentials@v1
-  delete-artifact:
+      if: matrix.registry == 'public.ecr.aws'
-    runs-on: ubuntu-20.04
-    needs: [prepare, mac, linux, docker]
-    steps:
-    - uses: geekyeggo/delete-artifact@v1
       with:
-        name: source
+        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        aws-region: ${{ secrets.AWS_DEFAULT_REGION }}
+    - name: Docker login to aws ecr
+      if: matrix.registry == 'public.ecr.aws'
+      run: aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws
+    - uses: docker/login-action@v1
+      if: matrix.registry == 'docker.io'
+      with:
+        username: ${{ secrets.DOCKER_HUB_USER }}
+        password: ${{ secrets.DOCKER_HUB_TOKEN }}
+    - uses: docker/metadata-action@v3
+      id: meta
+      with:
+        images: ${{ matrix.registry }}/${{ github.repository_owner }}/${{ matrix.profile }}
+        ## only 5.0 is latest
+        flavor: |
+          latest=false
+        tags: |
+          type=ref,event=branch
+          type=ref,event=pr
+          type=match,pattern=[v|e](.*),group=1
+        labels:
+          org.opencontainers.image.otp.version=${{ matrix.otp }}
+    - uses: docker/build-push-action@v2
+      if: matrix.profile != 'emqx-ee'
+      with:
+        ## only push when stable tag and rc tag
+        push: ${{ contains(github.ref, 'tags') && !contains(github.ref_name, 'beta') && !contains(github.ref_name, 'alpha') }}
+        pull: true
+        no-cache: true
+        platforms: linux/amd64,linux/arm64
+        tags: ${{ steps.meta.outputs.tags }}
+        labels: ${{ steps.meta.outputs.labels }}
+        build-args: |
+          BUILD_FROM=emqx/build-env:erl23.3.4.9-3-alpine
+          RUN_FROM=alpine:3.12
+          EMQX_NAME=${{ matrix.profile }}
+        file: source/deploy/docker/Dockerfile
+        context: source
+    - uses: docker/build-push-action@v2
+      if: matrix.profile == 'emqx-ee'
+      with:
+        ## only push when stable tag and rc tag
+        push: ${{ contains(github.ref, 'tags') && !contains(github.ref_name, 'beta') && !contains(github.ref_name, 'alpha') }}
+        pull: true
+        no-cache: true
+        platforms: linux/amd64,linux/arm64
+        tags: ${{ steps.meta.outputs.tags }}
+        labels: ${{ steps.meta.outputs.labels }}
+        build-args: |
+          BUILD_FROM=emqx/build-env:erl23.3.4.9-3-alpine
+          RUN_FROM=alpine:3.12
+          EMQX_NAME=${{ matrix.profile }}
+        file: source/deploy/docker/Dockerfile.enterprise
+        context: source
 
-  upload:
+  publish_artifacts:
     runs-on: ubuntu-20.04
 
     if: startsWith(github.ref, 'refs/tags/')
@@ -387,106 +417,37 @@ jobs:
     needs: [prepare, mac, linux, docker]
 
     strategy:
+      fail-fast: false
       matrix:
         profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
-
     steps:
-    - uses: actions/checkout@v2
-    - name: get_version
-      run: |
-        echo 'version<<EOF' >> $GITHUB_ENV
-        echo ${{ github.ref }} | sed -r  "s ^refs/heads/|^refs/tags/(.*) \1 g" >> $GITHUB_ENV
-        echo 'EOF' >> $GITHUB_ENV
     - uses: actions/download-artifact@v2
       with:
         name: ${{ matrix.profile }}
-        path: ./_packages/${{ matrix.profile }}
+        path: packages/${{ matrix.profile }}
     - name: install dos2unix
       run: sudo apt-get update && sudo apt install -y dos2unix
     - name: get packages
       run: |
         set -e -u
-        cd _packages/${{ matrix.profile }}
+        cd packages/${{ matrix.profile }}
         for var in $( ls |grep emqx |grep -v sha256); do
           dos2unix $var.sha256
           echo "$(cat $var.sha256) $var" | sha256sum -c || exit 1
         done
         cd -
-    - name: upload aws s3
+    - uses: aws-actions/configure-aws-credentials@v1
+      with:
+        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        aws-region: ${{ secrets.AWS_DEFAULT_REGION }}
+    - name: upload to aws s3
       run: |
         set -e -u
         if [ "${{ matrix.profile }}"  == "emqx" ];then
-            broker="emqx-ce"
+            s3dir="emqx-ce"
         else
-            broker=${{ matrix.profile }}
+            s3dir=${{ matrix.profile }}
         fi
-        aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID }}
+        aws s3 cp --recursive packages/${{ matrix.profile }} s3://${{ secrets.AWS_S3_BUCKET }}/${s3dir}/${{ github.ref_name }}
-        aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        aws cloudfront create-invalidation --distribution-id ${{ secrets.AWS_CLOUDFRONT_ID }} --paths "/${s3dir}/${{ github.ref_name }}/*"
-        aws configure set default.region ${{ secrets.AWS_DEFAULT_REGION }}
-
-        aws s3 cp --recursive _packages/${{ matrix.profile }} s3://${{ secrets.AWS_S3_BUCKET }}/$broker/${{ env.version }}
-        aws cloudfront create-invalidation --distribution-id ${{ secrets.AWS_CLOUDFRONT_ID }} --paths "/$broker/${{ env.version }}/*"
-    - uses: Rory-Z/upload-release-asset@v1
-      if: github.event_name == 'release' && matrix.profile != 'emqx-ee'
-      with:
-        repo: emqx
-        path: "_packages/${{ matrix.profile }}/emqx-*"
-        token: ${{ github.token }}
-    - uses: Rory-Z/upload-release-asset@v1
-      if: github.event_name == 'release' && matrix.profile == 'emqx-ee'
-      with:
-        repo: emqx-enterprise
-        path: "_packages/${{ matrix.profile }}/emqx-*"
-        token: ${{ github.token }}
-    - name: update to emqx.io
-      if: github.event_name == 'release'
-      run: |
-        set -e -x -u
-        curl -w %{http_code} \
-             --insecure \
-             -H "Content-Type: application/json" \
-             -H "token: ${{ secrets.EMQX_IO_TOKEN }}" \
-             -X POST \
-             -d "{\"repo\":\"emqx/emqx\", \"tag\": \"${{ env.version }}\" }" \
-             ${{ secrets.EMQX_IO_RELEASE_API }}
-    - name: push docker image to docker hub
-      if: github.event_name == 'release'
-      run: |
-        set -e -x -u
-        sudo make docker-prepare
-        cd _packages/${{ matrix.profile }} && for var in $(ls |grep docker |grep -v sha256); do unzip $var; sudo docker load < ${var%.*}; rm -f ${var%.*}; done && cd -
-        echo ${{ secrets.DOCKER_HUB_TOKEN }} |sudo docker login -u ${{ secrets.DOCKER_HUB_USER }} --password-stdin
-        sudo TARGET=emqx/${{ matrix.profile }} make docker-push
-        sudo TARGET=emqx/${{ matrix.profile }} make docker-manifest-list
-    - name: update repo.emqx.io
-      if: github.event_name == 'release' && endsWith(github.repository, 'enterprise') && matrix.profile == 'emqx-ee'
-      run: |
-        curl --silent --show-error \
-          -H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \
-          -H "Accept: application/vnd.github.v3+json" \
-          -X POST \
-          -d "{\"ref\":\"v1.0.1\",\"inputs\":{\"version\": \"${{ env.version }}\", \"emqx_ee\": \"true\"}}" \
-          "https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/update_emqx_repos.yaml/dispatches"
-    - name: update repo.emqx.io
-      if: github.event_name == 'release' && endsWith(github.repository, 'emqx') && matrix.profile == 'emqx'
-      run: |
-        curl --silent --show-error \
-          -H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \
-          -H "Accept: application/vnd.github.v3+json" \
-          -X POST \
-          -d "{\"ref\":\"v1.0.1\",\"inputs\":{\"version\": \"${{ env.version }}\", \"emqx_ce\": \"true\"}}" \
-          "https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/update_emqx_repos.yaml/dispatches"
-    - name: update homebrew packages
-      if: github.event_name == 'release' && endsWith(github.repository, 'emqx') && matrix.profile == 'emqx'
-      run: |
-        if [ -z $(echo $version | grep -oE "(alpha|beta|rc)\.[0-9]") ]; then
-            curl --silent --show-error \
-              -H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \
-              -H "Accept: application/vnd.github.v3+json" \
-              -X POST \
-              -d "{\"ref\":\"v1.0.1\",\"inputs\":{\"version\": \"${{ env.version }}\"}}" \
-              "https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/update_emqx_homebrew.yaml/dispatches"
-        fi
-    - uses: geekyeggo/delete-artifact@v1
-      with:
-        name: ${{ matrix.profile }}

.github/workflows/build_slim_packages.yaml

@@ -1,24 +1,30 @@
 name: Build slim packages
 
 on:
-  push:
-    tags:
-      - v*
-      - e*
   pull_request:
   workflow_dispatch:
 
 jobs:
   build:
-    runs-on: ubuntu-20.04
+    runs-on: ${{ matrix.runs-on }}
 
     strategy:
       matrix:
         erl_otp:
-        - erl23.2.7.2-emqx-2
+        - erl23.3.4.9-3
         os:
         - ubuntu20.04
         - centos7
+        runs-on:
+          - aws-amd64
+          - ubuntu-20.04
+        use-self-hosted:
+          - ${{ github.repository_owner == 'emqx' }}
+        exclude:
+          - runs-on: ubuntu-20.04
+            use-self-hosted: true
+          - runs-on: aws-amd64
+            use-self-hosted: false
 
     container: emqx/build-env:${{ matrix.erl_otp }}-${{ matrix.os }}
 
@@ -29,16 +35,31 @@ jobs:
         if make emqx-ee --dry-run > /dev/null 2>&1; then
           echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials
           git config --global credential.helper store
-          echo "${{ secrets.CI_GIT_TOKEN }}" >> ./scripts/git-token
           echo "EMQX_NAME=emqx-ee" >> $GITHUB_ENV
         else
           echo "EMQX_NAME=emqx" >> $GITHUB_ENV
         fi
+    - name: fix-git-unsafe-repository
+      run: git config --global --add safe.directory /__w/emqx/emqx
+    - uses: actions/cache@v2
+      with:
+        # dialyzer PLTs
+        path: ~/.cache/rebar3/
+        key: dialyer-${{ matrix.erl_otp }}
+    - name: make xref
+      run: make xref
+    - name: make dialyzer
+      run: make dialyzer
     - name: build zip packages
       run: make ${EMQX_NAME}-zip
     - name: build deb/rpm packages
       run: make ${EMQX_NAME}-pkg
-    - name: pakcages test
+    - uses: actions/upload-artifact@v1
+      if: failure()
+      with:
+        name: rebar3.crashdump
+        path: ./rebar3.crashdump
+    - name: packages test
       run: |
         export CODE_PATH=$GITHUB_WORKSPACE
         .ci/build_packages/tests.sh
@@ -48,13 +69,13 @@ jobs:
         path: _packages/**/*.zip
 
   mac:
-    runs-on: macos-10.15
-
     strategy:
       matrix:
         erl_otp:
-        - 23.2.7.2-emqx-2
+        - 23.3.4.9-3
-
+        macos:
+        - macos-11
+    runs-on: ${{ matrix.macos }}
     steps:
     - uses: actions/checkout@v1
     - name: prepare
@@ -62,7 +83,6 @@ jobs:
         if make emqx-ee --dry-run > /dev/null 2>&1; then
           echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials
           git config --global credential.helper store
-          echo "${{ secrets.CI_GIT_TOKEN }}" >> ./scripts/git-token
           echo "EMQX_NAME=emqx-ee" >> $GITHUB_ENV
         else
           echo "EMQX_NAME=emqx" >> $GITHUB_ENV
@@ -76,30 +96,47 @@ jobs:
     - uses: actions/cache@v2
       id: cache
       with:
-        path: ~/.kerl
+        path: ~/.kerl/${{ matrix.erl_otp }}
-        key: erl${{ matrix.erl_otp }}-macos10.15
+        key: otp-install-${{ matrix.erl_otp }}-${{ matrix.macos }}-static-ssl-disable-hipe-disable-jit
     - name: build erlang
       if: steps.cache.outputs.cache-hit != 'true'
       timeout-minutes: 60
       env:
         KERL_BUILD_BACKEND: git
         OTP_GITHUB_URL: https://github.com/emqx/otp
+        KERL_CONFIGURE_OPTIONS: --disable-dynamic-ssl-lib --with-ssl=/usr/local/opt/openssl@1.1 --disable-hipe --disable-jit
       run: |
         kerl update releases
         kerl build ${{ matrix.erl_otp }}
         kerl install ${{ matrix.erl_otp }} $HOME/.kerl/${{ matrix.erl_otp }}
     - name: build
+      env:
+        APPLE_SIGN_BINARIES: 1
+        APPLE_ID: developers@emqx.io
+        APPLE_TEAM_ID: 26N6HYJLZA
+        APPLE_ID_PASSWORD: ${{ secrets.APPLE_ID_PASSWORD }}
+        APPLE_DEVELOPER_IDENTITY: ${{ secrets.APPLE_DEVELOPER_IDENTITY }}
+        APPLE_DEVELOPER_ID_BUNDLE: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE }}
+        APPLE_DEVELOPER_ID_BUNDLE_PASSWORD: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE_PASSWORD }}
       run: |
         . $HOME/.kerl/${{ matrix.erl_otp }}/activate
         make ensure-rebar3
         sudo cp rebar3 /usr/local/bin/rebar3
         make ${EMQX_NAME}-zip
+    - uses: actions/upload-artifact@v1
+      if: failure()
+      with:
+        name: rebar3.crashdump
+        path: ./rebar3.crashdump
     - name: test
       run: |
         pkg_name=$(basename _packages/${EMQX_NAME}/emqx-*.zip)
         unzip -q _packages/${EMQX_NAME}/$pkg_name
         gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins
-        ./emqx/bin/emqx start || cat emqx/log/erlang.log.1
+        # test with a spaces in path
+        mv ./emqx "./emqx  home/"
+        cd "./emqx  home/"
+        ./bin/emqx start || cat log/erlang.log.1
         ready='no'
         for i in {1..10}; do
           if curl -fs 127.0.0.1:18083 > /dev/null; then
@@ -110,12 +147,13 @@ jobs:
         done
         if [ "$ready" != "yes" ]; then
           echo "Timed out waiting for emqx to be ready"
-          cat emqx/log/erlang.log.1
+          cat log/erlang.log.1
           exit 1
         fi
-        ./emqx/bin/emqx_ctl status
+        ./bin/emqx_ctl status
-        ./emqx/bin/emqx stop
+        ./bin/emqx stop
-        rm -rf emqx
+        cd ..
+        rm -rf "emqx  home"
     - uses: actions/upload-artifact@v2
       with:
         name: macos

.github/workflows/check_deps_integrity.yaml

@@ -5,7 +5,7 @@ on: [pull_request]
 jobs:
   check_deps_integrity:
     runs-on: ubuntu-20.04
-    container: emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04
+    container: emqx/build-env:erl23.3.4.9-3-ubuntu20.04
 
     steps:
       - uses: actions/checkout@v2

.github/workflows/elvis_lint.yaml

@@ -1,16 +0,0 @@
-name: Elvis Linter
-
-on: [pull_request]
-
-jobs:
-  build:
-    runs-on: ubuntu-20.04
-    steps:
-      - uses: actions/checkout@v2
-      - name: Set git token
-        if: endsWith(github.repository, 'enterprise')
-        run: |
-          echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials
-          git config --global credential.helper store
-      - run: |
-          ./scripts/elvis-check.sh $GITHUB_BASE_REF

.github/workflows/git_sync.yaml

@@ -3,7 +3,6 @@ name: Sync to enterprise
 on:
   push:
     branches:
-      - master
       - main-v*
 
 jobs:
@@ -23,11 +22,7 @@ jobs:
         id: create_pull_request
         run: |
           set -euo pipefail
-          if [ "$GITHUB_REF" = "refs/heads/master" ]; then
-            EE_REF="refs/heads/enterprise"
-          else
           EE_REF="${GITHUB_REF}-enterprise"
-          fi
           R=$(curl --silent --show-error \
             -H "Accept: application/vnd.github.v3+json" \
             -H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \

.github/workflows/release.yaml

@@ -0,0 +1,103 @@
+name: Upload release assets
+on:
+  release:
+    types:
+      - published
+
+jobs:
+  prepare:
+    runs-on: ubuntu-20.04
+    container: emqx/build-env:erl23.3.4.9-3-ubuntu20.04
+
+    outputs:
+      profiles: ${{ steps.set_profile.outputs.profiles}}
+      s3dir: ${{ steps.set_profile.outputs.s3dir}}
+
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          path: source
+          fetch-depth: 0
+      - name: set profile
+        id: set_profile
+        shell: bash
+        run: |
+          cd source
+          if make emqx-ee --dry-run > /dev/null 2>&1; then
+            echo "::set-output name=profiles::[\"emqx-ee\"]"
+          else
+            echo "::set-output name=profiles::[\"emqx\", \"emqx-edge\"]"
+          fi
+
+  upload:
+    runs-on: ubuntu-20.04
+    needs: prepare
+    strategy:
+      fail-fast: false
+      matrix:
+        profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
+
+    steps:
+      - uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ secrets.AWS_DEFAULT_REGION }}
+      - name: Get packages
+        run: |
+          if [ "${{ matrix.profile }}"  == "emqx" ];then
+              s3dir="emqx-ce"
+          else
+              s3dir=${{ matrix.profile }}
+          fi
+          aws s3 cp --recursive  s3://${{ secrets.AWS_S3_BUCKET }}/$s3dir/${{ github.ref_name }} packages
+      - uses: alexellis/upload-assets@0.2.2
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        with:
+          asset_paths: '["packages/*"]'
+      - name: update to emqx.io
+        run: |
+          set -e -x -u
+          curl -w %{http_code} \
+               --insecure \
+               -H "Content-Type: application/json" \
+               -H "token: ${{ secrets.EMQX_IO_TOKEN }}" \
+               -X POST \
+               -d "{\"repo\":\"emqx/emqx\", \"tag\": \"${{ github.ref_name }}\" }" \
+               ${{ secrets.EMQX_IO_RELEASE_API }}
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: get version
+        id: version
+        run: echo "::set-output name=version::$(./pkg-vsn.sh)"
+      - uses: emqx/push-helm-action@v1
+        if: github.event_name == 'release' && endsWith(github.repository, 'emqx') && matrix.profile == 'emqx'
+        with:
+          charts_dir: "${{ github.workspace }}/deploy/charts/emqx"
+          version: ${{ steps.version.outputs.version }}
+          aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws_region: "us-west-2"
+          aws_bucket_name: "repos-emqx-io"
+      - uses: emqx/push-helm-action@v1
+        if: github.event_name == 'release' && endsWith(github.repository, 'enterprise') && matrix.profile == 'emqx-ee'
+        with:
+          charts_dir: "${{ github.workspace }}/deploy/charts/emqx-ee"
+          version: ${{ steps.version.outputs.version }}
+          aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws_region: "us-west-2"
+          aws_bucket_name: "repos-emqx-io"
+      - name: update homebrew packages
+        if: github.event_name == 'release' && endsWith(github.repository, 'emqx') && matrix.profile == 'emqx'
+        run: |
+          if [ -z $(echo $version | grep -oE "(alpha|beta|rc)\.[0-9]") ]; then
+              curl --silent --show-error \
+                -H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \
+                -H "Accept: application/vnd.github.v3+json" \
+                -X POST \
+                -d "{\"ref\":\"v1.0.3\",\"inputs\":{\"version\": \"${{ github.ref_name }}\"}}" \
+                "https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/update_emqx_homebrew.yaml/dispatches"
+          fi

.github/workflows/run_acl_migration_tests.yaml

@@ -0,0 +1,22 @@
+name: ACL fix & migration integration tests
+
+on: workflow_dispatch
+
+jobs:
+    test:
+        runs-on: ubuntu-20.04
+        container: emqx/build-env:erl23.3.4.9-3-ubuntu20.04
+        strategy:
+            fail-fast: true
+        env:
+            BASE_VERSION: "4.3.0"
+        steps:
+        - uses: actions/checkout@v2
+          with:
+            path: emqx
+        - name: Prepare scripts
+          run: |
+            cp ./emqx/.ci/acl_migration_test/*.sh ./
+        - name: Run tests
+          run: |
+            ./suite.sh emqx "$BASE_VERSION"

.github/workflows/run_automate_tests.yaml

@@ -0,0 +1,445 @@
+name: Integration Test Suites
+
+on:
+  push:
+    tags:
+      - "v4.*"
+  pull_request:
+    branches:
+      - "main-v4.*"
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    outputs:
+      imgname: ${{ steps.build_docker.outputs.imgname}}
+      version: ${{ steps.build_docker.outputs.version}}
+    steps:
+    - name: download jmeter
+      id: dload_jmeter
+      timeout-minutes: 1
+      env:
+        JMETER_VERSION: 5.4.3
+      run: |
+        wget --no-verbose --no-check-certificate -O /tmp/apache-jmeter.tgz https://downloads.apache.org/jmeter/binaries/apache-jmeter-$JMETER_VERSION.tgz
+    - uses: actions/upload-artifact@v2
+      with:
+        name: apache-jmeter.tgz
+        path: /tmp/apache-jmeter.tgz
+    - uses: actions/checkout@v2
+    - uses: erlef/setup-beam@v1
+      with:
+        otp-version: "23.3.4.13"
+    - name: build docker
+      id: build_docker
+      run: |
+        if [ -f EMQX_ENTERPRISE ]; then
+          echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials
+          git config --global credential.helper store
+          make deps-emqx-ee
+          make clean
+          make emqx-ee-docker
+          echo "::set-output name=imgname::emqx-ee"
+          echo "::set-output name=version::$(./pkg-vsn.sh)"
+          docker save emqx/emqx-ee:$(./pkg-vsn.sh) -o emqx.tar
+        else
+          make emqx-docker
+          echo "::set-output name=imgname::emqx"
+          echo "::set-output name=version::$(./pkg-vsn.sh)"
+          docker save emqx/emqx:$(./pkg-vsn.sh) -o emqx.tar
+        fi
+    - uses: actions/upload-artifact@v2
+      with:
+        name: emqx-docker-image
+        path: emqx.tar
+
+  webhook:
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        webhook_type:
+        - webhook_data_bridge
+
+    needs: build
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/download-artifact@v2
+      with:
+        name: emqx-docker-image
+        path: /tmp
+    - name: load docker image
+      run: docker load < /tmp/emqx.tar
+    - name: docker compose up
+      timeout-minutes: 5
+      env:
+        TARGET: emqx/${{ needs.build.outputs.imgname }}
+        EMQX_TAG: ${{ needs.build.outputs.version }}
+      run: |
+        docker-compose \
+          -f .ci/docker-compose-file/docker-compose-emqx-cluster.yaml \
+          up -d --build
+    - uses: actions/checkout@v2
+      with:
+        repository: emqx/emqx-svt-web-server
+        ref: web-server-1.0
+        path: emqx-svt-web-server
+    - uses: actions/download-artifact@v2
+    - name: run webserver in docker
+      run: |
+        cd ./emqx-svt-web-server/svtserver
+        mvn clean package
+        cd target
+        docker run --name webserver --network emqx_bridge -d -v $(pwd)/svtserver-0.0.1.jar:/webserver/svtserver-0.0.1.jar --workdir /webserver openjdk:8-jdk bash \
+        -c "java -jar svtserver-0.0.1.jar"
+    - name: wait docker compose up
+      timeout-minutes: 5
+      run: |
+        while [ "$(docker inspect -f '{{ .State.Health.Status}}' node1.emqx.io)" != "healthy" ] || [ "$(docker inspect -f '{{ .State.Health.Status}}' node2.emqx.io)" != "healthy" ]; do
+          echo "['$(date -u +"%y-%m-%dt%h:%m:%sz")']:waiting emqx";
+          sleep 5;
+        done
+        docker ps -a
+        echo HAPROXY_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' haproxy) >> $GITHUB_ENV
+        echo WEB_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' webserver) >> $GITHUB_ENV
+    - uses: actions/checkout@v2
+      with:
+        repository: emqx/emqx-fvt
+        ref: v1.6.0
+        path: scripts
+    - uses: actions/setup-java@v1
+      with:
+        java-version: '8.0.282' # The JDK version to make available on the path.
+        java-package: jdk # (jre, jdk, or jdk+fx) - defaults to jdk
+        architecture: x64 # (x64 or x86) - defaults to x64
+    - uses: actions/download-artifact@v2
+      with:
+        name: apache-jmeter.tgz
+        path: /tmp
+    - name: install jmeter
+      timeout-minutes: 10
+      env:
+          JMETER_VERSION: 5.4.3
+      run: |
+        cd /tmp && tar -xvf apache-jmeter.tgz
+        echo "jmeter.save.saveservice.output_format=xml" >> /tmp/apache-jmeter-$JMETER_VERSION/user.properties
+        echo "jmeter.save.saveservice.response_data.on_error=true" >> /tmp/apache-jmeter-$JMETER_VERSION/user.properties
+        wget --no-verbose -O /tmp/apache-jmeter-$JMETER_VERSION/lib/ext/mqtt-xmeter-2.0.2-jar-with-dependencies.jar https://raw.githubusercontent.com/xmeter-net/mqtt-jmeter/master/Download/v2.0.2/mqtt-xmeter-2.0.2-jar-with-dependencies.jar
+        ln -s /tmp/apache-jmeter-$JMETER_VERSION /opt/jmeter
+    - name: run jmeter
+      run: |
+        /opt/jmeter/bin/jmeter.sh \
+          -Jjmeter.save.saveservice.output_format=xml -n \
+          -t scripts/automate-test-suite/${{ matrix.webhook_type }}.jmx \
+          -Demqx_ip=$HAPROXY_IP \
+          -Dweb_ip=$WEB_IP \
+          -l jmeter_logs/webhook_${{ matrix.webhook_type }}.jtl \
+          -j jmeter_logs/logs/webhook_${{ matrix.webhook_type }}.log
+    - name: check logs
+      run: |
+        if cat jmeter_logs/webhook_${{ matrix.webhook_type }}.jtl | grep -e '<failure>true</failure>' > /dev/null 2>&1; then
+          echo "check logs filed"
+          exit 1
+        fi
+    - uses: actions/upload-artifact@v1
+      if: always()
+      with:
+        name: jmeter_logs
+        path: ./jmeter_logs
+
+  mysql:
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        mysql_tag:
+        - 5.7
+        - 8
+        mysql_type:
+        - mysql_auth_acl
+
+    needs: build
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/download-artifact@v2
+      with:
+        name: emqx-docker-image
+        path: /tmp
+    - name: load docker image
+      run: docker load < /tmp/emqx.tar
+    - name: docker compose up
+      timeout-minutes: 5
+      env:
+        TARGET: emqx/${{ needs.build.outputs.imgname }}
+        EMQX_TAG: ${{ needs.build.outputs.version }}
+        MYSQL_TAG: ${{ matrix.mysql_tag }}
+      run: |
+        docker-compose \
+          -f .ci/docker-compose-file/docker-compose-emqx-cluster.yaml \
+          -f .ci/docker-compose-file/docker-compose-mysql-tls.yaml \
+          up -d --build
+    - name: wait docker compose up
+      timeout-minutes: 5
+      run: |
+        while [ "$(docker inspect -f '{{ .State.Health.Status}}' node1.emqx.io)" != "healthy" ] || [ "$(docker inspect -f '{{ .State.Health.Status}}' node2.emqx.io)" != "healthy" ]; do
+          echo "['$(date -u +"%y-%m-%dt%h:%m:%sz")']:waiting emqx";
+          sleep 5;
+        done
+        while [ $(docker ps -a --filter name=client --filter exited=0 | wc -l) \
+             != $(docker ps -a --filter name=client | wc -l) ]; do
+          sleep 1
+        done
+        docker ps -a
+        echo HAPROXY_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' haproxy) >> $GITHUB_ENV
+        echo MYSQL_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' mysql) >> $GITHUB_ENV
+    - uses: actions/checkout@v2
+      with:
+        repository: emqx/emqx-fvt
+        ref: v1.6.0
+        path: scripts
+    - uses: actions/setup-java@v1
+      with:
+        java-version: '8.0.282' # The JDK version to make available on the path.
+        java-package: jdk # (jre, jdk, or jdk+fx) - defaults to jdk
+        architecture: x64 # (x64 or x86) - defaults to x64
+    - uses: actions/download-artifact@v2
+      with:
+        name: apache-jmeter.tgz
+        path: /tmp
+    - name: install jmeter
+      timeout-minutes: 10
+      env:
+          JMETER_VERSION: 5.4.3
+      run: |
+        cd /tmp && tar -xvf apache-jmeter.tgz
+        echo "jmeter.save.saveservice.output_format=xml" >> /tmp/apache-jmeter-$JMETER_VERSION/user.properties
+        echo "jmeter.save.saveservice.response_data.on_error=true" >> /tmp/apache-jmeter-$JMETER_VERSION/user.properties
+        wget --no-verbose -O /tmp/apache-jmeter-$JMETER_VERSION/lib/ext/mqtt-xmeter-2.0.2-jar-with-dependencies.jar https://raw.githubusercontent.com/xmeter-net/mqtt-jmeter/master/Download/v2.0.2/mqtt-xmeter-2.0.2-jar-with-dependencies.jar
+        ln -s /tmp/apache-jmeter-$JMETER_VERSION /opt/jmeter
+    - name: install jmeter plugin
+      run: |
+        wget --no-verbose -O "/opt/jmeter/lib/mysql-connector-java-8.0.16.jar" https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar
+    - name: run jmeter
+      run: |
+        /opt/jmeter/bin/jmeter.sh \
+          -Jjmeter.save.saveservice.output_format=xml -n \
+          -t scripts/automate-test-suite/${{ matrix.mysql_type }}.jmx \
+          -Droute="apps/emqx_auth_mysql/test/emqx_auth_mysql_SUITE_data" \
+          -Dmysql_ip=$MYSQL_IP \
+          -Demqx_ip=$HAPROXY_IP \
+          -Ddbname="mqtt" \
+          -Dmysql_user="ssluser" \
+          -Ddb_user="root" \
+          -Dmysql_pwd="public" \
+          -Dconfig_path="/tmp/etc" \
+          -Ddocker_path=".ci/docker-compose-file" \
+          -l jmeter_logs/${{ matrix.mysql_type }}_${{ matrix.mysql_tag }}.jtl \
+          -j jmeter_logs/logs/${{ matrix.mysql_type }}_${{ matrix.mysql_tag }}.log
+    - name: check logs
+      run: |
+        if cat jmeter_logs/${{ matrix.mysql_type }}_${{ matrix.mysql_tag }}.jtl | grep -e '<failure>true</failure>' > /dev/null 2>&1; then
+          echo "check logs filed"
+          exit 1
+        fi
+    - uses: actions/upload-artifact@v1
+      if: always()
+      with:
+        name: jmeter_logs
+        path: ./jmeter_logs
+
+
+  postgresql:
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        pgsql_type:
+        - pgsql_auth_acl
+        pgsql_tag:
+        - 9
+        - 10
+        - 11
+        - 12
+        - 13
+
+    needs: build
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/download-artifact@v2
+      with:
+        name: emqx-docker-image
+        path: /tmp
+    - name: load docker image
+      run: docker load < /tmp/emqx.tar
+    - name: docker compose up
+      timeout-minutes: 5
+      env:
+        TARGET: emqx/${{ needs.build.outputs.imgname }}
+        EMQX_TAG: ${{ needs.build.outputs.version }}
+        PGSQL_TAG: ${{ matrix.pgsql_tag }}
+      run: |
+        docker-compose \
+          -f .ci/docker-compose-file/docker-compose-emqx-broker-cluster.yaml \
+          -f .ci/docker-compose-file/docker-compose-pgsql-tls.yaml \
+          up -d --build
+    - name: wait docker compose up
+      timeout-minutes: 5
+      run: |
+        while [ "$(docker inspect -f '{{ .State.Health.Status}}' node1.emqx.io)" != "healthy" ] || [ "$(docker inspect -f '{{ .State.Health.Status}}' node2.emqx.io)" != "healthy" ]; do
+          echo "['$(date -u +"%y-%m-%dt%h:%m:%sz")']:waiting emqx";
+          sleep 5;
+        done
+        docker ps -a
+        echo HAPROXY_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' haproxy) >> $GITHUB_ENV
+        echo PGSQL_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' pgsql) >> $GITHUB_ENV
+        echo CONFIG_PATH=$(docker inspect -f '{{ range .Mounts }}{{ if eq .Name "docker-compose-file_etc" }}{{ .Source }}{{ end }}{{ end }}' node1.emqx.io) >> $GITHUB_ENV
+    - uses: actions/checkout@v2
+      with:
+        repository: emqx/emqx-fvt
+        ref: v1.6.0
+        path: scripts
+    - uses: actions/setup-java@v1
+      with:
+        java-version: '8.0.282' # The JDK version to make available on the path.
+        java-package: jdk # (jre, jdk, or jdk+fx) - defaults to jdk
+        architecture: x64 # (x64 or x86) - defaults to x64
+    - uses: actions/download-artifact@v2
+      with:
+        name: apache-jmeter.tgz
+        path: /tmp
+    - name: install jmeter
+      timeout-minutes: 10
+      env:
+          JMETER_VERSION: 5.4.3
+      run: |
+        cd /tmp && tar -xvf apache-jmeter.tgz
+        echo "jmeter.save.saveservice.output_format=xml" >> /tmp/apache-jmeter-$JMETER_VERSION/user.properties
+        echo "jmeter.save.saveservice.response_data.on_error=true" >> /tmp/apache-jmeter-$JMETER_VERSION/user.properties
+        wget --no-verbose -O /tmp/apache-jmeter-$JMETER_VERSION/lib/ext/mqtt-xmeter-2.0.2-jar-with-dependencies.jar https://raw.githubusercontent.com/xmeter-net/mqtt-jmeter/master/Download/v2.0.2/mqtt-xmeter-2.0.2-jar-with-dependencies.jar
+        ln -s /tmp/apache-jmeter-$JMETER_VERSION /opt/jmeter
+    - name: install jmeter plugin
+      run: |
+        wget --no-verbose -O "/opt/jmeter/lib/postgresql-42.2.18.jar" https://repo1.maven.org/maven2/org/postgresql/postgresql/42.2.18/postgresql-42.2.18.jar
+    - name: run jmeter
+      run: |
+        sudo /opt/jmeter/bin/jmeter.sh \
+          -Jjmeter.save.saveservice.output_format=xml -n \
+          -t scripts/automate-test-suite/${{ matrix.pgsql_type }}.jmx \
+          -Droute="apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data" \
+          -Dca_name="ca.pem" \
+          -Dkey_name="client-key.pem" \
+          -Dcert_name="client-cert.pem" \
+          -Ddb_ip=$PGSQL_IP \
+          -Dpgsql_ip=$PGSQL_IP \
+          -Demqx_ip=$HAPROXY_IP \
+          -Dpgsql_user="root" \
+          -Dpgsql_pwd="public" \
+          -Ddbname="mqtt" \
+          -Dpgsql_db="mqtt" \
+          -Dport="5432" \
+          -Dconfig_path=$CONFIG_PATH \
+          -Ddocker_path=".ci/docker-compose-file" \
+          -l jmeter_logs/${{ matrix.pgsql_type }}_${{ matrix.pgsql_tag }}.jtl \
+          -j jmeter_logs/logs/${{ matrix.pgsql_type }}_${{ matrix.pgsql_tag }}.log
+    - name: check logs
+      run: |
+        if cat jmeter_logs/${{ matrix.pgsql_type }}_${{ matrix.pgsql_tag }}.jtl | grep -e '<failure>true</failure>' > /dev/null 2>&1; then
+          echo "check logs filed"
+          exit 1
+        fi
+    - uses: actions/upload-artifact@v1
+      if: always()
+      with:
+        name: jmeter_logs
+        path: ./jmeter_logs
+
+  http:
+    runs-on: ubuntu-latest
+
+    needs: build
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/download-artifact@v2
+      with:
+        name: emqx-docker-image
+        path: /tmp
+    - name: load docker image
+      run: docker load < /tmp/emqx.tar
+    - name: docker compose up
+      timeout-minutes: 5
+      env:
+        TARGET: emqx/${{ needs.build.outputs.imgname }}
+        EMQX_TAG: ${{ needs.build.outputs.version }}
+        MYSQL_TAG: 8
+      run: |
+        docker-compose \
+          -f .ci/docker-compose-file/docker-compose-emqx-broker-cluster.yaml \
+          -f .ci/docker-compose-file/docker-compose-mysql-tcp.yaml \
+          -f .ci/docker-compose-file/docker-compose-enterprise-tomcat-tcp.yaml \
+          up -d --build
+    - name: wait docker compose up
+      timeout-minutes: 5
+      run: |
+        while [ "$(docker inspect -f '{{ .State.Health.Status}}' node1.emqx.io)" != "healthy" ] || [ "$(docker inspect -f '{{ .State.Health.Status}}' node2.emqx.io)" != "healthy" ]; do
+          echo "['$(date -u +"%y-%m-%dt%h:%m:%sz")']:waiting emqx";
+          sleep 5;
+        done
+        docker ps -a
+        echo HAPROXY_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' haproxy) >> $GITHUB_ENV
+        echo HTTP_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' Tomcat) >> $GITHUB_ENV
+        echo MYSQL_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' mysql) >> $GITHUB_ENV
+        echo CONFIG_PATH=$(docker inspect -f '{{ range .Mounts }}{{ if eq .Name "docker-compose-file_etc" }}{{ .Source }}{{ end }}{{ end }}' node1.emqx.io) >> $GITHUB_ENV
+    - uses: actions/checkout@v2
+      with:
+        repository: emqx/emqx-fvt
+        ref: v1.6.0
+        path: scripts
+    - uses: actions/setup-java@v1
+      with:
+        java-version: '8.0.282' # The JDK version to make available on the path.
+        java-package: jdk # (jre, jdk, or jdk+fx) - defaults to jdk
+        architecture: x64 # (x64 or x86) - defaults to x64
+    - uses: actions/download-artifact@v2
+      with:
+        name: apache-jmeter.tgz
+        path: /tmp
+    - name: install jmeter
+      timeout-minutes: 10
+      env:
+          JMETER_VERSION: 5.4.3
+      run: |
+        cd /tmp && tar -xvf apache-jmeter.tgz
+        echo "jmeter.save.saveservice.output_format=xml" >> /tmp/apache-jmeter-$JMETER_VERSION/user.properties
+        echo "jmeter.save.saveservice.response_data.on_error=true" >> /tmp/apache-jmeter-$JMETER_VERSION/user.properties
+        wget --no-verbose -O /tmp/apache-jmeter-$JMETER_VERSION/lib/ext/mqtt-xmeter-2.0.2-jar-with-dependencies.jar https://raw.githubusercontent.com/xmeter-net/mqtt-jmeter/master/Download/v2.0.2/mqtt-xmeter-2.0.2-jar-with-dependencies.jar
+        ln -s /tmp/apache-jmeter-$JMETER_VERSION /opt/jmeter
+    - name: install jmeter plugin
+      run: |
+        wget --no-verbose -O "/opt/jmeter/lib/mysql-connector-java-8.0.16.jar" https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar
+    - name: run jmeter
+      run: |
+        sudo /opt/jmeter/bin/jmeter.sh \
+          -Jjmeter.save.saveservice.output_format=xml -n \
+          -t scripts/automate-test-suite/http_auth_acl.jmx \
+          -Dmysql_ip=$MYSQL_IP \
+          -Demqx_ip=$HAPROXY_IP \
+          -Dweb_server_ip=$HTTP_IP \
+          -Dconfig_path=$CONFIG_PATH \
+          -Ddocker_path=".ci/docker-compose-file" \
+          -l jmeter_logs/http_auth_acl.jtl \
+          -j jmeter_logs/logs/http_auth_acl.log
+    - name: check logs
+      run: |
+        if cat jmeter_logs/http_auth_acl.jtl | grep -e '<failure>true</failure>' > /dev/null 2>&1; then
+          echo "check logs filed"
+          sudo cat /var/lib/docker/volumes/docker-compose-file_etc/_data/emqx.conf
+          exit 1
+        fi
+    - uses: actions/upload-artifact@v1
+      if: always()
+      with:
+        name: jmeter_logs
+        path: ./jmeter_logs

.github/workflows/run_cts_tests.yaml

@@ -1,11 +1,13 @@
 name: Compatibility Test Suite
 
 on:
+  schedule:
+    - cron:  '0 */6 * * *'
+  pull_request:
   push:
     tags:
       - v*
       - e*
-  pull_request:
 
 jobs:
   ldap:
@@ -43,13 +45,16 @@ jobs:
           if make emqx-ee --dry-run > /dev/null 2>&1; then
             docker exec -i erlang bash -c "echo \"https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com\" > /root/.git-credentials && git config --global credential.helper store"
           fi
+      - name: fix-git-unsafe-repository
+        run: docker exec -i erlang sh -c "git config --global --add safe.directory /emqx"
       - name: run test cases
         run: |
-          export CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_
-          printenv > .env
           docker exec -i erlang sh -c "make ensure-rebar3"
-          docker exec -i erlang sh -c "./rebar3 eunit --dir apps/emqx_auth_ldap"
+          printenv | grep "^EMQX_" > .env
-          docker exec --env-file .env -i erlang sh -c "./rebar3 ct --dir apps/emqx_auth_ldap"
+          docker exec -i \
+            -e "CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_" \
+            --env-file .env \
+            erlang sh -c "make apps/emqx_auth_ldap-ct"
       - uses: actions/upload-artifact@v1
         if: failure()
         with:
@@ -112,13 +117,15 @@ jobs:
           if make emqx-ee --dry-run > /dev/null 2>&1; then
             docker exec -i erlang bash -c "echo \"https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com\" > /root/.git-credentials && git config --global credential.helper store"
           fi
+      - name: fix-git-unsafe-repository
+        run: docker exec -i erlang sh -c "git config --global --add safe.directory /emqx"
       - name: run test cases
         run: |
-          export CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_
+          printenv | grep "^EMQX_" > .env
-          printenv > .env
+          docker exec -i \
-          docker exec -i erlang sh -c "make ensure-rebar3"
+            -e "CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_" \
-          docker exec -i erlang sh -c "./rebar3 eunit --dir apps/emqx_auth_mongo"
+            --env-file .env \
-          docker exec --env-file .env -i erlang sh -c "./rebar3 ct --dir apps/emqx_auth_mongo"
+            erlang sh -c "make apps/emqx_auth_mongo-ct"
       - uses: actions/upload-artifact@v1
         if: failure()
         with:
@@ -194,13 +201,15 @@ jobs:
           if make emqx-ee --dry-run > /dev/null 2>&1; then
             docker exec -i erlang bash -c "echo \"https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com\" > /root/.git-credentials && git config --global credential.helper store"
           fi
+      - name: fix-git-unsafe-repository
+        run: docker exec -i erlang sh -c "git config --global --add safe.directory /emqx"
       - name: run test cases
         run: |
-          export CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_
+          printenv | grep "^EMQX_" > .env
-          printenv > .env
+          docker exec -i \
-          docker exec -i erlang sh -c "make ensure-rebar3"
+            -e "CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_" \
-          docker exec -i erlang sh -c "./rebar3 eunit --dir apps/emqx_auth_mysql"
+            --env-file .env \
-          docker exec --env-file .env -i erlang sh -c "./rebar3 ct --dir apps/emqx_auth_mysql"
+            erlang sh -c "make apps/emqx_auth_mysql-ct"
       - uses: actions/upload-artifact@v1
         if: failure()
         with:
@@ -265,16 +274,18 @@ jobs:
           if make emqx-ee --dry-run > /dev/null 2>&1; then
             docker exec -i erlang bash -c "echo \"https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com\" > /root/.git-credentials && git config --global credential.helper store"
           fi
+      - name: fix-git-unsafe-repository
+        run: docker exec -i erlang sh -c "git config --global --add safe.directory /emqx"
       - name: run test cases
         run: |
           export EMQX_AUTH__PGSQL__USERNAME=root \
                  EMQX_AUTH__PGSQL__PASSWORD=public \
-                 EMQX_AUTH__PGSQL__DATABASE=mqtt \
+                 EMQX_AUTH__PGSQL__DATABASE=mqtt
-                 CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_
+          printenv | grep "^EMQX_" > .env
-          printenv > .env
+          docker exec -i \
-          docker exec -i erlang sh -c "make ensure-rebar3"
+            -e "CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_" \
-          docker exec -i erlang sh -c "./rebar3 eunit --dir apps/emqx_auth_pgsql"
+            --env-file .env \
-          docker exec --env-file .env -i erlang sh -c "./rebar3 ct --dir apps/emqx_auth_pgsql"
+            erlang sh -c "make apps/emqx_auth_pgsql-ct"
       - uses: actions/upload-artifact@v1
         if: failure()
         with:
@@ -388,14 +399,16 @@ jobs:
           if make emqx-ee --dry-run > /dev/null 2>&1; then
             docker exec -i erlang bash -c "echo \"https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com\" > /root/.git-credentials && git config --global credential.helper store"
           fi
+      - name: fix-git-unsafe-repository
+        run: docker exec -i erlang sh -c "git config --global --add safe.directory /emqx"
       - name: run test cases
         run: |
-          export CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_
           export EMQX_AUTH__REIDS__PASSWORD=public
-          printenv > .env
+          printenv | grep "^EMQX_" > .env
-          docker exec -i erlang sh -c "make ensure-rebar3"
+          docker exec -i \
-          docker exec -i erlang sh -c "./rebar3 eunit --dir apps/emqx_auth_redis"
+            -e "CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_" \
-          docker exec --env-file .env -i erlang sh -c "./rebar3 ct --dir apps/emqx_auth_redis"
+            --env-file .env \
+            erlang sh -c "make apps/emqx_auth_redis-ct"
       - uses: actions/upload-artifact@v1
         if: failure()
         with:

.github/workflows/run_fvt_tests.yaml

@@ -13,25 +13,23 @@ jobs:
 
         steps:
         - uses: actions/checkout@v1
-        - uses: gleam-lang/setup-erlang@v1.1.2
+        - uses: erlef/setup-beam@v1
-          id: install_erlang
           with:
-            otp-version: 23.2
+            otp-version: "23.3.4.9"
-        - name: prepare
+        - name: make docker
           run: |
             if make emqx-ee --dry-run > /dev/null 2>&1; then
               echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials
               git config --global credential.helper store
-              echo "${{ secrets.CI_GIT_TOKEN }}" >> scripts/git-token
               make deps-emqx-ee
               echo "TARGET=emqx/emqx-ee" >> $GITHUB_ENV
               echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV
+              make emqx-ee-docker
             else
               echo "TARGET=emqx/emqx" >> $GITHUB_ENV
               echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV
+              make emqx-docker
             fi
-        - name: make emqx image
-          run: make docker
         - name: run emqx
           timeout-minutes: 5
           run: |
@@ -67,47 +65,33 @@ jobs:
 
         steps:
         - uses: actions/checkout@v1
-        - uses: gleam-lang/setup-erlang@v1.1.2
+        - uses: erlef/setup-beam@v1
-          id: install_erlang
           with:
-            otp-version: 23.2
+            otp-version: "23.3.4.9"
         - name: prepare
           run: |
             if make emqx-ee --dry-run > /dev/null 2>&1; then
               echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials
               git config --global credential.helper store
-              echo "${{ secrets.CI_GIT_TOKEN }}" >> scripts/git-token
               make deps-emqx-ee
               echo "TARGET=emqx/emqx-ee" >> $GITHUB_ENV
+              make emqx-ee-docker
             else
               echo "TARGET=emqx/emqx" >> $GITHUB_ENV
+              make emqx-docker
             fi
-        - name: make emqx image
+        - run: minikube start
-          run: make docker
-        - name: install k3s
-          env:
-            KUBECONFIG: "/etc/rancher/k3s/k3s.yaml"
-          run: |
-            sudo sh -c "echo \"127.0.0.1 $(hostname)\" >> /etc/hosts"
-            curl -sfL https://get.k3s.io | sh -
-            sudo chmod 644 /etc/rancher/k3s/k3s.yaml
-            kubectl cluster-info
         - name: install helm
-          env:
-            KUBECONFIG: "/etc/rancher/k3s/k3s.yaml"
           run: |
             curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3
             sudo chmod 700 get_helm.sh
             sudo ./get_helm.sh
             helm version
         - name: run emqx on chart
-          env:
-            KUBECONFIG: "/etc/rancher/k3s/k3s.yaml"
           timeout-minutes: 5
           run: |
             version=$(./pkg-vsn.sh)
-            sudo docker save ${TARGET}:$version -o emqx.tar.gz
+            minikube image load ${TARGET}:$version
-            sudo k3s ctr image import emqx.tar.gz
 
             sed -i -r "s/^appVersion: .*$/appVersion: \"${version}\"/g" deploy/charts/emqx/Chart.yaml
             sed -i '/emqx_telemetry/d' deploy/charts/emqx/values.yaml
@@ -130,11 +114,21 @@ jobs:
               echo "waiting emqx started";
               sleep 10;
             done
-        - name: get pods log
+        - name: get emqx-0 pods log
           if: failure()
-          env:
+          run: |
-            KUBECONFIG: "/etc/rancher/k3s/k3s.yaml"
+            kubectl describe pods emqx-0
-          run: kubectl describe pods emqx-0
+            kubectl logs emqx-0
+        - name: get emqx-1 pods log
+          if: failure()
+          run: |
+            kubectl describe pods emqx-1
+            kubectl logs emqx-1
+        - name: get emqx-2 pods log
+          if: failure()
+          run: |
+            kubectl describe pods emqx-2
+            kubectl logs emqx-2
         - uses: actions/checkout@v2
           with:
             repository: emqx/paho.mqtt.testing
@@ -145,16 +139,12 @@ jobs:
             pip install pytest
             echo "$HOME/.local/bin" >> $GITHUB_PATH
         - name: run paho test
-          env:
-            KUBECONFIG: "/etc/rancher/k3s/k3s.yaml"
           run: |
-            emqx_svc=$(kubectl get svc --namespace default emqx -o jsonpath="{.spec.clusterIP}")
+            nohup kubectl port-forward svc/emqx 1883:1883 &
-            emqx1=$(kubectl get pods emqx-1 -o jsonpath='{.status.podIP}')
-            emqx2=$(kubectl get pods emqx-2 -o jsonpath='{.status.podIP}')
 
-            pytest -v paho.mqtt.testing/interoperability/test_client/V5/test_connect.py -k test_basic --host $emqx_svc
+            pytest -v paho.mqtt.testing/interoperability/test_client/V5/test_connect.py -k test_basic
             RESULT=$?
-            pytest -v paho.mqtt.testing/interoperability/test_cluster --host1 $emqx1 --host2 $emqx2
+            pytest -v paho.mqtt.testing/interoperability/test_cluster
             RESULT=$((RESULT + $?))
             if [ 0 -ne $RESULT ]; then
                 kubectl logs emqx-1
@@ -162,117 +152,140 @@ jobs:
             fi
             exit $RESULT
 
-    relup_test:
+    relup_test_plan:
         runs-on: ubuntu-20.04
-        container: emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04
+        container: emqx/build-env:erl23.3.4.9-3-ubuntu20.04
+        outputs:
+          profile: ${{ steps.profile-and-versions.outputs.profile }}
+          vsn: ${{ steps.profile-and-versions.outputs.vsn }}
+          old_vsns: ${{ steps.profile-and-versions.outputs.old_vsns }}
+          broker: ${{ steps.profile-and-versions.outputs.broker }}
+          matrix: ${{ steps.generate-matrix.outputs.matrix }}
         defaults:
           run:
             shell: bash
         steps:
-        - uses: actions/setup-python@v2
-          with:
-            python-version: '3.8'
-            architecture: 'x64'
         - uses: actions/checkout@v2
+          name: Checkout
           with:
-            repository: emqx/paho.mqtt.testing
+            path: emqx
-            ref: develop-4.0
+            fetch-depth: 0
-            path: paho.mqtt.testing
+        - name: Get profile and version list
+          id: profile-and-versions
+          run: |
+            cd emqx
+            vsn="$(./pkg-vsn.sh)"
+            if make emqx-ee --dry-run > /dev/null 2>&1; then
+              profile="emqx-ee"
+              old_vsns="$(./scripts/relup-base-vsns.sh enterprise | xargs)"
+              broker="emqx-ee"
+            else
+              profile="emqx"
+              old_vsns="$(./scripts/relup-base-vsns.sh community | xargs)"
+              broker="emqx-ce"
+            fi
+
+            echo "OLD_VSNS=$old_vsns" >> $GITHUB_ENV
+
+            echo "::set-output name=vsn::$vsn"
+            echo "::set-output name=profile::$profile"
+            echo "::set-output name=broker::$broker"
+            echo "::set-output name=old_vsns::$old_vsns"
+        - name: Generate matrix
+          id: generate-matrix
+          run: |
+            matrix=$(echo -n "$OLD_VSNS" | sed 's/ $//g' | jq -R -s -c 'split(" ")')
+            echo "::set-output name=matrix::$matrix"
+
+    relup_test_build:
+        needs: relup_test_plan
+        runs-on: ubuntu-20.04
+        container: emqx/build-env:erl23.3.4.9-3-ubuntu20.04
+        defaults:
+          run:
+            shell: bash
+        env:
+          OLD_VSNS: "${{ needs.relup_test_plan.outputs.old_vsns }}"
+          PROFILE: "${{ needs.relup_test_plan.outputs.profile }}"
+          BROKER: "${{ needs.relup_test_plan.outputs.broker }}"
+        steps:
         - uses: actions/checkout@v2
+          name: Checkout
+          with:
+            path: emqx
+            fetch-depth: 0
+        - name: Prepare credentials
+          run: |
+            if [ "$PROFILE" = "emqx-ee" ]; then
+              echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials
+              git config --global credential.helper store
+            fi
+        - name: Build emqx
+          run: make -C emqx ${PROFILE}-zip
+        - uses: actions/upload-artifact@v2
+          name: Upload built emqx and test scenario
+          with:
+            name: emqx_built
+            path: |
+              emqx/_packages/*/*.zip
+              emqx/.ci/fvt_tests
+
+    relup_test_run:
+        needs:
+          - relup_test_plan
+          - relup_test_build
+        runs-on: ubuntu-20.04
+        container: emqx/relup-test-env:erl23.2.7.2-emqx-3-ubuntu20.04
+        strategy:
+          fail-fast: false
+          matrix:
+            old_vsn: ${{ fromJson(needs.relup_test_plan.outputs.matrix) }}
+        env:
+          OLD_VSN: "${{ matrix.old_vsn }}"
+          PROFILE: "${{ needs.relup_test_plan.outputs.profile }}"
+          VSN: "${{ needs.relup_test_plan.outputs.vsn }}"
+          BROKER: "${{ needs.relup_test_plan.outputs.broker }}"
+        defaults:
+          run:
+            shell: bash
+        steps:
+        - uses: actions/download-artifact@v2
+          name: Download built emqx and test scenario
+          with:
+            name: emqx_built
+            path: emqx_built
+        - uses: actions/checkout@v2
+          name: Checkout one_more_emqx
           with:
             repository: terry-xiaoyu/one_more_emqx
             ref: master
             path: one_more_emqx
-        - uses: actions/checkout@v2
+        - name: Prepare packages
-          with:
-            repository: emqx/emqtt-bench
-            ref: master
-            path: emqtt-bench
-        - uses: actions/checkout@v2
-          with:
-            repository: hawk/lux
-            ref: lux-2.6
-            path: lux
-        - uses: actions/checkout@v2
-          with:
-            repository: ${{ github.repository }}
-            path: emqx
-            fetch-depth: 0
-        - name: prepare
-          run: |
-            if make -C emqx emqx-ee --dry-run > /dev/null 2>&1; then
-              echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials
-              git config --global credential.helper store
-              echo "${{ secrets.CI_GIT_TOKEN }}" >> emqx/scripts/git-token
-              echo "PROFILE=emqx-ee" >> $GITHUB_ENV
-            else
-              echo "PROFILE=emqx" >> $GITHUB_ENV
-            fi
-        - name: get version
           run: |
             set -e -x -u
-            cd emqx
-            if [ $PROFILE = "emqx" ];then
-                broker="emqx-ce"
-                edition='opensource'
-            else
-                broker="emqx-ee"
-                edition='enterprise'
-            fi
-            echo "BROKER=$broker" >> $GITHUB_ENV
-
-            vsn="$(./pkg-vsn.sh)"
-            echo "VSN=$vsn" >> $GITHUB_ENV
-
-            pre_vsn="$(echo $vsn | grep -oE '^[0-9]+.[0-9]')"
-            if [ $PROFILE = "emqx" ]; then
-                old_vsns="$(git tag -l "v$pre_vsn.[0-9]" | xargs echo -n | sed "s/v$vsn//")"
-            else
-                old_vsns="$(git tag -l "e$pre_vsn.[0-9]" | xargs echo -n | sed "s/e$vsn//")"
-            fi
-            echo "OLD_VSNS=$old_vsns" >> $GITHUB_ENV
-        - name: download emqx
-          run: |
-            set -e -x -u
-            mkdir -p emqx/_upgrade_base
-            cd emqx/_upgrade_base
-            old_vsns=($(echo $OLD_VSNS | tr ' ' ' '))
-            for old_vsn in ${old_vsns[@]}; do
-              wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$BROKER/$old_vsn/$PROFILE-ubuntu20.04-${old_vsn#[e|v]}-amd64.zip
-            done
-        - name: build emqx
-          run: make -C emqx ${PROFILE}-zip
-        - name: build emqtt-bench
-          run: make -C emqtt-bench
-        - name: build lux
-          run: |
-            set -e -u -x
-            cd lux
-            autoconf
-            ./configure
-            make
-            make install
-        - name: run relup test
-          timeout-minutes: 20
-          run: |
-            set -e -x -u
-            if [ -n "$OLD_VSNS" ]; then
             mkdir -p packages
-                cp emqx/_packages/${PROFILE}/*.zip packages
+            cp emqx_built/_packages/*/*.zip packages
-                cp emqx/_upgrade_base/*.zip packages
+            cd packages
+            wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$BROKER/$OLD_VSN/$PROFILE-ubuntu20.04-${OLD_VSN#[e|v]}-amd64.zip
+        - name: Run relup test scenario
+          timeout-minutes: 5
+          run: |
             lux \
+            --progress verbose \
             --case_timeout infinity \
             --var PROFILE=$PROFILE \
             --var PACKAGE_PATH=$(pwd)/packages \
-                --var BENCH_PATH=$(pwd)/emqtt-bench \
             --var ONE_MORE_EMQX_PATH=$(pwd)/one_more_emqx \
             --var VSN="$VSN" \
-                --var OLD_VSNS="$OLD_VSNS" \
+            --var OLD_VSN="$OLD_VSN" \
-                emqx/.ci/fvt_tests/relup.lux
+            emqx_built/.ci/fvt_tests/relup.lux
-            fi
+        - uses: actions/upload-artifact@v2
-        - uses: actions/upload-artifact@v1
+          name: Save debug data
           if: failure()
           with:
-            name: lux_logs
+            name: debug_data
-            path: lux_logs
+            path: |
-
+              packages/emqx/log/emqx.log.1
+              packages/emqx2/log/emqx.log.1
+              packages/*.zip
+              lux_logs

.github/workflows/run_test_cases.yaml

@@ -5,29 +5,14 @@ on:
     tags:
       - v*
       - e*
+    branches:
+      - 'main-v4.[0-9]?'
   pull_request:
 
 jobs:
-    run_static_analysis:
-        runs-on: ubuntu-20.04
-        container: emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04
-
-        steps:
-        - uses: actions/checkout@v2
-        - name: set git credentials
-          run: |
-            if make emqx-ee --dry-run > /dev/null 2>&1; then
-              echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials
-              git config --global credential.helper store
-            fi
-        - name: xref
-          run: make xref
-        - name: dialyzer
-          run: make dialyzer
-
     run_proper_test:
         runs-on: ubuntu-20.04
-        container: emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04
+        container: emqx/build-env:erl23.3.4.9-3-ubuntu20.04
 
         steps:
         - uses: actions/checkout@v2
@@ -41,20 +26,24 @@ jobs:
           run: make proper
 
     run_common_test:
-        runs-on: ubuntu-20.04
+        runs-on: ${{ matrix.runs-on }}
-
+        strategy:
+          fail-fast: false
+          matrix:
+            runs-on:
+              - aws-amd64
+              - ubuntu-20.04
+            use-self-hosted:
+              - ${{ github.repository_owner == 'emqx' }}
+            exclude:
+              - runs-on: ubuntu-20.04
+                use-self-hosted: true
+              - runs-on: aws-amd64
+                use-self-hosted: false
         steps:
         - uses: actions/checkout@v2
-        - name: set edition
-          id: set_edition
-          run: |
-            if make emqx-ee --dry-run > /dev/null 2>&1; then
-                echo "EDITION=enterprise" >> $GITHUB_ENV
-            else
-                echo "EDITION=opensource" >> $GITHUB_ENV
-            fi
         - name: docker compose up
-          if: env.EDITION == 'opensource'
+          if: endsWith(github.repository, 'emqx')
           env:
             MYSQL_TAG: 8
             REDIS_TAG: 6
@@ -72,7 +61,7 @@ jobs:
                 -f .ci/docker-compose-file/docker-compose-redis-single-tcp.yaml \
                 up -d --build
         - name: docker compose up
-          if: env.EDITION == 'enterprise'
+          if: endsWith(github.repository, 'emqx-enterprise')
           env:
             MYSQL_TAG: 8
             REDIS_TAG: 6
@@ -110,6 +99,7 @@ jobs:
                 -f .ci/docker-compose-file/docker-compose-enterprise-pgsql-and-timescale-client.yaml \
                 up -d --build
             docker exec -i erlang bash -c "echo \"https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com\" > /root/.git-credentials && git config --global credential.helper store"
+            docker exec -i erlang bash -c "git config --global --add safe.directory /emqx"
             while [ $(docker ps -a --filter name=client --filter exited=0 | wc -l) \
                  != $(docker ps -a --filter name=client | wc -l) ]; do
               sleep 5
@@ -123,6 +113,7 @@ jobs:
         - name: run cover
           run: |
             printenv > .env
+            docker exec -i erlang bash -c "git config --global --add safe.directory /emqx"
             docker exec -i erlang bash -c "make cover"
             docker exec --env-file .env -i erlang bash -c "make coveralls"
         - name: cat rebar.crashdump

.gitignore

@@ -47,3 +47,16 @@ dist.zip
 scripts/git-token
 etc/*.seg
 _upgrade_base/
+erlang_ls.config
+.els_cache/
+# VSCode files
+.vs/
+.vscode/
+# Emacs Backup files
+*~
+# Emacs temporary files
+.#*
+*#
+# For direnv
+.envrc
+mix.lock

.tool-versions

@@ -1 +1 @@
-erlang 24.0.1-emqx-1
+erlang 23.3.4.9-3

CHANGES-4.3.md

@@ -0,0 +1,373 @@
+# EMQX 4.3 Changes
+
+Started tracking changes in CHANGE.md since EMQX v4.3.11
+
+NOTE: Keep prepending to the head of the file instead of the tail
+
+File format:
+
+- Use weight-2 heading for releases
+- One list item per change topic
+  Change log ends with a list of GitHub PRs
+
+## v4.3.22
+
+### Minor changes
+
+## v4.3.21
+
+### Enhancements
+
+- TLS listener memory usage optimization [#9005](https://github.com/emqx/emqx/pull/9005).
+  New config `listener.ssl.$NAME.hibernate_after` to hibernate TLS connection process after idling.
+  Hibernation can reduce RAM usage significantly, but may cost more CPU.
+  This configuration is by default disabled.
+  Our preliminary test shows a 50% of RAM usage decline when configured to '5s'.
+
+- TLS listener default buffer size to 4KB [#9007](https://github.com/emqx/emqx/pull/9007)
+  Eliminate uncertainty that the buffer size is set by OS default.
+
+- Disable authorization for `api/v4/emqx_prometheus` endpoint. [8955](https://github.com/emqx/emqx/pull/8955)
+
+- Added a test to prevent a last will testament message to be
+  published when a client is denied connection. [#8894](https://github.com/emqx/emqx/pull/8894)
+
+- QoS1 and QoS2 messages in session's buffer are re-dispatched to other members in the group
+  when the session terminates [#9094](https://github.com/emqx/emqx/pull/9094).
+  Prior to this enhancement, one would have to set `broker.shared_dispatch_ack_enabled` to true
+  to prevent sessions from buffering messages, however this acknowledgement comes with a cost.
+
+### Bug fixes
+
+- Fix delayed publish inaccurate caused by os time change. [#8908](https://github.com/emqx/emqx/pull/8908)
+
+- Hide redis password in error logs [#9071](https://github.com/emqx/emqx/pull/9071)
+  In this change, it also included more changes in redis client:
+  - Improve redis connection error logging [eredis:19](https://github.com/emqx/eredis/pull/19).
+    Also added support for eredis to accept an anonymous function as password instead of
+    passing around plaintext args which may get dumpped to crash logs (hard to predict where).
+    This change also added `format_status` callback for `gen_server` states which hold plaintext
+    password so the process termination log and `sys:get_status` will print '******' instead of
+    the password to console.
+  - Avoid pool name clashing [eredis_cluster#22](https://github.com/emqx/eredis_cluster/pull/22)
+    Same `format_status` callback is added here too for `gen_server`s which hold password in
+    their state.
+
+- Fix shared subscription message re-dispatches [#9094](https://github.com/emqx/emqx/pull/9094).
+  - When discarding QoS 2 inflight messages, there were excessive logs
+  - For wildcard deliveries, the re-dispatch used the wrong topic (the publishing topic,
+    but not the subscribing topic), caused messages to be lost when dispatching.
+
+## v4.3.20
+
+### Bug fixes
+
+- Fix rule-engine update behaviour which may initialize actions for disabled rules. [#8849](https://github.com/emqx/emqx/pull/8849)
+- Fix JWT plugin don't support non-integer timestamp claims. [#8862](https://github.com/emqx/emqx/pull/8862)
+- Fix a possible dead loop caused by shared subscriptions with `shared_dispatch_ack_enabled=true`. [#8918](https://github.com/emqx/emqx/pull/8918)
+- Fix dashboard binding IP address not working. [#8916](https://github.com/emqx/emqx/pull/8916)
+- Fix rule SQL topic matching to null values failed. [#8927](https://github.com/emqx/emqx/pull/8927)
+  The following SQL should not fail (crash) but return `{"r": false}`:
+  `SELECT topic =~ 't' as r FROM "$events/client_connected"`.
+  The topic is a null value as there's no such field in event `$events/client_connected`, so it
+  should return false if match it to a topic.
+
+## v4.3.19
+
+### Enhancements
+
+- Improve error message for LwM2M plugin when object ID is not valid. [#8654](https://github.com/emqx/emqx/pull/8654).
+- Add tzdata apk package to alpine docker image. [#8671](https://github.com/emqx/emqx/pull/8671)
+- Refine Rule Engine error log. RuleId will be logged when take action failed. [#8737](https://github.com/emqx/emqx/pull/8737)
+- Increases the latency interval for MQTT Bridge test connections to improve compatibility in high-latency environments. [#8745](https://github.com/emqx/emqx/pull/8745)
+- Close ExProto client process immediately if it's keepalive timeouted. [#8725](https://github.com/emqx/emqx/pull/8725)
+- Upgrade grpc-erl driver to 0.6.7 to support batch operation in sending stream. [#8725](https://github.com/emqx/emqx/pull/8725)
+- Improved jwt authentication module initialization process. [#8736](https://github.com/emqx/emqx/pull/8736)
+
+### Bug fixes
+
+- Fix rule SQL compare to null values always returns false. [#8743](https://github.com/emqx/emqx/pull/8743)
+  Before this change, the following SQL failed to match on the WHERE clause (`clientid != foo` returns false):
+  `SELECT 'some_var' as clientid FROM "t" WHERE clientid != foo`.
+  The `foo` variable is a null value, so `clientid != foo` should be evaluated as true.
+- Fix GET `/auth_clientid` and `/auth_username` counts. [#8655](https://github.com/emqx/emqx/pull/8655)
+- Add an idle timer for ExProto UDP client to avoid client leaking [#8628](https://github.com/emqx/emqx/pull/8628)
+- Fix ExHook can't be un-hooked if the grpc service stop first. [#8725](https://github.com/emqx/emqx/pull/8725)
+- Fix the problem that ExHook cannot continue hook chains execution for mismatched topics. [#8807](https://github.com/emqx/emqx/pull/8807)
+- Fix GET `/listeners/` crashes when listener is not ready. [#8752](https://github.com/emqx/emqx/pull/8752)
+- Fix repeated warning messages in bin/emqx [#8824](https://github.com/emqx/emqx/pull/8824)
+
+
+## v4.3.18
+
+### Enhancements
+
+- Upgrade Erlang/OTP from 23.2.7.2-emqx-3 to 23.3.4.9-3 [#8511](https://github.com/emqx/emqx/pull/8511)
+- Make possible to debug-print SSL handshake procedure by setting listener config `log_level=debug` [#8553](https://github.com/emqx/emqx/pull/8553)
+- Add option to perform GC on connection process after TLS/SSL handshake is performed. [#8649](https://github.com/emqx/emqx/pull/8649)
+  Expected to reduce around 35% memory consumption for each SSL connection. See [#8637](https://github.com/emqx/emqx/pull/8637) for more details.
+
+## v4.3.17
+
+### Bug fixes
+
+- Fixed issue where the dashboard APIs were being exposed under the
+  management listener. [#8411]
+
+- Fixed crash when shared persistent subscription [#8441]
+- Fixed issue in Lua hook that prevented messages from being
+  rejected [#8535]
+- Fix ExProto UDP client keepalive checking error.
+  This causes the clients to not expire as long as a new UDP packet arrives [#8575]
+
+### Enhancements
+
+- HTTP API(GET /rules/) support for pagination and fuzzy filtering. [#8450]
+- Add check_conf cli to check config format. [#8486]
+- Optimize performance of shared subscription
+
+## v4.3.16
+
+### Enhancements
+
+- Add the possibility of configuring the password for
+  password-protected private key files used for dashboard and
+  management HTTPS listeners. [#8129]
+- Add message republish supports using placeholder variables to specify QoS and Retain values. Set `${qos}` and `${flags.retain}` use the original QoS & Retain flag.
+- Add supports specifying the network interface address of the cluster listener & rcp call listener. Specify `0.0.0.0` use all network interfaces, or a particular network interface IP address.
+- ExHook supports to customize the socket parameters for gRPC client. [#8314]
+
+### Bug fixes
+
+- Avoid repeated writing `loaded_plugins` file if the plugin enable stauts has not changed [#8179]
+- Correctly tally `connack.auth_error` metrics when a client uses MQTT
+  3.1. [#8177]
+- Do not match ACL rules containing placeholders if there's no
+  information to fill them. [#8280]
+- Fixed issue in Lua hook that didn't prevent a topic from being
+  subscribed to. [#8288]
+- Ensuring that exhook dispatches the client events are sequential. [#8311]
+- Ensure start dashboard ok event if default_username is missing.
+- Fix key update from JWKS server by JWT auth. [#8337]
+- Better errors for JWT claim validations. [#8337]
+
+## v4.3.15
+
+### Enhancements
+
+* Refactored `bin/emqx` help messages.
+* Upgrade script refuses upgrade from incompatible versions. (e.g. hot upgrade from 4.3 to 4.4 will fail fast).
+* Made possible for EMQX to boot from a Linux directory which has white spaces in its path.
+* Add support for JWT authorization [#7596]
+  Now MQTT clients may be authorized with respect to a specific claim containing publish/subscribe topic whitelists.
+* Better randomisation of app screts (changed from timestamp seeded sha hash (uuid) to crypto:strong_rand_bytes)
+* Return a client_identifier_not_valid error when username is empty and username_as_clientid is set to true [#7862]
+* Add more rule engine date functions: format_date/3, format_date/4, date_to_unix_ts/3, date_to_unix_ts/4 [#7894]
+* Add proto_name and proto_ver fields for $event/client_disconnected event.
+* Mnesia auth/acl http api support multiple condition queries.
+* Inflight QoS1 Messages for shared topics are now redispatched to other alive subscribers upon chosen subscriber session termination.
+* Make auth metrics name more understandable.
+* Allow emqx_management http listener binding to specific interface [#8005]
+* Add rule-engine function float2str/2, user can specify the float output precision [#7991]
+
+### Bug fixes
+
+* List subscription topic (/api/v4/subscriptions), the result do not match with multiple conditions.
+* SSL closed error bug fixed for redis client.
+* Fix mqtt-sn client disconnected due to re-send a duplicated qos2 message
+* Rule-engine function hexstr2bin/1 support half byte [#7977]
+* Shared message delivery when all alive shared subs have full inflight [#7984]
+* Improved resilience against autocluster partitioning during cluster
+  startup. [#7876]
+  [ekka-158](https://github.com/emqx/ekka/pull/158)
+* Add regular expression check ^[0-9A-Za-z_\-]+$ for node name [#7979]
+* Fix `node_dump` variable sourcing. [#8026]
+* Fix heap size is growing too fast when trace large message.
+* Support customized timestamp format of the log messages.
+
+## v4.3.14
+
+### Enhancements
+
+* Add `RequestMeta` for exhook.proto in order to expose `cluster_name` of emqx in each gRPC request. [#7524]
+* Support customize emqx_exhook execution priority. [#7408]
+* add api: PUT /rules/{id}/reset_metrics.
+  This api reset the metrics of the rule engine of a rule, and reset the metrics of the action related to this rule. [#7474]
+* Enhanced rule engine error handling when json parsing error.
+* Add support for `RSA-PSK-AES256-GCM-SHA384`, `RSA-PSK-AES256-CBC-SHA384`,
+ `RSA-PSK-AES128-GCM-SHA256`, `RSA-PSK-AES128-CBC-SHA256` PSK ciphers, and remove `PSK-3DES-EDE-CBC-SHA`,
+ `PSK-RC4-SHA` from the default configuration. [#7427]
+* Diagnostic logging for mnesia `wait_for_table`
+  - prints check points of mnesia internal stats
+  - prints check points of per table loading stats
+  Help to locate the problem of long table loading time.
+* Add `local` strategy for Shared Subscription.
+  That will preferentially dispatch messages to a shared subscriber at the same
+  node. It will improves the efficiency of shared messages dispatching in certain
+  scenarios, especially when the emqx-bridge-mqtt plugin is configured as shared
+  subscription. [#7462]
+* Add some compression functions to rule-engine: gzip, gunzip, zip, unzip, zip_compress, zip_uncompress
+
+### Bug fixes
+
+* Prohibit empty topics in strict mode
+* Make sure ehttpc delete useless pool always succeed.
+* Update mongodb driver to fix potential process leak.
+* Fix a potential security issue #3155 with emqx-dashboard plugin.
+  In the earlier implementation, the Dashboard password is reset back to the
+  default value of emqx_dashboard.conf after the node left cluster.
+  Now we persist changed password to protect against reset. [#7518]
+* Silence grep/sed warnings in docker-entrypoint.sh. [#7520]
+* Generate `loaded_modules` and `loaded_plugins` files with default values when no such files exists. [#7520]
+* Fix the configuration `server_name_indication` set to disable does not take effect.
+* Fix backup files are not deleted and downloaded correctly when the API path has ISO8859-1 escape characters.
+
+## v4.3.13
+
+### Important changes
+
+* For docker image, /opt/emqx/etc has been removed from the VOLUME list,
+  this made it easier for the users to rebuild image on top with changed configs.
+* CentOS 7 Erlang runtime is rebuilt on OpenSSL-1.1.1n (previously on 1.0),
+  Prior to v4.3.13, EMQX pick certain cipher suites proposed by the clients,
+  but then fail to handshake resulting in a `malformed_handshake_data` exception.
+* CentOS 8 Erlang runtime is rebuilt on RockyLinux 8.
+  'centos8' will remain in the package name to keep it backward compatible.
+
+### Enhancements
+
+* CLI `emqx_ctl pem_cache clean` to force purge x509 certificate cache,
+  to force an immediate reload of all certificates after the files are updated on disk.
+* Refactor the ExProto so that anonymous clients can also be displayed on the dashboard [#6983]
+* Force shutdown of processes that cannot answer takeover event [#7026]
+* `topic` parameter in bridge configuration can have `${node}` substitution (just like in `clientid` parameter)
+* Add UTF-8 string validity check in `strict_mode` for MQTT packet.
+  When set to true, invalid UTF-8 strings will cause the client to be disconnected. i.e. client ID, topic name. [#7261]
+* Changed systemd service restart delay from 10 seconds to 60 seconds.
+* MQTT-SN gateway supports initiative to synchronize registered topics after session resumed. [#7300]
+* Add load control app for future development.
+* Change the precision of float to 17 digits after the decimal point when formatting a
+  float using payload templates of rule actions. The old precision is 10 digits before
+  this change. [#7336]
+* Return the cached resource status when querying a resource using HTTP APIs.
+  This is to avoid blocking the HTTP request if the resource is unavailable. [#7374]
+
+### Bug fixes
+
+* Fix the `{error,eexist}` error when do release upgrade again if last run failed. [#7121]
+* Fix case where publishing to a non-existent topic alias would crash the connection [#6979]
+* Fix HTTP-API 500 error on querying the lwm2m client list on the another node [#7009]
+* Fix the ExProto connection registry is not released after the client process abnormally exits [#6983]
+* Fix Server-KeepAlive wrongly applied on MQTT v3.0/v3.1 [#7085]
+* Fix Stomp client can not trigger `$event/client_connection` message [#7096]
+* Fix system memory false alarm at boot
+* Fix the MQTT-SN message replay when the topic is not registered to the client [#6970]
+* Fix rpc get node info maybe crash when other nodes is not ready.
+* Fix false alert level log “cannot_find_plugins” caused by duplicate plugin names in `loaded_plugins` files.
+* Prompt user how to change the dashboard's initial default password when emqx start.
+* Fix errno=13 'Permission denied' Cannot create FIFO boot error in Amazon Linux 2022 (el8 package)
+* Fix user or appid created, name only allow `^[A-Za-z]+[A-Za-z0-9-_]*$`
+* Fix subscribe http api crash by bad_qos `/mqtt/subscribe`,`/mqtt/subscribe_batch`.
+* Send DISCONNECT packet with reason code 0x98 if connection has been kicked [#7309]
+* Auto subscribe to an empty topic will be simply ignored now
+
+## v4.3.12
+### Important changes
+
+### Minor changes
+* Fix updating `emqx_auth_mnesia.conf` password and restarting the new password does not take effect [#6717]
+* Fix import data crash when emqx_auth_mnesia's record is not empty [#6717]
+* Fix `os_mon.sysmem_high_watermark` may not alert after reboot.
+* Enhancement: Log client status before killing it for holding the lock for too long.
+  [emqx-6959](https://github.com/emqx/emqx/pull/6959)
+  [ekka-144](https://github.com/emqx/ekka/pull/144)
+  [ekka-146](https://github.com/emqx/ekka/pull/146)
+
+## v4.3.11
+
+Important notes:
+
+- For Debian/Ubuntu users
+
+  We changed the package installed service from init.d to systemd.
+  The upgrade from init.d to systemd is verified, however it is
+  recommended to verify it before rolling out to production.
+  At least to ensure systemd is available in your system.
+
+- For Centos Users
+
+  RPM package now depends on `openssl11` which is NOT available
+  in certain centos distributions.
+  Please make sure the yum repo [epel-release](https://docs.fedoraproject.org/en-US/epel) is installed.
+
+### Important changes
+
+* Debian/Ubuntu package (deb) installed EMQX now runs on systemd [#6389]<br>
+  This is to take advantage of systemd's supervision functionality to ensure
+  EMQX service is restarted after crashes.
+
+### Minor changes
+
+* Clustering malfunction fixes [#6221, #6381]
+  Mostly changes made in [ekka](https://github.com/emqx/ekka/pull/134)<br>
+  From 0.8.1.4 to 0.8.1.6, fixes included intra-cluster RPC call timeouts,<br>
+  also fixed `ekka_locker` process crashed after killing a hanged lock owner.
+
+* Improved log message when TCP proxy is in use but proxy_protocol configuration is not turned on [#6416]<br>
+  "please check proxy_protocol config for specific listeners and zones" to hint a misconfiguration
+
+* Helm chart supports networking.k8s.io/v1 [#6368]
+
+* Fix session takeover race condition which may lead to message loss [#6396]
+
+* EMQX docker images are pushed to aws public ecr in an automated CI job [#6271]<br>
+  `docker pull public.ecr.aws/emqx/emqx:4.3.10`
+
+* Fix webhook URL path to allow rule-engine variable substitution [#6399]
+
+* Corrected RAM usage display [#6379]
+
+* Changed emqx_sn_registry table creation to runtime [#6357]<br>
+  This was a bug introduced in 4.3.3, in which the table is changed from ets to mnesia<br>
+  this will cause upgrade to fail when a later version node joins a 4.3.0-2 cluster<br>
+
+* Log level for normal termination changed from info to debug [#6358]
+
+* Added config `retainer.stop_publish_clear_msg` to enable/disable empty message retained message publish [#6343]<br>
+  In MQTT 3.1.1, it is unclear if a MQTT broker should publish the 'clear' (no payload) message<br>
+  to the subscribers, or just delete the retained message. So we have made it configurable
+
+* Fix mqtt bridge malfunction when remote host is unreachable (hangs the connection) [#6286, #6323]
+
+* System monitor now inspects `current_stacktrace` of suspicious process [#6290]<br>
+  `current_function` was not quite helpful
+
+* Changed default `max_topc_levels` config value to 128 [#6294, #6420]<br>
+  previously it has no limit (config value = 0), which can be a potential DoS threat
+
+* Collect only libcrypto and libtinfo so files for zip package [#6259]<br>
+  in 4.3.10 we tried to collect all so files, however glibc is not quite portable
+
+* Added openssl-1.1 to RPM dependency [#6239]
+
+* Http client duplicated header fix [#6195]
+
+* Fix `node_dump` issues when working with deb or rpm installation [#6209]
+
+* Pin Erlang/OTP 23.2.7.2-emqx-3 [#6246]<br>
+  4.3.10 is on 23.2.7.2-emqx-2, this bump is to fix an ECC signature name typo:
+  ecdsa_secp512r1_sha512 -> ecdsa_secp521r1_sha512
+
+* HTTP client performance improvement [#6474, #6414]<br>
+  The changes are mostly done in the dependency [repo](https://github.com/emqx/ehttpc).
+
+* For messages from gateways add message properties as MQTT message headers [#6142]<br>
+  e.g. messages from CoAP, LwM2M, Stomp, ExProto, when translated into MQTT message<br>
+  properties such as protocol name, protocol version, username (if any) peer-host<br>
+  etc. are filled as MQTT message headers.
+
+* Format the message id to hex strings in the log message [#6961]
+
+## v4.3.0~10
+
+Older version changes are not tracked here.

Makefile

@@ -3,11 +3,16 @@ REBAR_VERSION = 3.14.3-emqx-8
 REBAR = $(CURDIR)/rebar3
 BUILD = $(CURDIR)/build
 SCRIPTS = $(CURDIR)/scripts
+export EMQX_RELUP ?= true
+export EMQX_DEFAULT_BUILDER = emqx/build-env:erl23.3.4.9-3-alpine
+export EMQX_DEFAULT_RUNNER = alpine:3.12
 export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh)
-export EMQX_DESC ?= EMQ X
+export DOCKERFILE := deploy/docker/Dockerfile
-export EMQX_CE_DASHBOARD_VERSION ?= v4.3.1
 ifeq ($(OS),Windows_NT)
 	export REBAR_COLOR=none
+	FIND=/usr/bin/find
+else
+	FIND=find
 endif
 
 PROFILE ?= emqx
@@ -51,7 +56,7 @@ APPS=$(shell $(CURDIR)/scripts/find-apps.sh)
 ## app/name-ct targets are intended for local tests hence cover is not enabled
 .PHONY: $(APPS:%=%-ct)
 define gen-app-ct-target
-$1-ct:
+$1-ct: $(REBAR)
 	$(REBAR) ct --name 'test@127.0.0.1' -v --suite $(shell $(CURDIR)/scripts/find-suites.sh $1)
 endef
 $(foreach app,$(APPS),$(eval $(call gen-app-ct-target,$(app))))
@@ -85,16 +90,20 @@ $(REL_PROFILES:%=%): $(REBAR) get-dashboard
 clean: $(PROFILES:%=clean-%)
 $(PROFILES:%=clean-%):
 	@if [ -d _build/$(@:clean-%=%) ]; then \
+		rm -f rebar.lock; \
 		rm -rf _build/$(@:clean-%=%)/rel; \
-		find _build/$(@:clean-%=%) -name '*.beam' -o -name '*.so' -o -name '*.app' -o -name '*.appup' -o -name '*.o' -o -name '*.d' -type f | xargs rm -f; \
+		$(FIND) _build/$(@:clean-%=%) -name '*.beam' -o -name '*.so' -o -name '*.app' -o -name '*.appup' -o -name '*.o' -o -name '*.d' -type f | xargs rm -f; \
+		$(FIND) _build/$(@:clean-%=%) -type l -delete; \
 	fi
 
 .PHONY: clean-all
 clean-all:
 	@rm -rf _build
+	@rm -f rebar.lock
 
 .PHONY: deps-all
 deps-all: $(REBAR) $(PROFILES:%=deps-%)
+	@make clean # ensure clean at the end
 
 ## deps-<profile> is used in CI scripts to download deps and the
 ## share downloads between CI steps and/or copied into containers
@@ -102,10 +111,12 @@ deps-all: $(REBAR) $(PROFILES:%=deps-%)
 .PHONY: $(PROFILES:%=deps-%)
 $(PROFILES:%=deps-%): $(REBAR) get-dashboard
 	@$(REBAR) as $(@:deps-%=%) get-deps
+	@rm -f rebar.lock
 
 .PHONY: xref
-xref: $(REBAR)
+xref: $(REBAR) $(REL_PROFILES:%=%-rel)
 	@$(REBAR) as check xref
+	@scripts/xref-check.escript
 
 .PHONY: dialyzer
 dialyzer: $(REBAR)
@@ -118,10 +129,19 @@ COMMON_DEPS := $(REBAR) get-dashboard $(CONF_SEGS)
 $(REL_PROFILES:%=%-rel) $(PKG_PROFILES:%=%-rel): $(COMMON_DEPS)
 	@$(BUILD) $(subst -rel,,$(@)) rel
 
+## download relup base packages
+.PHONY: $(REL_PROFILES:%=%-relup-downloads)
+define download-relup-packages
+$1-relup-downloads:
+	@if [ "$${EMQX_RELUP}" = "true" ]; then $(CURDIR)/scripts/relup-base-packages.sh $1; fi
+endef
+ALL_ZIPS = $(REL_PROFILES)
+$(foreach zt,$(ALL_ZIPS),$(eval $(call download-relup-packages,$(zt))))
+
 ## relup target is to create relup instructions
 .PHONY: $(REL_PROFILES:%=%-relup)
 define gen-relup-target
-$1-relup: $(COMMON_DEPS)
+$1-relup: $1-relup-downloads $(COMMON_DEPS)
 	@$(BUILD) $1 relup
 endef
 ALL_ZIPS = $(REL_PROFILES)
@@ -144,11 +164,18 @@ $1: $1-rel
 endef
 $(foreach pt,$(PKG_PROFILES),$(eval $(call gen-pkg-target,$(pt))))
 
+## docker target is to create docker instructions
+.PHONY: $(REL_PROFILES:%=%-docker)
+define gen-docker-target
+$1-docker: $(COMMON_DEPS)
+	@$(BUILD) $1 docker
+endef
+ALL_ZIPS = $(REL_PROFILES)
+$(foreach zt,$(ALL_ZIPS),$(eval $(call gen-docker-target,$(zt))))
+
 .PHONY: run
 run: $(PROFILE) quickrun
 
 .PHONY: quickrun
 quickrun:
 	./_build/$(PROFILE)/rel/emqx/bin/emqx console
-
-include docker.mk

NOTICE

@@ -0,0 +1,5 @@
+EMQX, a highly scalable, highly available distributed MQTT messaging broker for IoT.
+Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+
+This product contains code developed at EMQ Technologies Co., Ltd.
+Visit https://www.emqx.come to learn more.

Windows.md

@@ -25,17 +25,17 @@ C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\Build
 
 Depending on your visual studio version and OS, the paths may differ.
 The first path is for rebar3 port compiler to find `cl.exe` and `link.exe`
-The second path is for Powershell or CMD to setup environment variables.
+The second path is for CMD to setup environment variables.
 
 ### Erlang/OTP
 
-Install Erlang/OTP 23.2 from https://www.erlang.org/downloads
+Install Erlang/OTP 23.3 from https://www.erlang.org/downloads
 You may need to edit the `Path` environment variable to allow running
-Erlang commands such as `erl` from powershell.
+Erlang commands such as `erl` from CMD.
 
-To validate Erlang installation in CMD or powershell:
+To validate Erlang installation in CMD :
 
-* Start (or restart) CMD or powershell
+* Start (or restart) CMD
 
 * Execute `erl` command to enter Erlang shell
 
@@ -63,7 +63,7 @@ Cygwin is what we tested with.
   to `Path` list.
 
 * Validate installation.
-  Start (restart) CMD or powershell console and execute `which bash`, it should
+  Start (restart) CMD console and execute `which bash`, it should
   print out `/usr/bin/bash`
 
 ### Other tools
@@ -88,7 +88,7 @@ scoop install git curl make jq zip unzip
 
 * Clone the repo: `git clone https://github.com/emqx/emqx.git`
 
-* Start CMD or Powershell
+* Start CMD
 
 * Execute `vcvarsall.bat x86_amd64` to load environment variables
 

apps/emqx_auth_http/etc/emqx_auth_http.conf

@@ -42,18 +42,18 @@ auth.http.auth_req.params = clientid=%c,username=%u,password=%P
 ## Value: URL
 ##
 ## Examples: http://127.0.0.1:80/mqtt/superuser, https://[::1]:80/mqtt/superuser
-auth.http.super_req.url = http://127.0.0.1:80/mqtt/superuser
+# auth.http.super_req.url = http://127.0.0.1:80/mqtt/superuser
 
 ## HTTP Request Method for SuperUser Request
 ##
 ## Value: post | get
-auth.http.super_req.method = post
+# auth.http.super_req.method = post
 
 ## HTTP Request Headers for SuperUser Request, Content-Type header is configured by default.
 ## The possible values of the Content-Type header: application/x-www-form-urlencoded, application/json
 ##
 ## Examples: auth.http.super_req.headers.accept = */*
-auth.http.super_req.headers.content-type = application/x-www-form-urlencoded
+# auth.http.super_req.headers.content-type = application/x-www-form-urlencoded
 
 ## Parameters used to construct the request body or query string parameters
 ## When the request method is GET, these parameters will be converted into query string parameters
@@ -70,7 +70,7 @@ auth.http.super_req.headers.content-type = application/x-www-form-urlencoded
 ##  - %d: subject of client TLS cert
 ##
 ## Value: <K1>=<V1>,<K2>=<V2>,...
-auth.http.super_req.params = clientid=%c,username=%u
+# auth.http.super_req.params = clientid=%c,username=%u
 
 ## HTTP URL API path for ACL Request
 ## Comment out this config to disable ACL checks
@@ -136,6 +136,11 @@ auth.http.connect_timeout = 5s
 ## Value: Number
 auth.http.pool_size = 32
 
+## Whether to enable HTTP Pipelining
+##
+## See: https://en.wikipedia.org/wiki/HTTP_pipelining
+auth.http.enable_pipelining = 100
+
 ##------------------------------------------------------------------------------
 ## SSL options
 

apps/emqx_auth_http/include/emqx_auth_http.hrl

@@ -1,23 +1 @@
-
 -define(APP, emqx_auth_http).
-
--record(auth_metrics, {
-        success = 'client.auth.success',
-        failure = 'client.auth.failure',
-        ignore = 'client.auth.ignore'
-    }).
-
--record(acl_metrics, {
-        allow = 'client.acl.allow',
-        deny = 'client.acl.deny',
-        ignore = 'client.acl.ignore'
-    }).
-
--define(METRICS(Type), tl(tuple_to_list(#Type{}))).
--define(METRICS(Type, K), #Type{}#Type.K).
-
--define(AUTH_METRICS, ?METRICS(auth_metrics)).
--define(AUTH_METRICS(K), ?METRICS(auth_metrics, K)).
-
--define(ACL_METRICS, ?METRICS(acl_metrics)).
--define(ACL_METRICS(K), ?METRICS(acl_metrics, K)).

apps/emqx_auth_http/priv/emqx_auth_http.schema

@@ -109,6 +109,30 @@ end}.
   {datatype, integer}
 ]}.
 
+{mapping, "auth.http.enable_pipelining", "emqx_auth_http.enable_pipelining", [
+  {default, "100"},
+  {datatype, string}
+]}.
+
+{translation, "emqx_auth_http.enable_pipelining", fun(Conf) ->
+  case cuttlefish:conf_get("auth.http.enable_pipelining", Conf, undefined) of
+    undefined -> 100;
+    Str ->
+      try
+        erlang:list_to_integer(Str)
+      catch _:_ ->
+        case erlang:list_to_atom(Str) of
+          true ->
+            100;
+          false ->
+            1;
+          _ ->
+            100
+        end
+      end
+  end
+end}.
+
 {mapping, "auth.http.ssl.cacertfile", "emqx_auth_http.cacertfile", [
   {datatype, string}
 ]}.

apps/emqx_auth_http/src/emqx_acl_http.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -29,26 +29,17 @@
         ]).
 
 %% ACL callbacks
--export([ register_metrics/0
+-export([ check_acl/5
-        , check_acl/5
         , description/0
         ]).
 
--spec(register_metrics() -> ok).
-register_metrics() ->
-    lists:foreach(fun emqx_metrics:ensure/1, ?ACL_METRICS).
-
 %%--------------------------------------------------------------------
 %% ACL callbacks
 %%--------------------------------------------------------------------
 
-check_acl(ClientInfo, PubSub, Topic, AclResult, Params) ->
+check_acl(#{username := <<$$, _/binary>>}, _PubSub, _Topic, _AclResult, _Params) ->
-    return_with(fun inc_metrics/1,
-                do_check_acl(ClientInfo, PubSub, Topic, AclResult, Params)).
-
-do_check_acl(#{username := <<$$, _/binary>>}, _PubSub, _Topic, _AclResult, _Params) ->
     ok;
-do_check_acl(ClientInfo, PubSub, Topic, _AclResult, #{acl := ACLParams = #{path := Path}}) ->
+check_acl(ClientInfo, PubSub, Topic, _AclResult, #{acl := ACLParams = #{path := Path}}) ->
     ClientInfo1 = ClientInfo#{access => access(PubSub), topic => Topic},
     case check_acl_request(ACLParams, ClientInfo1) of
         {ok, 200, <<"ignore">>} -> ok;
@@ -65,16 +56,6 @@ description() -> "ACL with HTTP API".
 %% Internal functions
 %%--------------------------------------------------------------------
 
-inc_metrics(ok) ->
-    emqx_metrics:inc(?ACL_METRICS(ignore));
-inc_metrics({stop, allow}) ->
-    emqx_metrics:inc(?ACL_METRICS(allow));
-inc_metrics({stop, deny}) ->
-    emqx_metrics:inc(?ACL_METRICS(deny)).
-
-return_with(Fun, Result) ->
-    Fun(Result), Result.
-
 check_acl_request(#{pool_name := PoolName,
                     path := Path,
                     method := Method,

apps/emqx_auth_http/src/emqx_auth_http.app.src

@@ -1,6 +1,6 @@
 {application, emqx_auth_http,
  [{description, "EMQ X Authentication/ACL with HTTP API"},
-  {vsn, "4.3.0"}, % strict semver, bump manually!
+  {vsn, "4.3.7"}, % strict semver, bump manually!
   {modules, []},
   {registered, [emqx_auth_http_sup]},
   {applications, [kernel,stdlib,ehttpc]},

apps/emqx_auth_http/src/emqx_auth_http.appup.src

@@ -0,0 +1,48 @@
+%% -*- mode: erlang -*-
+{VSN,
+  [{"4.3.6",
+    [ %% There are only changes to the schema file, so we don't need any
+      %% commands here
+    ]},
+   {"4.3.5",
+    [{load_module,emqx_auth_http_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_http,brutal_purge,soft_purge,[]}]},
+   {"4.3.4",
+    [{load_module,emqx_auth_http_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_http,brutal_purge,soft_purge,[]}]},
+   {"4.3.3",
+    [{load_module,emqx_auth_http_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_http,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_http,brutal_purge,soft_purge,[]}]},
+   {"4.3.2",
+    [{apply,{application,stop,[emqx_auth_http]}},
+     {load_module,emqx_auth_http_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_http,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_http,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_http_cli,brutal_purge,soft_purge,[]}]},
+   {<<"4.3.[0-1]">>,
+    [{restart_application,emqx_auth_http}]},
+   {<<".*">>,[]}],
+  [{"4.3.6",
+    [ %% There are only changes to the schema file, so we don't need any
+      %% commands here
+    ]},
+   {"4.3.5",
+    [{load_module,emqx_auth_http_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_http,brutal_purge,soft_purge,[]}]},
+   {"4.3.4",
+    [{load_module,emqx_auth_http_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_http,brutal_purge,soft_purge,[]}]},
+   {"4.3.3",
+    [{load_module,emqx_auth_http_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_http,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_http,brutal_purge,soft_purge,[]}]},
+   {"4.3.2",
+    [{apply,{application,stop,[emqx_auth_http]}},
+     {load_module,emqx_auth_http_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_http,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_http,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_http_cli,brutal_purge,soft_purge,[]}]},
+   {<<"4.3.[0-1]">>,
+    [{restart_application,emqx_auth_http}]},
+   {<<".*">>,[]}]}.

apps/emqx_auth_http/src/emqx_auth_http.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -30,22 +30,16 @@
         ]).
 
 %% Callbacks
--export([ register_metrics/0
+-export([ check/3
-        , check/3
         , description/0
         ]).
 
--spec(register_metrics() -> ok).
-register_metrics() ->
-    lists:foreach(fun emqx_metrics:ensure/1, ?AUTH_METRICS).
-
 check(ClientInfo, AuthResult, #{auth  := AuthParms = #{path := Path},
                                 super := SuperParams}) ->
     case authenticate(AuthParms, ClientInfo) of
         {ok, 200, <<"ignore">>} ->
-            emqx_metrics:inc(?AUTH_METRICS(ignore)), ok;
+            ok;
         {ok, 200, Body}  ->
-            emqx_metrics:inc(?AUTH_METRICS(success)),
             IsSuperuser = is_superuser(SuperParams, ClientInfo),
             {stop, AuthResult#{is_superuser => IsSuperuser,
                                 auth_result => success,
@@ -54,12 +48,10 @@ check(ClientInfo, AuthResult, #{auth  := AuthParms = #{path := Path},
         {ok, Code, _Body} ->
             ?LOG(error, "Deny connection from path: ~s, response http code: ~p",
                  [Path, Code]),
-            emqx_metrics:inc(?AUTH_METRICS(failure)),
             {stop, AuthResult#{auth_result => http_to_connack_error(Code),
                                anonymous   => false}};
         {error, Error} ->
             ?LOG(error, "Request auth path: ~s, error: ~p", [Path, Error]),
-            emqx_metrics:inc(?AUTH_METRICS(failure)),
             %%FIXME later: server_unavailable is not right.
             {stop, AuthResult#{auth_result => server_unavailable,
                                anonymous   => false}}

apps/emqx_auth_http/src/emqx_auth_http_app.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -50,14 +50,14 @@ translate_env(EnvName) ->
     case application:get_env(?APP, EnvName) of
         undefined -> ok;
         {ok, Req} ->
+            {ok, EnablePipelining} = application:get_env(?APP, enable_pipelining),
             {ok, PoolSize} = application:get_env(?APP, pool_size),
             {ok, ConnectTimeout} = application:get_env(?APP, connect_timeout),
             URL = proplists:get_value(url, Req),
             {ok, #{host := Host,
-                   path := Path0,
                    port := Port,
-                   scheme := Scheme}} = emqx_http_lib:uri_parse(URL),
+                   scheme := Scheme} = URIMap} = emqx_http_lib:uri_parse(URL),
-            Path = path(Path0),
+            Path = path(URIMap),
             MoreOpts = case Scheme of
                         http ->
                             [{transport_opts, emqx_misc:ipv6_probe([])}];
@@ -71,6 +71,7 @@ translate_env(EnvName) ->
                                      end,
                             SNI = case application:get_env(?APP, server_name_indication, undefined) of
                                     "disable" -> disable;
+                                    "" -> undefined;
                                     SNI0 -> SNI0
                                   end,
                             TLSOpts = lists:filter(
@@ -89,6 +90,7 @@ translate_env(EnvName) ->
                         end,
             PoolOpts = [{host, Host},
                         {port, Port},
+                        {enable_pipelining, EnablePipelining},
                         {pool_size, PoolSize},
                         {pool_type, random},
                         {connect_timeout, ConnectTimeout},
@@ -110,7 +112,6 @@ load_hooks() ->
     case application:get_env(?APP, auth_req) of
         undefined -> ok;
         {ok, AuthReq} ->
-            ok = emqx_auth_http:register_metrics(),
             PoolOpts = proplists:get_value(pool_opts, AuthReq),
             PoolName = proplists:get_value(pool_name, AuthReq),
             {ok, _} = ehttpc_sup:start_pool(PoolName, PoolOpts),
@@ -129,7 +130,6 @@ load_hooks() ->
     case application:get_env(?APP, acl_req) of
         undefined -> ok;
         {ok, ACLReq} ->
-            ok = emqx_acl_http:register_metrics(),
             PoolOpts2 = proplists:get_value(pool_opts, ACLReq),
             PoolName2 = proplists:get_value(pool_name, ACLReq),
             {ok, _} = ehttpc_sup:start_pool(PoolName2, PoolOpts2),
@@ -149,10 +149,13 @@ ensure_content_type_header(Method, Headers)
   when Method =:= post orelse Method =:= put ->
     Headers;
 ensure_content_type_header(_Method, Headers) ->
-    lists:keydelete("content-type", 1, Headers).
+    lists:keydelete(<<"content-type">>, 1, Headers).
 
-path("") ->
+path(#{path := "", 'query' := Query}) ->
+    "?" ++ Query;
+path(#{path := Path, 'query' := Query}) ->
+    Path ++ "?" ++ Query;
+path(#{path := ""}) ->
     "/";
-path(Path) ->
+path(#{path := Path}) ->
     Path.
-

apps/emqx_auth_http/src/emqx_auth_http_cli.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -29,16 +29,16 @@
 
 request(PoolName, get, Path, Headers, Params, Timeout) ->
     NewPath = Path ++ "?" ++ binary_to_list(cow_qs:qs(bin_kw(Params))),
-    reply(ehttpc:request(ehttpc_pool:pick_worker(PoolName), get, {NewPath, Headers}, Timeout));
+    reply(ehttpc:request(PoolName, get, {NewPath, Headers}, Timeout));
 
 request(PoolName, post, Path, Headers, Params, Timeout) ->
-    Body = case proplists:get_value("content-type", Headers) of
+    Body = case proplists:get_value(<<"content-type">>, Headers) of
                "application/x-www-form-urlencoded" ->
                    cow_qs:qs(bin_kw(Params));
                "application/json" -> 
                    emqx_json:encode(bin_kw(Params))
            end,
-    reply(ehttpc:request(ehttpc_pool:pick_worker(PoolName), post, {Path, Headers, Body}, Timeout)).
+    reply(ehttpc:request(PoolName, post, {Path, Headers, Body}, Timeout)).
 
 reply({ok, StatusCode, _Headers}) ->
     {ok, StatusCode, <<>>};

apps/emqx_auth_http/src/emqx_auth_http_sup.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

apps/emqx_auth_http/test/emqx_auth_http_SUITE.erl

@@ -1,4 +1,4 @@
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

apps/emqx_auth_jwt/README.md

@@ -46,6 +46,11 @@ auth.jwt.verify_claims = off
 ##  - %u: username
 ##  - %c: clientid
 # auth.jwt.verify_claims.username = %u
+
+## Name of the claim containg ACL rules
+##
+## Value: String
+#auth.jwt.acl_claim_name = acl
 ```
 
 Load the Plugin
@@ -62,6 +67,33 @@ Example
 mosquitto_pub -t 'pub' -m 'hello' -i test -u test -P eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJuYW1lIjoiYm9iIiwiYWdlIjoyOX0.bIV_ZQ8D5nQi0LT8AVkpM4Pd6wmlbpR9S8nOLJAsA8o
 ```
 
+ACL
+---
+JWT may contain lists of topics allowed for subscribing/publishing (ACL rules):
+
+Payload example:
+```json
+{
+  "sub": "emqx",
+  "name": "John Doe",
+  "iat": 1516239022,
+  "exp": 1516239122,
+  "acl": {
+    "sub": [
+      "a/b",
+      "c/+",
+      "%u/%c"
+    ],
+    "pub": [
+      "a/b",
+      "c/+",
+      "%u/%c"
+    ]
+  }
+}
+```
+
+
 Algorithms
 ----------
 

apps/emqx_auth_jwt/etc/emqx_auth_jwt.conf

@@ -47,3 +47,8 @@ auth.jwt.verify_claims = off
 ## For example, to verify that the username in the JWT payload is the same
 ## as the client (MQTT protocol) username
 #auth.jwt.verify_claims.username = %u
+
+## Name of the claim containg ACL rules
+##
+## Value: String
+#auth.jwt.acl_claim_name = acl

apps/emqx_auth_jwt/priv/emqx_auth_jwt.schema

@@ -47,3 +47,12 @@
               end, [], cuttlefish_variable:filter_by_prefix("auth.jwt.verify_claims", Conf))
    end
 end}.
+
+{mapping, "auth.jwt.acl_claim_name", "emqx_auth_jwt.acl_claim_name", [
+  {default, "acl"},
+  {datatype, string}
+]}.
+
+{translation, "emqx_auth_jwt.acl_claim_name", fun(Conf) ->
+    list_to_binary(cuttlefish:conf_get("auth.jwt.acl_claim_name", Conf))
+end}.

apps/emqx_auth_jwt/src/emqx_auth_jwt.app.src

@@ -1,6 +1,6 @@
 {application, emqx_auth_jwt,
  [{description, "EMQ X Authentication with JWT"},
-  {vsn, "4.3.1"}, % strict semver, bump manually!
+  {vsn, "4.3.7"}, % strict semver, bump manually!
   {modules, []},
   {registered, [emqx_auth_jwt_sup]},
   {applications, [kernel,stdlib,jose]},

apps/emqx_auth_jwt/src/emqx_auth_jwt.appup.src

@@ -1,15 +1,27 @@
-%% -*-: erlang -*-
+%% -*- mode: erlang -*-
+%% Unless you know what you are doing, DO NOT edit manually!!
 {VSN,
- [
+  [{"4.3.6",[{load_module,emqx_auth_jwt_svr,brutal_purge,soft_purge,[]}]},
-    {"4.3.0", [
+   {"4.3.5",
-      {load_module, emqx_auth_jwt_svr, brutal_purge, soft_purge, []}
+    [{load_module,emqx_auth_jwt,brutal_purge,soft_purge,[]},
-    ]},
+     {load_module,emqx_auth_jwt_svr,brutal_purge,soft_purge,[]}]},
-    {<<".*">>, []}
+   {"4.3.4",
- ],
+    [{load_module,emqx_auth_jwt,brutal_purge,soft_purge,[]},
- [
+     {load_module,emqx_auth_jwt_svr,brutal_purge,soft_purge,[]}]},
-    {"4.3.0", [
+   {"4.3.3",
-      {load_module, emqx_auth_jwt_svr, brutal_purge, soft_purge, []}
+    [{load_module,emqx_auth_jwt,brutal_purge,soft_purge,[]},
-    ]},
+     {load_module,emqx_auth_jwt_svr,brutal_purge,soft_purge,[]}]},
-    {<<".*">>, []}
+   {<<"4\\.3\\.[0-2]">>,[{restart_application,emqx_auth_jwt}]},
- ]
+   {<<".*">>,[]}],
-}.
+  [{"4.3.6",[{load_module,emqx_auth_jwt_svr,brutal_purge,soft_purge,[]}]},
+   {"4.3.5",
+    [{load_module,emqx_auth_jwt,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_jwt_svr,brutal_purge,soft_purge,[]}]},
+   {"4.3.4",
+    [{load_module,emqx_auth_jwt,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_jwt_svr,brutal_purge,soft_purge,[]}]},
+   {"4.3.3",
+    [{load_module,emqx_auth_jwt,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_jwt_svr,brutal_purge,soft_purge,[]}]},
+   {<<"4\\.3\\.[0-2]">>,[{restart_application,emqx_auth_jwt}]},
+   {<<".*">>,[]}]}.

apps/emqx_auth_jwt/src/emqx_auth_jwt.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -18,69 +18,110 @@
 
 -include_lib("emqx/include/emqx.hrl").
 -include_lib("emqx/include/logger.hrl").
+-include_lib("snabbkaffe/include/snabbkaffe.hrl").
 
 -logger_header("[JWT]").
 
--export([ register_metrics/0
+-export([ check_auth/3
-        , check/3
+        , check_acl/5
         , description/0
         ]).
 
--record(auth_metrics, {
+-export([string_to_number/1]).
-        success = 'client.auth.success',
-        failure = 'client.auth.failure',
-        ignore = 'client.auth.ignore'
-    }).
-
--define(METRICS(Type), tl(tuple_to_list(#Type{}))).
--define(METRICS(Type, K), #Type{}#Type.K).
-
--define(AUTH_METRICS, ?METRICS(auth_metrics)).
--define(AUTH_METRICS(K), ?METRICS(auth_metrics, K)).
-
--spec(register_metrics() -> ok).
-register_metrics() ->
-    lists:foreach(fun emqx_metrics:ensure/1, ?AUTH_METRICS).
 
 %%--------------------------------------------------------------------
 %% Authentication callbacks
 %%--------------------------------------------------------------------
 
-check(ClientInfo, AuthResult, #{pid := Pid,
+check_auth(ClientInfo, AuthResult, #{from := From, checklists := Checklists}) ->
-                                from := From,
-                                checklists := Checklists}) ->
     case maps:find(From, ClientInfo) of
         error ->
-            ok = emqx_metrics:inc(?AUTH_METRICS(ignore));
+            ok;
         {ok, undefined} ->
-            ok = emqx_metrics:inc(?AUTH_METRICS(ignore));
+            ok;
         {ok, Token} ->
-            case emqx_auth_jwt_svr:verify(Pid, Token) of
+            case emqx_auth_jwt_svr:verify(Token) of
                 {error, not_found} ->
-                    ok = emqx_metrics:inc(?AUTH_METRICS(ignore));
+                    ok;
                 {error, not_token} ->
-                    ok = emqx_metrics:inc(?AUTH_METRICS(ignore));
+                    ok;
                 {error, Reason} ->
-                    ok = emqx_metrics:inc(?AUTH_METRICS(failure)),
                     {stop, AuthResult#{auth_result => Reason, anonymous => false}};
                 {ok, Claims} ->
                     {stop, maps:merge(AuthResult, verify_claims(Checklists, Claims, ClientInfo))}
             end
     end.
 
+check_acl(ClientInfo = #{jwt_claims := Claims},
+          PubSub,
+          Topic,
+          _NoMatchAction,
+          #{acl_claim_name := AclClaimName}) ->
+    case Claims of
+        #{AclClaimName := Acl, <<"exp">> := Exp} ->
+            case is_expired(Exp) of
+                true ->
+                    ?DEBUG("acl_deny_due_to_jwt_expired", []),
+                    deny;
+                false ->
+                    verify_acl(ClientInfo, Acl, PubSub, Topic)
+            end;
+        #{AclClaimName := Acl} ->
+            verify_acl(ClientInfo, Acl, PubSub, Topic);
+        _ ->
+            ?DEBUG("no_acl_jwt_claim", []),
+            ignore
+    end;
+check_acl(_ClientInfo, _PubSub, _Topic, _NoMatchAction, _Env) ->
+    ?tp(debug, no_jwt_claim, #{}),
+    ignore.
+
+is_expired(Exp) when is_binary(Exp)  ->
+    case string_to_number(Exp) of
+        {ok, Val} ->
+            is_expired(Val);
+        _ ->
+            ?DEBUG("acl_deny_due_to_invalid_jwt_exp:~p", [Exp]),
+            true
+    end;
+is_expired(Exp) when is_number(Exp) ->
+    Now = erlang:system_time(second),
+    Now > Exp;
+is_expired(Exp) ->
+    ?DEBUG("acl_deny_due_to_invalid_jwt_exp:~p", [Exp]),
+    true.
+
 description() -> "Authentication with JWT".
 
+string_to_number(Bin) when is_binary(Bin) ->
+    string_to_number(Bin, fun erlang:binary_to_integer/1, fun erlang:binary_to_float/1);
+string_to_number(Str) when is_list(Str) ->
+    string_to_number(Str, fun erlang:list_to_integer/1, fun erlang:list_to_float/1);
+string_to_number(_) ->
+    false.
+
 %%------------------------------------------------------------------------------
 %% Verify Claims
 %%--------------------------------------------------------------------
 
+verify_acl(ClientInfo, #{<<"sub">> := SubTopics}, subscribe, Topic) when is_list(SubTopics) ->
+    verify_acl(ClientInfo, SubTopics, Topic);
+verify_acl(ClientInfo, #{<<"pub">> := PubTopics}, publish, Topic) when is_list(PubTopics) ->
+    verify_acl(ClientInfo, PubTopics, Topic);
+verify_acl(_ClientInfo, _Acl, _PubSub, _Topic) -> {stop, deny}.
+
+verify_acl(_ClientInfo, [], _Topic) -> {stop, deny};
+verify_acl(ClientInfo, [AclTopic | AclTopics], Topic) ->
+    case match_topic(ClientInfo, AclTopic, Topic) of
+        true -> {stop, allow};
+        false -> verify_acl(ClientInfo, AclTopics, Topic)
+    end.
+
 verify_claims(Checklists, Claims, ClientInfo) ->
     case do_verify_claims(feedvar(Checklists, ClientInfo), Claims) of
         {error, Reason} ->
-            ok = emqx_metrics:inc(?AUTH_METRICS(failure)),
             #{auth_result => Reason, anonymous => false};
         ok ->
-            ok = emqx_metrics:inc(?AUTH_METRICS(success)),
             #{auth_result => success, anonymous => false, jwt_claims => Claims}
     end.
 
@@ -97,3 +138,20 @@ feedvar(Checklists, #{username := Username, clientid := ClientId}) ->
                  ({K, <<"%c">>}) -> {K, ClientId};
                  ({K, Expected}) -> {K, Expected}
               end, Checklists).
+
+match_topic(ClientInfo, AclTopic, Topic) ->
+    AclTopicWords = emqx_topic:words(AclTopic),
+    TopicWords = emqx_topic:words(Topic),
+    AclTopicRendered = emqx_access_rule:feed_var(ClientInfo, AclTopicWords),
+    emqx_topic:match(TopicWords, AclTopicRendered).
+
+string_to_number(Str, IntFun, FloatFun) ->
+    try
+        {ok, IntFun(Str)}
+    catch _:_ ->
+        try
+            {ok, FloatFun(Str)}
+        catch _:_ ->
+            false
+        end
+    end.

apps/emqx_auth_jwt/src/emqx_auth_jwt_app.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -31,16 +31,19 @@
 start(_Type, _Args) ->
     {ok, Sup} = supervisor:start_link({local, ?MODULE}, ?MODULE, []),
 
-    {ok, Pid} = start_auth_server(jwks_svr_options()),
+    {ok, _} = start_auth_server(jwks_svr_options()),
-    ok = emqx_auth_jwt:register_metrics(),
-    AuthEnv0 = auth_env(),
-    AuthEnv1 = AuthEnv0#{pid => Pid},
 
-    _ = emqx:hook('client.authenticate', {emqx_auth_jwt, check, [AuthEnv1]}),
+    AuthEnv = auth_env(),
-    {ok, Sup, AuthEnv1}.
+    _ = emqx:hook('client.authenticate', {emqx_auth_jwt, check_auth, [AuthEnv]}),
 
-stop(AuthEnv) ->
+    AclEnv = acl_env(),
-    emqx:unhook('client.authenticate', {emqx_auth_jwt, check, [AuthEnv]}).
+    _ = emqx:hook('client.check_acl', {emqx_auth_jwt, check_acl, [AclEnv]}),
+
+    {ok, Sup}.
+
+stop(_State) ->
+    emqx:unhook('client.authenticate', {emqx_auth_jwt, check_auth}),
+    emqx:unhook('client.check_acl', {emqx_auth_jwt, check_acl}).
 
 %%--------------------------------------------------------------------
 %% Dummy supervisor
@@ -69,6 +72,9 @@ auth_env() ->
      , checklists => Checklists
      }.
 
+acl_env() ->
+    #{acl_claim_name => env(acl_claim_name, <<"acl">>)}.
+
 jwks_svr_options() ->
     [{K, V} || {K, V}
              <- [{secret, env(secret, undefined)},

apps/emqx_auth_jwt/src/emqx_auth_jwt_svr.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -20,13 +20,14 @@
 
 -include_lib("emqx/include/logger.hrl").
 -include_lib("jose/include/jose_jwk.hrl").
+-include_lib("snabbkaffe/include/snabbkaffe.hrl").
 
 -logger_header("[JWT-SVR]").
 
 %% APIs
 -export([start_link/1]).
 
--export([verify/2]).
+-export([verify/1]).
 
 %% gen_server callbacks
 -export([ init/1
@@ -44,8 +45,9 @@
                 | {interval, pos_integer()}.
 
 -define(INTERVAL, 300000).
+-define(TAB, ?MODULE).
 
--record(state, {static, remote, addr, tref, intv}).
+-record(state, {addr, tref, intv}).
 
 %%--------------------------------------------------------------------
 %% APIs
@@ -55,13 +57,13 @@
 start_link(Options) ->
     gen_server:start_link(?MODULE, [Options], []).
 
--spec verify(pid(), binary())
+-spec verify(binary())
     -> {error, term()}
      | {ok, Payload :: map()}.
-verify(S, JwsCompacted) when is_binary(JwsCompacted) ->
+verify(JwsCompacted) when is_binary(JwsCompacted) ->
     case catch jose_jws:peek(JwsCompacted) of
         {'EXIT', _} -> {error, not_token};
-        _ -> gen_server:call(S, {verify, JwsCompacted})
+        _ -> do_verify(JwsCompacted)
     end.
 
 %%--------------------------------------------------------------------
@@ -70,43 +72,25 @@ verify(S, JwsCompacted) when is_binary(JwsCompacted) ->
 
 init([Options]) ->
     ok = jose:json_module(jiffy),
-    {Static, Remote} = do_init_jwks(Options),
+    _ = ets:new(?TAB, [set, protected, named_table]),
+    Static = do_init_jwks(Options),
+    to_request_jwks(Options),
+    true = ets:insert(?TAB, [{static, Static}, {remote, undefined}]),
     Intv = proplists:get_value(interval, Options, ?INTERVAL),
     {ok, reset_timer(
            #state{
-              static = Static,
-              remote = Remote,
               addr = proplists:get_value(jwks_addr, Options),
               intv = Intv})}.
 
 %% @private
 do_init_jwks(Options) ->
-    K2J = fun(K, F) ->
+    OctJwk = key2jwt_value(secret,
-              case proplists:get_value(K, Options) of
+                           fun(V) ->
-                  undefined -> undefined;
-                  V ->
-                     try F(V) of
-                         {error, Reason} ->
-                             ?LOG(warning, "Build ~p JWK ~p failed: {error, ~p}~n",
-                                  [K, V, Reason]),
-                             undefined;
-                         J -> J
-                     catch T:R:_ ->
-                         ?LOG(warning, "Build ~p JWK ~p failed: {~p, ~p}~n",
-                              [K, V, T, R]),
-                         undefined
-                     end
-              end
-          end,
-    OctJwk = K2J(secret, fun(V) ->
                                    jose_jwk:from_oct(list_to_binary(V))
-                         end),
+                           end,
-    PemJwk = K2J(pubkey, fun jose_jwk:from_pem_file/1),
+                           Options),
-    Remote = K2J(jwks_addr, fun request_jwks/1),
+    PemJwk = key2jwt_value(pubkey, fun jose_jwk:from_pem_file/1, Options),
-    {[J ||J <- [OctJwk, PemJwk], J /= undefined], Remote}.
+    [J ||J <- [OctJwk, PemJwk], J /= undefined].
-
-handle_call({verify, JwsCompacted}, _From, State) ->
-    handle_verify(JwsCompacted, State);
 
 handle_call(_Req, _From, State) ->
     {reply, ok, State}.
@@ -116,12 +100,18 @@ handle_cast(_Msg, State) ->
 
 handle_info({timeout, _TRef, refresh}, State = #state{addr = Addr}) ->
     NState = try
-                 State#state{remote = request_jwks(Addr)}
+                 true = ets:insert(?TAB, {remote, request_jwks(Addr)}),
+                 State
              catch _:_ ->
                  State
              end,
     {noreply, reset_timer(NState)};
 
+handle_info({request_jwks, Options}, State) ->
+    Remote = key2jwt_value(jwks_addr, fun request_jwks/1, Options),
+    true = ets:insert(?TAB, {remote, Remote}),
+    {noreply, State};
+
 handle_info(_Info, State) ->
     {noreply, State}.
 
@@ -136,24 +126,10 @@ code_change(_OldVsn, State, _Extra) ->
 %% Internal funcs
 %%--------------------------------------------------------------------
 
-handle_verify(JwsCompacted,
+keys(Type) ->
-              State = #state{static = Static, remote = Remote}) ->
+    case ets:lookup(?TAB, Type) of
-    try
+        [{_, Keys}] -> Keys;
-        Jwks = case emqx_json:decode(jose_jws:peek_protected(JwsCompacted), [return_maps]) of
+        [] -> []
-                   #{<<"kid">> := Kid} when Remote /= undefined ->
-                       [J || J <- Remote, maps:get(<<"kid">>, J#jose_jwk.fields, undefined) =:= Kid];
-                   _ -> Static
-               end,
-        case Jwks of
-            [] -> {reply, {error, not_found}, State};
-            _ ->
-                {reply, do_verify(JwsCompacted, Jwks), State}
-        end
-    catch
-        Class : Reason : Stk ->
-            ?LOG(error, "Handle JWK crashed: ~p, ~p, stacktrace: ~p~n",
-                        [Class, Reason, Stk]),
-            {reply, {error, invalid_signature}, State}
     end.
 
 request_jwks(Addr) ->
@@ -163,7 +139,9 @@ request_jwks(Addr) ->
         {ok, {_Code, _Headers, Body}} ->
             try
                 JwkSet = jose_jwk:from(emqx_json:decode(Body, [return_maps])),
-                {_, Jwks} = JwkSet#jose_jwk.keys, Jwks
+                {_, Jwks} = JwkSet#jose_jwk.keys,
+                ?tp(debug, emqx_auth_jwt_svr_jwks_updated, #{jwks => Jwks, pid => self()}),
+                Jwks
             catch _:_ ->
                 ?LOG(error, "Invalid jwks server response: ~p~n", [Body]),
                 error(badarg)
@@ -181,6 +159,26 @@ cancel_timer(State = #state{tref = TRef}) ->
     _ = erlang:cancel_timer(TRef),
     State#state{tref = undefined}.
 
+do_verify(JwsCompacted) ->
+    try
+        Remote = keys(remote),
+        Jwks = case emqx_json:decode(jose_jws:peek_protected(JwsCompacted), [return_maps]) of
+                   #{<<"kid">> := Kid} when Remote /= undefined ->
+                       [J || J <- Remote, maps:get(<<"kid">>, J#jose_jwk.fields, undefined) =:= Kid];
+                   _ -> keys(static)
+               end,
+        case Jwks of
+            [] -> {error, not_found};
+            _ ->
+                do_verify(JwsCompacted, Jwks)
+        end
+    catch
+        Class : Reason : Stk ->
+            ?LOG(error, "verify JWK crashed: ~p, ~p, stacktrace: ~p~n",
+                        [Class, Reason, Stk]),
+            {error, invalid_signature}
+    end.
+
 do_verify(_JwsCompated, []) ->
     {error, invalid_signature};
 do_verify(JwsCompacted, [Jwk|More]) ->
@@ -190,7 +188,11 @@ do_verify(JwsCompacted, [Jwk|More]) ->
             case check_claims(Claims) of
                 {false, <<"exp">>} ->
                     {error, {invalid_signature, expired}};
-                NClaims ->
+                {false, <<"iat">>} ->
+                    {error, {invalid_signature, issued_in_future}};
+                {false, <<"nbf">>} ->
+                    {error, {invalid_signature, not_valid_yet}};
+                {true, NClaims} ->
                     {ok, NClaims}
             end;
         {false, _, _} ->
@@ -198,27 +200,62 @@ do_verify(JwsCompacted, [Jwk|More]) ->
     end.
 
 check_claims(Claims) ->
-    Now = os:system_time(seconds),
+    Now = erlang:system_time(seconds),
-    Checker = [{<<"exp">>, fun(ExpireTime) ->
+    Checker = [{<<"exp">>, with_num_value(
-                               Now < ExpireTime
+                             fun(ExpireTime) -> Now < ExpireTime end)},
-                           end},
+               {<<"iat">>, with_num_value(
-               {<<"iat">>, fun(IssueAt) ->
+                             fun(IssueAt) -> IssueAt =< Now end)},
-                               IssueAt =< Now
+               {<<"nbf">>, with_num_value(
-                           end},
+                             fun(NotBefore) -> NotBefore =< Now end)}
-               {<<"nbf">>, fun(NotBefore) ->
-                               NotBefore =< Now
-                           end}
               ],
     do_check_claim(Checker, Claims).
 
-do_check_claim([], Claims) ->
+with_num_value(Fun) ->
-    Claims;
+    fun(Value) ->
-do_check_claim([{K, F}|More], Claims) ->
+            case Value of
-    case maps:take(K, Claims) of
+                Num when is_number(Num) -> Fun(Num);
-        error -> do_check_claim(More, Claims);
+                Bin when is_binary(Bin) ->
-        {V, NClaims} ->
+                    case emqx_auth_jwt:string_to_number(Bin) of
-            case F(V) of
+                        {ok, Num} -> Fun(Num);
-                true -> do_check_claim(More, NClaims);
+                        _ -> false
-                _ -> {false, K}
+                    end;
+                Str when is_list(Str) ->
+                    case emqx_auth_jwt:string_to_number(Str) of
+                        {ok, Num} -> Fun(Num);
+                        _ -> false
+                    end
+            end
+    end.
+
+do_check_claim([], Claims) ->
+    {true, Claims};
+do_check_claim([{K, F}|More], Claims) ->
+    case Claims of
+        #{K := V} ->
+            case F(V) of
+                true -> do_check_claim(More, Claims);
+                _ -> {false, K}
+            end;
+        _ ->
+            do_check_claim(More, Claims)
+    end.
+
+to_request_jwks(Options) ->
+    erlang:send(self(), {request_jwks, Options}).
+
+key2jwt_value(Key, Func, Options) ->
+    case proplists:get_value(Key, Options) of
+        undefined -> undefined;
+        V ->
+            try Func(V) of
+                {error, Reason} ->
+                    ?LOG(warning, "Build ~p JWK ~p failed: {error, ~p}~n",
+                         [Key, V, Reason]),
+                    undefined;
+                J -> J
+            catch T:R ->
+                    ?LOG(warning, "Build ~p JWK ~p failed: {~p, ~p}~n",
+                         [Key, V, T, R]),
+                    undefined
             end
     end.

apps/emqx_auth_jwt/test/emqx_auth_jwt_SUITE.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -19,29 +19,20 @@
 -compile(export_all).
 -compile(nowarn_export_all).
 
--include_lib("emqx/include/emqx.hrl").
+-include_lib("emqx/include/emqx_mqtt.hrl").
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("common_test/include/ct.hrl").
+-include_lib("snabbkaffe/include/snabbkaffe.hrl").
 
--define(APP, emqx_auth_jwt).
+all() -> emqx_ct:all(?MODULE).
 
-all() ->
+init_per_testcase(TestCase, Config) ->
-    [{group, emqx_auth_jwt}].
+    ?MODULE:TestCase(init, Config),
-
-groups() ->
-    [{emqx_auth_jwt, [sequence], [ t_check_auth
-                                 , t_check_claims
-                                 , t_check_claims_clientid
-                                 , t_check_claims_username
-                                 , t_check_claims_kid_in_header
-                                 ]}
-    ].
-
-init_per_suite(Config) ->
     emqx_ct_helpers:start_apps([emqx_auth_jwt], fun set_special_configs/1),
     Config.
 
-end_per_suite(_Config) ->
+end_per_testcase(TestCase, Config) ->
+    try ?MODULE:TestCase('end', Config) catch _:_ -> ok end,
     emqx_ct_helpers:stop_apps([emqx_auth_jwt]).
 
 set_special_configs(emqx) ->
@@ -78,11 +69,13 @@ sign(Payload, Alg, Key) ->
 %% Testcases
 %%------------------------------------------------------------------------------
 
-t_check_auth(_) ->
+t_check_auth(init, _Config) ->
+    application:unset_env(emqx_auth_jwt, verify_claims).
+t_check_auth(_Config) ->
     Plain = #{clientid => <<"client1">>, username => <<"plain">>, zone => external},
     Jwt = sign([{clientid, <<"client1">>},
                 {username, <<"plain">>},
-                {exp, os:system_time(seconds) + 3}], <<"HS256">>, <<"emqxsecret">>),
+                {exp, erlang:system_time(seconds) + 2}], <<"HS256">>, <<"emqxsecret">>),
     ct:pal("Jwt: ~p~n", [Jwt]),
 
     Result0 = emqx_access_control:authenticate(Plain#{password => Jwt}),
@@ -91,7 +84,7 @@ t_check_auth(_) ->
 
     ct:sleep(3100),
     Result1 = emqx_access_control:authenticate(Plain#{password => Jwt}),
-    ct:pal("Auth result after 1000ms: ~p~n", [Result1]),
+    ct:pal("Auth result after 3100ms: ~p~n", [Result1]),
     ?assertMatch({error, _}, Result1),
 
     Jwt_Error = sign([{client_id, <<"client1">>},
@@ -102,15 +95,121 @@ t_check_auth(_) ->
     ?assertEqual({error, invalid_signature}, Result2),
     ?assertMatch({error, _}, emqx_access_control:authenticate(Plain#{password => <<"asd">>})).
 
-t_check_claims(_) ->
+t_check_nbf(init, _Config) ->
-    application:set_env(emqx_auth_jwt, verify_claims, [{sub, <<"value">>}]),
+    application:unset_env(emqx_auth_jwt, verify_claims).
-    application:stop(emqx_auth_jwt), application:start(emqx_auth_jwt),
+t_check_nbf(_Config) ->
+    Plain = #{clientid => <<"client1">>, username => <<"plain">>, zone => external},
+    Jwt = sign([{clientid, <<"client1">>},
+                {username, <<"plain">>},
+                {nbf, erlang:system_time(seconds) + 3}], <<"HS256">>, <<"emqxsecret">>),
+    ct:pal("Jwt: ~p~n", [Jwt]),
 
+    Result0 = emqx_access_control:authenticate(Plain#{password => Jwt}),
+    ct:pal("Auth result: ~p~n", [Result0]),
+    ?assertEqual({error, {invalid_signature, not_valid_yet}}, Result0).
+
+t_check_iat(init, _Config) ->
+    application:unset_env(emqx_auth_jwt, verify_claims).
+t_check_iat(_Config) ->
+    Plain = #{clientid => <<"client1">>, username => <<"plain">>, zone => external},
+    Jwt = sign([{clientid, <<"client1">>},
+                {username, <<"plain">>},
+                {iat, erlang:system_time(seconds) + 3}], <<"HS256">>, <<"emqxsecret">>),
+    ct:pal("Jwt: ~p~n", [Jwt]),
+
+    Result0 = emqx_access_control:authenticate(Plain#{password => Jwt}),
+    ct:pal("Auth result: ~p~n", [Result0]),
+    ?assertEqual({error, {invalid_signature, issued_in_future}}, Result0).
+
+t_check_auth_invalid_exp(init, _Config) ->
+    application:unset_env(emqx_auth_jwt, verify_claims).
+t_check_auth_invalid_exp(_Config) ->
+    Plain = #{clientid => <<"client1">>, username => <<"plain">>, zone => external},
+    Jwt0 = sign([{clientid, <<"client1">>},
+                 {username, <<"plain">>},
+                 {exp, [{foo, bar}]}], <<"HS256">>, <<"emqxsecret">>),
+    ct:pal("Jwt: ~p~n", [Jwt0]),
+
+    Result0 = emqx_access_control:authenticate(Plain#{password => Jwt0}),
+    ct:pal("Auth result: ~p~n", [Result0]),
+    ?assertMatch({error, _}, Result0),
+
+    Jwt1 = sign([{clientid, <<"client1">>},
+                 {username, <<"plain">>},
+                 {exp, <<"foobar">>}], <<"HS256">>, <<"emqxsecret">>),
+    ct:pal("Jwt: ~p~n", [Jwt1]),
+
+    Result1 = emqx_access_control:authenticate(Plain#{password => Jwt1}),
+    ct:pal("Auth result: ~p~n", [Result1]),
+    ?assertMatch({error, _}, Result1).
+
+t_check_auth_str_exp(init, _Config) ->
+    application:unset_env(emqx_auth_jwt, verify_claims).
+t_check_auth_str_exp(_Config) ->
+    Plain = #{clientid => <<"client1">>, username => <<"plain">>, zone => external},
+    Exp = integer_to_binary(erlang:system_time(seconds) + 3),
+
+    Jwt0 = sign([{clientid, <<"client1">>},
+                 {username, <<"plain">>},
+                 {exp, Exp}], <<"HS256">>, <<"emqxsecret">>),
+    ct:pal("Jwt: ~p~n", [Jwt0]),
+
+    Result0 = emqx_access_control:authenticate(Plain#{password => Jwt0}),
+    ct:pal("Auth result: ~p~n", [Result0]),
+    ?assertMatch({ok, #{auth_result := success, jwt_claims := _}}, Result0),
+
+    Jwt1 = sign([{clientid, <<"client1">>},
+                 {username, <<"plain">>},
+                 {exp, <<"0">>}], <<"HS256">>, <<"emqxsecret">>),
+    ct:pal("Jwt: ~p~n", [Jwt1]),
+
+    Result1 = emqx_access_control:authenticate(Plain#{password => Jwt1}),
+    ct:pal("Auth result: ~p~n", [Result1]),
+    ?assertMatch({error, _}, Result1),
+
+    Exp2 = float_to_binary(erlang:system_time(seconds) + 3.5),
+
+    Jwt2 = sign([{clientid, <<"client1">>},
+                 {username, <<"plain">>},
+                 {exp, Exp2}], <<"HS256">>, <<"emqxsecret">>),
+    ct:pal("Jwt: ~p~n", [Jwt2]),
+
+    Result2 = emqx_access_control:authenticate(Plain#{password => Jwt2}),
+    ct:pal("Auth result: ~p~n", [Result2]),
+    ?assertMatch({ok, #{auth_result := success, jwt_claims := _}}, Result2).
+
+t_check_auth_float_exp(init, _Config) ->
+    application:unset_env(emqx_auth_jwt, verify_claims).
+t_check_auth_float_exp(_Config) ->
+    Plain = #{clientid => <<"client1">>, username => <<"plain">>, zone => external},
+    Exp = erlang:system_time(seconds) + 3.5,
+
+    Jwt0 = sign([{clientid, <<"client1">>},
+                 {username, <<"plain">>},
+                 {exp, Exp}], <<"HS256">>, <<"emqxsecret">>),
+    ct:pal("Jwt: ~p~n", [Jwt0]),
+
+    Result0 = emqx_access_control:authenticate(Plain#{password => Jwt0}),
+    ct:pal("Auth result: ~p~n", [Result0]),
+    ?assertMatch({ok, #{auth_result := success, jwt_claims := _}}, Result0),
+
+    Jwt1 = sign([{clientid, <<"client1">>},
+                 {username, <<"plain">>},
+                 {exp, 1.5}], <<"HS256">>, <<"emqxsecret">>),
+    ct:pal("Jwt: ~p~n", [Jwt1]),
+
+    Result1 = emqx_access_control:authenticate(Plain#{password => Jwt1}),
+    ct:pal("Auth result: ~p~n", [Result1]),
+    ?assertMatch({error, _}, Result1).
+
+t_check_claims(init, _Config) ->
+    application:set_env(emqx_auth_jwt, verify_claims, [{sub, <<"value">>}]).
+t_check_claims(_Config) ->
     Plain = #{clientid => <<"client1">>, username => <<"plain">>, zone => external},
     Jwt = sign([{client_id, <<"client1">>},
                 {username, <<"plain">>},
                 {sub, value},
-                {exp, os:system_time(seconds) + 3}], <<"HS256">>, <<"emqxsecret">>),
+                {exp, erlang:system_time(seconds) + 3}], <<"HS256">>, <<"emqxsecret">>),
     Result0 = emqx_access_control:authenticate(Plain#{password => Jwt}),
     ct:pal("Auth result: ~p~n", [Result0]),
     ?assertMatch({ok, #{auth_result := success, jwt_claims := _}}, Result0),
@@ -120,13 +219,13 @@ t_check_claims(_) ->
     ct:pal("Auth result for the invalid jwt: ~p~n", [Result2]),
     ?assertEqual({error, invalid_signature}, Result2).
 
-t_check_claims_clientid(_) ->
+t_check_claims_clientid(init, _Config) ->
-    application:set_env(emqx_auth_jwt, verify_claims, [{clientid, <<"%c">>}]),
+    application:set_env(emqx_auth_jwt, verify_claims, [{clientid, <<"%c">>}]).
-    application:stop(emqx_auth_jwt), application:start(emqx_auth_jwt),
+t_check_claims_clientid(_Config) ->
     Plain = #{clientid => <<"client23">>, username => <<"plain">>, zone => external},
     Jwt = sign([{clientid, <<"client23">>},
                 {username, <<"plain">>},
-                {exp, os:system_time(seconds) + 3}], <<"HS256">>, <<"emqxsecret">>),
+                {exp, erlang:system_time(seconds) + 3}], <<"HS256">>, <<"emqxsecret">>),
     Result0 = emqx_access_control:authenticate(Plain#{password => Jwt}),
     ct:pal("Auth result: ~p~n", [Result0]),
     ?assertMatch({ok, #{auth_result := success, jwt_claims := _}}, Result0),
@@ -136,14 +235,13 @@ t_check_claims_clientid(_) ->
     ct:pal("Auth result for the invalid jwt: ~p~n", [Result2]),
     ?assertEqual({error, invalid_signature}, Result2).
 
-t_check_claims_username(_) ->
+t_check_claims_username(init, _Config) ->
-    application:set_env(emqx_auth_jwt, verify_claims, [{username, <<"%u">>}]),
+    application:set_env(emqx_auth_jwt, verify_claims, [{username, <<"%u">>}]).
-    application:stop(emqx_auth_jwt), application:start(emqx_auth_jwt),
+t_check_claims_username(_Config) ->
-
     Plain = #{clientid => <<"client23">>, username => <<"plain">>, zone => external},
     Jwt = sign([{client_id, <<"client23">>},
                 {username, <<"plain">>},
-                {exp, os:system_time(seconds) + 3}], <<"HS256">>, <<"emqxsecret">>),
+                {exp, erlang:system_time(seconds) + 3}], <<"HS256">>, <<"emqxsecret">>),
     Result0 = emqx_access_control:authenticate(Plain#{password => Jwt}),
     ct:pal("Auth result: ~p~n", [Result0]),
     ?assertMatch({ok, #{auth_result := success, jwt_claims := _}}, Result0),
@@ -153,14 +251,224 @@ t_check_claims_username(_) ->
     ct:pal("Auth result for the invalid jwt: ~p~n", [Result3]),
     ?assertEqual({error, invalid_signature}, Result3).
 
-t_check_claims_kid_in_header(_) ->
+t_check_claims_kid_in_header(init, _Config) ->
-    application:set_env(emqx_auth_jwt, verify_claims, []),
+    application:set_env(emqx_auth_jwt, verify_claims, []).
+t_check_claims_kid_in_header(_Config) ->
     Plain = #{clientid => <<"client23">>, username => <<"plain">>, zone => external},
     Jwt = sign([{clientid, <<"client23">>},
                 {username, <<"plain">>},
-                {exp, os:system_time(seconds) + 3}],
+                {exp, erlang:system_time(seconds) + 3}],
                #{<<"alg">> => <<"HS256">>,
                  <<"kid">> => <<"a_kid_str">>}, <<"emqxsecret">>),
     Result0 = emqx_access_control:authenticate(Plain#{password => Jwt}),
     ct:pal("Auth result: ~p~n", [Result0]),
     ?assertMatch({ok, #{auth_result := success, jwt_claims := _}}, Result0).
+
+t_keys_update(init, _Config) ->
+    ok = meck:new(httpc, [passthrough, no_history]),
+    ok = meck:expect(
+           httpc,
+           request,
+           fun(get, _, _, _) ->
+                   {ok,
+                    {200,
+                     [],
+                     jiffy:encode(#{<<"keys">> => []})}}
+           end),
+
+    application:set_env(emqx_auth_jwt, verify_claims, []),
+    application:set_env(emqx_auth_jwt, refresh_interval, 100),
+    application:set_env(emqx_auth_jwt, jwks, "http://localhost:4001/keys.json").
+t_keys_update(_Config) ->
+    ?check_trace(
+       snabbkaffe:block_until(
+         ?match_n_events(2, #{?snk_kind := emqx_auth_jwt_svr_jwks_updated}),
+         _Timeout    = infinity,
+         _BackInTIme = 0),
+       fun(_, Trace) ->
+               ?assertMatch([#{pid := Pid}, #{pid := Pid} |  _],
+                            ?of_kind(emqx_auth_jwt_svr_jwks_updated, Trace))
+       end).
+
+t_check_jwt_acl(init, _Config) ->
+    application:set_env(emqx_auth_jwt, verify_claims, [{sub, <<"value">>}]).
+t_check_jwt_acl(_Config) ->
+    Jwt = sign([{client_id, <<"client1">>},
+                {username, <<"plain">>},
+                {sub, value},
+                {acl, [{sub, [<<"a/b">>]},
+                       {pub, [<<"c/d">>]}]},
+                {exp, erlang:system_time(seconds) + 10}],
+               <<"HS256">>,
+               <<"emqxsecret">>),
+
+    {ok, C} = emqtt:start_link(
+                [{clean_start, true},
+                 {proto_ver, v5},
+                 {client_id, <<"client1">>},
+                 {password, Jwt}]),
+    {ok, _} = emqtt:connect(C),
+
+    ?assertMatch(
+       {ok, #{}, [0]},
+       emqtt:subscribe(C, <<"a/b">>, 0)),
+
+    ?assertMatch(
+       ok,
+       emqtt:publish(C, <<"c/d">>, <<"hi">>, 0)),
+
+    ?assertMatch(
+       {ok, #{}, [?RC_NOT_AUTHORIZED]},
+       emqtt:subscribe(C, <<"c/d">>, 0)),
+
+    ok = emqtt:publish(C, <<"a/b">>, <<"hi">>, 0),
+
+    receive
+        {publish, #{topic := <<"a/b">>}} ->
+            ?assert(false, "Publish to `a/b` should not be allowed")
+    after 100 -> ok
+    end,
+
+    ok = emqtt:disconnect(C).
+
+t_check_jwt_acl_no_recs(init, _Config) ->
+    application:set_env(emqx_auth_jwt, verify_claims, [{sub, <<"value">>}]).
+t_check_jwt_acl_no_recs(_Config) ->
+    Jwt = sign([{client_id, <<"client1">>},
+                {username, <<"plain">>},
+                {sub, value},
+                {acl, []},
+                {exp, erlang:system_time(seconds) + 10}],
+               <<"HS256">>,
+               <<"emqxsecret">>),
+
+    {ok, C} = emqtt:start_link(
+                [{clean_start, true},
+                 {proto_ver, v5},
+                 {client_id, <<"client1">>},
+                 {password, Jwt}]),
+    {ok, _} = emqtt:connect(C),
+
+    ?assertMatch(
+       {ok, #{}, [?RC_NOT_AUTHORIZED]},
+       emqtt:subscribe(C, <<"a/b">>, 0)),
+
+    ok = emqtt:disconnect(C).
+
+t_check_jwt_acl_no_acl_claim(init, _Config) ->
+    application:set_env(emqx_auth_jwt, verify_claims, [{sub, <<"value">>}]).
+t_check_jwt_acl_no_acl_claim(_Config) ->
+    Jwt = sign([{client_id, <<"client1">>},
+                {username, <<"plain">>},
+                {sub, value},
+                {exp, erlang:system_time(seconds) + 10}],
+               <<"HS256">>,
+               <<"emqxsecret">>),
+
+    {ok, C} = emqtt:start_link(
+                [{clean_start, true},
+                 {proto_ver, v5},
+                 {client_id, <<"client1">>},
+                 {password, Jwt}]),
+    {ok, _} = emqtt:connect(C),
+
+    ?assertMatch(
+       {ok, #{}, [?RC_NOT_AUTHORIZED]},
+       emqtt:subscribe(C, <<"a/b">>, 0)),
+
+    ok = emqtt:disconnect(C).
+
+t_check_jwt_acl_no_jwt_claims_helper(_ClientInfo, _LastAuthResult) ->
+    {stop, #{auth_result => success, anonymous => false}}.
+t_check_jwt_acl_no_jwt_claims(init, _Config) ->
+    ok;
+t_check_jwt_acl_no_jwt_claims('end', _Config) ->
+    ok = emqx_hooks:del(
+            'client.authenticate',
+            {?MODULE, t_check_jwt_acl_no_jwt_claims_helper, []}
+          ).
+t_check_jwt_acl_no_jwt_claims(_Config) ->
+    %% bypass the jwt authentication checking
+    ok = emqx_hooks:add(
+            'client.authenticate',
+            {?MODULE, t_check_jwt_acl_no_jwt_claims_helper, []},
+            _Priority = 99999
+          ),
+
+    {ok, C} = emqtt:start_link(
+                [{clean_start, true},
+                 {proto_ver, v5},
+                 {client_id, <<"client1">>},
+                 {username, <<"client1">>},
+                 {password, <<"password">>}]),
+    {ok, _} = emqtt:connect(C),
+
+    ok = snabbkaffe:start_trace(),
+
+    ?assertMatch(
+       {ok, #{}, [?RC_NOT_AUTHORIZED]},
+       emqtt:subscribe(C, <<"a/b">>, 0)),
+
+    {ok, _} = ?block_until(#{?snk_kind := no_jwt_claim}, 1000),
+    Trace = snabbkaffe:collect_trace(),
+    ?assertEqual(1, length(?of_kind(no_jwt_claim, Trace))),
+
+    snabbkaffe:stop(),
+    ok = emqtt:disconnect(C).
+
+t_check_jwt_acl_expire(init, _Config) ->
+    application:set_env(emqx_auth_jwt, verify_claims, [{sub, <<"value">>}]).
+t_check_jwt_acl_expire(_Config) ->
+    Jwt = sign([{client_id, <<"client1">>},
+                {username, <<"plain">>},
+                {sub, value},
+                {acl, [{sub, [<<"a/b">>]}]},
+                {exp, erlang:system_time(seconds) + 1}],
+               <<"HS256">>,
+               <<"emqxsecret">>),
+
+    {ok, C} = emqtt:start_link(
+                [{clean_start, true},
+                 {proto_ver, v5},
+                 {client_id, <<"client1">>},
+                 {password, Jwt}]),
+    {ok, _} = emqtt:connect(C),
+
+    ?assertMatch(
+       {ok, #{}, [0]},
+       emqtt:subscribe(C, <<"a/b">>, 0)),
+
+    ?assertMatch(
+       {ok, #{}, [0]},
+       emqtt:unsubscribe(C, <<"a/b">>)),
+
+    timer:sleep(2000),
+
+    ?assertMatch(
+       {ok, #{}, [?RC_NOT_AUTHORIZED]},
+       emqtt:subscribe(C, <<"a/b">>, 0)),
+
+    ok = emqtt:disconnect(C).
+
+t_check_jwt_acl_no_exp(init, _Config) ->
+    application:set_env(emqx_auth_jwt, verify_claims, [{sub, <<"value">>}]).
+t_check_jwt_acl_no_exp(_Config) ->
+    Jwt = sign([{client_id, <<"client1">>},
+                {username, <<"plain">>},
+                {sub, value},
+                {acl, [{sub, [<<"a/b">>]}]}],
+               <<"HS256">>,
+               <<"emqxsecret">>),
+
+    {ok, C} = emqtt:start_link(
+                [{clean_start, true},
+                 {proto_ver, v5},
+                 {client_id, <<"client1">>},
+                 {password, Jwt}]),
+    {ok, _} = emqtt:connect(C),
+
+    ?assertMatch(
+       {ok, #{}, [0]},
+       emqtt:subscribe(C, <<"a/b">>, 0)),
+
+    ok = emqtt:disconnect(C).

apps/emqx_auth_ldap/include/emqx_auth_ldap.hrl

@@ -1,23 +1 @@
-
 -define(APP, emqx_auth_ldap).
-
--record(auth_metrics, {
-        success = 'client.auth.success',
-        failure = 'client.auth.failure',
-        ignore = 'client.auth.ignore'
-    }).
-
--record(acl_metrics, {
-        allow = 'client.acl.allow',
-        deny = 'client.acl.deny',
-        ignore = 'client.acl.ignore'
-    }).
-
--define(METRICS(Type), tl(tuple_to_list(#Type{}))).
--define(METRICS(Type, K), #Type{}#Type.K).
-
--define(AUTH_METRICS, ?METRICS(auth_metrics)).
--define(AUTH_METRICS(K), ?METRICS(auth_metrics, K)).
-
--define(ACL_METRICS, ?METRICS(acl_metrics)).
--define(ACL_METRICS(K), ?METRICS(acl_metrics, K)).

apps/emqx_auth_ldap/priv/emqx_auth_ldap.schema

@@ -73,6 +73,7 @@
                  {verify, cuttlefish:conf_get("auth.ldap.ssl.verify", Conf, undefined)},
                  {server_name_indication, case cuttlefish:conf_get("auth.ldap.ssl.server_name_indication", Conf, undefined) of
                                             "disable" -> disable;
+                                            "" -> undefined;
                                             SNI -> SNI
                                           end}]
               end,

apps/emqx_auth_ldap/src/emqx_acl_ldap.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -22,24 +22,15 @@
 -include_lib("eldap/include/eldap.hrl").
 -include_lib("emqx/include/logger.hrl").
 
--export([ register_metrics/0
+-export([ check_acl/5
-        , check_acl/5
         , description/0
         ]).
 
--import(proplists, [get_value/2]).
-
--import(emqx_auth_ldap_cli, [search/4]).
-
--spec(register_metrics() -> ok).
-register_metrics() ->
-    lists:foreach(fun emqx_metrics:ensure/1, ?ACL_METRICS).
-
 check_acl(ClientInfo, PubSub, Topic, NoMatchAction, State) ->
     case do_check_acl(ClientInfo, PubSub, Topic, NoMatchAction, State) of
-        ok -> emqx_metrics:inc(?ACL_METRICS(ignore)), ok;
+        ok -> ok;
-        {stop, allow} -> emqx_metrics:inc(?ACL_METRICS(allow)), {stop, allow};
+        {stop, allow} -> {stop, allow};
-        {stop, deny} -> emqx_metrics:inc(?ACL_METRICS(deny)), {stop, deny}
+        {stop, deny} -> {stop, deny}
     end.
 
 do_check_acl(#{username := <<$$, _/binary>>}, _PubSub, _Topic, _NoMatchAction, _State) ->
@@ -70,14 +61,14 @@ do_check_acl(#{username := Username}, PubSub, Topic, _NoMatchAction,
 
     BaseDN = emqx_auth_ldap:replace_vars(CustomBaseDN, ReplaceRules),
 
-    case search(Pool, BaseDN, Filter, [Attribute, Attribute1]) of
+    case emqx_auth_ldap_cli:search(Pool, BaseDN, Filter, [Attribute, Attribute1]) of
         {error, noSuchObject} ->
             ok;
         {ok, #eldap_search_result{entries = []}} ->
             ok;
         {ok, #eldap_search_result{entries = [Entry]}} ->
-            Topics = get_value(Attribute, Entry#eldap_entry.attributes)
+            Topics = proplists:get_value(Attribute, Entry#eldap_entry.attributes, [])
-                ++ get_value(Attribute1, Entry#eldap_entry.attributes),
+                ++ proplists:get_value(Attribute1, Entry#eldap_entry.attributes, []),
             match(Topic, Topics);
         Error ->
             ?LOG(error, "[LDAP] search error:~p", [Error]),
@@ -95,4 +86,3 @@ match(Topic, [Filter | Topics]) ->
 
 description() ->
     "ACL with LDAP".
-

apps/emqx_auth_ldap/src/emqx_auth_ldap.app.src

@@ -1,6 +1,6 @@
 {application, emqx_auth_ldap,
  [{description, "EMQ X Authentication/ACL with LDAP"},
-  {vsn, "4.3.0"}, % strict semver, bump manually!
+  {vsn, "4.3.5"}, % strict semver, bump manually!
   {modules, []},
   {registered, [emqx_auth_ldap_sup]},
   {applications, [kernel,stdlib,eldap2,ecpool]},

apps/emqx_auth_ldap/src/emqx_auth_ldap.appup.src

@@ -0,0 +1,29 @@
+%% -*- mode: erlang -*-
+%% Unless you know what you are doing, DO NOT edit manually!!
+{VSN,
+  [{<<"4\\.3\\.[3-4]">>,
+    [{load_module,emqx_auth_ldap_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_ldap,brutal_purge,soft_purge,[]}]},
+   {"4.3.2",
+    [{load_module,emqx_auth_ldap_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_ldap,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_ldap,brutal_purge,soft_purge,[]}]},
+   {<<"4\\.3\\.[0-1]">>,
+    [{load_module,emqx_auth_ldap_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_ldap,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_ldap,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_ldap_cli,brutal_purge,soft_purge,[]}]},
+   {<<".*">>,[]}],
+  [{<<"4\\.3\\.[3-4]">>,
+    [{load_module,emqx_auth_ldap_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_ldap,brutal_purge,soft_purge,[]}]},
+   {"4.3.2",
+    [{load_module,emqx_auth_ldap_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_ldap,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_ldap,brutal_purge,soft_purge,[]}]},
+   {<<"4\\.3\\.[0-1]">>,
+    [{load_module,emqx_auth_ldap_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_ldap,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_ldap,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_ldap_cli,brutal_purge,soft_purge,[]}]},
+   {<<".*">>,[]}]}.

apps/emqx_auth_ldap/src/emqx_auth_ldap.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -26,17 +26,12 @@
 
 -import(emqx_auth_ldap_cli, [search/3]).
 
--export([ register_metrics/0
+-export([ check/3
-        , check/3
         , description/0
         , prepare_filter/4
         , replace_vars/2
         ]).
 
--spec(register_metrics() -> ok).
-register_metrics() ->
-    lists:foreach(fun emqx_metrics:ensure/1, ?AUTH_METRICS).
-
 check(ClientInfo = #{username := Username, password := Password}, AuthResult,
       State = #{password_attr := PasswdAttr, bind_as_user := BindAsUserRequired, pool := Pool}) ->
     CheckResult =
@@ -63,12 +58,10 @@ check(ClientInfo = #{username := Username, password := Password}, AuthResult,
         end,
     case CheckResult of
         ok ->
-            ok = emqx_metrics:inc(?AUTH_METRICS(success)),
             {stop, AuthResult#{auth_result => success, anonymous => false}};
         {error, not_found} ->
-            emqx_metrics:inc(?AUTH_METRICS(ignore));
+            ok;
         {error, ResultCode} ->
-            ok = emqx_metrics:inc(?AUTH_METRICS(failure)),
             ?LOG(error, "[LDAP] Auth from ldap failed: ~p", [ResultCode]),
             {stop, AuthResult#{auth_result => ResultCode, anonymous => false}}
     end.

apps/emqx_auth_ldap/src/emqx_auth_ldap_app.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -49,12 +49,10 @@ stop(_State) ->
     ok.
 
 load_auth_hook(DeviceDn) ->
-    ok = emqx_auth_ldap:register_metrics(),
     Params = maps:from_list(DeviceDn),
     emqx:hook('client.authenticate', fun emqx_auth_ldap:check/3, [Params#{pool => ?APP}]).
 
 load_acl_hook(DeviceDn) ->
-    ok = emqx_acl_ldap:register_metrics(),
     Params = maps:from_list(DeviceDn),
     emqx:hook('client.check_acl', fun emqx_acl_ldap:check_acl/5 , [Params#{pool => ?APP}]).
 

apps/emqx_auth_ldap/src/emqx_auth_ldap_cli.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -76,8 +76,8 @@ connect(Opts) ->
 search(Pool, Base, Filter) ->
     ecpool:with_client(Pool,
         fun(C) ->
-                case application:get_env(?APP, bind_as_user) of
+                case application:get_env(?APP, bind_as_user, false) of
-                    {ok, true} ->
+                    true ->
                         {ok, Opts} = application:get_env(?APP, ldap),
                         BindDn       = get_value(bind_dn, Opts),
                         BindPassword = get_value(bind_password, Opts),
@@ -91,7 +91,7 @@ search(Pool, Base, Filter) ->
                         catch
                             error:Reason -> {error, Reason}
                         end;
-                    {ok, false} ->
+                    false ->
                         eldap2:search(C, [{base, Base},
                                           {filter, Filter},
                                           {deref, eldap2:derefFindingBaseObj()}])
@@ -101,8 +101,8 @@ search(Pool, Base, Filter) ->
 search(Pool, Base, Filter, Attributes) ->
     ecpool:with_client(Pool,
         fun(C) ->
-                case application:get_env(?APP, bind_as_user) of
+                case application:get_env(?APP, bind_as_user, false) of
-                    {ok, true} ->
+                    true ->
                         {ok, Opts} = application:get_env(?APP, ldap),
                         BindDn       = get_value(bind_dn, Opts),
                         BindPassword = get_value(bind_password, Opts),
@@ -117,7 +117,7 @@ search(Pool, Base, Filter, Attributes) ->
                         catch
                             error:Reason -> {error, Reason}
                         end;
-                    {ok, false} ->
+                    false ->
                         eldap2:search(C, [{base, Base},
                                           {filter, Filter},
                                           {attributes, Attributes},

apps/emqx_auth_ldap/src/emqx_auth_ldap_sup.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

apps/emqx_auth_ldap/test/emqx_auth_ldap_SUITE.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -48,7 +48,9 @@ init_per_group(GrpName, Cfg) ->
     Cfg.
 
 end_per_group(_GrpName, _Cfg) ->
-    emqx_ct_helpers:stop_apps([emqx_auth_ldap]).
+    emqx_ct_helpers:stop_apps([emqx_auth_ldap]),
+    %% clear the application envs to avoid cross-suite testcase failure
+    application:unload(emqx_auth_ldap).
 
 %%--------------------------------------------------------------------
 %% Cases

apps/emqx_auth_ldap/test/emqx_auth_ldap_bind_as_user_SUITE.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -40,7 +40,9 @@ init_per_suite(Config) ->
     Config.
 
 end_per_suite(_Config) ->
-    emqx_ct_helpers:stop_apps([emqx_auth_ldap]).
+    emqx_ct_helpers:stop_apps([emqx_auth_ldap]),
+    %% clear the application envs to avoid cross-suite testcase failure
+    application:unload(emqx_auth_ldap).
 
 check_auth(_) ->
     MqttUser1 = #{clientid => <<"mqttuser1">>,

apps/emqx_auth_mnesia/include/emqx_auth_mnesia.hrl

@@ -3,36 +3,47 @@
 -type(login() :: {clientid, binary()}
               | {username, binary()}).
 
+-type(acl_target() :: login() | all).
+
+-type(acl_target_type() :: clientid | username | all).
+
+-type(access():: allow | deny).
+-type(action():: pub | sub).
+-type(legacy_action():: action() | pubsub).
+-type(created_at():: integer()).
+
 -record(emqx_user, {
+          login,
+          password,
+          created_at
+        }).
+
+-type(emqx_user() :: #emqx_user{
           login :: login(),
           password :: binary(),
-          created_at :: integer()
+          created_at :: created_at()
         }).
 
--record(emqx_acl, {
+-define(ACL_TABLE, emqx_acl).
-          filter:: {login() | all, emqx_topic:topic()},
+
-          action :: pub | sub | pubsub,
+-define(MIGRATION_MARK_KEY, emqx_acl2_migration_started).
-          access :: allow | deny,
+
-          created_at :: integer()
+-record(?ACL_TABLE, {
+          filter :: {acl_target(), emqx_topic:topic()} | ?MIGRATION_MARK_KEY,
+          action :: legacy_action(),
+          access :: access(),
+          created_at :: created_at()
          }).
 
--record(auth_metrics, {
+-define(MIGRATION_MARK_RECORD, #?ACL_TABLE{filter = ?MIGRATION_MARK_KEY, action = pub, access = deny, created_at = 0}).
-        success = 'client.auth.success',
+
-        failure = 'client.auth.failure',
+-type(rule() :: {access(), action(), emqx_topic:topic(), created_at()}).
-        ignore = 'client.auth.ignore'
+
+-define(ACL_TABLE2, emqx_acl2).
+
+-record(?ACL_TABLE2, {
+          who :: acl_target(),
+          rules :: [ rule() ]
          }).
 
--record(acl_metrics, {
+-type(acl_record() :: {acl_target(), emqx_topic:topic(), action(), access(), created_at()}).
-        allow = 'client.acl.allow',
-        deny = 'client.acl.deny',
-        ignore = 'client.acl.ignore'
-    }).
-
--define(METRICS(Type), tl(tuple_to_list(#Type{}))).
--define(METRICS(Type, K), #Type{}#Type.K).
-
--define(AUTH_METRICS, ?METRICS(auth_metrics)).
--define(AUTH_METRICS(K), ?METRICS(auth_metrics, K)).
-
--define(ACL_METRICS, ?METRICS(acl_metrics)).
--define(ACL_METRICS(K), ?METRICS(acl_metrics, K)).

apps/emqx_auth_mnesia/src/emqx_acl_mnesia.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -18,51 +18,35 @@
 
 -include("emqx_auth_mnesia.hrl").
 
--include_lib("stdlib/include/ms_transform.hrl").
-
--define(TABLE, emqx_acl).
-
 %% ACL Callbacks
 -export([ init/0
-        , register_metrics/0
         , check_acl/5
         , description/0
        ]).
 
 init() ->
-    ok = ekka_mnesia:create_table(emqx_acl, [
+    ok = emqx_acl_mnesia_db:create_table(),
-            {type, bag},
+    ok = emqx_acl_mnesia_db:create_table2().
-            {disc_copies, [node()]},
-            {attributes, record_info(fields, emqx_acl)},
-            {storage_properties, [{ets, [{read_concurrency, true}]}]}]),
-    ok = ekka_mnesia:copy_table(emqx_acl, disc_copies).
-
--spec(register_metrics() -> ok).
-register_metrics() ->
-    lists:foreach(fun emqx_metrics:ensure/1, ?ACL_METRICS).
 
 check_acl(ClientInfo = #{ clientid := Clientid }, PubSub, Topic, _NoMatchAction, _Params) ->
     Username = maps:get(username, ClientInfo, undefined),
 
     Acls = case Username of
                undefined ->
-                   emqx_acl_mnesia_cli:lookup_acl({clientid, Clientid}) ++
+                   emqx_acl_mnesia_db:lookup_acl({clientid, Clientid}) ++
-                   emqx_acl_mnesia_cli:lookup_acl(all);
+                   emqx_acl_mnesia_db:lookup_acl(all);
                _ ->
-                   emqx_acl_mnesia_cli:lookup_acl({clientid, Clientid}) ++
+                   emqx_acl_mnesia_db:lookup_acl({clientid, Clientid}) ++
-                   emqx_acl_mnesia_cli:lookup_acl({username, Username}) ++
+                   emqx_acl_mnesia_db:lookup_acl({username, Username}) ++
-                   emqx_acl_mnesia_cli:lookup_acl(all)
+                   emqx_acl_mnesia_db:lookup_acl(all)
            end,
 
     case match(ClientInfo, PubSub, Topic, Acls) of
         allow ->
-            emqx_metrics:inc(?ACL_METRICS(allow)),
             {stop, allow};
         deny ->
-            emqx_metrics:inc(?ACL_METRICS(deny)),
             {stop, deny};
         _ ->
-            emqx_metrics:inc(?ACL_METRICS(ignore)),
             ok
     end.
 
@@ -83,7 +67,6 @@ match(ClientInfo, PubSub, Topic, [ {_, ACLTopic, Action, Access, _} | Acls]) ->
 match_topic(ClientInfo, Topic, ACLTopic) when is_binary(Topic) ->
     emqx_topic:match(Topic, feed_var(ClientInfo, ACLTopic)).
 
-match_actions(_, pubsub) -> true;
 match_actions(subscribe, sub) -> true;
 match_actions(publish, pub) -> true;
 match_actions(_, _) -> false.

apps/emqx_auth_mnesia/src/emqx_acl_mnesia_api.erl

@@ -1,5 +1,5 @@
 %c%--------------------------------------------------------------------
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -16,8 +16,6 @@
 
 -module(emqx_acl_mnesia_api).
 
--include("emqx_auth_mnesia.hrl").
-
 -include_lib("stdlib/include/ms_transform.hrl").
 
 -import(proplists, [ get_value/2
@@ -98,27 +96,29 @@
         , delete/2
         ]).
 
+-define(CLIENTID_SCHEMA, [{<<"clientid">>, binary}, {<<"_like_clientid">>, binary}] ++ ?COMMON_SCHEMA).
+-define(USERNAME_SCHEMA, [{<<"username">>, binary}, {<<"_like_username">>, binary}] ++ ?COMMON_SCHEMA).
+-define(COMMON_SCHEMA, [{<<"topic">>, binary}, {<<"action">>, atom}, {<<"access">>, atom}]).
+
 list_clientid(_Bindings, Params) ->
-    MatchSpec = ets:fun2ms(
+    {_, Params1 = {_Qs, _Fuzzy}} = emqx_mgmt_api:params2qs(Params, ?CLIENTID_SCHEMA),
-                  fun({emqx_acl, {{clientid, Clientid}, Topic}, Action, Access, CreatedAt}) -> {{clientid,Clientid}, Topic, Action,Access, CreatedAt} end),
+    Table = emqx_acl_mnesia_db:login_acl_table(clientid, Params1),
-    return({ok, emqx_auth_mnesia_api:paginate(emqx_acl, MatchSpec, Params, fun emqx_acl_mnesia_cli:comparing/2, fun format/1)}).
+    return({ok, paginate_qh(Table, count(Table), Params, fun emqx_acl_mnesia_db:comparing/2, fun format/1)}).
 
 list_username(_Bindings, Params) ->
-    MatchSpec = ets:fun2ms(
+    {_, Params1 = {_Qs, _Fuzzy}} = emqx_mgmt_api:params2qs(Params, ?USERNAME_SCHEMA),
-                  fun({emqx_acl, {{username, Username}, Topic}, Action, Access, CreatedAt}) -> {{username, Username}, Topic, Action,Access, CreatedAt} end),
+    Table = emqx_acl_mnesia_db:login_acl_table(username, Params1),
-    return({ok, emqx_auth_mnesia_api:paginate(emqx_acl, MatchSpec, Params, fun emqx_acl_mnesia_cli:comparing/2, fun format/1)}).
+    return({ok, paginate_qh(Table, count(Table), Params, fun emqx_acl_mnesia_db:comparing/2, fun format/1)}).
 
 list_all(_Bindings, Params) ->
-    MatchSpec = ets:fun2ms(
+    {_, Params1 = {_Qs, _Fuzzy}} = emqx_mgmt_api:params2qs(Params, ?COMMON_SCHEMA),
-                  fun({emqx_acl, {all, Topic}, Action, Access, CreatedAt}) -> {all, Topic, Action,Access, CreatedAt}end
+    Table = emqx_acl_mnesia_db:login_acl_table(all, Params1),
-                 ),
+    return({ok, paginate_qh(Table, count(Table), Params, fun emqx_acl_mnesia_db:comparing/2, fun format/1)}).
-    return({ok, emqx_auth_mnesia_api:paginate(emqx_acl, MatchSpec, Params, fun emqx_acl_mnesia_cli:comparing/2, fun format/1)}).
-
 
 lookup(#{clientid := Clientid}, _Params) ->
-    return({ok, format(emqx_acl_mnesia_cli:lookup_acl({clientid, urldecode(Clientid)}))});
+    return({ok, format(emqx_acl_mnesia_db:lookup_acl({clientid, urldecode(Clientid)}))});
 lookup(#{username := Username}, _Params) ->
-    return({ok, format(emqx_acl_mnesia_cli:lookup_acl({username, urldecode(Username)}))}).
+    return({ok, format(emqx_acl_mnesia_db:lookup_acl({username, urldecode(Username)}))}).
 
 add(_Bindings, Params) ->
     [ P | _] = Params,
@@ -144,15 +144,15 @@ do_add(Params) ->
     Username = get_value(<<"username">>, Params, undefined),
     Login = case {Clientid, Username} of
                 {undefined, undefined} -> all;
-                {_, undefined} -> {clientid, urldecode(Clientid)};
+                {_, undefined} -> {clientid, Clientid};
-                {undefined, _} -> {username, urldecode(Username)}
+                {undefined, _} -> {username, Username}
             end,
-    Topic = urldecode(get_value(<<"topic">>, Params)),
+    Topic = get_value(<<"topic">>, Params),
-    Action = urldecode(get_value(<<"action">>, Params)),
+    Action = get_value(<<"action">>, Params),
-    Access = urldecode(get_value(<<"access">>, Params)),
+    Access = get_value(<<"access">>, Params),
     Re = case validate([login, topic, action, access], [Login, Topic, Action, Access]) of
         ok ->
-            emqx_acl_mnesia_cli:add_acl(Login, Topic, erlang:binary_to_atom(Action, utf8), erlang:binary_to_atom(Access, utf8));
+            emqx_acl_mnesia_db:add_acl(Login, Topic, erlang:binary_to_atom(Action, utf8), erlang:binary_to_atom(Access, utf8));
         Err -> Err
     end,
     maps:merge(#{topic => Topic,
@@ -165,15 +165,23 @@ do_add(Params) ->
                    end).
 
 delete(#{clientid := Clientid, topic := Topic}, _) ->
-    return(emqx_acl_mnesia_cli:remove_acl({clientid, urldecode(Clientid)}, urldecode(Topic)));
+    return(emqx_acl_mnesia_db:remove_acl({clientid, urldecode(Clientid)}, urldecode(Topic)));
 delete(#{username := Username, topic := Topic}, _) ->
-    return(emqx_acl_mnesia_cli:remove_acl({username, urldecode(Username)}, urldecode(Topic)));
+    return(emqx_acl_mnesia_db:remove_acl({username, urldecode(Username)}, urldecode(Topic)));
 delete(#{topic := Topic}, _) ->
-    return(emqx_acl_mnesia_cli:remove_acl(all, urldecode(Topic))).
+    return(emqx_acl_mnesia_db:remove_acl(all, urldecode(Topic))).
 
 %%------------------------------------------------------------------------------
 %% Interval Funcs
 %%------------------------------------------------------------------------------
+
+count(QH) ->
+    Count = qlc:fold(fun(_, Sum) -> Sum + 1 end, 0, QH),
+    case is_integer(Count) of
+        true -> Count;
+        false -> 0
+    end.
+
 format({{clientid, Clientid}, Topic, Action, Access, _CreatedAt}) ->
     #{clientid => Clientid, topic => Topic, action => Action, access => Access};
 format({{username, Username}, Topic, Action, Access, _CreatedAt}) ->
@@ -224,3 +232,27 @@ format_msg(Message) when is_tuple(Message) ->
 
 urldecode(S) ->
     emqx_http_lib:uri_decode(S).
+
+paginate_qh(Qh, Count, Params, ComparingFun, RowFun) ->
+    Page = page(Params),
+    Limit = limit(Params),
+    Cursor = qlc:cursor(Qh),
+    case Page > 1 of
+        true  ->
+            _ = qlc:next_answers(Cursor, (Page - 1) * Limit),
+            ok;
+        false -> ok
+    end,
+    Rows = qlc:next_answers(Cursor, Limit),
+    qlc:delete_cursor(Cursor),
+    #{meta  => #{page => Page, limit => Limit, count => Count},
+        data  => [RowFun(Row) || Row <- lists:sort(ComparingFun, Rows)]}.
+
+page(Params) ->
+    binary_to_integer(proplists:get_value(<<"_page">>, Params, <<"1">>)).
+
+limit(Params) ->
+    case proplists:get_value(<<"_limit">>, Params) of
+        undefined -> 50;
+        Size      -> binary_to_integer(Size)
+    end.

apps/emqx_auth_mnesia/src/emqx_acl_mnesia_cli.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -16,110 +16,28 @@
 
 -module(emqx_acl_mnesia_cli).
 
--include("emqx_auth_mnesia.hrl").
--include_lib("emqx/include/logger.hrl").
--include_lib("stdlib/include/ms_transform.hrl").
--define(TABLE, emqx_acl).
-
-%% Acl APIs
--export([ add_acl/4
-        , lookup_acl/1
-        , all_acls/0
-        , all_acls/1
-        , remove_acl/2
-        ]).
-
 -export([cli/1]).
--export([comparing/2]).
-%%--------------------------------------------------------------------
-%% Acl API
-%%--------------------------------------------------------------------
-
-%% @doc Add Acls
--spec(add_acl(login() | all, emqx_topic:topic(), pub | sub | pubsub, allow | deny) ->
-        ok | {error, any()}).
-add_acl(Login, Topic, Action, Access) ->
-    Filter = {Login, Topic},
-    Acl = #?TABLE{
-             filter = Filter,
-             action = Action,
-             access = Access,
-             created_at = erlang:system_time(millisecond)
-            },
-    ret(mnesia:transaction(
-          fun() ->
-                  OldRecords = mnesia:wread({?TABLE, Filter}),
-                  case Action of
-                      pubsub ->
-                          update_permission(pub, Acl, OldRecords),
-                          update_permission(sub, Acl, OldRecords);
-                      _ ->
-                          update_permission(Action, Acl, OldRecords)
-                  end
-          end)).
-
-%% @doc Lookup acl by login
--spec(lookup_acl(login() | all) -> list()).
-lookup_acl(undefined) -> [];
-lookup_acl(Login) ->
-    MatchSpec = ets:fun2ms(fun({?TABLE, {Filter, ACLTopic}, Action, Access, CreatedAt})
-                                 when Filter =:= Login ->
-                                   {Filter, ACLTopic, Action, Access, CreatedAt}
-                           end),
-    lists:sort(fun comparing/2, ets:select(?TABLE, MatchSpec)).
-
-%% @doc Remove acl
--spec(remove_acl(login() | all, emqx_topic:topic()) -> ok | {error, any()}).
-remove_acl(Login, Topic) ->
-    ret(mnesia:transaction(fun mnesia:delete/1, [{?TABLE, {Login, Topic}}])).
-
-%% @doc All logins
--spec(all_acls() -> list()).
-all_acls() ->
-    all_acls(clientid) ++
-    all_acls(username) ++
-    all_acls(all).
-
-all_acls(clientid) ->
-    MatchSpec = ets:fun2ms(
-                  fun({?TABLE, {{clientid, Clientid}, Topic}, Action, Access, CreatedAt}) ->
-                          {{clientid, Clientid}, Topic, Action, Access, CreatedAt}
-                  end),
-    lists:sort(fun comparing/2, ets:select(?TABLE, MatchSpec));
-all_acls(username) ->
-    MatchSpec = ets:fun2ms(
-                  fun({?TABLE, {{username, Username}, Topic}, Action, Access, CreatedAt}) ->
-                          {{username, Username}, Topic, Action, Access, CreatedAt}
-                  end),
-    lists:sort(fun comparing/2, ets:select(?TABLE, MatchSpec));
-all_acls(all) ->
-    MatchSpec = ets:fun2ms(
-                  fun({?TABLE, {all, Topic}, Action, Access, CreatedAt}) ->
-                          {all, Topic, Action, Access, CreatedAt}
-                  end
-                 ),
-    lists:sort(fun comparing/2, ets:select(?TABLE, MatchSpec)).
 
 %%--------------------------------------------------------------------
 %% ACL Cli
 %%--------------------------------------------------------------------
 
 cli(["list"]) ->
-    [print_acl(Acl) || Acl <- all_acls()];
+    [print_acl(Acl) || Acl <- emqx_acl_mnesia_db:all_acls()];
 
 cli(["list", "clientid"]) ->
-    [print_acl(Acl) || Acl <- all_acls(clientid)];
+    [print_acl(Acl) || Acl <- emqx_acl_mnesia_db:all_acls(clientid)];
 
 cli(["list", "username"]) ->
-    [print_acl(Acl) || Acl <- all_acls(username)];
+    [print_acl(Acl) || Acl <- emqx_acl_mnesia_db:all_acls(username)];
 
 cli(["list", "_all"]) ->
-    [print_acl(Acl) || Acl <- all_acls(all)];
+    [print_acl(Acl) || Acl <- emqx_acl_mnesia_db:all_acls(all)];
 
 cli(["add", "clientid", Clientid, Topic, Action, Access]) ->
     case validate(action, Action) andalso validate(access, Access) of
         true ->
-            case add_acl(
+            case emqx_acl_mnesia_db:add_acl(
                    {clientid, iolist_to_binary(Clientid)},
                    iolist_to_binary(Topic),
                    list_to_existing_atom(Action),
@@ -135,7 +53,7 @@ cli(["add", "clientid", Clientid, Topic, Action, Access]) ->
 cli(["add", "username", Username, Topic, Action, Access]) ->
     case validate(action, Action) andalso validate(access, Access) of
         true ->
-            case add_acl(
+            case emqx_acl_mnesia_db:add_acl(
                    {username, iolist_to_binary(Username)},
                    iolist_to_binary(Topic),
                    list_to_existing_atom(Action),
@@ -151,7 +69,7 @@ cli(["add", "username", Username, Topic, Action, Access]) ->
 cli(["add", "_all", Topic, Action, Access]) ->
     case validate(action, Action) andalso validate(access, Access) of
         true ->
-            case add_acl(
+            case emqx_acl_mnesia_db:add_acl(
                    all,
                    iolist_to_binary(Topic),
                    list_to_existing_atom(Action),
@@ -165,16 +83,16 @@ cli(["add", "_all", Topic, Action, Access]) ->
     end;
 
 cli(["show", "clientid", Clientid]) ->
-    [print_acl(Acl) || Acl <- lookup_acl({clientid, iolist_to_binary(Clientid)})];
+    [print_acl(Acl) || Acl <- emqx_acl_mnesia_db:lookup_acl({clientid, iolist_to_binary(Clientid)})];
 
 cli(["show", "username", Username]) ->
-    [print_acl(Acl) || Acl <- lookup_acl({username, iolist_to_binary(Username)})];
+    [print_acl(Acl) || Acl <- emqx_acl_mnesia_db:lookup_acl({username, iolist_to_binary(Username)})];
 
 cli(["del", "clientid", Clientid, Topic])->
     cli(["delete", "clientid", Clientid, Topic]);
 
 cli(["delete", "clientid", Clientid, Topic])->
-    case remove_acl({clientid, iolist_to_binary(Clientid)}, iolist_to_binary(Topic)) of
+    case emqx_acl_mnesia_db:remove_acl({clientid, iolist_to_binary(Clientid)}, iolist_to_binary(Topic)) of
          ok -> emqx_ctl:print("ok~n");
         {error, Reason} -> emqx_ctl:print("Error: ~p~n", [Reason])
     end;
@@ -183,7 +101,7 @@ cli(["del", "username", Username, Topic])->
     cli(["delete", "username", Username, Topic]);
 
 cli(["delete", "username", Username, Topic])->
-    case remove_acl({username, iolist_to_binary(Username)}, iolist_to_binary(Topic)) of
+    case emqx_acl_mnesia_db:remove_acl({username, iolist_to_binary(Username)}, iolist_to_binary(Topic)) of
          ok -> emqx_ctl:print("ok~n");
         {error, Reason} -> emqx_ctl:print("Error: ~p~n", [Reason])
     end;
@@ -192,7 +110,7 @@ cli(["del", "_all", Topic])->
     cli(["delete", "_all", Topic]);
 
 cli(["delete", "_all", Topic])->
-    case remove_acl(all, iolist_to_binary(Topic)) of
+    case emqx_acl_mnesia_db:remove_acl(all, iolist_to_binary(Topic)) of
          ok -> emqx_ctl:print("ok~n");
         {error, Reason} -> emqx_ctl:print("Error: ~p~n", [Reason])
     end;
@@ -201,6 +119,7 @@ cli(_) ->
     emqx_ctl:usage([ {"acl list clientid", "List clientid acls"}
                    , {"acl list username", "List username acls"}
                    , {"acl list _all", "List $all acls"}
+                   , {"acl list ", "List all acls"}
                    , {"acl show clientid <Clientid>", "Lookup clientid acl detail"}
                    , {"acl show username <Username>", "Lookup username acl detail"}
                    , {"acl aad clientid <Clientid> <Topic> <Action> <Access>", "Add clientid acl"}
@@ -215,13 +134,6 @@ cli(_) ->
 %% Internal functions
 %%--------------------------------------------------------------------
 
-comparing({_, _, _, _, CreatedAt1},
-          {_, _, _, _, CreatedAt2}) ->
-    CreatedAt1 >= CreatedAt2.
-
-ret({atomic, ok})     -> ok;
-ret({aborted, Error}) -> {error, Error}.
-
 validate(action, "pub") -> true;
 validate(action, "sub") -> true;
 validate(action, "pubsub") -> true;
@@ -244,27 +156,3 @@ print_acl({all, Topic, Action, Access, _}) ->
         "Acl($all topic = ~p action = ~p access = ~p)~n",
         [Topic, Action, Access]
      ).
-
-update_permission(Action, Acl0, OldRecords) ->
-    Acl = Acl0 #?TABLE{action = Action},
-    maybe_delete_shadowed_records(Action, OldRecords),
-    mnesia:write(Acl).
-
-maybe_delete_shadowed_records(_, []) ->
-    ok;
-maybe_delete_shadowed_records(Action1, [Rec = #emqx_acl{action = Action2} | Rest]) ->
-    if Action1 =:= Action2 ->
-            ok = mnesia:delete_object(Rec);
-       Action2 =:= pubsub ->
-            %% Perform migration from the old data format on the
-            %% fly. This is needed only for the enterprise version,
-            %% delete this branch on 5.0
-            mnesia:delete_object(Rec),
-            mnesia:write(Rec#?TABLE{action = other_action(Action1)});
-       true ->
-            ok
-    end,
-    maybe_delete_shadowed_records(Action1, Rest).
-
-other_action(pub) -> sub;
-other_action(sub) -> pub.

apps/emqx_auth_mnesia/src/emqx_acl_mnesia_db.erl

@@ -0,0 +1,388 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_acl_mnesia_db).
+
+-include("emqx_auth_mnesia.hrl").
+-include_lib("stdlib/include/ms_transform.hrl").
+-include_lib("stdlib/include/qlc.hrl").
+
+%% ACL APIs
+-export([ create_table/0
+        , create_table2/0
+        ]).
+
+-export([ add_acl/4
+        , lookup_acl/1
+        , all_acls_export/0
+        , all_acls/0
+        , all_acls/1
+        , remove_acl/2
+        , merge_acl_records/3
+        , login_acl_table/1
+        , login_acl_table/2
+        , is_migration_started/0
+        ]).
+
+-export([comparing/2]).
+
+%%--------------------------------------------------------------------
+%% ACL API
+%%--------------------------------------------------------------------
+
+%% @doc Create table `emqx_acl` of old format rules
+-spec(create_table() -> ok).
+create_table() ->
+    ok = ekka_mnesia:create_table(?ACL_TABLE, [
+        {type, bag},
+        {disc_copies, [node()]},
+        {attributes, record_info(fields, ?ACL_TABLE)},
+        {storage_properties, [{ets, [{read_concurrency, true}]}]}]),
+    ok = ekka_mnesia:copy_table(?ACL_TABLE, disc_copies).
+
+%% @doc Create table `emqx_acl2` of new format rules
+-spec(create_table2() -> ok).
+create_table2() ->
+    ok = ekka_mnesia:create_table(?ACL_TABLE2, [
+            {type, ordered_set},
+            {disc_copies, [node()]},
+            {attributes, record_info(fields, ?ACL_TABLE2)},
+            {storage_properties, [{ets, [{read_concurrency, true}]}]}]),
+    ok = ekka_mnesia:copy_table(?ACL_TABLE2, disc_copies).
+
+%% @doc Add Acls
+-spec(add_acl(acl_target(), emqx_topic:topic(), legacy_action(), access()) ->
+        ok | {error, any()}).
+add_acl(Login, Topic, Action, Access) ->
+    ret(mnesia:transaction(fun() ->
+                              case is_migration_started() of
+                                  true -> add_acl_new(Login, Topic, Action, Access);
+                                  false -> add_acl_old(Login, Topic, Action, Access)
+                              end
+                           end)).
+
+%% @doc Lookup acl by login
+-spec(lookup_acl(acl_target()) -> list(acl_record())).
+lookup_acl(undefined) -> [];
+lookup_acl(Login) ->
+    % After migration to ?ACL_TABLE2, ?ACL_TABLE never has any rules. This lookup should be removed later.
+    MatchSpec = ets:fun2ms(fun(#?ACL_TABLE{filter = {Filter, _}} = Rec)
+                                 when Filter =:= Login -> Rec
+                           end),
+    OldRecs = ets:select(?ACL_TABLE, MatchSpec),
+
+    NewAcls = ets:lookup(?ACL_TABLE2, Login),
+    MergedAcl = merge_acl_records(Login, OldRecs, NewAcls),
+    lists:sort(fun comparing/2, acl_to_list(MergedAcl)).
+
+%% @doc Remove ACL
+-spec remove_acl(acl_target(), emqx_topic:topic()) -> ok | {error, any()}.
+remove_acl(Login, Topic) ->
+    ret(mnesia:transaction(fun() ->
+                              mnesia:delete({?ACL_TABLE, {Login, Topic}}),
+                              case mnesia:wread({?ACL_TABLE2, Login}) of
+                                  [] -> ok;
+                                  [#?ACL_TABLE2{rules = Rules} = Acl] ->
+                                      case delete_topic_rules(Topic, Rules) of
+                                          [] -> mnesia:delete({?ACL_TABLE2, Login});
+                                          [_ | _] = RemainingRules ->
+                                              mnesia:write(Acl#?ACL_TABLE2{rules = RemainingRules})
+                                      end
+                              end
+                           end)).
+
+%% @doc All ACL rules
+-spec(all_acls() -> list(acl_record())).
+all_acls() ->
+    all_acls(username) ++
+    all_acls(clientid) ++
+    all_acls(all).
+
+%% @doc All ACL rules of specified type
+-spec(all_acls(acl_target_type()) -> list(acl_record())).
+all_acls(AclTargetType) ->
+    lists:sort(fun comparing/2, qlc:eval(login_acl_table(AclTargetType))).
+
+%% @doc All ACL rules fetched transactionally
+-spec(all_acls_export() -> list(acl_record())).
+all_acls_export() ->
+    AclTargetTypes = [username, clientid, all],
+    MatchSpecNew = lists:flatmap(fun login_match_spec_new/1, AclTargetTypes),
+    MatchSpecOld = lists:flatmap(fun login_match_spec_old/1, AclTargetTypes),
+
+    {atomic, Records} = mnesia:transaction(
+        fun() ->
+            QH = acl_table(MatchSpecNew, MatchSpecOld, {#{}, #{}}, fun mnesia:table/2, fun lookup_mnesia/2),
+            qlc:eval(QH)
+        end),
+    Records.
+
+%% @doc QLC table of logins matching spec
+-spec(login_acl_table(acl_target_type()) -> qlc:query_handle()).
+login_acl_table(AclTargetType) ->
+    login_acl_table(AclTargetType, {[], []}).
+
+login_acl_table(AclTargetType, {Qs, Fuzzy}) ->
+    ToMap = fun({Type, Symbol, Val}, Acc) -> Acc#{{Type, Symbol} => Val} end,
+    Qs1 = lists:foldl(ToMap, #{}, Qs),
+    Fuzzy1 = lists:foldl(ToMap, #{}, Fuzzy),
+    MatchSpecNew = login_match_spec_new(AclTargetType, Qs1),
+    MatchSpecOld = login_match_spec_old(AclTargetType, Qs1),
+    acl_table(MatchSpecNew, MatchSpecOld, {Qs1, Fuzzy1}, fun ets:table/2, fun lookup_ets/2).
+
+%% @doc Combine old `emqx_acl` ACL records with a new `emqx_acl2` ACL record for a given login
+-spec(merge_acl_records(acl_target(), [#?ACL_TABLE{}], [#?ACL_TABLE2{}]) -> #?ACL_TABLE2{}).
+merge_acl_records(Login, OldRecs, Acls) ->
+    OldRules = old_recs_to_rules(OldRecs),
+    NewRules = case Acls of
+        [] -> [];
+        [#?ACL_TABLE2{rules = Rules}] -> Rules
+    end,
+    #?ACL_TABLE2{who = Login, rules = merge_rules(NewRules, OldRules)}.
+
+%% @doc Checks if background migration of ACL rules from `emqx_acl` to `emqx_acl2` format started.
+%% Should be run in transaction
+-spec(is_migration_started() -> boolean()).
+is_migration_started() ->
+    case mnesia:read({?ACL_TABLE, ?MIGRATION_MARK_KEY}) of
+        [?MIGRATION_MARK_RECORD | _] -> true;
+        [] -> false
+    end.
+
+%%--------------------------------------------------------------------
+%% Internal functions
+%%--------------------------------------------------------------------
+
+add_acl_new(Login, Topic, Action, Access) ->
+    Rule = {Access, Action, Topic, erlang:system_time(millisecond)},
+    Rules = normalize_rule(Rule),
+    OldAcl = mnesia:wread({?ACL_TABLE2, Login}),
+    NewAcl = case OldAcl of
+        [#?ACL_TABLE2{rules = OldRules} = Acl] ->
+            Acl#?ACL_TABLE2{rules = merge_rules(Rules, OldRules)};
+        [] ->
+            #?ACL_TABLE2{who = Login, rules = Rules}
+    end,
+    mnesia:write(NewAcl).
+
+add_acl_old(Login, Topic, Action, Access) ->
+    Filter = {Login, Topic},
+    Acl = #?ACL_TABLE{
+             filter = Filter,
+             action = Action,
+             access = Access,
+             created_at = erlang:system_time(millisecond)
+            },
+    OldRecords = mnesia:wread({?ACL_TABLE, Filter}),
+    case Action of
+        pubsub ->
+            update_permission(pub, Acl, OldRecords),
+            update_permission(sub, Acl, OldRecords);
+        _ ->
+            update_permission(Action, Acl, OldRecords)
+    end.
+
+old_recs_to_rules(OldRecs) ->
+    lists:flatmap(fun old_rec_to_rules/1, OldRecs).
+
+old_rec_to_rules(#?ACL_TABLE{filter = {_, Topic}, action = Action, access = Access, created_at = CreatedAt}) ->
+    normalize_rule({Access, Action, Topic, CreatedAt}).
+
+normalize_rule({Access, pubsub, Topic, CreatedAt}) ->
+    [{Access, pub, Topic, CreatedAt}, {Access, sub, Topic, CreatedAt}];
+normalize_rule({Access, Action, Topic, CreatedAt}) ->
+    [{Access, Action, Topic, CreatedAt}].
+
+merge_rules([], OldRules) -> OldRules;
+merge_rules([NewRule | RestNewRules], OldRules) ->
+    merge_rules(RestNewRules, merge_rule(NewRule, OldRules)).
+
+merge_rule({_, Action, Topic, _ } = NewRule, OldRules) ->
+    [NewRule | lists:filter(
+        fun({_, OldAction, OldTopic, _}) ->
+            {Action, Topic} =/= {OldAction, OldTopic}
+        end, OldRules)].
+
+acl_to_list(#?ACL_TABLE2{who = Login, rules = Rules}) ->
+    [{Login, Topic, Action, Access, CreatedAt} || {Access, Action, Topic, CreatedAt} <- Rules].
+
+delete_topic_rules(Topic, Rules) ->
+    [Rule || {_, _, T, _} = Rule <- Rules, T =/= Topic].
+
+comparing({_, _, _, _, CreatedAt} = Rec1,
+          {_, _, _, _, CreatedAt} = Rec2) ->
+        Rec1 >= Rec2;
+
+comparing({_, _, _, _, CreatedAt1},
+          {_, _, _, _, CreatedAt2}) ->
+    CreatedAt1 >= CreatedAt2.
+
+login_match_spec_old(Type) -> login_match_spec_old(Type, #{}).
+
+login_match_spec_old(all, _) ->
+    ets:fun2ms(fun(#?ACL_TABLE{filter = {all, _}} = Record) ->
+                Record
+               end);
+
+login_match_spec_old(Type, Params) when (Type =:= username) orelse (Type =:= clientid) ->
+    case maps:get({Type, '=:='}, Params, undefined) of
+        undefined ->
+            ets:fun2ms(fun(#?ACL_TABLE{filter = {{RType, _}, _}} = Rec) when RType =:= Type -> Rec end);
+        Val ->
+            ets:fun2ms(fun(#?ACL_TABLE{filter = {{RType, RVal}, _}} = Rec)
+                when RType =:= Type andalso RVal =:= Val -> Rec end)
+    end.
+
+login_match_spec_new(Type) -> login_match_spec_new(Type, #{}).
+
+login_match_spec_new(all, _) ->
+    ets:fun2ms(fun(#?ACL_TABLE2{who = all} = Record) ->
+                Record
+               end);
+
+login_match_spec_new(Type, Params) when (Type =:= username) orelse (Type =:= clientid) ->
+    case maps:get({Type, '=:='}, Params, undefined) of
+        undefined ->
+            ets:fun2ms(fun(#?ACL_TABLE2{who = {RType, _}} = Rec) when RType =:= Type -> Rec end);
+        Val ->
+            ets:fun2ms(fun(#?ACL_TABLE2{who = {RType, RVal}} = Rec)
+                when RType =:= Type andalso RVal =:= Val  -> Rec end)
+    end.
+
+acl_table(MatchSpecNew, MatchSpecOld, Params, TableFun, LookupFun) ->
+    TraverseFun =
+        fun() ->
+           CursorNew =
+               qlc:cursor(
+                   TableFun(?ACL_TABLE2, [{traverse, {select, MatchSpecNew}}])),
+           CursorOld =
+               qlc:cursor(
+                   TableFun(?ACL_TABLE, [{traverse, {select, MatchSpecOld}}])),
+           traverse_new(CursorNew, CursorOld, Params, #{}, LookupFun)
+        end,
+
+    qlc:table(TraverseFun, []).
+
+
+% These are traverse funs for qlc table created by `acl_table/4`.
+% Traversing consumes memory: it collects logins present in `?ACL_TABLE` and
+% at the same time having rules in `?ACL_TABLE2`.
+% Such records appear if ACLs are inserted before migration started.
+% After migration, number of such logins is zero, so traversing starts working in
+% constant memory.
+
+traverse_new(CursorNew, CursorOld, Params, FoundKeys, LookupFun) ->
+    Acls = qlc:next_answers(CursorNew, 1),
+    case Acls of
+        [] ->
+            qlc:delete_cursor(CursorNew),
+            traverse_old(CursorOld, Params, FoundKeys);
+        [#?ACL_TABLE2{who = Login, rules = Rules} = Acl] ->
+            Keys = lists:usort([{Login, Topic} || {_, _, Topic, _} <- Rules]),
+            OldRecs = lists:flatmap(fun(Key) -> LookupFun(?ACL_TABLE, Key) end, Keys),
+            MergedAcl = merge_acl_records(Login, OldRecs, [Acl]),
+            NewFoundKeys =
+                lists:foldl(fun(#?ACL_TABLE{filter = Key}, Found) -> maps:put(Key, true, Found) end,
+                            FoundKeys,
+                            OldRecs),
+            case acl_to_list(MergedAcl) of
+                [] ->
+                    traverse_new(CursorNew, CursorOld, Params, NewFoundKeys, LookupFun);
+                List ->
+                    filter_params(List, Params) ++
+                    fun() -> traverse_new(CursorNew, CursorOld, Params, NewFoundKeys, LookupFun) end
+            end
+    end.
+
+filter_params(List, {Qs, Fuzzy}) ->
+    case maps:size(Qs) =:= 0 andalso maps:size(Fuzzy) =:= 0 of
+        false ->
+            Topic = maps:get({topic, '=:='}, Qs, undefined),
+            Action = maps:get({action, '=:='}, Qs, undefined),
+            Access = maps:get({access, '=:='}, Qs, undefined),
+            lists:filter(fun({Target, Topic0, Action0, Access0, _CreatedAt}) ->
+                CheckList = [{Topic, Topic0}, {Action, Action0}, {Access, Access0}],
+                case lists:all(fun is_match/1, CheckList) of
+                    true ->
+                        case Target of
+                            {Type, Login} ->
+                                case maps:get({Type, 'like'}, Fuzzy, <<>>) of
+                                    <<>> -> true;
+                                    LikeSchema -> binary:match(Login, LikeSchema) =/= nomatch
+                                end;
+                            all -> true
+                        end;
+                    false -> false
+                end
+                         end, List);
+        true -> List
+    end.
+
+is_match({Schema, Val}) ->
+    Schema =:= undefined orelse Schema =:= Val.
+
+traverse_old(CursorOld, Params, FoundKeys) ->
+    OldAcls = qlc:next_answers(CursorOld),
+    case OldAcls of
+        [] ->
+            qlc:delete_cursor(CursorOld),
+            [];
+        _ ->
+            Records = [{Login, Topic, Action, Access, CreatedAt}
+                || #?ACL_TABLE{filter = {Login, Topic}, action = LegacyAction, access = Access, created_at = CreatedAt} <- OldAcls,
+                {_, Action, _, _} <- normalize_rule({Access, LegacyAction, Topic, CreatedAt}),
+                not maps:is_key({Login, Topic}, FoundKeys)
+            ],
+            case Records of
+                [] -> traverse_old(CursorOld, Params, FoundKeys);
+                List ->
+                    filter_params(List, Params)
+                    ++ fun() -> traverse_old(CursorOld, Params, FoundKeys) end
+            end
+    end.
+
+lookup_mnesia(Tab, Key) ->
+    mnesia:read({Tab, Key}).
+
+lookup_ets(Tab, Key) ->
+    ets:lookup(Tab, Key).
+
+update_permission(Action, Acl0, OldRecords) ->
+    Acl = Acl0 #?ACL_TABLE{action = Action},
+    maybe_delete_shadowed_records(Action, OldRecords),
+    mnesia:write(Acl).
+
+maybe_delete_shadowed_records(_, []) ->
+    ok;
+maybe_delete_shadowed_records(Action1, [Rec = #emqx_acl{action = Action2} | Rest]) ->
+    if Action1 =:= Action2 ->
+            ok = mnesia:delete_object(Rec);
+       Action2 =:= pubsub ->
+            %% Perform migration from the old data format on the
+            %% fly. This is needed only for the enterprise version,
+            %% delete this branch on 5.0
+            mnesia:delete_object(Rec),
+            mnesia:write(Rec#?ACL_TABLE{action = other_action(Action1)});
+       true ->
+            ok
+    end,
+    maybe_delete_shadowed_records(Action1, Rest).
+
+other_action(pub) -> sub;
+other_action(sub) -> pub.
+
+ret({atomic, ok})     -> ok;
+ret({aborted, Error}) -> {error, Error}.

apps/emqx_auth_mnesia/src/emqx_acl_mnesia_migrator.erl

@@ -0,0 +1,215 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_acl_mnesia_migrator).
+
+-include("emqx_auth_mnesia.hrl").
+-include_lib("emqx/include/logger.hrl").
+-include_lib("snabbkaffe/include/snabbkaffe.hrl").
+
+-behaviour(gen_statem).
+
+-define(CHECK_ALL_NODES_INTERVAL, 60000).
+
+-type(migration_delay_reason() :: old_nodes | bad_nodes).
+
+-export([
+    callback_mode/0,
+    init/1
+]).
+
+-export([
+    waiting_all_nodes/3,
+    checking_old_table/3,
+    migrating/3
+]).
+
+-export([
+    start_link/0,
+    start_link/1,
+    start_supervised/0,
+    stop_supervised/0,
+    migrate_records/0,
+    is_migrating_on_node/1,
+    is_old_table_migrated/0
+]).
+
+%%--------------------------------------------------------------------
+%% External interface
+%%--------------------------------------------------------------------
+
+start_link() ->
+    start_link(?MODULE).
+
+start_link(Name) when is_atom(Name) ->
+    start_link(#{
+        name => Name
+    });
+
+start_link(#{name := Name} = Opts) ->
+    gen_statem:start_link({local, Name}, ?MODULE, Opts, []).
+
+start_supervised() ->
+    try
+        {ok, _} = supervisor:restart_child(emqx_auth_mnesia_sup, ?MODULE),
+        ok
+    catch
+        exit:{noproc, _} -> ok
+    end.
+
+stop_supervised() ->
+    try
+        ok = supervisor:terminate_child(emqx_auth_mnesia_sup, ?MODULE),
+        ok = supervisor:delete_child(emqx_auth_mnesia_sup, ?MODULE)
+    catch
+        exit:{noproc, _} -> ok
+    end.
+
+%%--------------------------------------------------------------------
+%% gen_statem callbacks
+%%--------------------------------------------------------------------
+
+callback_mode() -> state_functions.
+
+init(Opts) ->
+    ok = emqx_acl_mnesia_db:create_table(),
+    ok = emqx_acl_mnesia_db:create_table2(),
+    Name = maps:get(name, Opts, ?MODULE),
+    CheckNodesInterval = maps:get(check_nodes_interval, Opts, ?CHECK_ALL_NODES_INTERVAL),
+    GetNodes = maps:get(get_nodes, Opts, fun all_nodes/0),
+    Data =
+        #{name => Name,
+          check_nodes_interval => CheckNodesInterval,
+          get_nodes => GetNodes},
+    {ok, waiting_all_nodes, Data, [{state_timeout, 0, check_nodes}]}.
+
+%%--------------------------------------------------------------------
+%% state callbacks
+%%--------------------------------------------------------------------
+
+waiting_all_nodes(state_timeout, check_nodes, Data) ->
+    #{name := Name, check_nodes_interval := CheckNodesInterval, get_nodes := GetNodes} = Data,
+    case is_all_nodes_migrating(Name, GetNodes()) of
+        true ->
+            ?tp(info, emqx_acl_mnesia_migrator_check_old_table, #{}),
+            {next_state, checking_old_table, Data, [{next_event, internal, check_old_table}]};
+        {false, Reason, Nodes} ->
+            ?tp(info,
+                emqx_acl_mnesia_migrator_bad_nodes_delay,
+                #{delay => CheckNodesInterval,
+                  reason => Reason,
+                  name => Name,
+                  nodes => Nodes}),
+            {keep_state_and_data, [{state_timeout, CheckNodesInterval, check_nodes}]}
+    end.
+
+checking_old_table(internal, check_old_table, Data) ->
+    case is_old_table_migrated() of
+        true ->
+            ?tp(info, emqx_acl_mnesia_migrator_finish, #{}),
+            {next_state, finished, Data, [{hibernate, true}]};
+        false ->
+            ?tp(info, emqx_acl_mnesia_migrator_start_migration, #{}),
+            {next_state, migrating, Data, [{next_event, internal, start_migration}]}
+    end.
+
+migrating(internal, start_migration, Data) ->
+    ok = migrate_records(),
+    {next_state, checking_old_table, Data, [{next_event, internal, check_old_table}]}.
+
+%% @doc Returns `true` if migration is started in the local node, otherwise crash.
+-spec(is_migrating_on_node(atom()) -> true).
+is_migrating_on_node(Name) ->
+    true = is_pid(erlang:whereis(Name)).
+
+%% @doc Run migration of records
+-spec(migrate_records() -> ok).
+migrate_records() ->
+    ok = add_migration_mark(),
+    Key = peek_record(),
+    do_migrate_records(Key).
+
+%% @doc Run migration of records
+-spec(is_all_nodes_migrating(atom(), list(node())) -> true | {false, migration_delay_reason(), list(node())}).
+is_all_nodes_migrating(Name, Nodes) ->
+    case rpc:multicall(Nodes, ?MODULE, is_migrating_on_node, [Name]) of
+        {Results, []} ->
+            OldNodes = [ Node || {Node, Result} <- lists:zip(Nodes, Results), Result =/= true ],
+            case OldNodes of
+                [] -> true;
+                _ -> {false, old_nodes, OldNodes}
+            end;
+        {_, [_BadNode | _] = BadNodes} ->
+            {false, bad_nodes, BadNodes}
+    end.
+
+%%--------------------------------------------------------------------
+%% Internal functions
+%%--------------------------------------------------------------------
+
+all_nodes() ->
+    ekka_mnesia:cluster_nodes(all).
+
+is_old_table_migrated() ->
+    Result =
+        mnesia:transaction(fun() ->
+                              case mnesia:first(?ACL_TABLE) of
+                                  ?MIGRATION_MARK_KEY ->
+                                      case mnesia:next(?ACL_TABLE, ?MIGRATION_MARK_KEY) of
+                                          '$end_of_table' -> true;
+                                          _OtherKey -> false
+                                      end;
+                                  '$end_of_table' -> false;
+                                  _OtherKey -> false
+                              end
+                           end),
+    case Result of
+        {atomic, true} ->
+            true;
+        _ ->
+            false
+    end.
+
+add_migration_mark() ->
+    {atomic, ok} = mnesia:transaction(fun() -> mnesia:write(?MIGRATION_MARK_RECORD) end),
+    ok.
+
+peek_record() ->
+    Key = mnesia:dirty_first(?ACL_TABLE),
+    case Key of
+        ?MIGRATION_MARK_KEY ->
+            mnesia:dirty_next(?ACL_TABLE, Key);
+        _ -> Key
+    end.
+
+do_migrate_records('$end_of_table') -> ok;
+do_migrate_records({_Login, _Topic} = Key) ->
+    ?tp(emqx_acl_mnesia_migrator_record_selected, #{key => Key}),
+    _ = mnesia:transaction(fun migrate_one_record/1, [Key]),
+    do_migrate_records(peek_record()).
+
+migrate_one_record({Login, _Topic} = Key) ->
+    case mnesia:wread({?ACL_TABLE, Key}) of
+        [] ->
+            ?tp(emqx_acl_mnesia_migrator_record_missed, #{key => Key}),
+            record_missing;
+        OldRecs ->
+            Acls = mnesia:wread({?ACL_TABLE2, Login}),
+            UpdatedAcl = emqx_acl_mnesia_db:merge_acl_records(Login, OldRecs, Acls),
+            ok = mnesia:write(UpdatedAcl),
+            ok = mnesia:delete({?ACL_TABLE, Key}),
+            ?tp(emqx_acl_mnesia_migrator_record_migrated, #{key => Key})
+    end.

apps/emqx_auth_mnesia/src/emqx_auth_mnesia.app.src

@@ -1,6 +1,6 @@
 {application, emqx_auth_mnesia,
  [{description, "EMQ X Authentication with Mnesia"},
-  {vsn, "4.3.0"}, % strict semver, bump manually
+  {vsn, "4.3.9"}, % strict semver, bump manually
   {modules, []},
   {registered, []},
   {applications, [kernel,stdlib,mnesia]},

apps/emqx_auth_mnesia/src/emqx_auth_mnesia.appup.src

@@ -0,0 +1,67 @@
+%% -*- mode: erlang -*-
+%% Unless you know what you are doing, DO NOT edit manually!!
+{VSN,
+  [{"4.3.7",
+    [{load_module,emqx_auth_mnesia_api,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_mnesia_cli,brutal_purge,soft_purge,[]}]},
+   {<<"4\\.3\\.[5-6]">>,
+    [{load_module,emqx_auth_mnesia_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_mnesia,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_mnesia_api,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_mnesia_db,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_mnesia_cli,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_mnesia_api,brutal_purge,soft_purge,[]}]},
+   {<<"4\\.3\\.[0-3]">>,
+    [{load_module,emqx_auth_mnesia_cli,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_mnesia,brutal_purge,soft_purge,[]},
+     {add_module,emqx_acl_mnesia_db},
+     {add_module,emqx_acl_mnesia_migrator,[emqx_acl_mnesia_db]},
+     {update,emqx_auth_mnesia_sup,supervisor},
+     {apply,{emqx_acl_mnesia_migrator,start_supervised,[]}},
+     {load_module,emqx_auth_mnesia_api,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_mnesia,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_mnesia_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_mnesia_api,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_mnesia_cli,brutal_purge,soft_purge,[]}]},
+   {"4.3.4",
+    [{load_module,emqx_auth_mnesia_api,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_mnesia,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_mnesia_db,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_mnesia_api,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_mnesia_cli,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_mnesia,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_mnesia_cli,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_mnesia_app,brutal_purge,soft_purge,[]}]},
+   {<<".*">>,[]}],
+  [{"4.3.7",
+    [{load_module,emqx_auth_mnesia_api,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_mnesia_cli,brutal_purge,soft_purge,[]}]},
+   {<<"4\\.3\\.[5-6]">>,
+    [{load_module,emqx_auth_mnesia_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_mnesia,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_mnesia_api,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_mnesia_db,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_mnesia_cli,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_mnesia_api,brutal_purge,soft_purge,[]}]},
+   {<<"4\\.3\\.[0-3]">>,
+    [{load_module,emqx_auth_mnesia_cli,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_mnesia,brutal_purge,soft_purge,[]},
+     {apply,{emqx_acl_mnesia_migrator,stop_supervised,[]}},
+     {update,emqx_auth_mnesia_sup,supervisor},
+     {load_module,emqx_acl_mnesia_cli,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_mnesia_api,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_mnesia_api,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_mnesia,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_mnesia_app,brutal_purge,soft_purge,[]},
+     {delete_module,emqx_acl_mnesia_migrator},
+     {delete_module,emqx_acl_mnesia_db}]},
+   {"4.3.4",
+    [{load_module,emqx_auth_mnesia_api,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_mnesia_api,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_mnesia,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_mnesia_cli,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_mnesia_db,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_mnesia,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_mnesia_cli,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_mnesia_app,brutal_purge,soft_purge,[]}]},
+   {<<".*">>,[]}]}.

apps/emqx_auth_mnesia/src/emqx_auth_mnesia.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -27,29 +27,31 @@
 -define(TABLE, emqx_user).
 %% Auth callbacks
 -export([ init/1
-        , register_metrics/0
         , check/3
         , description/0
         ]).
 
+-export([ match_password/3
+        , hash_type/0
+        ]).
+
 init(#{clientid_list := ClientidList, username_list := UsernameList}) ->
     ok = ekka_mnesia:create_table(?TABLE, [
             {disc_copies, [node()]},
             {attributes, record_info(fields, emqx_user)},
             {storage_properties, [{ets, [{read_concurrency, true}]}]}]),
-    _ = [ add_default_user({{clientid, iolist_to_binary(Clientid)}, iolist_to_binary(Password)})
+    lists:foreach(fun({Clientid, Password}) ->
-      || {Clientid, Password} <- ClientidList],
+        emqx_auth_mnesia_cli:add_default_user(clientid, iolist_to_binary(Clientid), iolist_to_binary(Password))
-    _ = [ add_default_user({{username, iolist_to_binary(Username)}, iolist_to_binary(Password)})
+    end, ClientidList),
-      || {Username, Password} <- UsernameList],
+
+    lists:foreach(fun({Username, Password}) ->
+        emqx_auth_mnesia_cli:add_default_user(username, iolist_to_binary(Username), iolist_to_binary(Password))
+    end, UsernameList),
+
     ok = ekka_mnesia:copy_table(?TABLE, disc_copies).
 
-%% @private
+hash_type() ->
-add_default_user({Login, Password}) when is_tuple(Login) ->
+    application:get_env(emqx_auth_mnesia, password_hash, sha256).
-    emqx_auth_mnesia_cli:add_user(Login, Password).
-
--spec(register_metrics() -> ok).
-register_metrics() ->
-    lists:foreach(fun emqx_metrics:ensure/1, ?AUTH_METRICS).
 
 check(ClientInfo = #{ clientid := Clientid
                     , password := NPassword
@@ -60,16 +62,14 @@ check(ClientInfo = #{ clientid := Clientid
                            end),
     case ets:select(?TABLE, MatchSpec) of
         [] ->
-            emqx_metrics:inc(?AUTH_METRICS(ignore)),
             ok;
         List ->
             case match_password(NPassword, HashType, List)  of
                 false ->
-                    ?LOG(error, "[Mnesia] Auth from mnesia failed: ~p", [ClientInfo]),
+                    Info = maps:without([password], ClientInfo),
-                    emqx_metrics:inc(?AUTH_METRICS(failure)),
+                    ?LOG(info, "[Mnesia] Auth from mnesia failed: ~p", [Info]),
                     {stop, AuthResult#{anonymous => false, auth_result => password_error}};
                 _ ->
-                    emqx_metrics:inc(?AUTH_METRICS(success)),
                     {stop, AuthResult#{anonymous => false, auth_result => success}}
             end
     end.

apps/emqx_auth_mnesia/src/emqx_auth_mnesia_api.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -18,18 +18,20 @@
 
 -include_lib("stdlib/include/qlc.hrl").
 -include_lib("stdlib/include/ms_transform.hrl").
+-include("emqx_auth_mnesia.hrl").
 
 -define(TABLE, emqx_user).
 
 -import(proplists, [get_value/2]).
 -import(minirest,  [return/1]).
--export([paginate/5]).
 
 -export([ list_clientid/2
         , lookup_clientid/2
         , add_clientid/2
         , update_clientid/2
         , delete_clientid/2
+        , query_clientid/3
+        , query_username/3
         ]).
 
 -rest_api(#{name   => list_clientid,
@@ -109,13 +111,32 @@
             descr  => "Delete username in the cluster"
            }).
 
+-define(CLIENTID_SCHEMA, {?TABLE,
+    [
+        {<<"clientid">>, binary},
+        {<<"_like_clientid">>, binary}
+    ]}).
+
+-define(USERNAME_SCHEMA, {?TABLE,
+    [
+        {<<"username">>, binary},
+        {<<"_like_username">>, binary}
+    ]}).
+
+-define(query_clientid, {?MODULE, query_clientid}).
+-define(query_username, {?MODULE, query_username}).
+
 %%------------------------------------------------------------------------------
 %% Auth Clientid Api
 %%------------------------------------------------------------------------------
 
 list_clientid(_Bindings, Params) ->
-    MatchSpec = ets:fun2ms(fun({?TABLE, {clientid, Clientid}, Password, CreatedAt}) -> {?TABLE, {clientid, Clientid}, Password, CreatedAt} end),
+    SortFun = fun(#{created_at := C1}, #{created_at := C2}) -> C1 > C2 end,
-    return({ok, paginate(?TABLE, MatchSpec, Params, fun emqx_auth_mnesia_cli:comparing/2, fun({?TABLE, {clientid, X}, _, _}) -> #{clientid => X} end)}).
+    CountFun = fun() ->
+        MatchSpec = [{{?TABLE, {clientid, '_'}, '_', '_'}, [], [true]}],
+        ets:select_count(?TABLE, MatchSpec)
+               end,
+    return({ok, emqx_mgmt_api:node_query(node(), Params, ?CLIENTID_SCHEMA, ?query_clientid, SortFun, CountFun)}).
 
 lookup_clientid(#{clientid := Clientid}, _Params) ->
     return({ok, format(emqx_auth_mnesia_cli:lookup_user({clientid, urldecode(Clientid)}))}).
@@ -133,15 +154,15 @@ add_clientid(_Bindings, Params) ->
     end.
 
 do_add_clientid([ Params | ParamsN ], ReList ) ->
-    Clientid = urldecode(get_value(<<"clientid">>, Params)),
+    Clientid = get_value(<<"clientid">>, Params),
     do_add_clientid(ParamsN, [{Clientid, format_msg(do_add_clientid(Params))} | ReList]);
 
 do_add_clientid([], ReList) ->
     {ok, ReList}.
 
 do_add_clientid(Params) ->
-    Clientid = urldecode(get_value(<<"clientid">>, Params)),
+    Clientid = get_value(<<"clientid">>, Params),
-    Password = urldecode(get_value(<<"password">>, Params)),
+    Password = get_value(<<"password">>, Params),
     Login = {clientid, Clientid},
     case validate([login, password], [Login, Password]) of
         ok ->
@@ -152,7 +173,7 @@ do_add_clientid(Params) ->
 update_clientid(#{clientid := Clientid}, Params) ->
     Password = get_value(<<"password">>, Params),
     case validate([password], [Password]) of
-        ok -> return(emqx_auth_mnesia_cli:update_user({clientid, urldecode(Clientid)}, urldecode(Password)));
+        ok -> return(emqx_auth_mnesia_cli:update_user({clientid, urldecode(Clientid)}, Password));
         Err -> return(Err)
     end.
 
@@ -164,8 +185,12 @@ delete_clientid(#{clientid := Clientid}, _) ->
 %%------------------------------------------------------------------------------
 
 list_username(_Bindings, Params) ->
-    MatchSpec = ets:fun2ms(fun({?TABLE, {username, Username}, Password, CreatedAt}) -> {?TABLE, {username, Username}, Password, CreatedAt} end),
+    SortFun = fun(#{created_at := C1}, #{created_at := C2}) -> C1 > C2 end,
-    return({ok, paginate(?TABLE, MatchSpec, Params, fun emqx_auth_mnesia_cli:comparing/2, fun({?TABLE, {username, X}, _, _}) -> #{username => X} end)}).
+    CountFun = fun() ->
+        MatchSpec = [{{?TABLE, {username, '_'}, '_', '_'}, [], [true]}],
+        ets:select_count(?TABLE, MatchSpec)
+               end,
+    return({ok, emqx_mgmt_api:node_query(node(), Params, ?USERNAME_SCHEMA, ?query_username, SortFun, CountFun)}).
 
 lookup_username(#{username := Username}, _Params) ->
     return({ok, format(emqx_auth_mnesia_cli:lookup_user({username, urldecode(Username)}))}).
@@ -182,15 +207,15 @@ add_username(_Bindings, Params) ->
     end.
 
 do_add_username([ Params | ParamsN ], ReList ) ->
-    Username = urldecode(get_value(<<"username">>, Params)),
+    Username = get_value(<<"username">>, Params),
     do_add_username(ParamsN, [{Username, format_msg(do_add_username(Params))} | ReList]);
 
 do_add_username([], ReList) ->
     {ok, ReList}.
 
 do_add_username(Params) ->
-    Username = urldecode(get_value(<<"username">>, Params)),
+    Username = get_value(<<"username">>, Params),
-    Password = urldecode(get_value(<<"password">>, Params)),
+    Password = get_value(<<"password">>, Params),
     Login = {username, Username},
     case validate([login, password], [Login, Password]) of
         ok ->
@@ -201,7 +226,7 @@ do_add_username(Params) ->
 update_username(#{username := Username}, Params) ->
     Password = get_value(<<"password">>, Params),
     case validate([password], [Password]) of
-        ok -> return(emqx_auth_mnesia_cli:update_user({username, urldecode(Username)}, urldecode(Password)));
+        ok -> return(emqx_auth_mnesia_cli:update_user({username, urldecode(Username)}, Password));
         Err -> return(Err)
     end.
 
@@ -211,68 +236,52 @@ delete_username(#{username := Username}, _) ->
 %%------------------------------------------------------------------------------
 %% Paging Query
 %%------------------------------------------------------------------------------
+query_clientid(Qs, Start, Limit) -> query(clientid, Qs, Start, Limit).
+query_username(Qs, Start, Limit) -> query(username, Qs, Start, Limit).
 
-paginate(Tables, MatchSpec, Params, ComparingFun, RowFun) ->
+query(Type, {Qs, []}, Start, Limit) ->
-    Qh = query_handle(Tables, MatchSpec),
+    Ms = qs2ms(Type, Qs),
-    Count = count(Tables, MatchSpec),
+    emqx_mgmt_api:select_table(?TABLE, Ms, Start, Limit, fun format/1);
-    Page = page(Params),
-    Limit = limit(Params),
-    Cursor = qlc:cursor(Qh),
-    case Page > 1 of
-        true  ->
-            _ = qlc:next_answers(Cursor, (Page - 1) * Limit),
-            ok;
-        false -> ok
-    end,
-    Rows = qlc:next_answers(Cursor, Limit),
-    qlc:delete_cursor(Cursor),
-    #{meta  => #{page => Page, limit => Limit, count => Count},
-      data  => [RowFun(Row) || Row <- lists:sort(ComparingFun, Rows)]}.
 
-query_handle(Table, MatchSpec) when is_atom(Table) ->
+query(Type, {Qs, Fuzzy}, Start, Limit) ->
-    Options = {traverse, {select, MatchSpec}},
+    Ms = qs2ms(Type, Qs),
-    qlc:q([R|| R <- ets:table(Table, Options)]);
+    MatchFun = match_fun(Ms, Fuzzy),
-query_handle([Table], MatchSpec) when is_atom(Table) ->
+    emqx_mgmt_api:traverse_table(?TABLE, MatchFun, Start, Limit, fun format/1).
-    Options = {traverse, {select, MatchSpec}},
-    qlc:q([R|| R <- ets:table(Table, Options)]);
-query_handle(Tables, MatchSpec) ->
-    Options = {traverse, {select, MatchSpec}},
-    qlc:append([qlc:q([E || E <- ets:table(T, Options)]) || T <- Tables]).
 
-count(Table, MatchSpec) when is_atom(Table) ->
+-spec qs2ms(clientid | username, list()) -> ets:match_spec().
-    [{MatchPattern, Where, _Re}] = MatchSpec,
+qs2ms(Type, Qs) ->
-    NMatchSpec = [{MatchPattern, Where, [true]}],
+    Init = #?TABLE{login = {Type, '_'}, password  = '_', created_at = '_'},
-    ets:select_count(Table, NMatchSpec);
+    MatchHead = lists:foldl(fun(Q, Acc) ->  match_ms(Q, Acc) end, Init, Qs),
-count([Table], MatchSpec) when is_atom(Table) ->
+    [{MatchHead, [], ['$_']}].
-    [{MatchPattern, Where, _Re}] = MatchSpec,
-    NMatchSpec = [{MatchPattern, Where, [true]}],
-    ets:select_count(Table, NMatchSpec);
-count(Tables, MatchSpec) ->
-    lists:sum([count(T, MatchSpec) || T <- Tables]).
 
-page(Params) ->
+match_ms({Type, '=:=', Value}, MatchHead) -> MatchHead#?TABLE{login = {Type, Value}};
-    binary_to_integer(proplists:get_value(<<"_page">>, Params, <<"1">>)).
+match_ms(_, MatchHead) -> MatchHead.
 
-limit(Params) ->
+match_fun(Ms, Fuzzy) ->
-    case proplists:get_value(<<"_limit">>, Params) of
+    MsC = ets:match_spec_compile(Ms),
-        undefined -> 10;
+    fun(Rows) ->
-        Size      -> binary_to_integer(Size)
+        Ls = ets:match_spec_run(Rows, MsC),
+        lists:filter(fun(E) -> run_fuzzy_match(E, Fuzzy) end, Ls)
     end.
 
+run_fuzzy_match(_, []) -> true;
+run_fuzzy_match(E = #?TABLE{login = {Key, Str}}, [{Key, like, SubStr}|Fuzzy]) ->
+    binary:match(Str, SubStr) /= nomatch andalso run_fuzzy_match(E, Fuzzy);
+run_fuzzy_match(_E, [{_Key, like, _SubStr}| _Fuzzy]) -> false.
+
 %%------------------------------------------------------------------------------
 %% Interval Funcs
 %%------------------------------------------------------------------------------
 
-format([{?TABLE, {clientid, ClientId}, Password, _InterTime}]) ->
+format([{?TABLE, {clientid, ClientId}, _Password, CreatedAt}]) ->
-    #{clientid => ClientId,
+    #{clientid => ClientId, created_at => CreatedAt};
-      password => Password};
 
-format([{?TABLE, {username, Username}, Password, _InterTime}]) ->
+format([{?TABLE, {username, Username}, _Password, CreatedAt}]) ->
-    #{username => Username,
+    #{username => Username, created_at => CreatedAt};
-      password => Password};
 
 format([]) ->
-    #{}.
+    #{};
+format(User) -> format([User]).
 
 validate([], []) ->
     ok;

apps/emqx_auth_mnesia/src/emqx_auth_mnesia_app.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -56,13 +56,9 @@ load_auth_hook() ->
     ClientidList = application:get_env(?APP, clientid_list, []),
     UsernameList = application:get_env(?APP, username_list, []),
     ok = emqx_auth_mnesia:init(#{clientid_list => ClientidList, username_list => UsernameList}),
-    ok = emqx_auth_mnesia:register_metrics(),
+    Params = #{hash_type => emqx_auth_mnesia:hash_type()},
-    Params = #{
-            hash_type => application:get_env(emqx_auth_mnesia, password_hash, sha256)
-            },
     emqx:hook('client.authenticate', fun emqx_auth_mnesia:check/3, [Params]).
 
 load_acl_hook() ->
     ok = emqx_acl_mnesia:init(),
-    ok = emqx_acl_mnesia:register_metrics(),
     emqx:hook('client.check_acl', fun emqx_acl_mnesia:check_acl/5, [#{}]).

apps/emqx_auth_mnesia/src/emqx_auth_mnesia_cli.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -22,6 +22,8 @@
 -define(TABLE, emqx_user).
 %% Auth APIs
 -export([ add_user/2
+        , force_add_user/2
+        , add_default_user/3
         , update_user/2
         , remove_user/1
         , lookup_user/1
@@ -56,6 +58,65 @@ insert_user(User = #emqx_user{login = Login}) ->
         [_|_] -> mnesia:abort(existed)
     end.
 
+-spec(add_default_user(clientid | username, binary(), binary()) -> ok | {error, any()}).
+add_default_user(Type, Key, Password) ->
+    Login = {Type, Key},
+    case add_user(Login, Password) of
+        ok -> ok;
+        {error, existed} ->
+            NewPwd = encrypted_data(Password),
+            [#emqx_user{password = OldPwd}] = emqx_auth_mnesia_cli:lookup_user(Login),
+            HashType = emqx_auth_mnesia:hash_type(),
+            case emqx_auth_mnesia:match_password(NewPwd, HashType, [OldPwd])  of
+                true -> ok;
+                false ->
+                    %% We can't force add default,
+                    %% otherwise passwords that have been updated via HTTP API will be reset after reboot.
+                    TypeCtl =
+                        case Type of
+                            clientid -> clientid;
+                            username -> user
+                        end,
+                    ?LOG(warning,
+                        "[Auth Mnesia] auth.client.x.~p=~s password in the emqx_auth_mnesia.conf\n"
+                        "does not match the password in the database(mnesia).\n"
+                        "1. If you have already changed the password via the HTTP API, this warning has no effect.\n"
+                        "You can remove the `auth.client.x.~p=~s` from emqx_auth_mnesia.conf to resolve this warning.\n"
+                        "2. If you just want to update the password by manually changing the configuration file,\n"
+                        "you need to delete the old user and password using `emqx_ctl ~p delete ~s` first\n"
+                        "the new password in emqx_auth_mnesia.conf can take effect after reboot.",
+                        [Type, Key, Type, Key, TypeCtl, Key]),
+                    ok
+            end;
+        Error -> Error
+    end.
+
+force_add_user(Login, Password) ->
+    User = #emqx_user{
+        login = Login,
+        password = encrypted_data(Password),
+        created_at = erlang:system_time(millisecond)
+    },
+    case ret(mnesia:transaction(fun insert_or_update_user/2, [Password, User])) of
+        {ok, override} ->
+            ?LOG(warning, "[Mnesia] (~p)'s password has be updated.", [Login]),
+            ok;
+        Other -> Other
+    end.
+
+insert_or_update_user(NewPwd, User = #emqx_user{login = Login}) ->
+    case mnesia:read(?TABLE, Login) of
+        []    -> mnesia:write(User);
+        [#emqx_user{password = Pwd}] ->
+            case emqx_auth_mnesia:match_password(NewPwd, emqx_auth_mnesia:hash_type(), [Pwd])  of
+                true -> ok;
+                false ->
+                    ok = mnesia:write(User),
+                    {ok, override}
+            end
+    end.
+
+
 %% @doc Update User
 -spec(update_user(tuple(), binary()) -> ok | {error, any()}).
 update_user(Login, NewPassword) ->
@@ -105,11 +166,11 @@ comparing({?TABLE, _, _, CreatedAt1},
           {?TABLE, _, _, CreatedAt2}) ->
     CreatedAt1 >= CreatedAt2.
 
-ret({atomic, ok})     -> ok;
+ret({atomic, Res}) -> Res;
 ret({aborted, Error}) -> {error, Error}.
 
 encrypted_data(Password) ->
-    HashType = application:get_env(emqx_auth_mnesia, password_hash, sha256),
+    HashType = emqx_auth_mnesia:hash_type(),
     SaltBin = salt(),
     <<SaltBin/binary, (hash(Password, SaltBin, HashType))/binary>>.
 

apps/emqx_auth_mnesia/src/emqx_auth_mnesia_sup.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -33,4 +33,16 @@ start_link() ->
 %%--------------------------------------------------------------------
 
 init([]) ->
-    {ok, {{one_for_one, 10, 100}, []}}.
+    {ok, {{one_for_one, 10, 100}, [
+        child_spec(emqx_acl_mnesia_migrator, worker, [])
+    ]}}.
+
+child_spec(M, worker, Args) ->
+    #{id       => M,
+      start    => {M, start_link, Args},
+      restart  => permanent,
+      shutdown => 5000,
+      type     => worker,
+      modules  => [M]
+     }.
+

apps/emqx_auth_mnesia/test/emqx_acl_mnesia_SUITE.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -20,10 +20,13 @@
 -compile(export_all).
 
 -include("emqx_auth_mnesia.hrl").
+-include_lib("emqx/include/emqx.hrl").
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("common_test/include/ct.hrl").
+-include_lib("snabbkaffe/include/snabbkaffe.hrl").
 
 -import(emqx_ct_http, [ request_api/3
+                      , request_api/4
                       , request_api/5
                       , get_http_data/1
                       , create_default_app/0
@@ -39,10 +42,17 @@ all() ->
     emqx_ct:all(?MODULE).
 
 groups() ->
-    [].
+    [{async_migration_tests, [sequence], [
+        t_old_and_new_acl_migration_by_migrator,
+        t_old_and_new_acl_migration_repeated_by_migrator,
+        t_migration_concurrency
+    ]}].
 
 init_per_suite(Config) ->
-    emqx_ct_helpers:start_apps([emqx_modules, emqx_management, emqx_auth_mnesia], fun set_special_configs/1),
+    emqx_ct_helpers:start_apps( [emqx_modules, emqx_management, emqx_auth_mnesia]
+                              , fun set_special_configs/1
+                              ),
+    supervisor:terminate_child(emqx_auth_mnesia_sup, emqx_acl_mnesia_migrator),
     create_default_app(),
     Config.
 
@@ -50,15 +60,55 @@ end_per_suite(_Config) ->
     delete_default_app(),
     emqx_ct_helpers:stop_apps([emqx_modules, emqx_management, emqx_auth_mnesia]).
 
-init_per_testcase(t_check_acl_as_clientid, Config) ->
+init_per_testcase_clean(_, Config) ->
+    mnesia:clear_table(?ACL_TABLE),
+    mnesia:clear_table(?ACL_TABLE2),
+    Config.
+
+init_per_testcase_emqx_hook(t_check_acl_as_clientid, Config) ->
     emqx:hook('client.check_acl', fun emqx_acl_mnesia:check_acl/5, [#{key_as => clientid}]),
     Config;
-
+init_per_testcase_emqx_hook(_, Config) ->
-init_per_testcase(_, Config) ->
     emqx:hook('client.check_acl', fun emqx_acl_mnesia:check_acl/5, [#{key_as => username}]),
     Config.
 
-end_per_testcase(_, Config) ->
+init_per_testcase_migration(t_management_before_migration, Config) ->
+    Config;
+init_per_testcase_migration(_, Config) ->
+    emqx_acl_mnesia_migrator:migrate_records(),
+    Config.
+
+init_per_testcase_other(t_last_will_testament_message_check_acl, Config) ->
+    OriginalACLNoMatch = application:get_env(emqx, acl_nomatch),
+    application:set_env(emqx, acl_nomatch, deny),
+    emqx_mod_acl_internal:unload([]),
+    %% deny all for this client
+    ClientID = <<"lwt_client">>,
+    ok = emqx_acl_mnesia_db:add_acl({clientid, ClientID}, <<"#">>, pubsub, deny),
+    [ {original_acl_nomatch, OriginalACLNoMatch}
+    , {clientid, ClientID}
+    | Config];
+init_per_testcase_other(_TestCase, Config) ->
+    Config.
+
+init_per_testcase(Case, Config) ->
+    PerTestInitializers = [
+        fun init_per_testcase_clean/2,
+        fun init_per_testcase_migration/2,
+        fun init_per_testcase_emqx_hook/2,
+        fun init_per_testcase_other/2
+    ],
+    lists:foldl(fun(Init, Conf) -> Init(Case, Conf) end, Config, PerTestInitializers).
+
+end_per_testcase(t_last_will_testament_message_check_acl, Config) ->
+    emqx:unhook('client.check_acl', fun emqx_acl_mnesia:check_acl/5),
+    case ?config(original_acl_nomatch, Config) of
+        {ok, Original} -> application:set_env(emqx, acl_nomatch, Original);
+        _ -> ok
+    end,
+    emqx_mod_acl_internal:load([]),
+    ok;
+end_per_testcase(_TestCase, Config) ->
     emqx:unhook('client.check_acl', fun emqx_acl_mnesia:check_acl/5),
     Config.
 
@@ -76,25 +126,34 @@ set_special_configs(_App) ->
 %% Testcases
 %%------------------------------------------------------------------------------
 
-t_management(_Config) ->
+t_management_before_migration(_Config) ->
-    clean_all_acls(),
+    {atomic, IsStarted} = mnesia:transaction(fun emqx_acl_mnesia_db:is_migration_started/0),
-    ?assertEqual("Acl with Mnesia", emqx_acl_mnesia:description()),
+    ?assertNot(IsStarted),
-    ?assertEqual([], emqx_acl_mnesia_cli:all_acls()),
+    run_acl_tests().
 
-    ok = emqx_acl_mnesia_cli:add_acl({clientid, <<"test_clientid">>}, <<"topic/%c">>, sub, allow),
+t_management_after_migration(_Config) ->
-    ok = emqx_acl_mnesia_cli:add_acl({clientid, <<"test_clientid">>}, <<"topic/+">>, pub, deny),
+    {atomic, IsStarted} = mnesia:transaction(fun emqx_acl_mnesia_db:is_migration_started/0),
-    ok = emqx_acl_mnesia_cli:add_acl({username, <<"test_username">>}, <<"topic/%u">>, sub, deny),
+    ?assert(IsStarted),
-    ok = emqx_acl_mnesia_cli:add_acl({username, <<"test_username">>}, <<"topic/+">>, pub, allow),
+    run_acl_tests().
-    ok = emqx_acl_mnesia_cli:add_acl(all, <<"#">>, pubsub, deny),
+
+run_acl_tests() ->
+    ?assertEqual("Acl with Mnesia", emqx_acl_mnesia:description()),
+    ?assertEqual([], emqx_acl_mnesia_db:all_acls()),
+
+    ok = emqx_acl_mnesia_db:add_acl({clientid, <<"test_clientid">>}, <<"topic/%c">>, sub, allow),
+    ok = emqx_acl_mnesia_db:add_acl({clientid, <<"test_clientid">>}, <<"topic/+">>, pub, deny),
+    ok = emqx_acl_mnesia_db:add_acl({username, <<"test_username">>}, <<"topic/%u">>, sub, deny),
+    ok = emqx_acl_mnesia_db:add_acl({username, <<"test_username">>}, <<"topic/+">>, pub, allow),
+    ok = emqx_acl_mnesia_db:add_acl(all, <<"#">>, pubsub, deny),
     %% Sleeps below are needed to hide the race condition between
     %% mnesia and ets dirty select in check_acl, that make this test
     %% flaky
     timer:sleep(100),
 
-    ?assertEqual(2, length(emqx_acl_mnesia_cli:lookup_acl({clientid, <<"test_clientid">>}))),
+    ?assertEqual(2, length(emqx_acl_mnesia_db:lookup_acl({clientid, <<"test_clientid">>}))),
-    ?assertEqual(2, length(emqx_acl_mnesia_cli:lookup_acl({username, <<"test_username">>}))),
+    ?assertEqual(2, length(emqx_acl_mnesia_db:lookup_acl({username, <<"test_username">>}))),
-    ?assertEqual(2, length(emqx_acl_mnesia_cli:lookup_acl(all))),
+    ?assertEqual(2, length(emqx_acl_mnesia_db:lookup_acl(all))),
-    ?assertEqual(6, length(emqx_acl_mnesia_cli:all_acls())),
+    ?assertEqual(6, length(emqx_acl_mnesia_db:all_acls())),
 
     User1 = #{zone => external, clientid => <<"test_clientid">>},
     User2 = #{zone => external, clientid => <<"no_exist">>, username => <<"test_username">>},
@@ -110,30 +169,32 @@ t_management(_Config) ->
     deny  = emqx_access_control:check_acl(User3, publish,   <<"topic/A/B">>),
 
     %% Test merging of pubsub capability:
-    ok = emqx_acl_mnesia_cli:add_acl({clientid, <<"test_clientid">>}, <<"topic/mix">>, pubsub, deny),
+    ok = emqx_acl_mnesia_db:add_acl({clientid, <<"test_clientid">>}, <<"topic/mix">>, pubsub, deny),
     timer:sleep(100),
     deny  = emqx_access_control:check_acl(User1, subscribe,   <<"topic/mix">>),
     deny  = emqx_access_control:check_acl(User1, publish,     <<"topic/mix">>),
-    ok = emqx_acl_mnesia_cli:add_acl({clientid, <<"test_clientid">>}, <<"topic/mix">>, pub, allow),
+    ok = emqx_acl_mnesia_db:add_acl({clientid, <<"test_clientid">>}, <<"topic/mix">>, pub, allow),
     timer:sleep(100),
     deny  = emqx_access_control:check_acl(User1, subscribe,   <<"topic/mix">>),
     allow = emqx_access_control:check_acl(User1, publish,     <<"topic/mix">>),
-    ok = emqx_acl_mnesia_cli:add_acl({clientid, <<"test_clientid">>}, <<"topic/mix">>, pubsub, allow),
+    ok = emqx_acl_mnesia_db:add_acl( {clientid, <<"test_clientid">>}
+                                   , <<"topic/mix">>, pubsub, allow
+                                   ),
     timer:sleep(100),
     allow = emqx_access_control:check_acl(User1, subscribe,   <<"topic/mix">>),
     allow = emqx_access_control:check_acl(User1, publish,     <<"topic/mix">>),
-    ok = emqx_acl_mnesia_cli:add_acl({clientid, <<"test_clientid">>}, <<"topic/mix">>, sub, deny),
+    ok = emqx_acl_mnesia_db:add_acl({clientid, <<"test_clientid">>}, <<"topic/mix">>, sub, deny),
     timer:sleep(100),
     deny  = emqx_access_control:check_acl(User1, subscribe,   <<"topic/mix">>),
     allow = emqx_access_control:check_acl(User1, publish,     <<"topic/mix">>),
-    ok = emqx_acl_mnesia_cli:add_acl({clientid, <<"test_clientid">>}, <<"topic/mix">>, pub, deny),
+    ok = emqx_acl_mnesia_db:add_acl({clientid, <<"test_clientid">>}, <<"topic/mix">>, pub, deny),
     timer:sleep(100),
     deny  = emqx_access_control:check_acl(User1, subscribe,   <<"topic/mix">>),
     deny  = emqx_access_control:check_acl(User1, publish,     <<"topic/mix">>),
 
     %% Test implicit migration of pubsub to pub and sub:
-    ok = emqx_acl_mnesia_cli:remove_acl({clientid, <<"test_clientid">>}, <<"topic/mix">>),
+    ok = emqx_acl_mnesia_db:remove_acl({clientid, <<"test_clientid">>}, <<"topic/mix">>),
-    ok = mnesia:dirty_write(#emqx_acl{
+    ok = mnesia:dirty_write(#?ACL_TABLE{
                                filter = {{clientid, <<"test_clientid">>}, <<"topic/mix">>},
                                action = pubsub,
                                access = allow,
@@ -142,34 +203,138 @@ t_management(_Config) ->
     timer:sleep(100),
     allow = emqx_access_control:check_acl(User1, subscribe,   <<"topic/mix">>),
     allow = emqx_access_control:check_acl(User1, publish,     <<"topic/mix">>),
-    ok = emqx_acl_mnesia_cli:add_acl({clientid, <<"test_clientid">>}, <<"topic/mix">>, pub, deny),
+    ok = emqx_acl_mnesia_db:add_acl({clientid, <<"test_clientid">>}, <<"topic/mix">>, pub, deny),
     timer:sleep(100),
     allow = emqx_access_control:check_acl(User1, subscribe,   <<"topic/mix">>),
     deny  = emqx_access_control:check_acl(User1, publish,     <<"topic/mix">>),
-    ok = emqx_acl_mnesia_cli:add_acl({clientid, <<"test_clientid">>}, <<"topic/mix">>, sub, deny),
+    ok = emqx_acl_mnesia_db:add_acl({clientid, <<"test_clientid">>}, <<"topic/mix">>, sub, deny),
     timer:sleep(100),
     deny  = emqx_access_control:check_acl(User1, subscribe,   <<"topic/mix">>),
     deny  = emqx_access_control:check_acl(User1, publish,     <<"topic/mix">>),
 
-    ok = emqx_acl_mnesia_cli:remove_acl({clientid, <<"test_clientid">>}, <<"topic/%c">>),
+    ok = emqx_acl_mnesia_db:remove_acl({clientid, <<"test_clientid">>}, <<"topic/%c">>),
-    ok = emqx_acl_mnesia_cli:remove_acl({clientid, <<"test_clientid">>}, <<"topic/+">>),
+    ok = emqx_acl_mnesia_db:remove_acl({clientid, <<"test_clientid">>}, <<"topic/+">>),
-    ok = emqx_acl_mnesia_cli:remove_acl({clientid, <<"test_clientid">>}, <<"topic/mix">>),
+    ok = emqx_acl_mnesia_db:remove_acl({clientid, <<"test_clientid">>}, <<"topic/mix">>),
-    ok = emqx_acl_mnesia_cli:remove_acl({username, <<"test_username">>}, <<"topic/%u">>),
+    ok = emqx_acl_mnesia_db:remove_acl({username, <<"test_username">>}, <<"topic/%u">>),
-    ok = emqx_acl_mnesia_cli:remove_acl({username, <<"test_username">>}, <<"topic/+">>),
+    ok = emqx_acl_mnesia_db:remove_acl({username, <<"test_username">>}, <<"topic/+">>),
-    ok = emqx_acl_mnesia_cli:remove_acl(all, <<"#">>),
+    ok = emqx_acl_mnesia_db:remove_acl(all, <<"#">>),
     timer:sleep(100),
 
-    ?assertEqual([], emqx_acl_mnesia_cli:all_acls()).
+    ?assertEqual([], emqx_acl_mnesia_db:all_acls()).
+
+t_old_and_new_acl_combination(_Config) ->
+    create_conflicting_records(),
+
+    ?assertEqual(combined_conflicting_records(), emqx_acl_mnesia_db:all_acls()),
+    ?assertEqual(
+        lists:usort(combined_conflicting_records()),
+        lists:usort(emqx_acl_mnesia_db:all_acls_export())).
+
+t_old_and_new_acl_migration(_Config) ->
+    create_conflicting_records(),
+    emqx_acl_mnesia_migrator:migrate_records(),
+
+    ?assertEqual(combined_conflicting_records(), emqx_acl_mnesia_db:all_acls()),
+    ?assertEqual(
+        lists:usort(combined_conflicting_records()),
+        lists:usort(emqx_acl_mnesia_db:all_acls_export())),
+
+    % check that old table is not popoulated anymore
+    ok = emqx_acl_mnesia_db:add_acl({clientid, <<"test_clientid">>}, <<"topic/%c">>, sub, allow),
+    ?assert(emqx_acl_mnesia_migrator:is_old_table_migrated()).
+
+
+t_migration_concurrency(_Config) ->
+    Key = {{clientid,<<"client6">>}, <<"t">>},
+    Record = #?ACL_TABLE{filter = Key, action = pubsub, access = deny, created_at = 0},
+    {atomic, ok} = mnesia:transaction(fun mnesia:write/1, [Record]),
+
+    LockWaitAndDelete =
+        fun() ->
+           [_Rec] = mnesia:wread({?ACL_TABLE, Key}),
+           {{Pid, Ref}, _} =
+               ?wait_async_action(spawn_monitor(fun emqx_acl_mnesia_migrator:migrate_records/0),
+                                  #{?snk_kind := emqx_acl_mnesia_migrator_record_selected},
+                                  1000),
+           mnesia:delete({?ACL_TABLE, Key}),
+           {Pid, Ref}
+        end,
+
+    ?check_trace(
+        begin
+            {atomic, {Pid, Ref}} = mnesia:transaction(LockWaitAndDelete),
+            receive {'DOWN', Ref, process, Pid, _} -> ok end
+        end,
+        fun(_, Trace) ->
+            ?assertMatch([_], ?of_kind(emqx_acl_mnesia_migrator_record_missed, Trace))
+        end),
+
+    ?assert(emqx_acl_mnesia_migrator:is_old_table_migrated()),
+    ?assertEqual([], emqx_acl_mnesia_db:all_acls()).
+
+
+t_old_and_new_acl_migration_by_migrator(_Config) ->
+    create_conflicting_records(),
+
+    meck:new(fake_nodes, [non_strict]),
+    meck:expect(fake_nodes, all, fun() -> [node(), 'somebadnode@127.0.0.1'] end),
+
+    ?check_trace(
+        begin
+            % check all nodes every 30 ms
+            {ok, _} = emqx_acl_mnesia_migrator:start_link(#{
+                name => ct_migrator,
+                check_nodes_interval => 30,
+                get_nodes => fun fake_nodes:all/0
+            }),
+            timer:sleep(100)
+        end,
+        fun(_, Trace) ->
+            ?assertEqual([], ?of_kind(emqx_acl_mnesia_migrator_start_migration, Trace))
+        end),
+
+    ?check_trace(
+        begin
+            meck:expect(fake_nodes, all, fun() -> [node()] end),
+            timer:sleep(100)
+        end,
+        fun(_, Trace) ->
+            ?assertMatch([_], ?of_kind(emqx_acl_mnesia_migrator_finish, Trace))
+        end),
+
+    meck:unload(fake_nodes),
+
+    ?assertEqual(combined_conflicting_records(), emqx_acl_mnesia_db:all_acls()),
+    ?assert(emqx_acl_mnesia_migrator:is_old_table_migrated()).
+
+t_old_and_new_acl_migration_repeated_by_migrator(_Config) ->
+    create_conflicting_records(),
+    emqx_acl_mnesia_migrator:migrate_records(),
+
+    ?check_trace(
+        begin
+            {ok, _} = emqx_acl_mnesia_migrator:start_link(ct_migrator),
+            timer:sleep(100)
+        end,
+        fun(_, Trace) ->
+            ?assertEqual([], ?of_kind(emqx_acl_mnesia_migrator_start_migration, Trace)),
+            ?assertMatch([_], ?of_kind(emqx_acl_mnesia_migrator_finish, Trace))
+        end).
+
+t_start_stop_supervised(_Config) ->
+    ?assertEqual(undefined, whereis(emqx_acl_mnesia_migrator)),
+    ok = emqx_acl_mnesia_migrator:start_supervised(),
+    ?assert(is_pid(whereis(emqx_acl_mnesia_migrator))),
+    ok = emqx_acl_mnesia_migrator:stop_supervised(),
+    ?assertEqual(undefined, whereis(emqx_acl_mnesia_migrator)).
 
 t_acl_cli(_Config) ->
     meck:new(emqx_ctl, [non_strict, passthrough]),
-    meck:expect(emqx_ctl, print, fun(Arg) -> emqx_ctl:format(Arg) end),
+    meck:expect(emqx_ctl, print, fun(Arg) -> emqx_ctl:format(Arg, []) end),
     meck:expect(emqx_ctl, print, fun(Msg, Arg) -> emqx_ctl:format(Msg, Arg) end),
     meck:expect(emqx_ctl, usage, fun(Usages) -> emqx_ctl:format_usage(Usages) end),
     meck:expect(emqx_ctl, usage, fun(Cmd, Descr) -> emqx_ctl:format_usage(Cmd, Descr) end),
 
-    clean_all_acls(),
-
     ?assertEqual(0, length(emqx_acl_mnesia_cli:cli(["list"]))),
 
     emqx_acl_mnesia_cli:cli(["add", "clientid", "test_clientid", "topic/A", "pub", "deny"]),
@@ -202,8 +367,6 @@ t_acl_cli(_Config) ->
     meck:unload(emqx_ctl).
 
 t_rest_api(_Config) ->
-    clean_all_acls(),
-
     Params1 = [#{<<"clientid">> => <<"test_clientid">>,
                  <<"topic">> => <<"topic/A">>,
                  <<"action">> => <<"pub">>,
@@ -218,13 +381,31 @@ t_rest_api(_Config) ->
                  <<"topic">> => <<"topic/C">>,
                  <<"action">> => <<"pubsub">>,
                  <<"access">> => <<"deny">>
+                },
+               #{<<"clientid">> => <<"good_clientid1">>,
+                 <<"topic">> => <<"topic/D">>,
+                 <<"action">> => <<"pubsub">>,
+                 <<"access">> => <<"deny">>
                 }],
     {ok, _} = request_http_rest_add([], Params1),
+
     {ok, Re1} = request_http_rest_list(["clientid", "test_clientid"]),
     ?assertMatch(4, length(get_http_data(Re1))),
+    {ok, Re11} = request_http_rest_list(["clientid"], "_like_clientid=good"),
+    ?assertMatch(2, length(get_http_data(Re11))),
+    {ok, Re12} = request_http_rest_list(["clientid"], "_like_clientid=clientid"),
+    ?assertMatch(6, length(get_http_data(Re12))),
+    {ok, Re13} = request_http_rest_list(["clientid"], "_like_clientid=clientid&action=pub"),
+    ?assertMatch(3, length(get_http_data(Re13))),
+    {ok, Re14} = request_http_rest_list(["clientid"], "_like_clientid=clientid&access=deny"),
+    ?assertMatch(4, length(get_http_data(Re14))),
+    {ok, Re15} = request_http_rest_list(["clientid"], "_like_clientid=clientid&topic=topic/A"),
+    ?assertMatch(1, length(get_http_data(Re15))),
+
     {ok, _} = request_http_rest_delete(["clientid", "test_clientid", "topic", "topic/A"]),
     {ok, _} = request_http_rest_delete(["clientid", "test_clientid", "topic", "topic/B"]),
     {ok, _} = request_http_rest_delete(["clientid", "test_clientid", "topic", "topic/C"]),
+    {ok, _} = request_http_rest_delete(["clientid", "good_clientid1", "topic", "topic/D"]),
     {ok, Res1} = request_http_rest_list(["clientid"]),
     ?assertMatch([], get_http_data(Res1)),
 
@@ -242,13 +423,30 @@ t_rest_api(_Config) ->
                  <<"topic">> => <<"topic/C">>,
                  <<"action">> => <<"pubsub">>,
                  <<"access">> => <<"deny">>
+                },
+               #{<<"username">> => <<"good_username">>,
+                 <<"topic">> => <<"topic/D">>,
+                 <<"action">> => <<"pubsub">>,
+                 <<"access">> => <<"deny">>
                 }],
     {ok, _} = request_http_rest_add([], Params2),
     {ok, Re2} = request_http_rest_list(["username", "test_username"]),
     ?assertMatch(4, length(get_http_data(Re2))),
+    {ok, Re21} = request_http_rest_list(["username"], "_like_username=good"),
+    ?assertMatch(2, length(get_http_data(Re21))),
+    {ok, Re22} = request_http_rest_list(["username"], "_like_username=username"),
+    ?assertMatch(6, length(get_http_data(Re22))),
+    {ok, Re23} = request_http_rest_list(["username"], "_like_username=username&action=pub"),
+    ?assertMatch(3, length(get_http_data(Re23))),
+    {ok, Re24} = request_http_rest_list(["username"], "_like_username=username&access=deny"),
+    ?assertMatch(4, length(get_http_data(Re24))),
+    {ok, Re25} = request_http_rest_list(["username"], "_like_username=username&topic=topic/A"),
+    ?assertMatch(1, length(get_http_data(Re25))),
+
     {ok, _} = request_http_rest_delete(["username", "test_username", "topic", "topic/A"]),
     {ok, _} = request_http_rest_delete(["username", "test_username", "topic", "topic/B"]),
     {ok, _} = request_http_rest_delete(["username", "test_username", "topic", "topic/C"]),
+    {ok, _} = request_http_rest_delete(["username", "good_username", "topic", "topic/D"]),
     {ok, Res2} = request_http_rest_list(["username"]),
     ?assertMatch([], get_http_data(Res2)),
 
@@ -263,23 +461,83 @@ t_rest_api(_Config) ->
                #{<<"topic">> => <<"topic/C">>,
                  <<"action">> => <<"pubsub">>,
                  <<"access">> => <<"deny">>
-                }],
+                },
+               #{<<"topic">> => <<"topic/D">>,
+                 <<"action">> => <<"pubsub">>,
+                 <<"access">> => <<"deny">>
+                }
+        ],
     {ok, _} = request_http_rest_add([], Params3),
+
     {ok, Re3} = request_http_rest_list(["$all"]),
-    ?assertMatch(4, length(get_http_data(Re3))),
+    ?assertMatch(6, length(get_http_data(Re3))),
+    {ok, Re31} = request_http_rest_list(["$all"], "topic=topic/A"),
+    ?assertMatch(1, length(get_http_data(Re31))),
+    {ok, Re32} = request_http_rest_list(["$all"], "action=sub"),
+    ?assertMatch(3, length(get_http_data(Re32))),
+    {ok, Re33} = request_http_rest_list(["$all"], "access=deny"),
+    ?assertMatch(4, length(get_http_data(Re33))),
+    {ok, Re34} = request_http_rest_list(["$all"], "action=sub&access=deny"),
+    ?assertMatch(2, length(get_http_data(Re34))),
+
     {ok, _} = request_http_rest_delete(["$all", "topic", "topic/A"]),
     {ok, _} = request_http_rest_delete(["$all", "topic", "topic/B"]),
     {ok, _} = request_http_rest_delete(["$all", "topic", "topic/C"]),
+    {ok, _} = request_http_rest_delete(["$all", "topic", "topic/D"]),
     {ok, Res3} = request_http_rest_list(["$all"]),
     ?assertMatch([], get_http_data(Res3)).
 
-%%------------------------------------------------------------------------------
+%% asserts that we check ACL for the LWT topic before publishing the
-%% Helpers
+%% LWT.
-%%------------------------------------------------------------------------------
+t_last_will_testament_message_check_acl(Config) ->
+    ClientID = ?config(clientid, Config),
+    {ok, C} = emqtt:start_link([
+        {clientid, ClientID},
+        {will_topic, <<"$SYS/lwt">>},
+        {will_payload, <<"should not be published">>}
+    ]),
+    {ok, _} = emqtt:connect(C),
+    ok = emqx:subscribe(<<"$SYS/lwt">>),
+    unlink(C),
+    ok = snabbkaffe:start_trace(),
+    {true, {ok, _}} =
+        ?wait_async_action(
+        exit(C, kill),
+        #{?snk_kind := last_will_testament_publish_denied},
+        1_000
+    ),
+    ok = snabbkaffe:stop(),
 
-clean_all_acls() ->
+    receive
-    [ mnesia:dirty_delete({emqx_acl, Login})
+        {deliver, <<"$SYS/lwt">>, #message{payload = <<"should not be published">>}} ->
-      || Login <- mnesia:dirty_all_keys(emqx_acl)].
+            error(lwt_should_not_be_published_to_forbidden_topic)
+    after 1_000 ->
+        ok
+    end,
+
+    ok.
+
+create_conflicting_records() ->
+    Records = [
+        #?ACL_TABLE{ filter = {{clientid,<<"client6">>}, <<"t">>}
+                   , action = pubsub, access = deny, created_at = 0
+                   },
+        #?ACL_TABLE{ filter = {{clientid,<<"client5">>}, <<"t">>}
+                   , action = pubsub, access = deny, created_at = 1
+                   },
+        #?ACL_TABLE2{who = {clientid,<<"client5">>}, rules = [{allow, sub, <<"t">>, 2}]}
+    ],
+    mnesia:transaction(fun() -> lists:foreach(fun mnesia:write/1, Records) end).
+
+
+combined_conflicting_records() ->
+    % pubsub's are split, ACL_TABLE2 rules shadow ACL_TABLE rules
+    [
+        {{clientid,<<"client5">>},<<"t">>,sub,allow,2},
+        {{clientid,<<"client5">>},<<"t">>,pub,deny,1},
+        {{clientid,<<"client6">>},<<"t">>,sub,deny,0},
+        {{clientid,<<"client6">>},<<"t">>,pub,deny,0}
+    ].
 
 %%--------------------------------------------------------------------
 %% HTTP Request
@@ -288,6 +546,9 @@ clean_all_acls() ->
 request_http_rest_list(Path) ->
     request_api(get, uri(Path), default_auth_header()).
 
+request_http_rest_list(Path, Qs) ->
+    request_api(get, uri(Path), Qs, default_auth_header()).
+
 request_http_rest_lookup(Path) ->
     request_api(get, uri(Path), default_auth_header()).
 

apps/emqx_auth_mnesia/test/emqx_auth_mnesia_SUITE.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -21,6 +21,9 @@
 -include("emqx_auth_mnesia.hrl").
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("common_test/include/ct.hrl").
+-include_lib("snabbkaffe/include/snabbkaffe.hrl").
+
+-include_lib("emqx/include/emqx_mqtt.hrl").
 
 -import(emqx_ct_http, [ request_api/3
                       , request_api/5
@@ -46,11 +49,15 @@ all() ->
 groups() ->
     [].
 
+init_per_suite(t_boot) ->
+    ok;
 init_per_suite(Config) ->
     ok = emqx_ct_helpers:start_apps([emqx_management, emqx_auth_mnesia], fun set_special_configs/1),
     create_default_app(),
     Config.
 
+end_per_suite(t_boot) ->
+    ok;
 end_per_suite(_Config) ->
     delete_default_app(),
     emqx_ct_helpers:stop_apps([emqx_management, emqx_auth_mnesia]).
@@ -65,10 +72,99 @@ set_special_configs(emqx) ->
 set_special_configs(_App) ->
     ok.
 
+set_default(ClientId, UserName, Pwd, HashType) ->
+    application:set_env(emqx_auth_mnesia, clientid_list, [{ClientId, Pwd}]),
+    application:set_env(emqx_auth_mnesia, username_list, [{UserName, Pwd}]),
+    application:set_env(emqx_auth_mnesia, password_hash, HashType),
+    ok.
+
+init_per_testcase(t_will_message_connection_denied, Config) ->
+    emqx_zone:set_env(external, allow_anonymous, false),
+    Config;
+init_per_testcase(_TestCase, Config) ->
+    Config.
+
+end_per_testcase(t_will_message_connection_denied, _Config) ->
+    emqx_zone:unset_env(external, allow_anonymous),
+    application:stop(emqx_auth_mnesia),
+    ok;
+end_per_testcase(_TestCase, _Config) ->
+    ok.
+
 %%------------------------------------------------------------------------------
 %% Testcases
 %%------------------------------------------------------------------------------
 
+t_boot(_Config) ->
+    clean_all_users(),
+    emqx_ct_helpers:stop_apps([emqx_auth_mnesia]),
+    ClientId = <<"clientid-test">>,
+    UserName = <<"username-test">>,
+    Pwd = <<"emqx123456">>,
+    ok = emqx_ct_helpers:start_apps([emqx_auth_mnesia],
+        fun(_) -> set_default(ClientId, UserName, Pwd, sha256) end),
+    Ok = {stop, #{anonymous => false, auth_result => success}},
+    Failed = {stop, #{anonymous => false, auth_result => password_error}},
+    ?assertEqual(Ok,
+        emqx_auth_mnesia:check(#{clientid => ClientId, password => Pwd}, #{}, #{hash_type => sha256})),
+    ?assertEqual(Ok,
+        emqx_auth_mnesia:check(#{clientid => <<"NotExited">>, username => UserName, password => Pwd},
+            #{}, #{hash_type => sha256})),
+    ?assertEqual(Failed,
+        emqx_auth_mnesia:check(#{clientid => ClientId, password => <<Pwd/binary, "bad">>},
+            #{}, #{hash_type => sha256})),
+    ?assertEqual(Failed,
+        emqx_auth_mnesia:check(#{clientid => ClientId, username => UserName, password => <<Pwd/binary, "bad">>},
+            #{}, #{hash_type => sha256})),
+    emqx_ct_helpers:stop_apps([emqx_auth_mnesia]),
+
+    %% change default pwd
+    NewPwd = <<"emqx654321">>,
+    ok = emqx_ct_helpers:start_apps([emqx_auth_mnesia],
+        fun(_) -> set_default(ClientId, UserName, NewPwd, sha256) end),
+    ?assertEqual(Failed,
+        emqx_auth_mnesia:check(#{clientid => ClientId, password => NewPwd},
+            #{}, #{hash_type => sha256})),
+    ?assertEqual(Failed,
+        emqx_auth_mnesia:check(#{clientid => <<"NotExited">>, username => UserName, password => NewPwd},
+            #{}, #{hash_type => sha256})),
+    clean_all_users(),
+    emqx_ct_helpers:stop_apps([emqx_auth_mnesia]),
+
+    ok = emqx_ct_helpers:start_apps([emqx_auth_mnesia],
+        fun(_) -> set_default(ClientId, UserName, NewPwd, sha256) end),
+    ?assertEqual(Ok,
+        emqx_auth_mnesia:check(#{clientid => ClientId, password => NewPwd},
+            #{}, #{hash_type => sha256})),
+    ?assertEqual(Ok,
+        emqx_auth_mnesia:check(#{clientid => <<"NotExited">>, username => UserName, password => NewPwd},
+            #{}, #{hash_type => sha256})),
+    emqx_ct_helpers:stop_apps([emqx_auth_mnesia]),
+
+    %% change hash_type
+    NewPwd2 = <<"emqx6543210">>,
+    ok = emqx_ct_helpers:start_apps([emqx_auth_mnesia],
+        fun(_) -> set_default(ClientId, UserName, NewPwd2, plain) end),
+    ?assertEqual(Failed,
+        emqx_auth_mnesia:check(#{clientid => ClientId, password => NewPwd2},
+            #{}, #{hash_type => plain})),
+    ?assertEqual(Failed,
+        emqx_auth_mnesia:check(#{clientid => <<"NotExited">>, username => UserName, password => NewPwd2},
+            #{}, #{hash_type => plain})),
+    clean_all_users(),
+    emqx_ct_helpers:stop_apps([emqx_auth_mnesia]),
+
+    ok = emqx_ct_helpers:start_apps([emqx_auth_mnesia],
+        fun(_) -> set_default(ClientId, UserName, NewPwd2, plain) end),
+    ?assertEqual(Ok,
+        emqx_auth_mnesia:check(#{clientid => ClientId, password => NewPwd2},
+            #{}, #{hash_type => plain})),
+    ?assertEqual(Ok,
+        emqx_auth_mnesia:check(#{clientid => <<"NotExited">>, username => UserName, password => NewPwd2},
+            #{}, #{hash_type => plain})),
+    clean_all_users(),
+    ok.
+
 t_management(_Config) ->
     clean_all_users(),
 
@@ -193,7 +289,8 @@ t_clientid_rest_api(_Config) ->
     clean_all_users(),
 
     {ok, Result1} = request_http_rest_list(["auth_clientid"]),
-    [] = get_http_data(Result1),
+    ?assertMatch(#{<<"data">> := [], <<"meta">> := #{<<"count">> := 0}},
+        emqx_json:decode(Result1, [return_maps])),
 
     Params1 = #{<<"clientid">> => ?CLIENTID, <<"password">> => ?PASSWORD},
     {ok, _} = request_http_rest_add(["auth_clientid"], Params1),
@@ -207,20 +304,46 @@ t_clientid_rest_api(_Config) ->
 
     Params3 = [ #{<<"clientid">> => ?CLIENTID, <<"password">> => ?PASSWORD}
               , #{<<"clientid">> => <<"clientid1">>, <<"password">> => ?PASSWORD}
-              , #{<<"clientid">> => <<"clientid2">>, <<"password">> => ?PASSWORD}
+              , #{<<"clientid">> => <<"client2">>, <<"password">> => ?PASSWORD}
               ],
     {ok, Result3} = request_http_rest_add(["auth_clientid"], Params3),
     ?assertMatch(#{ ?CLIENTID := <<"{error,existed}">>
                   , <<"clientid1">> := <<"ok">>
-                  , <<"clientid2">> := <<"ok">>
+                  , <<"client2">> := <<"ok">>
                   }, get_http_data(Result3)),
 
     {ok, Result4} = request_http_rest_list(["auth_clientid"]),
-    ?assertEqual(3, length(get_http_data(Result4))),
+    #{<<"data">> := Data4, <<"meta">> := #{<<"count">> := Count4}}
+        = emqx_json:decode(Result4, [return_maps]),
+
+    ?assertEqual(3, Count4),
+    ?assertEqual([<<"client2">>, <<"clientid1">>, ?CLIENTID],
+        lists:sort(lists:map(fun(#{<<"clientid">> := C}) -> C end, Data4))),
+
+    UserNameParams = [#{<<"username">> => <<"username1">>, <<"password">> => ?PASSWORD}
+        , #{<<"username">> => <<"username2">>, <<"password">> => ?PASSWORD}
+    ],
+    {ok, _} = request_http_rest_add(["auth_username"], UserNameParams),
+
+    {ok, Result41} = request_http_rest_list(["auth_clientid"]),
+    %% the count clientid is not affected by username count.
+    ?assertEqual(Result4, Result41),
+
+    {ok, Result42} = request_http_rest_list(["auth_username"]),
+    #{<<"data">> := Data42, <<"meta">> := #{<<"count">> := Count42}}
+        = emqx_json:decode(Result42, [return_maps]),
+    ?assertEqual(2, Count42),
+    ?assertEqual([<<"username1">>, <<"username2">>],
+        lists:sort(lists:map(fun(#{<<"username">> := U}) -> U end, Data42))),
+
+    {ok, Result5} = request_http_rest_list(["auth_clientid?_like_clientid=id"]),
+    ?assertEqual(2, length(get_http_data(Result5))),
+    {ok, Result6} = request_http_rest_list(["auth_clientid?_like_clientid=x"]),
+    ?assertEqual(0, length(get_http_data(Result6))),
 
     {ok, _} = request_http_rest_delete(Path),
-    {ok, Result5} = request_http_rest_lookup(Path),
+    {ok, Result7} = request_http_rest_lookup(Path),
-    ?assertMatch(#{}, get_http_data(Result5)).
+    ?assertMatch(#{}, get_http_data(Result7)).
 
 t_username_rest_api(_Config) ->
     clean_all_users(),
@@ -251,9 +374,14 @@ t_username_rest_api(_Config) ->
     {ok, Result4} = request_http_rest_list(["auth_username"]),
     ?assertEqual(3, length(get_http_data(Result4))),
 
+    {ok, Result5} = request_http_rest_list(["auth_username?_like_username=for"]),
+    ?assertEqual(1, length(get_http_data(Result5))),
+    {ok, Result6} = request_http_rest_list(["auth_username?_like_username=x"]),
+    ?assertEqual(0, length(get_http_data(Result6))),
+
     {ok, _} = request_http_rest_delete(Path),
-    {ok, Result5} = request_http_rest_lookup([Path]),
+    {ok, Result7} = request_http_rest_lookup([Path]),
-    ?assertMatch(#{}, get_http_data(Result5)).
+    ?assertMatch(#{}, get_http_data(Result7)).
 
 t_password_hash(_) ->
     clean_all_users(),
@@ -279,6 +407,48 @@ t_password_hash(_) ->
     application:stop(emqx_auth_mnesia),
     ok = application:start(emqx_auth_mnesia).
 
+t_will_message_connection_denied(Config) when is_list(Config) ->
+    ClientId = Username = <<"subscriber">>,
+    Password = <<"p">>,
+    application:stop(emqx_auth_mnesia),
+    ok = emqx_ct_helpers:start_apps([emqx_auth_mnesia]),
+    ok = emqx_auth_mnesia_cli:add_user({clientid, ClientId}, Password),
+
+    {ok, Subscriber} = emqtt:start_link([
+        {clientid, ClientId},
+        {password, Password}
+    ]),
+    {ok, _} = emqtt:connect(Subscriber),
+    {ok, _, [?RC_SUCCESS]} = emqtt:subscribe(Subscriber, <<"lwt">>),
+
+    process_flag(trap_exit, true),
+
+    {ok, Publisher} = emqtt:start_link([
+        {clientid, <<"publisher">>},
+        {will_topic, <<"lwt">>},
+        {will_payload, <<"should not be published">>}
+    ]),
+    snabbkaffe:start_trace(),
+    ?wait_async_action(
+        {error, _} = emqtt:connect(Publisher),
+        #{?snk_kind := channel_terminated}
+    ),
+    snabbkaffe:stop(),
+
+    timer:sleep(1000),
+
+    receive
+        {publish, #{
+            topic := <<"lwt">>,
+            payload := <<"should not be published">>
+        }} ->
+            ct:fail("should not publish will message")
+    after 0 ->
+        ok
+    end,
+
+    ok.
+
 %%------------------------------------------------------------------------------
 %% Helpers
 %%------------------------------------------------------------------------------

apps/emqx_auth_mongo/include/emqx_auth_mongo.hrl

@@ -1,4 +1,3 @@
-
 -define(APP, emqx_auth_mongo).
 
 -define(DEFAULT_SELECTORS, [{<<"username">>, <<"%u">>}]).
@@ -14,24 +13,3 @@
 
 -record(aclquery, {collection = <<"mqtt_acl">>,
                    selector   = {<<"username">>, <<"%u">>}}).
-
--record(auth_metrics, {
-        success = 'client.auth.success',
-        failure = 'client.auth.failure',
-        ignore  = 'client.auth.ignore'
-    }).
-
--record(acl_metrics, {
-        allow = 'client.acl.allow',
-        deny = 'client.acl.deny',
-        ignore = 'client.acl.ignore'
-    }).
-
--define(METRICS(Type), tl(tuple_to_list(#Type{}))).
--define(METRICS(Type, K), #Type{}#Type.K).
-
--define(AUTH_METRICS, ?METRICS(auth_metrics)).
--define(AUTH_METRICS(K), ?METRICS(auth_metrics, K)).
-
--define(ACL_METRICS, ?METRICS(acl_metrics)).
--define(ACL_METRICS(K), ?METRICS(acl_metrics, K)).

apps/emqx_auth_mongo/priv/emqx_auth_mongo.schema

@@ -141,6 +141,7 @@
                 Filter([{verify, Verify},
                         {server_name_indication, case cuttlefish:conf_get(Prefix ++ ".server_name_indication", Conf, undefined) of
                                                    "disable" -> disable;
+                                                   "" -> undefined;
                                                    SNI -> SNI
                                                  end},
                         {keyfile, cuttlefish:conf_get(Prefix ++ ".keyfile", Conf, undefined)},

apps/emqx_auth_mongo/rebar.config

@@ -1,7 +1,6 @@
 {deps,
   %% NOTE: mind poolboy version when updating mongodb-erlang version
- [{mongodb, {git,"https://github.com/emqx/mongodb-erlang", {tag, "v3.0.7"}}},
+ [%% mongodb-erlang uses a special fork https://github.com/comtihon/poolboy.git
-  %% mongodb-erlang uses a special fork https://github.com/comtihon/poolboy.git
   %% (which has overflow_ttl feature added).
   %% However, it references `{branch, "master}` (commit 9c06a9a on 2021-04-07).
   %% By accident, We have always been using the upstream fork due to
@@ -29,4 +28,3 @@
 {cover_enabled, true}.
 {cover_opts, [verbose]}.
 {cover_export_enabled, true}.
-

apps/emqx_auth_mongo/src/emqx_acl_mongo.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -21,17 +21,12 @@
 -include_lib("emqx/include/logger.hrl").
 
 %% ACL callbacks
--export([ register_metrics/0
+-export([ check_acl/5
-        , check_acl/5
         , description/0
         ]).
--spec(register_metrics() -> ok).
-register_metrics() ->
-    lists:foreach(fun emqx_metrics:ensure/1, ?ACL_METRICS).
 
 check_acl(#{username := <<$$, _/binary>>}, _PubSub, _Topic, _AclResult, _State) ->
     ok;
-
 check_acl(ClientInfo, PubSub, Topic, _AclResult, Env = #{aclquery := AclQuery}) ->
     #aclquery{collection = Coll, selector = SelectorList} = AclQuery,
     Pool = maps:get(pool, Env, ?APP),
@@ -43,20 +38,16 @@ check_acl(ClientInfo, PubSub, Topic, _AclResult, Env = #{aclquery := AclQuery})
         [] -> ok;
         Rows ->
             try match(ClientInfo, Topic, topics(PubSub, Rows)) of
-                matched -> emqx_metrics:inc(?ACL_METRICS(allow)),
+                matched -> {stop, allow};
-                           {stop, allow};
+                nomatch -> {stop, deny}
-                nomatch -> emqx_metrics:inc(?ACL_METRICS(deny)),
-                           {stop, deny}
             catch
                 _Err:Reason->
                     ?LOG(error, "[MongoDB] Check mongo ~p ACL failed, got ACL config: ~p, error: :~p",
                                 [PubSub, Rows, Reason]),
-                    emqx_metrics:inc(?ACL_METRICS(ignore)),
                     ignore
             end
     end.
 
-
 match(_ClientInfo, _Topic, []) ->
     nomatch;
 match(ClientInfo, Topic, [TopicFilter|More]) ->

apps/emqx_auth_mongo/src/emqx_auth_mongo.app.src

@@ -1,6 +1,6 @@
 {application, emqx_auth_mongo,
  [{description, "EMQ X Authentication/ACL with MongoDB"},
-  {vsn, "4.3.0"}, % strict semver, bump manually!
+  {vsn, "4.3.4"}, % strict semver, bump manually!
   {modules, []},
   {registered, [emqx_auth_mongo_sup]},
   {applications, [kernel,stdlib,mongodb,ecpool]},

apps/emqx_auth_mongo/src/emqx_auth_mongo.appup.src

@@ -0,0 +1,19 @@
+%% -*- mode: erlang -*-
+%% Unless you know what you are doing, DO NOT edit manually!!
+{VSN,
+  [{<<"4\\.3\\.[1-3]">>,
+    [{load_module,emqx_auth_mongo_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_mongo,brutal_purge,soft_purge,[]}]},
+   {"4.3.0",
+    [{load_module,emqx_auth_mongo_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_mongo,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_mongo,brutal_purge,soft_purge,[]}]},
+   {<<".*">>,[]}],
+  [{<<"4\\.3\\.[1-3]">>,
+    [{load_module,emqx_auth_mongo_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_mongo,brutal_purge,soft_purge,[]}]},
+   {"4.3.0",
+    [{load_module,emqx_auth_mongo_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_mongo,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_mongo,brutal_purge,soft_purge,[]}]},
+   {<<".*">>,[]}]}.

apps/emqx_auth_mongo/src/emqx_auth_mongo.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -23,8 +23,7 @@
 -include_lib("emqx/include/logger.hrl").
 -include_lib("emqx/include/types.hrl").
 
--export([ register_metrics/0
+-export([ check/3
-        , check/3
         , description/0
         ]).
 
@@ -35,9 +34,9 @@
         , query_multi/3
         ]).
 
--spec(register_metrics() -> ok).
+-export([ available/2
-register_metrics() ->
+        , available/3
-    lists:foreach(fun emqx_metrics:ensure/1, ?AUTH_METRICS).
+        ]).
 
 check(ClientInfo = #{password := Password}, AuthResult,
       Env = #{authquery := AuthQuery, superquery := SuperQuery}) ->
@@ -45,10 +44,9 @@ check(ClientInfo = #{password := Password}, AuthResult,
                hash = HashType, selector = Selector} = AuthQuery,
     Pool = maps:get(pool, Env, ?APP),
     case query(Pool, Collection, maps:from_list(replvars(Selector, ClientInfo))) of
-        undefined -> emqx_metrics:inc(?AUTH_METRICS(ignore));
+        undefined -> ok;
         {error, Reason} ->
             ?LOG(error, "[MongoDB] Can't connect to MongoDB server: ~0p", [Reason]),
-            ok = emqx_metrics:inc(?AUTH_METRICS(failure)),
             {stop, AuthResult#{auth_result => not_authorized, anonymous => false}};
         UserMap ->
             Result = case [maps:get(Field, UserMap, undefined) || Field <- Fields] of
@@ -60,13 +58,11 @@ check(ClientInfo = #{password := Password}, AuthResult,
                      end,
             case Result of
                 ok ->
-                    ok = emqx_metrics:inc(?AUTH_METRICS(success)),
                     {stop, AuthResult#{is_superuser => is_superuser(Pool, SuperQuery, ClientInfo),
                                        anonymous => false,
                                        auth_result => success}};
                 {error, Error} ->
                     ?LOG(error, "[MongoDB] check auth fail: ~p", [Error]),
-                    ok = emqx_metrics:inc(?AUTH_METRICS(failure)),
                     {stop, AuthResult#{auth_result => Error, anonymous => false}}
             end
     end.
@@ -97,6 +93,56 @@ is_superuser(Pool, #superquery{collection = Coll, field = Field, selector = Sele
             end
     end.
 
+%%--------------------------------------------------------------------
+%% Availability Test
+%%--------------------------------------------------------------------
+
+available(Pool, #superquery{collection = Collection, selector = Selector}) ->
+    available(Pool, Collection, maps:from_list(replvars(Selector, test_client_info())));
+available(Pool, #authquery{collection = Collection, selector = Selector}) ->
+    available(Pool, Collection, maps:from_list(replvars(Selector, test_client_info())));
+available(Pool, #aclquery{collection = Collection, selector = Selectors}) ->
+    Fun =
+        fun(Selector) ->
+            maps:from_list(emqx_auth_mongo:replvars(Selector, test_client_info()))
+        end,
+    available(Pool, Collection, lists:map(Fun, Selectors), fun query_multi/3).
+
+available(Pool, Collection, Query) ->
+    available(Pool, Collection, Query, fun query/3).
+
+available(Pool, Collection, Query, Fun) ->
+    try Fun(Pool, Collection, Query) of
+        {error, Reason} ->
+            ?LOG(error, "[MongoDB] ~p availability test error: ~0p", [Collection, Reason]),
+            {error, Reason};
+        Error = #{<<"code">> := Code} ->
+            CodeName = maps:get(<<"codeName">>, Error, undefined),
+            ErrorMessage = maps:get(<<"errmsg">>, Error, undefined),
+            ?LOG(error, "[MongoDB] ~p availability test error, code: ~p Name: ~0p Message: ~0p",
+                [Collection, Code, CodeName, ErrorMessage]),
+            {error, {mongo_error, Code}};
+        _Return ->
+            %% Any success result is fine.
+            ok
+    catch E:R:S ->
+        ?LOG(error, "[MongoDB] ~p availability test error, ~p: ~0p: ~0p", [Collection, E, R, S]),
+        {error, R}
+    end.
+
+%% Test client info
+test_client_info() ->
+    #{
+        clientid => <<"EMQX_availability_test_client">>,
+        username => <<"EMQX_availability_test_username">>,
+        cn => <<"EMQX_availability_test_cn">>,
+        dn => <<"EMQX_availability_test_dn">>
+    }.
+
+%%--------------------------------------------------------------------
+%% Internal func
+%%--------------------------------------------------------------------
+
 replvars(VarList, ClientInfo) ->
     lists:map(fun(Var) -> replvar(Var, ClientInfo) end, VarList).
 
@@ -129,6 +175,9 @@ query_multi(Pool, Collection, SelectorList) ->
     lists:reverse(lists:flatten(lists:foldl(fun(Selector, Acc1) ->
         Batch = ecpool:with_client(Pool, fun(Conn) ->
                   case mongo_api:find(Conn, Collection, Selector, #{}) of
+                      {error, Reason} ->
+                          ?LOG(error, "[MongoDB] query_multi failed, got error: ~p", [Reason]),
+                          [];
                       [] -> [];
                       {ok, Cursor} ->
                           mc_cursor:foldl(fun(O, Acc2) -> [O|Acc2] end, [], Cursor, 1000)

apps/emqx_auth_mongo/src/emqx_auth_mongo_app.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -36,27 +36,64 @@
 
 start(_StartType, _StartArgs) ->
     {ok, Sup} = emqx_auth_mongo_sup:start_link(),
-    with_env(auth_query, fun reg_authmod/1),
+    ok = safe_start(),
-    with_env(acl_query,  fun reg_aclmod/1),
     {ok, Sup}.
 
 prep_stop(State) ->
-    ok = emqx:unhook('client.authenticate', fun emqx_auth_mongo:check/3),
+    ok = unload_hook(),
-    ok = emqx:unhook('client.check_acl', fun emqx_acl_mongo:check_acl/5),
+    _ = stop_pool(),
     State.
 
 stop(_State) ->
     ok.
 
+unload_hook() ->
+    ok = emqx:unhook('client.authenticate', fun emqx_auth_mongo:check/3),
+    ok = emqx:unhook('client.check_acl', fun emqx_acl_mongo:check_acl/5).
+
+stop_pool() ->
+    ecpool:stop_sup_pool(?APP).
+
+safe_start() ->
+    try
+        ok = with_env(auth_query, fun reg_authmod/1),
+        ok = with_env(acl_query,  fun reg_aclmod/1),
+        ok
+    catch _E:R:_S ->
+        unload_hook(),
+        _ = stop_pool(),
+        {error, R}
+    end.
+
 reg_authmod(AuthQuery) ->
-    emqx_auth_mongo:register_metrics(),
+    case emqx_auth_mongo:available(?APP, AuthQuery) of
-    SuperQuery = r(super_query, application:get_env(?APP, super_query, undefined)),
+        ok ->
-    ok = emqx:hook('client.authenticate', fun emqx_auth_mongo:check/3,
+            HookFun = fun emqx_auth_mongo:check/3,
-                   [#{authquery => AuthQuery, superquery => SuperQuery, pool => ?APP}]).
+            HookOptions = #{authquery => AuthQuery, superquery => undefined, pool => ?APP},
+            case r(super_query, application:get_env(?APP, super_query, undefined)) of
+                undefined ->
+                    ok = emqx:hook('client.authenticate', HookFun, [HookOptions]);
+                SuperQuery ->
+                    case emqx_auth_mongo:available(?APP, SuperQuery) of
+                        ok ->
+                            ok = emqx:hook('client.authenticate', HookFun,
+                                     [HookOptions#{superquery => SuperQuery}]);
+                        {error, Reason} ->
+                            {error, Reason}
+                    end
+            end;
+        {error, Reason} ->
+            {error, Reason}
+    end.
 
 reg_aclmod(AclQuery) ->
-    emqx_acl_mongo:register_metrics(),
+    case emqx_auth_mongo:available(?APP, AclQuery) of
-    ok = emqx:hook('client.check_acl', fun emqx_acl_mongo:check_acl/5, [#{aclquery => AclQuery, pool => ?APP}]).
+        ok ->
+            ok = emqx:hook('client.check_acl', fun emqx_acl_mongo:check_acl/5,
+                     [#{aclquery => AclQuery, pool => ?APP}]);
+        {error, Reason} ->
+            {error, Reason}
+    end.
 
 %%--------------------------------------------------------------------
 %% Internal functions
@@ -84,4 +121,3 @@ r(auth_query, Config) ->
 r(acl_query, Config) ->
     #aclquery{collection = list_to_binary(get_value(collection, Config, "mqtt_acl")),
               selector   = get_value(selector, Config, [?DEFAULT_SELECTORS])}.
-

apps/emqx_auth_mongo/src/emqx_auth_mongo_sup.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -52,10 +52,14 @@ all() ->
 init_per_suite(Cfg) ->
     emqx_ct_helpers:start_apps([emqx_auth_mongo], fun set_special_confs/1),
     init_mongo_data(),
+    %% avoid inter-suite flakiness
+    ok = emqx_mod_acl_internal:unload([]),
     Cfg.
 
 end_per_suite(_Cfg) ->
     deinit_mongo_data(),
+    %% avoid inter-suite flakiness
+    ok = emqx_mod_acl_internal:load([]),
     emqx_ct_helpers:stop_apps([emqx_auth_mongo]).
 
 set_special_confs(emqx) ->

apps/emqx_auth_mysql/include/emqx_auth_mysql.hrl

@@ -1,23 +1 @@
-
 -define(APP, emqx_auth_mysql).
-
--record(auth_metrics, {
-        success = 'client.auth.success',
-        failure = 'client.auth.failure',
-        ignore = 'client.auth.ignore'
-    }).
-
--record(acl_metrics, {
-        allow = 'client.acl.allow',
-        deny = 'client.acl.deny',
-        ignore = 'client.acl.ignore'
-    }).
-
--define(METRICS(Type), tl(tuple_to_list(#Type{}))).
--define(METRICS(Type, K), #Type{}#Type.K).
-
--define(AUTH_METRICS, ?METRICS(auth_metrics)).
--define(AUTH_METRICS(K), ?METRICS(auth_metrics, K)).
-
--define(ACL_METRICS, ?METRICS(acl_metrics)).
--define(ACL_METRICS(K), ?METRICS(acl_metrics, K)).