Merge pull request #9610 from thalesmg/fix-upgrade-unpacked-v43

fix(upgrade): fix start script existence check (v4.3)

.ci/acl_migration_test/build.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -xe
+
+cd "$EMQX_PATH"
+
+rm -rf _build _upgrade_base
+
+mkdir _upgrade_base
+pushd _upgrade_base
+    wget "https://s3-us-west-2.amazonaws.com/packages.emqx/emqx-ce/v${EMQX_BASE}/emqx-ubuntu20.04-${EMQX_BASE}-amd64.zip"
+popd
+
+make emqx-zip

.ci/acl_migration_test/prepare.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -xe
+
+mkdir -p "$TEST_PATH"
+cd "$TEST_PATH"
+
+cp ../"$EMQX_PATH"/_upgrade_base/*.zip ./
+unzip ./*.zip
+
+cp ../"$EMQX_PATH"/_packages/emqx/*.zip ./emqx/releases/
+
+git clone --depth 1 https://github.com/terry-xiaoyu/one_more_emqx.git
+
+./one_more_emqx/one_more_emqx.sh emqx2

.ci/acl_migration_test/suite.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+set -xe
+
+export EMQX_PATH="$1"
+export EMQX_BASE="$2"
+
+export TEST_PATH="emqx_test"
+
+./build.sh
+
+VERSION=$("$EMQX_PATH"/pkg-vsn.sh)
+export VERSION
+
+./prepare.sh
+
+./test.sh

.ci/acl_migration_test/test.sh

@@ -0,0 +1,121 @@
+#!/bin/bash
+
+set -e
+
+EMQX_ENDPOINT="http://localhost:8081/api/v4/acl"
+EMQX2_ENDPOINT="http://localhost:8917/api/v4/acl"
+
+function run() {
+    emqx="$1"
+    shift
+
+    echo "[$emqx]" "$@"
+
+    pushd "$TEST_PATH/$emqx"
+        "$@"
+    popd
+}
+
+function post_rule() {
+    endpoint="$1"
+    rule="$2"
+    echo -n "->($endpoint) "
+    curl -s -u admin:public -X POST "$endpoint" -d "$rule"
+    echo
+}
+
+function verify_clientid_rule() {
+    endpoint="$1"
+    id="$2"
+    echo -n "<-($endpoint) "
+    curl -s -u admin:public "$endpoint/clientid/$id" | grep "$id" || (echo "verify rule for client $id failed" && return 1)
+}
+
+# Run nodes
+
+run emqx ./bin/emqx start
+run emqx2 ./bin/emqx start
+
+run emqx ./bin/emqx_ctl plugins load emqx_auth_mnesia
+run emqx2 ./bin/emqx_ctl plugins load emqx_auth_mnesia
+
+run emqx2 ./bin/emqx_ctl cluster join 'emqx@127.0.0.1'
+
+# Add ACL rule to unupgraded EMQX nodes
+
+post_rule "$EMQX_ENDPOINT" '{"clientid": "CLIENT1_A","topic": "t", "action": "pub", "access": "allow"}'
+post_rule "$EMQX2_ENDPOINT" '{"clientid": "CLIENT1_B","topic": "t", "action": "pub", "access": "allow"}'
+
+# Upgrade emqx2 node
+
+run emqx2 ./bin/emqx install "$VERSION"
+sleep 60
+
+# Verify upgrade blocked
+
+run emqx2 ./bin/emqx eval 'emqx_acl_mnesia_migrator:is_old_table_migrated().' | grep false || (echo "emqx2 shouldn't have migrated" && exit 1)
+
+# Verify old rules on both nodes
+
+verify_clientid_rule "$EMQX_ENDPOINT" 'CLIENT1_A'
+verify_clientid_rule "$EMQX2_ENDPOINT" 'CLIENT1_A'
+
+verify_clientid_rule "$EMQX_ENDPOINT" 'CLIENT1_B'
+verify_clientid_rule "$EMQX2_ENDPOINT" 'CLIENT1_B'
+
+# Add ACL on OLD and NEW node, verify on all nodes
+
+post_rule "$EMQX_ENDPOINT" '{"clientid": "CLIENT2_A","topic": "t", "action": "pub", "access": "allow"}'
+post_rule "$EMQX2_ENDPOINT" '{"clientid": "CLIENT2_B","topic": "t", "action": "pub", "access": "allow"}'
+
+verify_clientid_rule "$EMQX_ENDPOINT" 'CLIENT2_A'
+verify_clientid_rule "$EMQX2_ENDPOINT" 'CLIENT2_A'
+
+verify_clientid_rule "$EMQX_ENDPOINT" 'CLIENT2_B'
+verify_clientid_rule "$EMQX2_ENDPOINT" 'CLIENT2_B'
+
+# Upgrade emqx node
+
+run emqx ./bin/emqx install "$VERSION"
+
+# Wait for upgrade
+
+sleep 60
+
+# Verify if upgrade occured
+
+run emqx ./bin/emqx eval 'emqx_acl_mnesia_migrator:is_old_table_migrated().' | grep true || (echo "emqx should have migrated" && exit 1)
+run emqx2 ./bin/emqx eval 'emqx_acl_mnesia_migrator:is_old_table_migrated().' | grep true || (echo "emqx2 should have migrated" && exit 1)
+
+# Verify rules are kept
+
+verify_clientid_rule "$EMQX_ENDPOINT" 'CLIENT1_A'
+verify_clientid_rule "$EMQX2_ENDPOINT" 'CLIENT1_A'
+
+verify_clientid_rule "$EMQX_ENDPOINT" 'CLIENT1_B'
+verify_clientid_rule "$EMQX2_ENDPOINT" 'CLIENT1_B'
+
+verify_clientid_rule "$EMQX_ENDPOINT" 'CLIENT2_A'
+verify_clientid_rule "$EMQX2_ENDPOINT" 'CLIENT2_A'
+
+verify_clientid_rule "$EMQX_ENDPOINT" 'CLIENT2_B'
+verify_clientid_rule "$EMQX2_ENDPOINT" 'CLIENT2_B'
+
+# Add ACL on OLD and NEW node, verify on all nodes
+
+post_rule "$EMQX_ENDPOINT" '{"clientid": "CLIENT3_A","topic": "t", "action": "pub", "access": "allow"}'
+post_rule "$EMQX2_ENDPOINT" '{"clientid": "CLIENT3_B","topic": "t", "action": "pub", "access": "allow"}'
+
+verify_clientid_rule "$EMQX_ENDPOINT" 'CLIENT3_A'
+verify_clientid_rule "$EMQX2_ENDPOINT" 'CLIENT3_A'
+
+verify_clientid_rule "$EMQX_ENDPOINT" 'CLIENT3_B'
+verify_clientid_rule "$EMQX2_ENDPOINT" 'CLIENT3_B'
+
+# Stop nodes
+
+run emqx ./bin/emqx stop
+run emqx2 ./bin/emqx stop
+
+echo "Success!"
+

.ci/build_packages/Dockerfile

@@ -1,4 +1,5 @@
-ARG BUILD_FROM=emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04
+ARG BUILD_FROM=emqx/build-env:erl23.3.4.9-3-ubuntu20.04
+# This Dockerfile is only used for EMQX 4.3, no need to update for 4.4 or later
 FROM ${BUILD_FROM}
 
 ARG EMQX_NAME=emqx

.ci/build_packages/tests.sh

@@ -88,6 +88,7 @@ emqx_test(){
             ;;
             "rpm")
                 packagename=$(basename "${PACKAGE_PATH}/${EMQX_NAME}"-*.rpm)
+
                 rpm -ivh "${PACKAGE_PATH}/${packagename}"
                 if ! rpm -q emqx | grep -q emqx; then
                     echo "package install error"
@@ -129,23 +130,6 @@ running_test(){
     pytest -v /paho-mqtt-testing/interoperability/test_client/V5/test_connect.py::test_basic
     # shellcheck disable=SC2009 # pgrep does not support Extended Regular Expressions
     emqx stop || kill "$(ps -ef | grep -E '\-progname\s.+emqx\s' |awk '{print $2}')"
-
-    if [ "$(sed -n '/^ID=/p' /etc/os-release | sed -r 's/ID=(.*)/\1/g' | sed 's/"//g')" = ubuntu ] \
-    || [ "$(sed -n '/^ID=/p' /etc/os-release | sed -r 's/ID=(.*)/\1/g' | sed 's/"//g')" = debian ] ;then
-        service emqx start || ( tail /var/log/emqx/emqx.log.1 && exit 1 )
-        IDLE_TIME=0
-        while ! emqx_ctl status | grep -E 'Node\s.*@.*\sis\sstarted'
-        do
-            if [ $IDLE_TIME -gt 10 ]
-            then
-                echo "emqx service error"
-                exit 1
-            fi
-            sleep 10
-            IDLE_TIME=$((IDLE_TIME+1))
-        done
-        service emqx stop
-    fi
 }
 
 relup_test(){
@@ -155,16 +139,37 @@ relup_test(){
 
         find . -maxdepth 1 -name "${EMQX_NAME}-*-${ARCH}.zip" |
             while read -r pkg; do
+                if [[ "${pkg}" == *4.3.13* ]]; then
+                    echo "skipping upgrade test from 4.3.13 because this release had crypto linked with openssl 1.1.1n, it was in later version rolled back (to default 1.1.1k)."
+                    continue
+                fi
                 packagename=$(basename "${pkg}")
-                unzip "$packagename"
+                unzip -q "$packagename"
                 ./emqx/bin/emqx start || ( tail emqx/log/emqx.log.1 && exit 1 )
                 ./emqx/bin/emqx_ctl status
                 ./emqx/bin/emqx versions
+                OldVsn="$(./emqx/bin/emqx eval 'Versions=[{S, V} ||  {_,V,_, S} <- release_handler:which_releases()],
+                            Current = proplists:get_value(current, Versions, proplists:get_value(permanent, Versions)),
+                            io:format("~s", [Current])')"
                 cp "${PACKAGE_PATH}/${EMQX_NAME}"-*-"${TARGET_VERSION}-${ARCH}".zip ./emqx/releases
                 ./emqx/bin/emqx install "${TARGET_VERSION}"
                 [ "$(./emqx/bin/emqx versions |grep permanent | awk '{print $2}')" = "${TARGET_VERSION}" ] || exit 1
+                export EMQX_WAIT_FOR_STOP=300
                 ./emqx/bin/emqx_ctl status
-                ./emqx/bin/emqx stop
+
+                # also test remove old rel
+                ./emqx/bin/emqx uninstall "$OldVsn"
+
+                # check emqx still runs
+                ./emqx/bin/emqx ping
+
+                if ! ./emqx/bin/emqx stop; then
+                    cat emqx/log/erlang.log.1 || true
+                    cat emqx/log/emqx.log.1 || true
+                    echo "failed to stop emqx"
+                    exit 1
+                fi
+
                 rm -rf emqx
             done
    fi

.ci/docker-compose-file/conf.env

@@ -1,5 +1,5 @@
 EMQX_AUTH__LDAP__SERVERS=ldap_server
-EMQX_AUTH__MONGO__SERVER=mongo_server:27017
+EMQX_AUTH__MONGO__SERVER=toxiproxy:27017
 EMQX_AUTH__MYSQL__SERVER=mysql_server:3306
 EMQX_AUTH__MYSQL__USERNAME=root
 EMQX_AUTH__MYSQL__PASSWORD=public

.ci/docker-compose-file/docker-compose-emqx-broker-cluster.yaml

@@ -0,0 +1,99 @@
+version: '3.9'
+
+services:
+  haproxy:
+    container_name: haproxy
+    image: haproxy:2.3
+    depends_on:
+      - emqx1
+      - emqx2
+    volumes:
+      - ./haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg
+      - ../../etc/certs:/usr/local/etc/haproxy/certs
+    ports:
+      - "18083:18083"
+    #  - "1883:1883"
+    #  - "8883:8883"
+    #  - "8083:8083"
+    #  - "5683:5683/udp"
+    #  - "9999:9999"
+    #  - "8084:8084"
+    networks:
+      - emqx_bridge
+    working_dir: /usr/local/etc/haproxy
+    command:
+      - bash
+      - -c
+      - |
+        cat /usr/local/etc/haproxy/certs/cert.pem /usr/local/etc/haproxy/certs/key.pem > /usr/local/etc/haproxy/certs/emqx.pem
+        haproxy -f /usr/local/etc/haproxy/haproxy.cfg
+
+  emqx1:
+    restart: always
+    container_name: node1.emqx.io
+    image: $TARGET:$EMQX_TAG
+    env_file:
+      - conf.cluster.env
+    volumes:
+      - etc:/opt/emqx/etc
+    environment:
+      - "EMQX_HOST=node1.emqx.io"
+    ports:
+      - "11881:18083"
+#      - "1883:1883"
+    command:
+        - /bin/sh
+        - -c
+        - |
+          sed -i "s 127.0.0.1 $$(ip route show |grep "link" |awk '{print $$1}') g" /opt/emqx/etc/acl.conf
+          sed -i '/emqx_telemetry/d' /opt/emqx/data/loaded_plugins
+          /opt/emqx/bin/emqx foreground
+    healthcheck:
+      test: ["CMD", "/opt/emqx/bin/emqx_ctl", "status"]
+      interval: 5s
+      timeout: 25s
+      retries: 5
+    networks:
+      emqx_bridge:
+        aliases:
+        - node1.emqx.io
+
+  emqx2:
+    restart: always
+    container_name: node2.emqx.io
+    image: $TARGET:$EMQX_TAG
+    env_file:
+      - conf.cluster.env
+    volumes:
+      - etc:/opt/emqx/etc
+    environment:
+      - "EMQX_HOST=node2.emqx.io"
+    ports:
+      - "11882:18083"
+    command:
+      - /bin/sh
+      - -c
+      - |
+        sed -i "s 127.0.0.1 $$(ip route show |grep "link" |awk '{print $$1}') g" /opt/emqx/etc/acl.conf
+        sed -i '/emqx_telemetry/d' /opt/emqx/data/loaded_plugins
+        /opt/emqx/bin/emqx foreground
+    healthcheck:
+      test: ["CMD", "/opt/emqx/bin/emqx", "ping"]
+      interval: 5s
+      timeout: 25s
+      retries: 5
+    networks:
+      emqx_bridge:
+        aliases:
+        - node2.emqx.io
+volumes:
+  etc:
+networks:
+  emqx_bridge:
+    driver: bridge
+    name: emqx_bridge
+    ipam:
+      driver: default
+      config:
+        - subnet: 172.100.239.0/24
+          gateway: 172.100.239.1

.ci/docker-compose-file/docker-compose-emqx-cluster.yaml

@@ -27,6 +27,7 @@ services:
         haproxy -f /usr/local/etc/haproxy/haproxy.cfg
 
   emqx1:
+    restart: always
     container_name: node1.emqx.io
     image: $TARGET:$EMQX_TAG
     env_file:
@@ -51,6 +52,7 @@ services:
         - node1.emqx.io
 
   emqx2:
+    restart: always
     container_name: node2.emqx.io
     image: $TARGET:$EMQX_TAG
     env_file:

.ci/docker-compose-file/docker-compose-enterprise-tomcat-tcp.yaml

@@ -0,0 +1,10 @@
+version: '3.9'
+
+services:
+  web_server:
+    container_name: Tomcat
+    build:
+      context: ./http-service
+    image: web-server
+    networks:
+      - emqx_bridge

.ci/docker-compose-file/docker-compose-toxiproxy.yaml

@@ -0,0 +1,16 @@
+version: '3.9'
+
+services:
+  toxiproxy:
+    container_name: toxiproxy
+    image: ghcr.io/shopify/toxiproxy:2.5.0
+    restart: always
+    networks:
+      - emqx_bridge
+    volumes:
+      - "./toxiproxy.json:/config/toxiproxy.json"
+    ports:
+      - 8474:8474
+    command:
+      - "-host=0.0.0.0"
+      - "-config=/config/toxiproxy.json"

.ci/docker-compose-file/docker-compose.yaml

@@ -3,7 +3,7 @@ version: '3.9'
 services:
   erlang:
     container_name: erlang
-    image: emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04
+    image: emqx/build-env:erl23.3.4.9-3-ubuntu20.04
     env_file:
       - conf.env
     environment:

.ci/docker-compose-file/http-service/Dockerfile

@@ -0,0 +1,15 @@
+FROM tomcat:10.0.5
+
+RUN wget https://downloads.apache.org/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.zip \
+	&& unzip -q apache-maven-3.6.3-bin.zip \
+	&& mv apache-maven-3.6.3 /opt/apache-maven-3.6.3/ \
+	&& ln -s /opt/apache-maven-3.6.3/ /opt/maven
+ENV M2_HOME=/opt/maven
+ENV M2=$M2_HOME/bin
+ENV PATH=$M2:$PATH
+COPY ./web-server /code
+WORKDIR /code
+RUN mvn package -Dmaven.skip.test=true
+RUN mv ./target/emqx-web-0.0.1.war /usr/local/tomcat/webapps/emqx-web.war
+EXPOSE 8080
+CMD ["/usr/local/tomcat/bin/catalina.sh","run"]

.ci/docker-compose-file/http-service/web-server/pom.xml

@@ -0,0 +1,65 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <groupId>emqx-web</groupId>
+  <artifactId>emqx-web</artifactId>
+  <version>0.0.1</version>
+  <packaging>war</packaging>
+  <dependencies>
+	<dependency>
+    	<groupId>mysql</groupId>
+    	<artifactId>mysql-connector-java</artifactId>
+    	<version>8.0.16</version>
+    </dependency>
+    <dependency>
+    	<groupId>commons-dbutils</groupId>
+    	<artifactId>commons-dbutils</artifactId>
+    	<version>1.7</version>
+	</dependency>
+	<dependency>
+  	  <groupId>commons-logging</groupId>
+   	 <artifactId>commons-logging</artifactId>
+    	<version>1.2</version>
+	</dependency>
+	<dependency>
+    	<groupId>commons-dbcp</groupId>
+    	<artifactId>commons-dbcp</artifactId>
+   		<version>1.4</version>
+	</dependency>
+	<dependency>
+    	<groupId>commons-pool</groupId>
+    	<artifactId>commons-pool</artifactId>
+    	<version>1.6</version>
+	</dependency>
+	<dependency>
+     	<groupId>jakarta.servlet</groupId>
+     	<artifactId>jakarta.servlet-api</artifactId>
+     	<version>5.0.0</version>
+     	<scope>provided</scope>
+  	</dependency>
+  </dependencies>
+  <build>
+    <resources>
+      <resource>
+        <directory>src/main/reousrce</directory>
+        <excludes>
+          <exclude>**/*.java</exclude>
+        </excludes>
+      </resource>
+    </resources>
+    <plugins>
+      <plugin>
+        <artifactId>maven-compiler-plugin</artifactId>
+        <version>3.8.1</version>
+        <configuration>
+          <source>1.8</source>
+          <target>1.8</target>
+        </configuration>
+      </plugin>
+      <plugin>
+        <artifactId>maven-war-plugin</artifactId>
+        <version>3.2.3</version>
+      </plugin>
+    </plugins>
+  </build>
+
+</project>

.ci/docker-compose-file/http-service/web-server/src/main/java/com/emqx/dao/AuthDAO.java

@@ -0,0 +1,54 @@
+package com.emqx.dao;
+
+import java.io.IOException;
+import java.sql.SQLException;
+
+import org.apache.commons.dbutils.QueryRunner;
+import org.apache.commons.dbutils.handlers.ScalarHandler;
+
+import com.emqx.util.EmqxDatabaseUtil;
+
+public class AuthDAO {
+
+	public String getUserName(String userName) throws IOException, SQLException {
+		QueryRunner runner = new QueryRunner(EmqxDatabaseUtil.getDataSource());
+		String sql = "select password from http_user where username='"+userName+"'";
+		String password =runner.query(sql, new ScalarHandler<String>());
+		return password;
+	}
+	
+	public String getClient(String clientid) throws IOException, SQLException {
+		QueryRunner runner = new QueryRunner(EmqxDatabaseUtil.getDataSource());
+		String sql = "select password from http_user where clientid='"+clientid+"'";
+		String password =runner.query(sql, new ScalarHandler<String>());
+		return password;
+	}
+	
+	public String getUserAccess(String userName) throws IOException, SQLException {
+		QueryRunner runner = new QueryRunner(EmqxDatabaseUtil.getDataSource());
+		String sql = "select access from http_acl where username='"+userName+"'";
+		String access =runner.query(sql, new ScalarHandler<String>());
+		return access;
+	}
+	
+	public String getUserTopic(String userName) throws IOException, SQLException {
+		QueryRunner runner = new QueryRunner(EmqxDatabaseUtil.getDataSource());
+		String sql = "select topic from http_acl where username='"+userName+"'";
+		String topic =runner.query(sql, new ScalarHandler<String>());
+		return topic;
+	}
+	
+	public String getClientAccess(String clientid) throws IOException, SQLException {
+		QueryRunner runner = new QueryRunner(EmqxDatabaseUtil.getDataSource());
+		String sql = "select access from http_acl where clientid='"+clientid+"'";
+		String access =runner.query(sql, new ScalarHandler<String>());
+		return access;
+	}
+	
+	public String getClientTopic(String clientid) throws IOException, SQLException {
+		QueryRunner runner = new QueryRunner(EmqxDatabaseUtil.getDataSource());
+		String sql = "select topic from http_acl where clientid='"+clientid+"'";
+		String topic =runner.query(sql, new ScalarHandler<String>());
+		return topic;
+	}
+}

.ci/docker-compose-file/http-service/web-server/src/main/java/com/emqx/dao/DBUtilsTest.java

@@ -0,0 +1,45 @@
+package com.emqx.dao;
+
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.sql.SQLException;
+import java.util.Properties;
+
+import org.apache.commons.dbcp.BasicDataSource;
+import org.apache.commons.dbutils.QueryRunner;
+import org.apache.commons.dbutils.handlers.ColumnListHandler;
+import org.apache.commons.dbutils.handlers.ScalarHandler;
+import org.apache.commons.dbutils.handlers.columns.StringColumnHandler;
+
+
+public class DBUtilsTest {
+
+	public static void main(String args[]) throws FileNotFoundException, IOException, SQLException {
+		Properties property = new Properties();//流文件  
+		
+		property.load(DBUtilsTest.class.getClassLoader().getResourceAsStream("database.properties"));
+		
+		BasicDataSource dataSource = new BasicDataSource();
+		dataSource.setDriverClassName(property.getProperty("jdbc.driver"));
+		dataSource.setUrl(property.getProperty("jdbc.url"));
+		dataSource.setUsername(property.getProperty("jdbc.username"));
+		dataSource.setPassword(property.getProperty("jdbc.password"));
+
+		// 初始化连接数 if(initialSize!=null)
+		//dataSource.setInitialSize(Integer.parseInt(initialSize));
+
+		// 最小空闲连接 if(minIdle!=null)
+		//dataSource.setMinIdle(Integer.parseInt(minIdle));
+
+		// 最大空闲连接 if(maxIdle!=null)
+		//dataSource.setMaxIdle(Integer.parseInt(maxIdle));
+
+		QueryRunner runner = new QueryRunner(dataSource);
+		String sql="select username from mqtt_user where id=1";
+		String result = runner.query(sql, new ScalarHandler<String>());
+		
+		System.out.println(result);
+				
+	}
+}

.ci/docker-compose-file/http-service/web-server/src/main/java/com/emqx/servlet/AclServlet.java

@@ -0,0 +1,103 @@
+package com.emqx.servlet;
+
+import java.io.IOException;
+import java.sql.SQLException;
+
+import com.emqx.dao.AuthDAO;
+
+import jakarta.servlet.ServletException;
+import jakarta.servlet.http.HttpServlet;
+import jakarta.servlet.http.HttpServletRequest;
+import jakarta.servlet.http.HttpServletResponse;
+
+public class AclServlet extends HttpServlet {
+
+	@Override
+	protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException {
+		// TODO Auto-generated method stub
+		doPost(req, resp);
+	}
+	@Override
+	protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException {
+		String clientid = req.getParameter("clientid");
+		String username = req.getParameter("username");
+		String access = req.getParameter("access");
+		String topic = req.getParameter("topic");
+		//String password = req.getParameter("password");
+		
+		//step0: password is not null, or not pass.
+		
+		AuthDAO dao = new AuthDAO();
+		try {
+			//step1: check username access&topic
+			if(username != null) {
+				String access_1 = dao.getUserAccess(username);
+				String topic_1 = dao.getUserTopic(username);
+				
+				if(access.equals(access_1)) {
+					if(topic.equals(topic_1)) {
+						resp.setStatus(200);
+					}
+					else {
+						if(clientid != null){
+							String access_2 = dao.getClientAccess(clientid);
+							String topic_2 = dao.getClientTopic(clientid);
+								if(access.equals(access_2)) {
+									if(topic.equals(topic_2)) {
+										resp.setStatus(200);
+									}
+									else {
+										resp.setStatus(400);
+									}
+								}else {
+									resp.setStatus(400);
+								}
+						}else {
+							resp.setStatus(400);
+						}
+					}
+				}else {//step2.1: username password is not match, then check clientid password
+					if(clientid != null){
+						String access_3 = dao.getClientAccess(clientid);
+						String topic_3 = dao.getClientTopic(clientid);
+							if(access.equals(access_3)) {
+								if(topic.equals(topic_3)) {
+									resp.setStatus(200);
+								}
+								else {
+									resp.setStatus(400);
+								}
+							}else {
+								resp.setStatus(400);
+							}
+					}else {
+						resp.setStatus(400);
+					}
+				}
+			}else {//step2.2: username is null, then check clientid password
+				if(clientid != null){
+					String access_4 = dao.getClientAccess(clientid);
+					String topic_4 = dao.getClientTopic(clientid);
+						if(access.equals(access_4)) {
+							if(topic.equals(topic_4)) {
+								resp.setStatus(200);
+							}
+							else {
+								resp.setStatus(400);
+							}
+						}else {
+							resp.setStatus(400);
+						}
+				}else {
+					resp.setStatus(400);
+				}
+			}
+		} catch (IOException e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+		} catch (SQLException e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+		}
+	}
+}

.ci/docker-compose-file/http-service/web-server/src/main/java/com/emqx/servlet/AuthServlet.java

@@ -0,0 +1,72 @@
+package com.emqx.servlet;
+
+import java.io.IOException;
+import java.sql.SQLException;
+
+import com.emqx.dao.AuthDAO;
+
+import jakarta.servlet.ServletException;
+import jakarta.servlet.http.HttpServlet;
+import jakarta.servlet.http.HttpServletRequest;
+import jakarta.servlet.http.HttpServletResponse;
+
+public class AuthServlet extends HttpServlet {
+
+	@Override
+	protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException {
+		// TODO Auto-generated method stub
+		doPost(req, resp);
+	}
+	@Override
+	protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException {
+		String clientid = req.getParameter("clientid");
+		String username =req.getParameter("username");
+		String password = req.getParameter("password");
+		
+		//step0: password is not null, or not pass.
+		if(password == null) {
+			resp.setStatus(400);
+			return;
+		}
+		AuthDAO dao = new AuthDAO();
+		try {
+			//step1: check username password
+			if(username != null) {
+				String password_d = dao.getUserName(username);
+				
+				if(password.equals(password_d)) {
+					resp.setStatus(200);
+					//200
+				}else {//step2.1: username password is not match, then check clientid password
+					if(clientid != null){
+						String password_c = dao.getClient(clientid);
+							if(password.equals(password_c)) {
+								resp.setStatus(200);
+							}else {
+								resp.setStatus(400);
+							}
+					}else {
+						resp.setStatus(400);
+					}
+				}
+			}else {//step2.2: username is null, then check clientid password
+				if(clientid != null){
+					String password_c = dao.getClient(clientid);
+						if(password.equals(password_c)) {
+							resp.setStatus(200);
+						}else {
+							resp.setStatus(400);
+						}
+				}else {
+					resp.setStatus(400);
+				}
+			}
+		} catch (IOException e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+		} catch (SQLException e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+		}
+	}
+}

.ci/docker-compose-file/http-service/web-server/src/main/java/com/emqx/util/EmqxDatabaseUtil.java

@@ -0,0 +1,27 @@
+package com.emqx.util;
+
+import java.io.IOException;
+import java.util.Properties;
+
+import javax.sql.DataSource;
+
+import org.apache.commons.dbcp.BasicDataSource;
+
+import com.emqx.dao.DBUtilsTest;
+
+public class EmqxDatabaseUtil {
+
+	public static DataSource getDataSource() throws IOException {
+		Properties property = new Properties();// 流文件
+
+		property.load(EmqxDatabaseUtil.class.getClassLoader().getResourceAsStream("database.properties"));
+
+		BasicDataSource dataSource = new BasicDataSource();
+		dataSource.setDriverClassName(property.getProperty("jdbc.driver"));
+		dataSource.setUrl(property.getProperty("jdbc.url"));
+		dataSource.setUsername(property.getProperty("jdbc.username"));
+		dataSource.setPassword(property.getProperty("jdbc.password"));
+		
+		return dataSource;
+	}
+}

.ci/docker-compose-file/http-service/web-server/src/main/reousrce/database.properties

@@ -0,0 +1,4 @@
+jdbc.driver= com.mysql.jdbc.Driver
+jdbc.url= jdbc:mysql://mysql_server:3306/mqtt
+jdbc.username= root
+jdbc.password= public

.ci/docker-compose-file/http-service/web-server/src/main/webapp/META-INF/MANIFEST.MF

@@ -0,0 +1,3 @@
+Manifest-Version: 1.0
+Class-Path: 
+

.ci/docker-compose-file/http-service/web-server/src/main/webapp/WEB-INF/web.xml

@@ -0,0 +1,31 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<web-app xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xmlns="http://JAVA.sun.com/xml/ns/javaee"
+	xsi:schemaLocation="http://java.sun.com/xml/ns/javaee http://java.sun.com/xml/ns/javaee/web-app_2_5.xsd"
+	id="WebApp_ID" version="2.5">
+	<display-name>emqx-web</display-name>
+	<servlet>
+		<servlet-name>Auth</servlet-name>
+		<servlet-class>com.emqx.servlet.AuthServlet</servlet-class>
+	</servlet>
+	<servlet>
+		<servlet-name>Acl</servlet-name>
+		<servlet-class>com.emqx.servlet.AclServlet</servlet-class>
+	</servlet>
+	<servlet-mapping>
+		<servlet-name>Auth</servlet-name>
+		<url-pattern>/auth</url-pattern>
+	</servlet-mapping>
+	<servlet-mapping>
+		<servlet-name>Acl</servlet-name>
+		<url-pattern>/acl</url-pattern>
+	</servlet-mapping>
+	<welcome-file-list>
+		<welcome-file>index.html</welcome-file>
+		<welcome-file>index.htm</welcome-file>
+		<welcome-file>index.jsp</welcome-file>
+		<welcome-file>default.html</welcome-file>
+		<welcome-file>default.htm</welcome-file>
+		<welcome-file>default.jsp</welcome-file>
+	</welcome-file-list>
+</web-app>

.ci/docker-compose-file/http-service/web-server/src/main/webapp/index.html

@@ -0,0 +1,10 @@
+<!DOCTYPE html>
+<html>
+<head>
+<meta charset="UTF-8">
+<title>love</title>
+</head>
+<body>
+It's lucky, jiabanxiang.
+</body>
+</html>

.ci/docker-compose-file/python/pytest.sh

@@ -10,7 +10,7 @@ LB="haproxy"
 
 apk update && apk add git curl
 git clone -b develop-4.0 https://github.com/emqx/paho.mqtt.testing.git /paho.mqtt.testing
-pip install pytest
+pip install pytest==6.2.5
 
 pytest -v /paho.mqtt.testing/interoperability/test_client/V5/test_connect.py -k test_basic --host "$LB"
 RESULT=$?

.ci/docker-compose-file/toxiproxy.json

@@ -0,0 +1,8 @@
+[
+  {
+    "name": "mongo_single",
+    "listen": "0.0.0.0:27017",
+    "upstream": "mongo:27017",
+    "enabled": true
+  }
+]

.ci/fvt_tests/http_server/rebar.config

@@ -1,7 +1,7 @@
 {erl_opts, [debug_info]}.
 {deps, 
  [
-    {minirest, {git, "https://github.com/emqx/minirest.git", {tag, "0.3.5"}}}
+    {minirest, {git, "https://github.com/emqx/minirest.git", {tag, "0.3.6"}}}
  ]}.
 
 {shell, [

.ci/fvt_tests/local_relup_test_run.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+USAGE="$0 profile vsn old_vsn package_path"
+EXAMPLE="$0 emqx 4.3.8-b3bb6075 v4.3.2 /home/alice/relup_dubug/downloaded_packages"
+
+if [[ $# -ne 4 ]]; then
+    echo "$USAGE"
+    echo "$EXAMPLE"
+    exit 1
+fi
+
+set -ex
+
+PROFILE="$1"
+VSN="$2"
+OLD_VSN="$3"
+PACKAGE_PATH="$4"
+
+TEMPDIR=$(mktemp -d)
+trap '{ rm -rf -- "$TEMPDIR"; }' EXIT
+
+git clone --branch=master "https://github.com/terry-xiaoyu/one_more_emqx.git" "$TEMPDIR/one_more_emqx"
+cp -r "$PACKAGE_PATH" "$TEMPDIR/packages"
+cp relup.lux "$TEMPDIR/"
+cp -r http_server "$TEMPDIR/http_server"
+
+exec docker run \
+    -v "$TEMPDIR:/relup_test" \
+    -w "/relup_test" \
+    -e REBAR_COLOR=none \
+    -it emqx/relup-test-env:erl23.2.7.2-emqx-3-ubuntu20.04 \
+        lux \
+        --progress verbose \
+        --case_timeout infinity \
+        --var PROFILE="$PROFILE" \
+        --var PACKAGE_PATH="/relup_test/packages" \
+        --var ONE_MORE_EMQX_PATH="/relup_test/one_more_emqx" \
+        --var VSN="$VSN" \
+        --var OLD_VSN="$OLD_VSN" \
+        relup.lux

.ci/fvt_tests/relup.lux

@@ -1,15 +1,12 @@
 [config var=PROFILE]
 [config var=PACKAGE_PATH]
-[config var=BENCH_PATH]
 [config var=ONE_MORE_EMQX_PATH]
 [config var=VSN]
-[config var=OLD_VSNS]
+[config var=OLD_VSN]
 
 [config shell_cmd=/bin/bash]
 [config timeout=600000]
 
-[loop old_vsn $OLD_VSNS]
-
 [shell http_server]
     !cd http_server
     !rebar3 shell
@@ -21,29 +18,30 @@
     ?>
 
 [shell emqx]
+    !OLD_VSN=$(echo $OLD_VSN | sed  -r 's/[v|e]//g')
     !cd $PACKAGE_PATH
-    !unzip -q -o $PROFILE-ubuntu20.04-$(echo $old_vsn | sed  -r 's/[v|e]//g')-amd64.zip
+    !unzip -q -o $PROFILE-ubuntu20.04-$${OLD_VSN}-amd64.zip
     ?SH-PROMPT
 
     !cd emqx
-    !sed -i 's|listener.wss.external[ \t]*=.*|listener.wss.external = 8085|g' etc/emqx.conf
+    !export EMQX_LOG__LEVEL=debug
-    !sed -i '/emqx_telemetry/d' data/loaded_plugins
 
     !./bin/emqx start
     ?EMQ X .* is started successfully!
     ?SH-PROMPT
 
 [shell emqx2]
+    !OLD_VSN=$(echo $OLD_VSN | sed  -r 's/[v|e]//g')
     !cd $PACKAGE_PATH
     !cp -f $ONE_MORE_EMQX_PATH/one_more_$(echo $PROFILE | sed 's/-/_/g').sh .
     !./one_more_$(echo $PROFILE | sed 's/-/_/g').sh emqx2
     ?SH-PROMPT
     !cd emqx2
 
-    !sed -i '/emqx_telemetry/d' data/loaded_plugins
+    !export EMQX_LOG__LEVEL=debug
 
     !./bin/emqx start
-    ?EMQ X (.*) is started successfully!
+    ?EMQ X .* is started successfully!
     ?SH-PROMPT
 
     !./bin/emqx_ctl cluster join emqx@127.0.0.1
@@ -63,6 +61,8 @@
     !./bin/emqx_ctl rules create 'SELECT * FROM "t/#"' '[{"name":"data_to_webserver", "params": {"$$resource":  "resource:691c29ba"}}]'
     ?created
     ?SH-PROMPT
+    !sleep 5
+    ?SH-PROMPT
 
 [shell emqx]
     !./bin/emqx_ctl resources list
@@ -71,11 +71,11 @@
     !./bin/emqx_ctl rules list
     ?691c29ba
     ?SH-PROMPT
+    !./bin/emqx_ctl broker metrics | grep "messages.publish"
+    ???SH-PROMPT
 
 [shell bench]
-    !cd $BENCH_PATH
+    !emqtt_bench pub -c 10 -I 1000 -t t/%i -s 64 -L 300
-
-    !./emqtt_bench pub -c 10 -I 1000 -t t/%i -s 64 -L 300
     ???sent
 
 [shell emqx]
@@ -84,6 +84,27 @@
 
     !cp -f ../$PROFILE-ubuntu20.04-$VSN-amd64.zip releases/
 
+    ## upgrade to the new version
+    !./bin/emqx install $VSN
+    ?Made release permanent: "$VSN"
+    ?SH-PROMPT
+
+    !./bin/emqx versions |grep permanent
+    ?(.*)$VSN
+    ?SH-PROMPT
+
+    ## downgrade to the old version
+    !./bin/emqx install $${OLD_VSN}
+    ?Made release permanent:.*
+    ?SH-PROMPT
+
+    !./bin/emqx versions |grep permanent | grep -qs "$${OLD_VSN}"
+    ?SH-PROMPT:
+    !echo ==$$?==
+    ?^==0==
+    ?SH-PROMPT:
+
+    ## again, upgrade to the new version
     !./bin/emqx install $VSN
     ?Made release permanent: "$VSN"
     ?SH-PROMPT
@@ -99,12 +120,37 @@
     """
     ?SH-PROMPT
 
+    !./bin/emqx_ctl plugins list | grep --color=never emqx_management
+    ?Plugin\(emqx_management.*active=true\)
+    ?SH-PROMPT
+
 [shell emqx2]
     !echo "" > log/emqx.log.1
     ?SH-PROMPT
 
     !cp -f ../$PROFILE-ubuntu20.04-$VSN-amd64.zip releases/
 
+    ## upgrade to the new version
+    !./bin/emqx install $VSN
+    ?Made release permanent: "$VSN"
+    ?SH-PROMPT
+
+    !./bin/emqx versions |grep permanent
+    ?(.*)$VSN
+    ?SH-PROMPT
+
+    ## downgrade to the old version
+    !./bin/emqx install $${OLD_VSN}
+    ?Made release permanent:.*
+    ?SH-PROMPT
+
+    !./bin/emqx versions |grep permanent | grep -qs "$${OLD_VSN}"
+    ?SH-PROMPT:
+    !echo ==$$?==
+    ?^==0==
+    ?SH-PROMPT:
+
+    ## again, upgrade to the new version
     !./bin/emqx install $VSN
     ?Made release permanent: "$VSN"
     ?SH-PROMPT
@@ -120,11 +166,34 @@
     """
     ?SH-PROMPT
 
+    !./bin/emqx_ctl plugins list | grep --color=never emqx_management
+    ?Plugin\(emqx_management.*active=true\)
+    ?SH-PROMPT
+
 [shell bench]
     ???publish complete
     ??SH-PROMPT:
+    !sleep 30
+    ?SH-PROMPT
+
+[shell emqx]
+    !./bin/emqx_ctl broker metrics | grep "messages.publish"
+    ???SH-PROMPT
+
+## We don't guarantee not to lose a single message!
+## So even if we received 290~300 messages, we consider it as success
+[shell bench]
+    !curl --user admin:public --silent --show-error http://localhost:8081/api/v4/rules | jq -M --raw-output ".data[0].metrics[] | select(.node==\"emqx@127.0.0.1\").matched"
+    ?(29[0-9])|(300)
+    ?SH-PROMPT
+
+    !curl --user admin:public --silent --show-error http://localhost:8081/api/v4/rules | jq -M --raw-output ".data[0].actions[0].metrics[] | select(.node==\"emqx@127.0.0.1\").success"
+    ?(29[0-9])|(300)
+    ?SH-PROMPT
+
+    ## The /counter API is provided by .ci/fvt_test/http_server
     !curl http://127.0.0.1:8080/counter
-    ???{"data":300,"code":0}
+    ?\{"data":(29[0-9])|(300),"code":0\}
     ?SH-PROMPT
 
 [shell emqx2]
@@ -158,8 +227,6 @@
     !halt(3).
     ?SH-PROMPT:
 
-[endloop]
-
 [cleanup]
     !echo ==$$?==
     ?==0==

.github/CODEOWNERS

@@ -0,0 +1,43 @@
+## MQTT & Core
+/src/ @qzhuyan
+/include/ @qzhuyan
+/etc/ @qzhuyan
+/test/ @qzhuyan
+
+## CI
+/.github/ @id
+/.ci/ @id
+/scripts/ @id
+/build @id
+/deploy/ @id
+
+## Authenticatio & ACL
+/apps/emqx_auth_*/ @savonarola
+/apps/emqx_psk_file/ @savonarola
+/apps/emqx_retainer/ @savonarola
+/apps/emqx_sasl/ @savonarola
+
+## Gateway
+/apps/emqx_coap/ @HJianBo
+/apps/emqx_exhook/ @HJianBo
+/apps/emqx_exproto/ @HJianBo
+/apps/emqx_lua_hook/ @HJianBo
+/apps/emqx_lwm2m/ @HJianBo
+
+## OPs
+/apps/emqx_management/ @zhongwencool
+/apps/emqx_recon/ @zhongwencool
+/apps/emqx_plugin_libs/ @zhongwencool
+/apps/emqx_prometheus/ @zhongwencool
+/apps/emqx_recon/ @zhongwencool
+
+
+## Data integration
+/apps/emqx_rule_engine/ @thalesmg
+/apps/emqx_web_hook/ @thalesmg
+
+## External Plugins
+/lib-extra/ @zmstone
+
+## Default
+* @zmstone

.github/ISSUE_TEMPLATE/bug-report.md

@@ -12,7 +12,7 @@ assignees: tigercl
 
 **Environment**:
 
-- EMQ X version (e.g. `emqx_ctl status`): 
+- EMQX version (e.g. `emqx_ctl status`):
 - Hardware configuration (e.g. `lscpu`):
 - OS (e.g. `cat /etc/os-release`):
 - Kernel (e.g. `uname -a`):

.github/actions/detect-profiles/action.yaml

@@ -0,0 +1,27 @@
+name: 'Detect profiles'
+outputs:
+  profiles:
+    description: 'Detected profiles'
+    value: ${{ steps.detect-profiles.outputs.profiles}}
+
+runs:
+  using: composite
+  steps:
+    - id: detect-profiles
+      shell: bash
+      run: |
+        if [ -d source ]; then
+          ## source code downloaded
+          cd source
+        fi
+        if [ ! -d .git ]; then
+          echo "Not git dir, $(pwd)"
+          exit 1
+        fi
+        if [ -f 'EMQX_ENTERPRISE' ]; then
+          echo "profiles=[\"emqx-ee\"]" >> $GITHUB_OUTPUT
+          echo "EMQX_NAME=emqx-ee" >> $GITHUB_ENV
+        else
+          echo "profiles=[\"emqx\", \"emqx-edge\"]" >> $GITHUB_OUTPUT
+          echo "EMQX_NAME=emqx" >> $GITHUB_ENV
+        fi

.github/actions/package-macos/action.yaml

@@ -0,0 +1,95 @@
+name: 'Create MacOS package'
+inputs:
+  otp: # 24.2.1-1, 23.3.4.9-3
+    required: true
+    type: string
+  os:
+    required: false
+    type: string
+    default: macos-11
+  apple_id_password:
+    required: true
+    type: string
+  apple_developer_identity:
+    required: true
+    type: string
+  apple_developer_id_bundle:
+    required: true
+    type: string
+  apple_developer_id_bundle_password:
+    required: true
+    type: string
+
+runs:
+  using: composite
+  steps:
+    - name: prepare
+      shell: bash
+      run: |
+        brew update
+        brew install curl zip unzip gnu-sed coreutils unixodbc freetds openssl@1.1
+        echo "/usr/local/opt/bison/bin" >> $GITHUB_PATH
+        echo "/usr/local/bin" >> $GITHUB_PATH
+    - uses: actions/cache@v3
+      id: cache
+      with:
+        path: /opt/erlang/${{ inputs.otp }}
+        key: otp-install-${{ inputs.otp }}-${{ inputs.os }}-static-ssl-disable-hipe-disable-jit
+    - name: build erlang
+      if: steps.cache.outputs.cache-hit != 'true'
+      shell: bash
+      run: |
+        git clone --depth 1 --branch OTP-${{ inputs.otp }} https://github.com/emqx/otp.git $HOME/otp-${{ inputs.otp }}
+        cd $HOME/otp-${{ inputs.otp }}
+        ./configure --disable-dynamic-ssl-lib --with-ssl=/usr/local/opt/openssl@1.1 --disable-hipe --disable-jit --prefix=/opt/erlang/${{ inputs.otp }}
+        make -j$(nproc)
+        sudo make install
+    - name: build
+      env:
+        AUTO_INSTALL_BUILD_DEPS: 1
+        APPLE_SIGN_BINARIES: 1
+        APPLE_ID: developers@emqx.io
+        APPLE_TEAM_ID: 26N6HYJLZA
+        APPLE_ID_PASSWORD: ${{ inputs.apple_id_password }}
+        APPLE_DEVELOPER_IDENTITY: ${{ inputs.apple_developer_identity }}
+        APPLE_DEVELOPER_ID_BUNDLE: ${{ inputs.apple_developer_id_bundle }}
+        APPLE_DEVELOPER_ID_BUNDLE_PASSWORD: ${{ inputs.apple_developer_id_bundle_password }}
+      shell: bash
+      run: |
+        export PATH="/opt/erlang/${{ inputs.otp }}/bin:$PATH"
+        make ensure-rebar3
+        sudo cp rebar3 /usr/local/bin/rebar3
+        make ${EMQX_NAME}-zip
+    - name: test
+      shell: bash
+      run: |
+        pkg_name=$(basename _packages/${EMQX_NAME}/${EMQX_NAME}-*.zip)
+        unzip -q _packages/${EMQX_NAME}/$pkg_name
+        # test with a spaces in path
+        mv ./emqx "./emqx  home/"
+        cd "./emqx  home/"
+        gsed -i '/emqx_telemetry/d' data/loaded_plugins
+        ./bin/emqx start || cat log/erlang.log.1
+        ready='no'
+        for i in {1..10}; do
+          if curl -fs 127.0.0.1:18083 > /dev/null; then
+            ready='yes'
+            break
+          fi
+          sleep 1
+        done
+        if [ "$ready" != "yes" ]; then
+          echo "Timed out waiting for emqx to be ready"
+          cat log/erlang.log.1
+          exit 1
+        fi
+        ./bin/emqx_ctl status
+        if ! ./bin/emqx stop; then
+          cat log/erlang.log.1 || true
+          cat log/emqx.log.1 || true
+          echo "failed to stop emqx"
+          exit 1
+        fi
+        cd ..
+        # test with a spaces in path
+        rm -rf "emqx  home"

.github/workflows/apps_version_check.yaml

@@ -3,10 +3,41 @@ name: Check Apps Version
 on: [pull_request]
 
 jobs:
-  check_apps_version:
+  check_apps_version_4_x:
     runs-on: ubuntu-20.04
 
+    strategy:
+      matrix:
+        erl_otp:
+        - erl23.3.4.18-1
+        os:
+        - ubuntu20.04
+
+    container: emqx/build-env:${{ matrix.erl_otp }}-${{ matrix.os }}
+
     steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0 # need full history
+      - name: fix-git-unsafe-repository
+        run: git config --global --add safe.directory /__w/emqx/emqx
+      - name: Check relup (ce)
+        if: endsWith(github.repository, 'emqx')
+        run: ./scripts/update-appup.sh emqx --check
+      - name: Check relup (ee)
+        if: endsWith(github.repository, 'enterprise')
+        run: ./scripts/update-appup.sh emqx-ee --check
       - name: Check apps version
-        run: ./scripts/apps-version-check.sh
+        run: ./scripts/check-apps-vsn.sh
+      - name: Check chart versions
+        run: ./scripts/check-chart-vsn.sh
+      - uses: actions/upload-artifact@v3.1.0
+        if: failure()
+        with:
+          name: expected_appup_files
+          path: |
+            apps/*/src/*.appup.src
+            src/*.appup.src
+            lib-ce/*/src/*.appup.src
+            lib-ee/*/src/*.appup.src
+          retention-days: 1

.github/workflows/build_packages.yaml

@@ -1,43 +1,38 @@
 name: Cross build packages
 
+concurrency:
+  group: build-${{ github.event_name }}-${{ github.ref }}
+  cancel-in-progress: true
+
 on:
+  push:
+    branches:
+      - 'main-v4.**'
+    tags:
+      - v*
+      - e*
   schedule:
     - cron:  '0 */6 * * *'
-  release:
-    types:
-      - published
   workflow_dispatch:
 
 jobs:
   prepare:
+    # avoid building when syncing to ee repo
+    if: endsWith(github.repository, 'emqx')
     runs-on: ubuntu-20.04
-    container: emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04
+    container: emqx/build-env:erl23.3.4.18-1-ubuntu20.04
 
     outputs:
-      profiles: ${{ steps.set_profile.outputs.profiles}}
+      profiles: ${{ steps.detect-profiles.outputs.profiles}}
-      old_vsns: ${{ steps.set_profile.outputs.old_vsns}}
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           path: source
-          fetch-depth: 0
+          fetch-depth: 0 # clone full git history
-      - name: set profile
+      - name: detect-profiles
-        id: set_profile
+        id: detect-profiles
-        shell: bash
+        uses: ./source/.github/actions/detect-profiles
-        run: |
-          cd source
-          vsn="$(./pkg-vsn.sh)"
-          pre_vsn="$(echo $vsn | grep -oE '^[0-9]+.[0-9]')"
-          if make emqx-ee --dry-run > /dev/null 2>&1; then
-            old_vsns="$(git tag -l "e$pre_vsn.[0-9]" | xargs echo -n | sed "s/e$vsn//")"
-            echo "::set-output name=old_vsns::$old_vsns"
-            echo "::set-output name=profiles::[\"emqx-ee\"]"
-          else
-            old_vsns="$(git tag -l "v$pre_vsn.[0-9]" | xargs echo -n | sed "s/v$vsn//")"
-            echo "::set-output name=old_vsns::$old_vsns"
-            echo "::set-output name=profiles::[\"emqx\", \"emqx-edge\"]"
-          fi
       - name: get_all_deps
         if: endsWith(github.repository, 'emqx')
         run: |
@@ -48,7 +43,6 @@ jobs:
         run: |
           echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials
           git config --global credential.helper store
-          echo "${{ secrets.CI_GIT_TOKEN }}" >> source/scripts/git-token
           make -C source deps-all
           zip -ryq source.zip source/* source/.[^.]*
       - uses: actions/upload-artifact@v2
@@ -58,13 +52,13 @@ jobs:
 
   windows:
     runs-on: windows-2019
-
     needs: prepare
     if: endsWith(github.repository, 'emqx')
-
     strategy:
       matrix:
         profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
+        otp:
+          - 23.3.4.17
         exclude:
           - profile: emqx-edge
 
@@ -76,16 +70,15 @@ jobs:
     - name: unzip source code
       run: Expand-Archive -Path source.zip -DestinationPath ./
     - uses: ilammy/msvc-dev-cmd@v1
-    - uses: gleam-lang/setup-erlang@v1.1.0
+    - uses: erlef/setup-beam@v1
-      id: install_erlang
       with:
-        otp-version: 23.2
+        otp-version: ${{ matrix.otp }}
     - name: build
       env:
         PYTHON: python
+        DIAGNOSTIC: 1
       run: |
-        $env:PATH = "${{ steps.install_erlang.outputs.erlpath }}\bin;$env:PATH"
+        erl -eval "erlang:display(crypto:info_lib())" -s init stop
-
         $version = $( "${{ github.ref }}" -replace "^(.*)/(.*)/" )
         if ($version -match "^v[0-9]+\.[0-9]+(\.[0-9]+)?") {
           $regex = "[0-9]+\.[0-9]+(-alpha|-beta|-rc)?\.[0-9]+"
@@ -101,14 +94,11 @@ jobs:
             Remove-Item -Force -Path rebar.lock
         }
         make ensure-rebar3
-        copy rebar3 "${{ steps.install_erlang.outputs.erlpath }}\bin"
-        ls "${{ steps.install_erlang.outputs.erlpath }}\bin"
-        rebar3 --help
         make ${{ matrix.profile }}
         mkdir -p _packages/${{ matrix.profile }}
         Compress-Archive -Path _build/${{ matrix.profile }}/rel/emqx -DestinationPath _build/${{ matrix.profile }}/rel/$pkg_name
         mv _build/${{ matrix.profile }}/rel/$pkg_name _packages/${{ matrix.profile }}
-        Get-FileHash -Path "_packages/${{ matrix.profile }}/$pkg_name" | Format-List | grep 'Hash' | awk '{print $3}'  > _packages/${{ matrix.profile }}/$pkg_name.sha256
+        sha256sum "_packages/${{ matrix.profile }}/$pkg_name" | head -c 64 > "_packages/${{ matrix.profile }}/${pkg_name}.sha256"
     - name: run emqx
       timeout-minutes: 1
       run: |
@@ -118,88 +108,45 @@ jobs:
         ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx stop
         ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx install
         ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx uninstall
-    - uses: actions/upload-artifact@v1
+    - uses: actions/upload-artifact@v3
-      if: startsWith(github.ref, 'refs/tags/')
       with:
         name: ${{ matrix.profile }}
-        path: source/_packages/${{ matrix.profile }}/.
+        path: source/_packages/${{ matrix.profile }}/
 
   mac:
-    runs-on: macos-10.15
-
     needs: prepare
 
     strategy:
       matrix:
-        profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
+        otp:
-        erl_otp:
+          - 23.3.4.18-1
-          - 23.2.7.2-emqx-2
+        os:
-        exclude:
+          - macos-11
-          - profile: emqx-edge
+    runs-on: ${{ matrix.os }}
 
     steps:
-    - uses: actions/download-artifact@v2
+    - uses: actions/download-artifact@v3
       with:
         name: source
         path: .
     - name: unzip source code
-      run: unzip -q source.zip
-    - name: prepare
       run: |
-        brew update
+        ln -s . source
-        brew install curl zip unzip gnu-sed kerl unixodbc freetds
+        unzip -q source.zip
-        echo "/usr/local/bin" >> $GITHUB_PATH
+        rm source source.zip
-        git config --global credential.helper store
+    - uses: ./.github/actions/detect-profiles
-    - uses: actions/cache@v2
+    - uses: ./.github/actions/package-macos
-      id: cache
       with:
-        path: ~/.kerl
+        otp: ${{ matrix.otp }}
-        key: erl${{ matrix.erl_otp }}-macos10.15
+        os: ${{ matrix.os }}
-    - name: build erlang
+        apple_id_password: ${{ secrets.APPLE_ID_PASSWORD }}
-      if: steps.cache.outputs.cache-hit != 'true'
+        apple_developer_identity: ${{ secrets.APPLE_DEVELOPER_IDENTITY }}
-      timeout-minutes: 60
+        apple_developer_id_bundle: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE }}
-      env:
+        apple_developer_id_bundle_password: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE_PASSWORD }}
-        KERL_BUILD_BACKEND: git
+    - uses: actions/upload-artifact@v3
-        OTP_GITHUB_URL: https://github.com/emqx/otp
-      run: |
-        kerl update releases
-        kerl build ${{ matrix.erl_otp }}
-        kerl install ${{ matrix.erl_otp }} $HOME/.kerl/${{ matrix.erl_otp }}
-    - name: build
-      run: |
-        . $HOME/.kerl/${{ matrix.erl_otp }}/activate
-        make -C source ensure-rebar3
-        sudo cp source/rebar3 /usr/local/bin/rebar3
-        make -C source ${{ matrix.profile }}-zip
-    - name: test
-      run: |
-        cd source
-        pkg_name=$(basename _packages/${{ matrix.profile }}/${{ matrix.profile }}-*.zip)
-        unzip -q _packages/${{ matrix.profile }}/$pkg_name
-        gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins
-        ./emqx/bin/emqx start || cat emqx/log/erlang.log.1
-        ready='no'
-        for i in {1..10}; do
-          if curl -fs 127.0.0.1:18083 > /dev/null; then
-            ready='yes'
-            break
-          fi
-          sleep 1
-        done
-        if [ "$ready" != "yes" ]; then
-          echo "Timed out waiting for emqx to be ready"
-          cat emqx/log/erlang.log.1
-          exit 1
-        fi
-        ./emqx/bin/emqx_ctl status
-        ./emqx/bin/emqx stop
-        rm -rf emqx
-        openssl dgst -sha256 ./_packages/${{ matrix.profile }}/$pkg_name | awk '{print $2}'  > ./_packages/${{ matrix.profile }}/$pkg_name.sha256
-    - uses: actions/upload-artifact@v1
-      if: startsWith(github.ref, 'refs/tags/')
       with:
-        name: ${{ matrix.profile }}
+        name: ${{ env.EMQX_NAME }}
-        path: source/_packages/${{ matrix.profile }}/.
+        path: _packages/${{ env.EMQX_NAME }}/
 
   linux:
     runs-on: ubuntu-20.04
@@ -207,6 +154,7 @@ jobs:
     needs: prepare
 
     strategy:
+      fail-fast: false
       matrix:
         profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
         arch:
@@ -222,8 +170,11 @@ jobs:
           - centos8
           - centos7
           - centos6
-          - raspbian10
+          # - raspbian10 #armv6l is too slow to emulate
           # - raspbian9
+        otp_version:
+          #- 23.2.7.2-emqx-3
+          - 23.3.4.18-1
         exclude:
         - os: centos6
           arch: arm64
@@ -260,35 +211,9 @@ jobs:
         path: .
     - name: unzip source code
       run: unzip -q source.zip
-    - name: downloads old emqx zip packages
-      env:
-        PROFILE: ${{ matrix.profile }}
-        ARCH: ${{ matrix.arch }}
-        SYSTEM: ${{ matrix.os }}
-        OLD_VSNS: ${{ needs.prepare.outputs.old_vsns }}
-      run: |
-        set -e -x -u
-        broker=$PROFILE
-        if [ $PROFILE = "emqx" ];then
-            broker="emqx-ce"
-        fi
-        if [ ! -z "$(echo $SYSTEM | grep -oE 'raspbian')" ]; then
-            export ARCH="arm"
-        fi
-
-        mkdir -p source/_upgrade_base
-        cd source/_upgrade_base
-        old_vsns=($(echo $OLD_VSNS | tr ' ' ' '))
-        for tag in ${old_vsns[@]}; do
-          if [ ! -z "$(echo $(curl -I -m 10 -o /dev/null -s -w %{http_code} https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip) | grep -oE "^[23]+")" ];then
-            wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip
-            wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip.sha256
-            echo "$(cat $PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip.sha256) $PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip" | sha256sum -c || exit 1
-          fi
-        done
     - name: build emqx packages
       env:
-        ERL_OTP: erl23.2.7.2-emqx-2
+        ERL_OTP: erl${{ matrix.otp_version }}
         PROFILE: ${{ matrix.profile }}
         ARCH: ${{ matrix.arch }}
         SYSTEM: ${{ matrix.os }}
@@ -312,20 +237,29 @@ jobs:
         docker volume prune -f
     - name: create sha256
       env:
-        PROFILE: ${{ matrix.profile}}
+        PROFILE: ${{ matrix.profile }}
+        ERL_OTP: erl${{ matrix.otp_version }}
+        ARCH: ${{ matrix.arch }}
       run: |
         if [ -d /tmp/packages/$PROFILE ]; then
           cd /tmp/packages/$PROFILE
             for var in $(ls emqx-* ); do
+              if [[ $ERL_OTP == erl23.2* ]]; then
+                # Keep package with new OTP as default
+                # But move package with old otp to track 2
+                echo "rename track 2 package"
+                oldfile="$var"
+                var="${var/${ARCH}/otp23.2-${ARCH}}"
+                mv "$oldfile" "$var"
+              fi
               bash -c "echo $(sha256sum $var | awk '{print $1}') > $var.sha256"
             done
           cd -
         fi
     - uses: actions/upload-artifact@v1
-      if: startsWith(github.ref, 'refs/tags/')
       with:
         name: ${{ matrix.profile }}
-        path: /tmp/packages/${{ matrix.profile }}/.
+        path: /tmp/packages/${{ matrix.profile }}/
 
   docker:
     runs-on: ubuntu-20.04
@@ -333,19 +267,18 @@ jobs:
     needs: prepare
 
     strategy:
+      fail-fast: false
       matrix:
         profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
-        arch:
+        registry:
-          - [amd64, x86_64]
+          - 'docker.io'
-          - [arm64v8, aarch64]
+          - 'public.ecr.aws'
-          - [arm32v7, arm]
-          - [i386, i386]
-          - [s390x, s390x]
         exclude:
+          # we don't have an aws ecr repo for enterprise and edge yet
+          - profile: emqx-edge
+            registry: 'public.ecr.aws'
           - profile: emqx-ee
-            arch: [i386, i386]
+            registry: 'public.ecr.aws'
-          - profile: emqx-ee
-            arch: [s390x, s390x]
 
     steps:
     - uses: actions/download-artifact@v2
@@ -354,32 +287,72 @@ jobs:
         path: .
     - name: unzip source code
       run: unzip -q source.zip
-    - name: build emqx docker image
+    - uses: docker/setup-buildx-action@v1
-      env:
+    - uses: docker/setup-qemu-action@v1
-        PROFILE: ${{ matrix.profile }}
-        ARCH: ${{ matrix.arch[0] }}
-        QEMU_ARCH: ${{ matrix.arch[1] }}
-      run: |
-        sudo docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
-
-        cd source
-        sudo TARGET=emqx/$PROFILE ARCH=$ARCH QEMU_ARCH=$QEMU_ARCH  make docker
-        cd _packages/$PROFILE && for var in $(ls ${PROFILE}-docker-* ); do sudo bash -c "echo $(sha256sum $var | awk '{print $1}') > $var.sha256"; done && cd -
-    - uses: actions/upload-artifact@v1
-      if: startsWith(github.ref, 'refs/tags/')
       with:
-        name: ${{ matrix.profile }}
+        image: tonistiigi/binfmt:latest
-        path: source/_packages/${{ matrix.profile }}/.
+        platforms: all
-
+    - uses: aws-actions/configure-aws-credentials@v1
-  delete-artifact:
+      if: matrix.registry == 'public.ecr.aws'
-    runs-on: ubuntu-20.04
-    needs: [prepare, mac, linux, docker]
-    steps:
-    - uses: geekyeggo/delete-artifact@v1
       with:
-        name: source
+        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        aws-region: ${{ secrets.AWS_DEFAULT_REGION }}
+    - name: Docker login to aws ecr
+      if: matrix.registry == 'public.ecr.aws'
+      run: aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws
+    - uses: docker/login-action@v1
+      if: matrix.registry == 'docker.io'
+      with:
+        username: ${{ secrets.DOCKER_HUB_USER }}
+        password: ${{ secrets.DOCKER_HUB_TOKEN }}
+    - uses: docker/metadata-action@v3
+      id: meta
+      with:
+        images: ${{ matrix.registry }}/${{ github.repository_owner }}/${{ matrix.profile }}
+        ## only 5.0 is latest
+        flavor: |
+          latest=false
+        tags: |
+          type=ref,event=branch
+          type=ref,event=pr
+          type=match,pattern=[v|e](.*),group=1
+        labels:
+          org.opencontainers.image.otp.version=${{ matrix.otp }}
+    - uses: docker/build-push-action@v2
+      if: matrix.profile != 'emqx-ee'
+      with:
+        ## only push when stable tag and rc tag
+        push: ${{ contains(github.ref, 'tags') && !contains(github.ref_name, 'beta') && !contains(github.ref_name, 'alpha') }}
+        pull: true
+        no-cache: true
+        platforms: linux/amd64,linux/arm64
+        tags: ${{ steps.meta.outputs.tags }}
+        labels: ${{ steps.meta.outputs.labels }}
+        build-args: |
+          BUILD_FROM=emqx/build-env:erl23.3.4.18-1-alpine
+          RUN_FROM=alpine:3.12
+          EMQX_NAME=${{ matrix.profile }}
+        file: source/deploy/docker/Dockerfile
+        context: source
+    - uses: docker/build-push-action@v2
+      if: matrix.profile == 'emqx-ee'
+      with:
+        ## only push when stable tag and rc tag
+        push: ${{ contains(github.ref, 'tags') && !contains(github.ref_name, 'beta') && !contains(github.ref_name, 'alpha') }}
+        pull: true
+        no-cache: true
+        platforms: linux/amd64,linux/arm64
+        tags: ${{ steps.meta.outputs.tags }}
+        labels: ${{ steps.meta.outputs.labels }}
+        build-args: |
+          BUILD_FROM=emqx/build-env:erl23.3.4.18-1-alpine
+          RUN_FROM=alpine:3.12
+          EMQX_NAME=${{ matrix.profile }}
+        file: source/deploy/docker/Dockerfile.enterprise
+        context: source
 
-  upload:
+  publish_artifacts:
     runs-on: ubuntu-20.04
 
     if: startsWith(github.ref, 'refs/tags/')
@@ -387,106 +360,37 @@ jobs:
     needs: [prepare, mac, linux, docker]
 
     strategy:
+      fail-fast: false
       matrix:
         profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
-
     steps:
-    - uses: actions/checkout@v2
-    - name: get_version
-      run: |
-        echo 'version<<EOF' >> $GITHUB_ENV
-        echo ${{ github.ref }} | sed -r  "s ^refs/heads/|^refs/tags/(.*) \1 g" >> $GITHUB_ENV
-        echo 'EOF' >> $GITHUB_ENV
     - uses: actions/download-artifact@v2
       with:
         name: ${{ matrix.profile }}
-        path: ./_packages/${{ matrix.profile }}
+        path: packages/${{ matrix.profile }}
     - name: install dos2unix
       run: sudo apt-get update && sudo apt install -y dos2unix
     - name: get packages
       run: |
         set -e -u
-        cd _packages/${{ matrix.profile }}
+        cd packages/${{ matrix.profile }}
         for var in $( ls |grep emqx |grep -v sha256); do
           dos2unix $var.sha256
           echo "$(cat $var.sha256) $var" | sha256sum -c || exit 1
         done
         cd -
-    - name: upload aws s3
+    - uses: aws-actions/configure-aws-credentials@v1
+      with:
+        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        aws-region: ${{ secrets.AWS_DEFAULT_REGION }}
+    - name: upload to aws s3
       run: |
         set -e -u
         if [ "${{ matrix.profile }}"  == "emqx" ];then
-            broker="emqx-ce"
+            s3dir="emqx-ce"
         else
-            broker=${{ matrix.profile }}
+            s3dir=${{ matrix.profile }}
         fi
-        aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID }}
+        aws s3 cp --recursive packages/${{ matrix.profile }} s3://${{ secrets.AWS_S3_BUCKET }}/${s3dir}/${{ github.ref_name }}
-        aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        aws cloudfront create-invalidation --distribution-id ${{ secrets.AWS_CLOUDFRONT_ID }} --paths "/${s3dir}/${{ github.ref_name }}/*"
-        aws configure set default.region ${{ secrets.AWS_DEFAULT_REGION }}
-
-        aws s3 cp --recursive _packages/${{ matrix.profile }} s3://${{ secrets.AWS_S3_BUCKET }}/$broker/${{ env.version }}
-        aws cloudfront create-invalidation --distribution-id ${{ secrets.AWS_CLOUDFRONT_ID }} --paths "/$broker/${{ env.version }}/*"
-    - uses: Rory-Z/upload-release-asset@v1
-      if: github.event_name == 'release' && matrix.profile != 'emqx-ee'
-      with:
-        repo: emqx
-        path: "_packages/${{ matrix.profile }}/emqx-*"
-        token: ${{ github.token }}
-    - uses: Rory-Z/upload-release-asset@v1
-      if: github.event_name == 'release' && matrix.profile == 'emqx-ee'
-      with:
-        repo: emqx-enterprise
-        path: "_packages/${{ matrix.profile }}/emqx-*"
-        token: ${{ github.token }}
-    - name: update to emqx.io
-      if: github.event_name == 'release'
-      run: |
-        set -e -x -u
-        curl -w %{http_code} \
-             --insecure \
-             -H "Content-Type: application/json" \
-             -H "token: ${{ secrets.EMQX_IO_TOKEN }}" \
-             -X POST \
-             -d "{\"repo\":\"emqx/emqx\", \"tag\": \"${{ env.version }}\" }" \
-             ${{ secrets.EMQX_IO_RELEASE_API }}
-    - name: push docker image to docker hub
-      if: github.event_name == 'release'
-      run: |
-        set -e -x -u
-        sudo make docker-prepare
-        cd _packages/${{ matrix.profile }} && for var in $(ls |grep docker |grep -v sha256); do unzip $var; sudo docker load < ${var%.*}; rm -f ${var%.*}; done && cd -
-        echo ${{ secrets.DOCKER_HUB_TOKEN }} |sudo docker login -u ${{ secrets.DOCKER_HUB_USER }} --password-stdin
-        sudo TARGET=emqx/${{ matrix.profile }} make docker-push
-        sudo TARGET=emqx/${{ matrix.profile }} make docker-manifest-list
-    - name: update repo.emqx.io
-      if: github.event_name == 'release' && endsWith(github.repository, 'enterprise') && matrix.profile == 'emqx-ee'
-      run: |
-        curl --silent --show-error \
-          -H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \
-          -H "Accept: application/vnd.github.v3+json" \
-          -X POST \
-          -d "{\"ref\":\"v1.0.1\",\"inputs\":{\"version\": \"${{ env.version }}\", \"emqx_ee\": \"true\"}}" \
-          "https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/update_emqx_repos.yaml/dispatches"
-    - name: update repo.emqx.io
-      if: github.event_name == 'release' && endsWith(github.repository, 'emqx') && matrix.profile == 'emqx'
-      run: |
-        curl --silent --show-error \
-          -H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \
-          -H "Accept: application/vnd.github.v3+json" \
-          -X POST \
-          -d "{\"ref\":\"v1.0.1\",\"inputs\":{\"version\": \"${{ env.version }}\", \"emqx_ce\": \"true\"}}" \
-          "https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/update_emqx_repos.yaml/dispatches"
-    - name: update homebrew packages
-      if: github.event_name == 'release' && endsWith(github.repository, 'emqx') && matrix.profile == 'emqx'
-      run: |
-        if [ -z $(echo $version | grep -oE "(alpha|beta|rc)\.[0-9]") ]; then
-            curl --silent --show-error \
-              -H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \
-              -H "Accept: application/vnd.github.v3+json" \
-              -X POST \
-              -d "{\"ref\":\"v1.0.1\",\"inputs\":{\"version\": \"${{ env.version }}\"}}" \
-              "https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/update_emqx_homebrew.yaml/dispatches"
-        fi
-    - uses: geekyeggo/delete-artifact@v1
-      with:
-        name: ${{ matrix.profile }}

.github/workflows/build_slim_packages.yaml

@@ -1,122 +1,106 @@
 name: Build slim packages
 
 on:
-  push:
-    tags:
-      - v*
-      - e*
   pull_request:
   workflow_dispatch:
 
 jobs:
   build:
-    runs-on: ubuntu-20.04
+    runs-on: ${{ matrix.runs-on }}
 
     strategy:
+      fail-fast: false
       matrix:
-        erl_otp:
+        otp:
-        - erl23.2.7.2-emqx-2
+          - erl23.3.4.18-1
         os:
           - ubuntu20.04
-        - centos7
+          - centos8
+        runs-on:
+          - aws-amd64
+          - ubuntu-20.04
+        use-self-hosted:
+          - ${{ github.repository_owner == 'emqx' }}
+        exclude:
+          - runs-on: ubuntu-20.04
+            use-self-hosted: true
+          - runs-on: aws-amd64
+            use-self-hosted: false
 
-    container: emqx/build-env:${{ matrix.erl_otp }}-${{ matrix.os }}
+    container: emqx/build-env:${{ matrix.otp }}-${{ matrix.os }}
 
     steps:
-    - uses: actions/checkout@v1
+    - uses: AutoModality/action-clean@v1
-    - name: prepare
+    - uses: actions/checkout@v3
+      with:
+          fetch-depth: 0 # clone full git history
+    - name: fix-git-unsafe-repository
+      run: git config --global --add safe.directory /__w/emqx/emqx
+    - uses: ./.github/actions/detect-profiles
+    - name: ensure access to github
+      if: endsWith(github.repository, 'enterprise')
       run: |
-        if make emqx-ee --dry-run > /dev/null 2>&1; then
         echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials
         git config --global credential.helper store
-          echo "${{ secrets.CI_GIT_TOKEN }}" >> ./scripts/git-token
+    - uses: actions/cache@v3
-          echo "EMQX_NAME=emqx-ee" >> $GITHUB_ENV
+      with:
-        else
+        # dialyzer PLTs
-          echo "EMQX_NAME=emqx" >> $GITHUB_ENV
+        path: ~/.cache/rebar3/
-        fi
+        key: dialyzer-${{ matrix.otp }}
+    - name: make xref
+      run: make xref
+    - name: make dialyzer
+      run: make dialyzer
     - name: build zip packages
       run: make ${EMQX_NAME}-zip
     - name: build deb/rpm packages
       run: make ${EMQX_NAME}-pkg
-    - name: pakcages test
+    - uses: actions/upload-artifact@v3
+      if: failure()
+      with:
+        name: rebar3.crashdump
+        path: ./rebar3.crashdump
+    - name: packages test
       run: |
         export CODE_PATH=$GITHUB_WORKSPACE
         .ci/build_packages/tests.sh
-    - uses: actions/upload-artifact@v2
+    - uses: actions/upload-artifact@v3
       with:
         name: ${{ matrix.os }}
         path: _packages/**/*.zip
 
   mac:
-    runs-on: macos-10.15
-
     strategy:
       matrix:
-        erl_otp:
+        otp:
-        - 23.2.7.2-emqx-2
+          - 23.3.4.18-1
-
+        os:
+          - macos-11
+    runs-on: ${{ matrix.os }}
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v3
-    - name: prepare
+      with:
+          fetch-depth: 0 # clone full git history
+    - name: ensure access to github
+      if: endsWith(github.repository, 'enterprise')
       run: |
-        if make emqx-ee --dry-run > /dev/null 2>&1; then
         echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials
         git config --global credential.helper store
-          echo "${{ secrets.CI_GIT_TOKEN }}" >> ./scripts/git-token
+    - uses: ./.github/actions/detect-profiles
-          echo "EMQX_NAME=emqx-ee" >> $GITHUB_ENV
+    - uses: ./.github/actions/package-macos
-        else
-          echo "EMQX_NAME=emqx" >> $GITHUB_ENV
-        fi
-    - name: prepare
-      run: |
-        brew update
-        brew install curl zip unzip gnu-sed kerl unixodbc freetds
-        echo "/usr/local/bin" >> $GITHUB_PATH
-        git config --global credential.helper store
-    - uses: actions/cache@v2
-      id: cache
       with:
-        path: ~/.kerl
+        otp: ${{ matrix.otp }}
-        key: erl${{ matrix.erl_otp }}-macos10.15
+        os: ${{ matrix.os }}
-    - name: build erlang
+        apple_id_password: ${{ secrets.APPLE_ID_PASSWORD }}
-      if: steps.cache.outputs.cache-hit != 'true'
+        apple_developer_identity: ${{ secrets.APPLE_DEVELOPER_IDENTITY }}
-      timeout-minutes: 60
+        apple_developer_id_bundle: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE }}
-      env:
+        apple_developer_id_bundle_password: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE_PASSWORD }}
-        KERL_BUILD_BACKEND: git
+    - uses: actions/upload-artifact@v3
-        OTP_GITHUB_URL: https://github.com/emqx/otp
+      if: failure()
-      run: |
+      with:
-        kerl update releases
+        name: rebar3.crashdump
-        kerl build ${{ matrix.erl_otp }}
+        path: ./rebar3.crashdump
-        kerl install ${{ matrix.erl_otp }} $HOME/.kerl/${{ matrix.erl_otp }}
+    - uses: actions/upload-artifact@v3
-    - name: build
-      run: |
-        . $HOME/.kerl/${{ matrix.erl_otp }}/activate
-        make ensure-rebar3
-        sudo cp rebar3 /usr/local/bin/rebar3
-        make ${EMQX_NAME}-zip
-    - name: test
-      run: |
-        pkg_name=$(basename _packages/${EMQX_NAME}/emqx-*.zip)
-        unzip -q _packages/${EMQX_NAME}/$pkg_name
-        gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins
-        ./emqx/bin/emqx start || cat emqx/log/erlang.log.1
-        ready='no'
-        for i in {1..10}; do
-          if curl -fs 127.0.0.1:18083 > /dev/null; then
-            ready='yes'
-            break
-          fi
-          sleep 1
-        done
-        if [ "$ready" != "yes" ]; then
-          echo "Timed out waiting for emqx to be ready"
-          cat emqx/log/erlang.log.1
-          exit 1
-        fi
-        ./emqx/bin/emqx_ctl status
-        ./emqx/bin/emqx stop
-        rm -rf emqx
-    - uses: actions/upload-artifact@v2
       with:
         name: macos
         path: _packages/**/*.zip

.github/workflows/check_deps_integrity.yaml

@@ -5,7 +5,7 @@ on: [pull_request]
 jobs:
   check_deps_integrity:
     runs-on: ubuntu-20.04
-    container: emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04
+    container: emqx/build-env:erl23.3.4.18-1-ubuntu20.04
 
     steps:
       - uses: actions/checkout@v2

.github/workflows/elvis_lint.yaml

@@ -1,16 +0,0 @@
-name: Elvis Linter
-
-on: [pull_request]
-
-jobs:
-  build:
-    runs-on: ubuntu-20.04
-    steps:
-      - uses: actions/checkout@v2
-      - name: Set git token
-        if: endsWith(github.repository, 'enterprise')
-        run: |
-          echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials
-          git config --global credential.helper store
-      - run: |
-          ./scripts/elvis-check.sh $GITHUB_BASE_REF

.github/workflows/git_sync.yaml

@@ -3,7 +3,6 @@ name: Sync to enterprise
 on:
   push:
     branches:
-      - master
       - main-v*
 
 jobs:
@@ -23,11 +22,7 @@ jobs:
         id: create_pull_request
         run: |
           set -euo pipefail
-          if [ "$GITHUB_REF" = "refs/heads/master" ]; then
-            EE_REF="refs/heads/enterprise"
-          else
           EE_REF="${GITHUB_REF}-enterprise"
-          fi
           R=$(curl --silent --show-error \
             -H "Accept: application/vnd.github.v3+json" \
             -H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \

.github/workflows/release.yaml

@@ -0,0 +1,93 @@
+name: Upload release assets
+on:
+  release:
+    types:
+      - published
+
+jobs:
+  prepare:
+    runs-on: ubuntu-20.04
+    container: emqx/build-env:erl23.3.4.18-1-ubuntu20.04
+
+    outputs:
+      profiles: ${{ steps.detect-profiles.outputs.profiles}}
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0 # clone full git history
+      - id: detect-profiles
+        uses: ./.github/actions/detect-profiles
+
+  upload:
+    runs-on: ubuntu-20.04
+    needs: prepare
+    strategy:
+      fail-fast: false
+      matrix:
+        profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
+
+    steps:
+      - uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ secrets.AWS_DEFAULT_REGION }}
+      - name: Get packages
+        run: |
+          if [ "${{ matrix.profile }}"  == "emqx" ];then
+              s3dir="emqx-ce"
+          else
+              s3dir=${{ matrix.profile }}
+          fi
+          aws s3 cp --recursive  s3://${{ secrets.AWS_S3_BUCKET }}/$s3dir/${{ github.ref_name }} packages
+      - uses: alexellis/upload-assets@0.2.2
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        with:
+          asset_paths: '["packages/*"]'
+      - name: update to emqx.io
+        run: |
+          set -e -x -u
+          curl -w %{http_code} \
+               --insecure \
+               -H "Content-Type: application/json" \
+               -H "token: ${{ secrets.EMQX_IO_TOKEN }}" \
+               -X POST \
+               -d "{\"repo\":\"emqx/emqx\", \"tag\": \"${{ github.ref_name }}\" }" \
+               ${{ secrets.EMQX_IO_RELEASE_API }}
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0 # clone full git history
+      - name: get version
+        id: version
+        run: echo "version=$(./pkg-vsn.sh)" >> $GITHUB_OUTPUT
+      - uses: emqx/push-helm-action@v1
+        if: github.event_name == 'release' && endsWith(github.repository, 'emqx') && matrix.profile == 'emqx'
+        with:
+          charts_dir: "${{ github.workspace }}/deploy/charts/emqx"
+          version: ${{ steps.version.outputs.version }}
+          aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws_region: "us-west-2"
+          aws_bucket_name: "repos-emqx-io"
+      - uses: emqx/push-helm-action@v1
+        if: github.event_name == 'release' && endsWith(github.repository, 'enterprise') && matrix.profile == 'emqx-ee'
+        with:
+          charts_dir: "${{ github.workspace }}/deploy/charts/emqx-ee"
+          version: ${{ steps.version.outputs.version }}
+          aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws_region: "us-west-2"
+          aws_bucket_name: "repos-emqx-io"
+      - name: update homebrew packages
+        if: github.event_name == 'release' && endsWith(github.repository, 'emqx') && matrix.profile == 'emqx'
+        run: |
+          if [ -z $(echo $version | grep -oE "(alpha|beta|rc)\.[0-9]") ]; then
+              curl --silent --show-error \
+                -H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \
+                -H "Accept: application/vnd.github.v3+json" \
+                -X POST \
+                -d "{\"ref\":\"v1.0.3\",\"inputs\":{\"version\": \"${{ github.ref_name }}\"}}" \
+                "https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/update_emqx_homebrew.yaml/dispatches"
+          fi

.github/workflows/run_acl_migration_tests.yaml

@@ -0,0 +1,22 @@
+name: ACL fix & migration integration tests
+
+on: workflow_dispatch
+
+jobs:
+    test:
+        runs-on: ubuntu-20.04
+        container: emqx/build-env:erl23.3.4.18-1-ubuntu20.04
+        strategy:
+            fail-fast: true
+        env:
+            BASE_VERSION: "4.3.0"
+        steps:
+        - uses: actions/checkout@v2
+          with:
+            path: emqx
+        - name: Prepare scripts
+          run: |
+            cp ./emqx/.ci/acl_migration_test/*.sh ./
+        - name: Run tests
+          run: |
+            ./suite.sh emqx "$BASE_VERSION"

.github/workflows/run_automate_tests.yaml

@@ -0,0 +1,445 @@
+name: Integration Test Suites
+
+on:
+  push:
+    tags:
+      - "v4.*"
+  pull_request:
+    branches:
+      - "main-v4.*"
+
+jobs:
+  build:
+    runs-on: ubuntu-20.04
+    outputs:
+      imgname: ${{ steps.build_docker.outputs.imgname}}
+      version: ${{ steps.build_docker.outputs.version}}
+    steps:
+    - name: download jmeter
+      id: dload_jmeter
+      timeout-minutes: 1
+      env:
+        JMETER_VERSION: 5.4.3
+      run: |
+        wget --no-verbose --no-check-certificate -O /tmp/apache-jmeter.tgz https://downloads.apache.org/jmeter/binaries/apache-jmeter-$JMETER_VERSION.tgz
+    - uses: actions/upload-artifact@v2
+      with:
+        name: apache-jmeter.tgz
+        path: /tmp/apache-jmeter.tgz
+    - uses: actions/checkout@v2
+    - uses: erlef/setup-beam@v1
+      with:
+        otp-version: "23.3.4.17"
+    - name: build docker
+      id: build_docker
+      run: |
+        if [ -f EMQX_ENTERPRISE ]; then
+          echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials
+          git config --global credential.helper store
+          make deps-emqx-ee
+          make clean
+          make emqx-ee-docker
+          echo "::set-output name=imgname::emqx-ee"
+          echo "::set-output name=version::$(./pkg-vsn.sh)"
+          docker save emqx/emqx-ee:$(./pkg-vsn.sh) -o emqx.tar
+        else
+          make emqx-docker
+          echo "::set-output name=imgname::emqx"
+          echo "::set-output name=version::$(./pkg-vsn.sh)"
+          docker save emqx/emqx:$(./pkg-vsn.sh) -o emqx.tar
+        fi
+    - uses: actions/upload-artifact@v2
+      with:
+        name: emqx-docker-image
+        path: emqx.tar
+
+  webhook:
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        webhook_type:
+        - webhook_data_bridge
+
+    needs: build
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/download-artifact@v2
+      with:
+        name: emqx-docker-image
+        path: /tmp
+    - name: load docker image
+      run: docker load < /tmp/emqx.tar
+    - name: docker compose up
+      timeout-minutes: 5
+      env:
+        TARGET: emqx/${{ needs.build.outputs.imgname }}
+        EMQX_TAG: ${{ needs.build.outputs.version }}
+      run: |
+        docker-compose \
+          -f .ci/docker-compose-file/docker-compose-emqx-cluster.yaml \
+          up -d --build
+    - uses: actions/checkout@v2
+      with:
+        repository: emqx/emqx-svt-web-server
+        ref: web-server-1.0
+        path: emqx-svt-web-server
+    - uses: actions/download-artifact@v2
+    - name: run webserver in docker
+      run: |
+        cd ./emqx-svt-web-server/svtserver
+        mvn clean package
+        cd target
+        docker run --name webserver --network emqx_bridge -d -v $(pwd)/svtserver-0.0.1.jar:/webserver/svtserver-0.0.1.jar --workdir /webserver openjdk:8-jdk bash \
+        -c "java -jar svtserver-0.0.1.jar"
+    - name: wait docker compose up
+      timeout-minutes: 5
+      run: |
+        while [ "$(docker inspect -f '{{ .State.Health.Status}}' node1.emqx.io)" != "healthy" ] || [ "$(docker inspect -f '{{ .State.Health.Status}}' node2.emqx.io)" != "healthy" ]; do
+          echo "['$(date -u +"%y-%m-%dt%h:%m:%sz")']:waiting emqx";
+          sleep 5;
+        done
+        docker ps -a
+        echo HAPROXY_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' haproxy) >> $GITHUB_ENV
+        echo WEB_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' webserver) >> $GITHUB_ENV
+    - uses: actions/checkout@v2
+      with:
+        repository: emqx/emqx-fvt
+        ref: v1.6.0
+        path: scripts
+    - uses: actions/setup-java@v1
+      with:
+        java-version: '8.0.282' # The JDK version to make available on the path.
+        java-package: jdk # (jre, jdk, or jdk+fx) - defaults to jdk
+        architecture: x64 # (x64 or x86) - defaults to x64
+    - uses: actions/download-artifact@v2
+      with:
+        name: apache-jmeter.tgz
+        path: /tmp
+    - name: install jmeter
+      timeout-minutes: 10
+      env:
+          JMETER_VERSION: 5.4.3
+      run: |
+        cd /tmp && tar -xvf apache-jmeter.tgz
+        echo "jmeter.save.saveservice.output_format=xml" >> /tmp/apache-jmeter-$JMETER_VERSION/user.properties
+        echo "jmeter.save.saveservice.response_data.on_error=true" >> /tmp/apache-jmeter-$JMETER_VERSION/user.properties
+        wget --no-verbose -O /tmp/apache-jmeter-$JMETER_VERSION/lib/ext/mqtt-xmeter-2.0.2-jar-with-dependencies.jar https://raw.githubusercontent.com/xmeter-net/mqtt-jmeter/master/Download/v2.0.2/mqtt-xmeter-2.0.2-jar-with-dependencies.jar
+        ln -s /tmp/apache-jmeter-$JMETER_VERSION /opt/jmeter
+    - name: run jmeter
+      run: |
+        /opt/jmeter/bin/jmeter.sh \
+          -Jjmeter.save.saveservice.output_format=xml -n \
+          -t scripts/automate-test-suite/${{ matrix.webhook_type }}.jmx \
+          -Demqx_ip=$HAPROXY_IP \
+          -Dweb_ip=$WEB_IP \
+          -l jmeter_logs/webhook_${{ matrix.webhook_type }}.jtl \
+          -j jmeter_logs/logs/webhook_${{ matrix.webhook_type }}.log
+    - name: check logs
+      run: |
+        if cat jmeter_logs/webhook_${{ matrix.webhook_type }}.jtl | grep -e '<failure>true</failure>' > /dev/null 2>&1; then
+          echo "check logs filed"
+          exit 1
+        fi
+    - uses: actions/upload-artifact@v1
+      if: always()
+      with:
+        name: jmeter_logs
+        path: ./jmeter_logs
+
+  mysql:
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        mysql_tag:
+        - 5.7
+        - 8
+        mysql_type:
+        - mysql_auth_acl
+
+    needs: build
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/download-artifact@v2
+      with:
+        name: emqx-docker-image
+        path: /tmp
+    - name: load docker image
+      run: docker load < /tmp/emqx.tar
+    - name: docker compose up
+      timeout-minutes: 5
+      env:
+        TARGET: emqx/${{ needs.build.outputs.imgname }}
+        EMQX_TAG: ${{ needs.build.outputs.version }}
+        MYSQL_TAG: ${{ matrix.mysql_tag }}
+      run: |
+        docker-compose \
+          -f .ci/docker-compose-file/docker-compose-emqx-cluster.yaml \
+          -f .ci/docker-compose-file/docker-compose-mysql-tls.yaml \
+          up -d --build
+    - name: wait docker compose up
+      timeout-minutes: 5
+      run: |
+        while [ "$(docker inspect -f '{{ .State.Health.Status}}' node1.emqx.io)" != "healthy" ] || [ "$(docker inspect -f '{{ .State.Health.Status}}' node2.emqx.io)" != "healthy" ]; do
+          echo "['$(date -u +"%y-%m-%dt%h:%m:%sz")']:waiting emqx";
+          sleep 5;
+        done
+        while [ $(docker ps -a --filter name=client --filter exited=0 | wc -l) \
+             != $(docker ps -a --filter name=client | wc -l) ]; do
+          sleep 1
+        done
+        docker ps -a
+        echo HAPROXY_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' haproxy) >> $GITHUB_ENV
+        echo MYSQL_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' mysql) >> $GITHUB_ENV
+    - uses: actions/checkout@v2
+      with:
+        repository: emqx/emqx-fvt
+        ref: v1.6.0
+        path: scripts
+    - uses: actions/setup-java@v1
+      with:
+        java-version: '8.0.282' # The JDK version to make available on the path.
+        java-package: jdk # (jre, jdk, or jdk+fx) - defaults to jdk
+        architecture: x64 # (x64 or x86) - defaults to x64
+    - uses: actions/download-artifact@v2
+      with:
+        name: apache-jmeter.tgz
+        path: /tmp
+    - name: install jmeter
+      timeout-minutes: 10
+      env:
+          JMETER_VERSION: 5.4.3
+      run: |
+        cd /tmp && tar -xvf apache-jmeter.tgz
+        echo "jmeter.save.saveservice.output_format=xml" >> /tmp/apache-jmeter-$JMETER_VERSION/user.properties
+        echo "jmeter.save.saveservice.response_data.on_error=true" >> /tmp/apache-jmeter-$JMETER_VERSION/user.properties
+        wget --no-verbose -O /tmp/apache-jmeter-$JMETER_VERSION/lib/ext/mqtt-xmeter-2.0.2-jar-with-dependencies.jar https://raw.githubusercontent.com/xmeter-net/mqtt-jmeter/master/Download/v2.0.2/mqtt-xmeter-2.0.2-jar-with-dependencies.jar
+        ln -s /tmp/apache-jmeter-$JMETER_VERSION /opt/jmeter
+    - name: install jmeter plugin
+      run: |
+        wget --no-verbose -O "/opt/jmeter/lib/mysql-connector-java-8.0.16.jar" https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar
+    - name: run jmeter
+      run: |
+        /opt/jmeter/bin/jmeter.sh \
+          -Jjmeter.save.saveservice.output_format=xml -n \
+          -t scripts/automate-test-suite/${{ matrix.mysql_type }}.jmx \
+          -Droute="apps/emqx_auth_mysql/test/emqx_auth_mysql_SUITE_data" \
+          -Dmysql_ip=$MYSQL_IP \
+          -Demqx_ip=$HAPROXY_IP \
+          -Ddbname="mqtt" \
+          -Dmysql_user="ssluser" \
+          -Ddb_user="root" \
+          -Dmysql_pwd="public" \
+          -Dconfig_path="/tmp/etc" \
+          -Ddocker_path=".ci/docker-compose-file" \
+          -l jmeter_logs/${{ matrix.mysql_type }}_${{ matrix.mysql_tag }}.jtl \
+          -j jmeter_logs/logs/${{ matrix.mysql_type }}_${{ matrix.mysql_tag }}.log
+    - name: check logs
+      run: |
+        if cat jmeter_logs/${{ matrix.mysql_type }}_${{ matrix.mysql_tag }}.jtl | grep -e '<failure>true</failure>' > /dev/null 2>&1; then
+          echo "check logs filed"
+          exit 1
+        fi
+    - uses: actions/upload-artifact@v1
+      if: always()
+      with:
+        name: jmeter_logs
+        path: ./jmeter_logs
+
+
+  postgresql:
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        pgsql_type:
+        - pgsql_auth_acl
+        pgsql_tag:
+        - 9
+        - 10
+        - 11
+        - 12
+        - 13
+
+    needs: build
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/download-artifact@v2
+      with:
+        name: emqx-docker-image
+        path: /tmp
+    - name: load docker image
+      run: docker load < /tmp/emqx.tar
+    - name: docker compose up
+      timeout-minutes: 5
+      env:
+        TARGET: emqx/${{ needs.build.outputs.imgname }}
+        EMQX_TAG: ${{ needs.build.outputs.version }}
+        PGSQL_TAG: ${{ matrix.pgsql_tag }}
+      run: |
+        docker-compose \
+          -f .ci/docker-compose-file/docker-compose-emqx-broker-cluster.yaml \
+          -f .ci/docker-compose-file/docker-compose-pgsql-tls.yaml \
+          up -d --build
+    - name: wait docker compose up
+      timeout-minutes: 5
+      run: |
+        while [ "$(docker inspect -f '{{ .State.Health.Status}}' node1.emqx.io)" != "healthy" ] || [ "$(docker inspect -f '{{ .State.Health.Status}}' node2.emqx.io)" != "healthy" ]; do
+          echo "['$(date -u +"%y-%m-%dt%h:%m:%sz")']:waiting emqx";
+          sleep 5;
+        done
+        docker ps -a
+        echo HAPROXY_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' haproxy) >> $GITHUB_ENV
+        echo PGSQL_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' pgsql) >> $GITHUB_ENV
+        echo CONFIG_PATH=$(docker inspect -f '{{ range .Mounts }}{{ if eq .Name "docker-compose-file_etc" }}{{ .Source }}{{ end }}{{ end }}' node1.emqx.io) >> $GITHUB_ENV
+    - uses: actions/checkout@v2
+      with:
+        repository: emqx/emqx-fvt
+        ref: v1.6.0
+        path: scripts
+    - uses: actions/setup-java@v1
+      with:
+        java-version: '8.0.282' # The JDK version to make available on the path.
+        java-package: jdk # (jre, jdk, or jdk+fx) - defaults to jdk
+        architecture: x64 # (x64 or x86) - defaults to x64
+    - uses: actions/download-artifact@v2
+      with:
+        name: apache-jmeter.tgz
+        path: /tmp
+    - name: install jmeter
+      timeout-minutes: 10
+      env:
+          JMETER_VERSION: 5.4.3
+      run: |
+        cd /tmp && tar -xvf apache-jmeter.tgz
+        echo "jmeter.save.saveservice.output_format=xml" >> /tmp/apache-jmeter-$JMETER_VERSION/user.properties
+        echo "jmeter.save.saveservice.response_data.on_error=true" >> /tmp/apache-jmeter-$JMETER_VERSION/user.properties
+        wget --no-verbose -O /tmp/apache-jmeter-$JMETER_VERSION/lib/ext/mqtt-xmeter-2.0.2-jar-with-dependencies.jar https://raw.githubusercontent.com/xmeter-net/mqtt-jmeter/master/Download/v2.0.2/mqtt-xmeter-2.0.2-jar-with-dependencies.jar
+        ln -s /tmp/apache-jmeter-$JMETER_VERSION /opt/jmeter
+    - name: install jmeter plugin
+      run: |
+        wget --no-verbose -O "/opt/jmeter/lib/postgresql-42.2.18.jar" https://repo1.maven.org/maven2/org/postgresql/postgresql/42.2.18/postgresql-42.2.18.jar
+    - name: run jmeter
+      run: |
+        sudo /opt/jmeter/bin/jmeter.sh \
+          -Jjmeter.save.saveservice.output_format=xml -n \
+          -t scripts/automate-test-suite/${{ matrix.pgsql_type }}.jmx \
+          -Droute="apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data" \
+          -Dca_name="ca.pem" \
+          -Dkey_name="client-key.pem" \
+          -Dcert_name="client-cert.pem" \
+          -Ddb_ip=$PGSQL_IP \
+          -Dpgsql_ip=$PGSQL_IP \
+          -Demqx_ip=$HAPROXY_IP \
+          -Dpgsql_user="root" \
+          -Dpgsql_pwd="public" \
+          -Ddbname="mqtt" \
+          -Dpgsql_db="mqtt" \
+          -Dport="5432" \
+          -Dconfig_path=$CONFIG_PATH \
+          -Ddocker_path=".ci/docker-compose-file" \
+          -l jmeter_logs/${{ matrix.pgsql_type }}_${{ matrix.pgsql_tag }}.jtl \
+          -j jmeter_logs/logs/${{ matrix.pgsql_type }}_${{ matrix.pgsql_tag }}.log
+    - name: check logs
+      run: |
+        if cat jmeter_logs/${{ matrix.pgsql_type }}_${{ matrix.pgsql_tag }}.jtl | grep -e '<failure>true</failure>' > /dev/null 2>&1; then
+          echo "check logs filed"
+          exit 1
+        fi
+    - uses: actions/upload-artifact@v1
+      if: always()
+      with:
+        name: jmeter_logs
+        path: ./jmeter_logs
+
+  http:
+    runs-on: ubuntu-latest
+
+    needs: build
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/download-artifact@v2
+      with:
+        name: emqx-docker-image
+        path: /tmp
+    - name: load docker image
+      run: docker load < /tmp/emqx.tar
+    - name: docker compose up
+      timeout-minutes: 5
+      env:
+        TARGET: emqx/${{ needs.build.outputs.imgname }}
+        EMQX_TAG: ${{ needs.build.outputs.version }}
+        MYSQL_TAG: 8
+      run: |
+        docker-compose \
+          -f .ci/docker-compose-file/docker-compose-emqx-broker-cluster.yaml \
+          -f .ci/docker-compose-file/docker-compose-mysql-tcp.yaml \
+          -f .ci/docker-compose-file/docker-compose-enterprise-tomcat-tcp.yaml \
+          up -d --build
+    - name: wait docker compose up
+      timeout-minutes: 5
+      run: |
+        while [ "$(docker inspect -f '{{ .State.Health.Status}}' node1.emqx.io)" != "healthy" ] || [ "$(docker inspect -f '{{ .State.Health.Status}}' node2.emqx.io)" != "healthy" ]; do
+          echo "['$(date -u +"%y-%m-%dt%h:%m:%sz")']:waiting emqx";
+          sleep 5;
+        done
+        docker ps -a
+        echo HAPROXY_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' haproxy) >> $GITHUB_ENV
+        echo HTTP_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' Tomcat) >> $GITHUB_ENV
+        echo MYSQL_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' mysql) >> $GITHUB_ENV
+        echo CONFIG_PATH=$(docker inspect -f '{{ range .Mounts }}{{ if eq .Name "docker-compose-file_etc" }}{{ .Source }}{{ end }}{{ end }}' node1.emqx.io) >> $GITHUB_ENV
+    - uses: actions/checkout@v2
+      with:
+        repository: emqx/emqx-fvt
+        ref: v1.6.0
+        path: scripts
+    - uses: actions/setup-java@v1
+      with:
+        java-version: '8.0.282' # The JDK version to make available on the path.
+        java-package: jdk # (jre, jdk, or jdk+fx) - defaults to jdk
+        architecture: x64 # (x64 or x86) - defaults to x64
+    - uses: actions/download-artifact@v2
+      with:
+        name: apache-jmeter.tgz
+        path: /tmp
+    - name: install jmeter
+      timeout-minutes: 10
+      env:
+          JMETER_VERSION: 5.4.3
+      run: |
+        cd /tmp && tar -xvf apache-jmeter.tgz
+        echo "jmeter.save.saveservice.output_format=xml" >> /tmp/apache-jmeter-$JMETER_VERSION/user.properties
+        echo "jmeter.save.saveservice.response_data.on_error=true" >> /tmp/apache-jmeter-$JMETER_VERSION/user.properties
+        wget --no-verbose -O /tmp/apache-jmeter-$JMETER_VERSION/lib/ext/mqtt-xmeter-2.0.2-jar-with-dependencies.jar https://raw.githubusercontent.com/xmeter-net/mqtt-jmeter/master/Download/v2.0.2/mqtt-xmeter-2.0.2-jar-with-dependencies.jar
+        ln -s /tmp/apache-jmeter-$JMETER_VERSION /opt/jmeter
+    - name: install jmeter plugin
+      run: |
+        wget --no-verbose -O "/opt/jmeter/lib/mysql-connector-java-8.0.16.jar" https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar
+    - name: run jmeter
+      run: |
+        sudo /opt/jmeter/bin/jmeter.sh \
+          -Jjmeter.save.saveservice.output_format=xml -n \
+          -t scripts/automate-test-suite/http_auth_acl.jmx \
+          -Dmysql_ip=$MYSQL_IP \
+          -Demqx_ip=$HAPROXY_IP \
+          -Dweb_server_ip=$HTTP_IP \
+          -Dconfig_path=$CONFIG_PATH \
+          -Ddocker_path=".ci/docker-compose-file" \
+          -l jmeter_logs/http_auth_acl.jtl \
+          -j jmeter_logs/logs/http_auth_acl.log
+    - name: check logs
+      run: |
+        if cat jmeter_logs/http_auth_acl.jtl | grep -e '<failure>true</failure>' > /dev/null 2>&1; then
+          echo "check logs filed"
+          sudo cat /var/lib/docker/volumes/docker-compose-file_etc/_data/emqx.conf
+          exit 1
+        fi
+    - uses: actions/upload-artifact@v1
+      if: always()
+      with:
+        name: jmeter_logs
+        path: ./jmeter_logs

.github/workflows/run_cts_tests.yaml

@@ -1,11 +1,13 @@
 name: Compatibility Test Suite
 
 on:
+  schedule:
+    - cron:  '0 */6 * * *'
+  pull_request:
   push:
     tags:
       - v*
       - e*
-  pull_request:
 
 jobs:
   ldap:
@@ -22,6 +24,11 @@ jobs:
 
     steps:
       - uses: actions/checkout@v1
+        # to avoid dirty self-hosted runners
+      - name: stop containers
+        run: |
+          docker rm -f $(docker ps -qa) || true
+          docker network rm $(docker network ls -q) || true
       - name: docker compose up
         env:
           LDAP_TAG: ${{ matrix.ldap_tag }}
@@ -43,13 +50,16 @@ jobs:
           if make emqx-ee --dry-run > /dev/null 2>&1; then
             docker exec -i erlang bash -c "echo \"https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com\" > /root/.git-credentials && git config --global credential.helper store"
           fi
+      - name: fix-git-unsafe-repository
+        run: docker exec -i erlang sh -c "git config --global --add safe.directory /emqx"
       - name: run test cases
         run: |
-          export CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_
-          printenv > .env
           docker exec -i erlang sh -c "make ensure-rebar3"
-          docker exec -i erlang sh -c "./rebar3 eunit --dir apps/emqx_auth_ldap"
+          printenv | grep "^EMQX_" > .env
-          docker exec --env-file .env -i erlang sh -c "./rebar3 ct --dir apps/emqx_auth_ldap"
+          docker exec -i \
+            -e "CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_" \
+            --env-file .env \
+            erlang sh -c "make apps/emqx_auth_ldap-ct"
       - uses: actions/upload-artifact@v1
         if: failure()
         with:
@@ -74,9 +84,14 @@ jobs:
 
     steps:
       - uses: actions/checkout@v1
+      - name: stop containers
+        run: |
+          docker rm -f $(docker ps -qa) || true
+          docker network rm $(docker network ls -q) || true
       - name: docker-compose up
         run: |
           docker-compose \
+            -f .ci/docker-compose-file/docker-compose-toxiproxy.yaml \
             -f .ci/docker-compose-file/docker-compose-mongo-${{ matrix.connect_type }}.yaml \
             -f .ci/docker-compose-file/docker-compose.yaml \
             up -d --build
@@ -112,13 +127,15 @@ jobs:
           if make emqx-ee --dry-run > /dev/null 2>&1; then
             docker exec -i erlang bash -c "echo \"https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com\" > /root/.git-credentials && git config --global credential.helper store"
           fi
+      - name: fix-git-unsafe-repository
+        run: docker exec -i erlang sh -c "git config --global --add safe.directory /emqx"
       - name: run test cases
         run: |
-          export CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_
+          printenv | grep "^EMQX_" > .env
-          printenv > .env
+          docker exec -i \
-          docker exec -i erlang sh -c "make ensure-rebar3"
+            -e "CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_" \
-          docker exec -i erlang sh -c "./rebar3 eunit --dir apps/emqx_auth_mongo"
+            --env-file .env \
-          docker exec --env-file .env -i erlang sh -c "./rebar3 ct --dir apps/emqx_auth_mongo"
+            erlang sh -c "make apps/emqx_auth_mongo-ct"
       - uses: actions/upload-artifact@v1
         if: failure()
         with:
@@ -143,6 +160,10 @@ jobs:
 
     steps:
       - uses: actions/checkout@v1
+      - name: stop containers
+        run: |
+          docker rm -f $(docker ps -qa) || true
+          docker network rm $(docker network ls -q) || true
       - name: docker-compose up
         timeout-minutes: 5
         run: |
@@ -194,13 +215,15 @@ jobs:
           if make emqx-ee --dry-run > /dev/null 2>&1; then
             docker exec -i erlang bash -c "echo \"https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com\" > /root/.git-credentials && git config --global credential.helper store"
           fi
+      - name: fix-git-unsafe-repository
+        run: docker exec -i erlang sh -c "git config --global --add safe.directory /emqx"
       - name: run test cases
         run: |
-          export CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_
+          printenv | grep "^EMQX_" > .env
-          printenv > .env
+          docker exec -i \
-          docker exec -i erlang sh -c "make ensure-rebar3"
+            -e "CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_" \
-          docker exec -i erlang sh -c "./rebar3 eunit --dir apps/emqx_auth_mysql"
+            --env-file .env \
-          docker exec --env-file .env -i erlang sh -c "./rebar3 ct --dir apps/emqx_auth_mysql"
+            erlang sh -c "make apps/emqx_auth_mysql-ct"
       - uses: actions/upload-artifact@v1
         if: failure()
         with:
@@ -227,6 +250,10 @@ jobs:
         - tcp
     steps:
       - uses: actions/checkout@v1
+      - name: stop containers
+        run: |
+          docker rm -f $(docker ps -qa) || true
+          docker network rm $(docker network ls -q) || true
       - name: docker-compose up
         run: |
           docker-compose \
@@ -265,16 +292,18 @@ jobs:
           if make emqx-ee --dry-run > /dev/null 2>&1; then
             docker exec -i erlang bash -c "echo \"https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com\" > /root/.git-credentials && git config --global credential.helper store"
           fi
+      - name: fix-git-unsafe-repository
+        run: docker exec -i erlang sh -c "git config --global --add safe.directory /emqx"
       - name: run test cases
         run: |
           export EMQX_AUTH__PGSQL__USERNAME=root \
                  EMQX_AUTH__PGSQL__PASSWORD=public \
-                 EMQX_AUTH__PGSQL__DATABASE=mqtt \
+                 EMQX_AUTH__PGSQL__DATABASE=mqtt
-                 CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_
+          printenv | grep "^EMQX_" > .env
-          printenv > .env
+          docker exec -i \
-          docker exec -i erlang sh -c "make ensure-rebar3"
+            -e "CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_" \
-          docker exec -i erlang sh -c "./rebar3 eunit --dir apps/emqx_auth_pgsql"
+            --env-file .env \
-          docker exec --env-file .env -i erlang sh -c "./rebar3 ct --dir apps/emqx_auth_pgsql"
+            erlang sh -c "make apps/emqx_auth_pgsql-ct"
       - uses: actions/upload-artifact@v1
         if: failure()
         with:
@@ -306,6 +335,10 @@ jobs:
 
     steps:
       - uses: actions/checkout@v1
+      - name: stop containers
+        run: |
+          docker rm -f $(docker ps -qa) || true
+          docker network rm $(docker network ls -q) || true
       - name: docker-compose up
         run: |
           docker-compose \
@@ -388,14 +421,16 @@ jobs:
           if make emqx-ee --dry-run > /dev/null 2>&1; then
             docker exec -i erlang bash -c "echo \"https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com\" > /root/.git-credentials && git config --global credential.helper store"
           fi
+      - name: fix-git-unsafe-repository
+        run: docker exec -i erlang sh -c "git config --global --add safe.directory /emqx"
       - name: run test cases
         run: |
-          export CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_
           export EMQX_AUTH__REIDS__PASSWORD=public
-          printenv > .env
+          printenv | grep "^EMQX_" > .env
-          docker exec -i erlang sh -c "make ensure-rebar3"
+          docker exec -i \
-          docker exec -i erlang sh -c "./rebar3 eunit --dir apps/emqx_auth_redis"
+            -e "CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_" \
-          docker exec --env-file .env -i erlang sh -c "./rebar3 ct --dir apps/emqx_auth_redis"
+            --env-file .env \
+            erlang sh -c "make apps/emqx_auth_redis-ct"
       - uses: actions/upload-artifact@v1
         if: failure()
         with:

.github/workflows/run_fvt_tests.yaml

@@ -13,25 +13,23 @@ jobs:
 
         steps:
         - uses: actions/checkout@v1
-        - uses: gleam-lang/setup-erlang@v1.1.2
+        - uses: erlef/setup-beam@v1
-          id: install_erlang
           with:
-            otp-version: 23.2
+            otp-version: "23.3.4.17"
-        - name: prepare
+        - name: make docker
           run: |
             if make emqx-ee --dry-run > /dev/null 2>&1; then
               echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials
               git config --global credential.helper store
-              echo "${{ secrets.CI_GIT_TOKEN }}" >> scripts/git-token
               make deps-emqx-ee
               echo "TARGET=emqx/emqx-ee" >> $GITHUB_ENV
               echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV
+              make emqx-ee-docker
             else
               echo "TARGET=emqx/emqx" >> $GITHUB_ENV
               echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV
+              make emqx-docker
             fi
-        - name: make emqx image
-          run: make docker
         - name: run emqx
           timeout-minutes: 5
           run: |
@@ -67,47 +65,33 @@ jobs:
 
         steps:
         - uses: actions/checkout@v1
-        - uses: gleam-lang/setup-erlang@v1.1.2
+        - uses: erlef/setup-beam@v1
-          id: install_erlang
           with:
-            otp-version: 23.2
+            otp-version: "23.3.4.17"
         - name: prepare
           run: |
             if make emqx-ee --dry-run > /dev/null 2>&1; then
               echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials
               git config --global credential.helper store
-              echo "${{ secrets.CI_GIT_TOKEN }}" >> scripts/git-token
               make deps-emqx-ee
               echo "TARGET=emqx/emqx-ee" >> $GITHUB_ENV
+              make emqx-ee-docker
             else
               echo "TARGET=emqx/emqx" >> $GITHUB_ENV
+              make emqx-docker
             fi
-        - name: make emqx image
+        - run: minikube start
-          run: make docker
-        - name: install k3s
-          env:
-            KUBECONFIG: "/etc/rancher/k3s/k3s.yaml"
-          run: |
-            sudo sh -c "echo \"127.0.0.1 $(hostname)\" >> /etc/hosts"
-            curl -sfL https://get.k3s.io | sh -
-            sudo chmod 644 /etc/rancher/k3s/k3s.yaml
-            kubectl cluster-info
         - name: install helm
-          env:
-            KUBECONFIG: "/etc/rancher/k3s/k3s.yaml"
           run: |
             curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3
             sudo chmod 700 get_helm.sh
             sudo ./get_helm.sh
             helm version
         - name: run emqx on chart
-          env:
-            KUBECONFIG: "/etc/rancher/k3s/k3s.yaml"
           timeout-minutes: 5
           run: |
             version=$(./pkg-vsn.sh)
-            sudo docker save ${TARGET}:$version -o emqx.tar.gz
+            minikube image load ${TARGET}:$version
-            sudo k3s ctr image import emqx.tar.gz
 
             sed -i -r "s/^appVersion: .*$/appVersion: \"${version}\"/g" deploy/charts/emqx/Chart.yaml
             sed -i '/emqx_telemetry/d' deploy/charts/emqx/values.yaml
@@ -130,11 +114,21 @@ jobs:
               echo "waiting emqx started";
               sleep 10;
             done
-        - name: get pods log
+        - name: get emqx-0 pods log
           if: failure()
-          env:
+          run: |
-            KUBECONFIG: "/etc/rancher/k3s/k3s.yaml"
+            kubectl describe pods emqx-0
-          run: kubectl describe pods emqx-0
+            kubectl logs emqx-0
+        - name: get emqx-1 pods log
+          if: failure()
+          run: |
+            kubectl describe pods emqx-1
+            kubectl logs emqx-1
+        - name: get emqx-2 pods log
+          if: failure()
+          run: |
+            kubectl describe pods emqx-2
+            kubectl logs emqx-2
         - uses: actions/checkout@v2
           with:
             repository: emqx/paho.mqtt.testing
@@ -145,16 +139,12 @@ jobs:
             pip install pytest
             echo "$HOME/.local/bin" >> $GITHUB_PATH
         - name: run paho test
-          env:
-            KUBECONFIG: "/etc/rancher/k3s/k3s.yaml"
           run: |
-            emqx_svc=$(kubectl get svc --namespace default emqx -o jsonpath="{.spec.clusterIP}")
+            nohup kubectl port-forward svc/emqx 1883:1883 &
-            emqx1=$(kubectl get pods emqx-1 -o jsonpath='{.status.podIP}')
-            emqx2=$(kubectl get pods emqx-2 -o jsonpath='{.status.podIP}')
 
-            pytest -v paho.mqtt.testing/interoperability/test_client/V5/test_connect.py -k test_basic --host $emqx_svc
+            pytest -v paho.mqtt.testing/interoperability/test_client/V5/test_connect.py -k test_basic
             RESULT=$?
-            pytest -v paho.mqtt.testing/interoperability/test_cluster --host1 $emqx1 --host2 $emqx2
+            pytest -v paho.mqtt.testing/interoperability/test_cluster
             RESULT=$((RESULT + $?))
             if [ 0 -ne $RESULT ]; then
                 kubectl logs emqx-1
@@ -162,117 +152,140 @@ jobs:
             fi
             exit $RESULT
 
-    relup_test:
+    relup_test_plan:
         runs-on: ubuntu-20.04
-        container: emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04
+        container: emqx/build-env:erl23.3.4.18-1-ubuntu20.04
+        outputs:
+          profile: ${{ steps.profile-and-versions.outputs.profile }}
+          vsn: ${{ steps.profile-and-versions.outputs.vsn }}
+          old_vsns: ${{ steps.profile-and-versions.outputs.old_vsns }}
+          broker: ${{ steps.profile-and-versions.outputs.broker }}
+          matrix: ${{ steps.generate-matrix.outputs.matrix }}
         defaults:
           run:
             shell: bash
         steps:
-        - uses: actions/setup-python@v2
-          with:
-            python-version: '3.8'
-            architecture: 'x64'
         - uses: actions/checkout@v2
+          name: Checkout
           with:
-            repository: emqx/paho.mqtt.testing
+            path: emqx
-            ref: develop-4.0
+            fetch-depth: 0
-            path: paho.mqtt.testing
+        - name: Get profile and version list
+          id: profile-and-versions
+          run: |
+            cd emqx
+            vsn="$(./pkg-vsn.sh)"
+            if make emqx-ee --dry-run > /dev/null 2>&1; then
+              profile="emqx-ee"
+              old_vsns="$(./scripts/relup-base-vsns.sh enterprise | xargs)"
+              broker="emqx-ee"
+            else
+              profile="emqx"
+              old_vsns="$(./scripts/relup-base-vsns.sh community | xargs)"
+              broker="emqx-ce"
+            fi
+
+            echo "OLD_VSNS=$old_vsns" >> $GITHUB_ENV
+
+            echo "::set-output name=vsn::$vsn"
+            echo "::set-output name=profile::$profile"
+            echo "::set-output name=broker::$broker"
+            echo "::set-output name=old_vsns::$old_vsns"
+        - name: Generate matrix
+          id: generate-matrix
+          run: |
+            matrix=$(echo -n "$OLD_VSNS" | sed 's/ $//g' | jq -R -s -c 'split(" ")')
+            echo "::set-output name=matrix::$matrix"
+
+    relup_test_build:
+        needs: relup_test_plan
+        runs-on: ubuntu-20.04
+        container: emqx/build-env:erl23.3.4.18-1-ubuntu20.04
+        defaults:
+          run:
+            shell: bash
+        env:
+          OLD_VSNS: "${{ needs.relup_test_plan.outputs.old_vsns }}"
+          PROFILE: "${{ needs.relup_test_plan.outputs.profile }}"
+          BROKER: "${{ needs.relup_test_plan.outputs.broker }}"
+        steps:
         - uses: actions/checkout@v2
+          name: Checkout
+          with:
+            path: emqx
+            fetch-depth: 0
+        - name: Prepare credentials
+          run: |
+            if [ "$PROFILE" = "emqx-ee" ]; then
+              echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials
+              git config --global credential.helper store
+            fi
+        - name: Build emqx
+          run: make -C emqx ${PROFILE}-zip
+        - uses: actions/upload-artifact@v2
+          name: Upload built emqx and test scenario
+          with:
+            name: emqx_built
+            path: |
+              emqx/_packages/*/*.zip
+              emqx/.ci/fvt_tests
+
+    relup_test_run:
+        needs:
+          - relup_test_plan
+          - relup_test_build
+        runs-on: ubuntu-20.04
+        container: emqx/relup-test-env:erl23.3.4.18-1-ubuntu20.04
+        strategy:
+          fail-fast: false
+          matrix:
+            old_vsn: ${{ fromJson(needs.relup_test_plan.outputs.matrix) }}
+        env:
+          OLD_VSN: "${{ matrix.old_vsn }}"
+          PROFILE: "${{ needs.relup_test_plan.outputs.profile }}"
+          VSN: "${{ needs.relup_test_plan.outputs.vsn }}"
+          BROKER: "${{ needs.relup_test_plan.outputs.broker }}"
+        defaults:
+          run:
+            shell: bash
+        steps:
+        - uses: actions/download-artifact@v2
+          name: Download built emqx and test scenario
+          with:
+            name: emqx_built
+            path: emqx_built
+        - uses: actions/checkout@v2
+          name: Checkout one_more_emqx
           with:
             repository: terry-xiaoyu/one_more_emqx
             ref: master
             path: one_more_emqx
-        - uses: actions/checkout@v2
+        - name: Prepare packages
-          with:
-            repository: emqx/emqtt-bench
-            ref: master
-            path: emqtt-bench
-        - uses: actions/checkout@v2
-          with:
-            repository: hawk/lux
-            ref: lux-2.6
-            path: lux
-        - uses: actions/checkout@v2
-          with:
-            repository: ${{ github.repository }}
-            path: emqx
-            fetch-depth: 0
-        - name: prepare
-          run: |
-            if make -C emqx emqx-ee --dry-run > /dev/null 2>&1; then
-              echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials
-              git config --global credential.helper store
-              echo "${{ secrets.CI_GIT_TOKEN }}" >> emqx/scripts/git-token
-              echo "PROFILE=emqx-ee" >> $GITHUB_ENV
-            else
-              echo "PROFILE=emqx" >> $GITHUB_ENV
-            fi
-        - name: get version
           run: |
             set -e -x -u
-            cd emqx
-            if [ $PROFILE = "emqx" ];then
-                broker="emqx-ce"
-                edition='opensource'
-            else
-                broker="emqx-ee"
-                edition='enterprise'
-            fi
-            echo "BROKER=$broker" >> $GITHUB_ENV
-
-            vsn="$(./pkg-vsn.sh)"
-            echo "VSN=$vsn" >> $GITHUB_ENV
-
-            pre_vsn="$(echo $vsn | grep -oE '^[0-9]+.[0-9]')"
-            if [ $PROFILE = "emqx" ]; then
-                old_vsns="$(git tag -l "v$pre_vsn.[0-9]" | xargs echo -n | sed "s/v$vsn//")"
-            else
-                old_vsns="$(git tag -l "e$pre_vsn.[0-9]" | xargs echo -n | sed "s/e$vsn//")"
-            fi
-            echo "OLD_VSNS=$old_vsns" >> $GITHUB_ENV
-        - name: download emqx
-          run: |
-            set -e -x -u
-            mkdir -p emqx/_upgrade_base
-            cd emqx/_upgrade_base
-            old_vsns=($(echo $OLD_VSNS | tr ' ' ' '))
-            for old_vsn in ${old_vsns[@]}; do
-              wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$BROKER/$old_vsn/$PROFILE-ubuntu20.04-${old_vsn#[e|v]}-amd64.zip
-            done
-        - name: build emqx
-          run: make -C emqx ${PROFILE}-zip
-        - name: build emqtt-bench
-          run: make -C emqtt-bench
-        - name: build lux
-          run: |
-            set -e -u -x
-            cd lux
-            autoconf
-            ./configure
-            make
-            make install
-        - name: run relup test
-          timeout-minutes: 20
-          run: |
-            set -e -x -u
-            if [ -n "$OLD_VSNS" ]; then
             mkdir -p packages
-                cp emqx/_packages/${PROFILE}/*.zip packages
+            cp emqx_built/_packages/*/*.zip packages
-                cp emqx/_upgrade_base/*.zip packages
+            cd packages
+            wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$BROKER/$OLD_VSN/$PROFILE-ubuntu20.04-${OLD_VSN#[e|v]}-amd64.zip
+        - name: Run relup test scenario
+          timeout-minutes: 5
+          run: |
             lux \
+            --progress verbose \
             --case_timeout infinity \
             --var PROFILE=$PROFILE \
             --var PACKAGE_PATH=$(pwd)/packages \
-                --var BENCH_PATH=$(pwd)/emqtt-bench \
             --var ONE_MORE_EMQX_PATH=$(pwd)/one_more_emqx \
             --var VSN="$VSN" \
-                --var OLD_VSNS="$OLD_VSNS" \
+            --var OLD_VSN="$OLD_VSN" \
-                emqx/.ci/fvt_tests/relup.lux
+            emqx_built/.ci/fvt_tests/relup.lux
-            fi
+        - uses: actions/upload-artifact@v2
-        - uses: actions/upload-artifact@v1
+          name: Save debug data
           if: failure()
           with:
-            name: lux_logs
+            name: debug_data
-            path: lux_logs
+            path: |
-
+              packages/emqx/log/emqx.log.1
+              packages/emqx2/log/emqx.log.1
+              packages/*.zip
+              lux_logs

.github/workflows/run_test_cases.yaml

@@ -5,74 +5,156 @@ on:
     tags:
       - v*
       - e*
+    branches:
+      - 'main-v4.[0-9]?'
   pull_request:
 
 jobs:
-    run_static_analysis:
+  prepare:
-        runs-on: ubuntu-20.04
+    runs-on: aws-amd64
-        container: emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04
+    container: emqx/build-env:erl23.3.4.18-1-ubuntu20.04
-
+    outputs:
+      fast_ct_apps: ${{ steps.run_find_apps.outputs.fast_ct_apps }}
+      docker_ct_apps: ${{ steps.run_find_apps.outputs.docker_ct_apps }}
     steps:
-        - uses: actions/checkout@v2
+      - uses: AutoModality/action-clean@v1
-        - name: set git credentials
+      - uses: actions/checkout@v3
+        with:
+          path: source
+          fetch-depth: 0
+      - name: git credentials
+        if: endsWith(github.repository, 'emqx-enterprise')
         run: |
-            if make emqx-ee --dry-run > /dev/null 2>&1; then
           echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials
           git config --global credential.helper store
-            fi
+      - name: find_ct_apps
-        - name: xref
+        working-directory: source
-          run: make xref
+        id: run_find_apps
-        - name: dialyzer
+        # emqx_plugin_libs doesn't have a test suite -> excluded from app list
-          run: make dialyzer
+        # emqx ct is run independently -> exclude it from the app list (regular ct)
-
+        # lib-ee/emqx_node_rebalance -> exclude it from app list (regular ct)
-    run_proper_test:
+        run: |
-        runs-on: ubuntu-20.04
+          fast_ct_apps="$(./scripts/find-apps.sh --ct fast --json)"
-        container: emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04
+          docker_ct_apps="$(./scripts/find-apps.sh --ct docker --json)"
+          echo "fast_ct_apps=$fast_ct_apps" | tee -a $GITHUB_OUTPUT
+          echo "docker_ct_apps=$docker_ct_apps" | tee -a $GITHUB_OUTPUT
+      - name: get_all_deps
+        working-directory: source
+        run: |
+          git config --global --add safe.directory $(pwd)
+          # build the default profile for two purposes
+          # 1. download all dependencies (so the individual app runs do not depend on github credentials)
+          # 2. some of the files such as segmented config files are not created when compiling only the test profile
+          make
+          # compile test profile to speed up the individual runs
+          ./rebar3 as test compile
+          cd ..
+          zip -ryq source.zip source/* source/.[^.]*
+      - uses: actions/upload-artifact@v3
+        with:
+          name: source
+          path: source.zip
 
+  eunit_and_proper:
+    needs: prepare
+    runs-on: aws-amd64
+    container: emqx/build-env:erl23.3.4.18-1-ubuntu20.04
+    strategy:
+      fail-fast: false
+      matrix:
+        task:
+          - eunit
+          - proper
     steps:
-        - uses: actions/checkout@v2
+      - uses: AutoModality/action-clean@v1
-        - name: set git credentials
+      - uses: actions/download-artifact@v3
-          run: |
+        with:
-            if make emqx-ee --dry-run > /dev/null 2>&1; then
+          name: source
-              echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials
+          path: .
-              git config --global credential.helper store
+      - name: unzip source code
-            fi
+        run: unzip -o -q source.zip
-        - name: proper
+      # produces eunit.coverdata and proper.coverdata
-          run: make proper
+      - name: eunit and proper
-
+        working-directory: source
-    run_common_test:
+        run: make ${{ matrix.task }}
-        runs-on: ubuntu-20.04
+      - uses: actions/upload-artifact@v3
+        with:
+          name: cover
+          path: source/_build/test/cover
+          if-no-files-found: warn
 
+  fast_ct:
+    needs: prepare
+    runs-on: ${{ matrix.runs-on }}
+    container: emqx/build-env:erl23.3.4.18-1-ubuntu20.04
+    strategy:
+      fail-fast: false
+      matrix:
+        app_name: ${{ fromJson(needs.prepare.outputs.fast_ct_apps) }}
+        runs-on:
+          - aws-amd64
+          - ubuntu-20.04
+        use-self-hosted:
+          - ${{ github.repository_owner == 'emqx' }}
+        exclude:
+          - runs-on: ubuntu-20.04
+            use-self-hosted: true
+          - runs-on: aws-amd64
+            use-self-hosted: false
     steps:
-        - uses: actions/checkout@v2
+      - uses: AutoModality/action-clean@v1
-        - name: set edition
+      - uses: actions/download-artifact@v3
-          id: set_edition
+        with:
+          name: source
+          path: .
+      - name: unzip source code
+        run: unzip -o -q source.zip
+      # produces emqx-<appname>.coverdata
+      - name: make-app-ct-pipeline
+        working-directory: source
         run: |
-            if make emqx-ee --dry-run > /dev/null 2>&1; then
+          export CT_READABLE=true
-                echo "EDITION=enterprise" >> $GITHUB_ENV
+          make ${{ matrix.app_name }}-ct-pipeline
-            else
+      - uses: actions/upload-artifact@v3
-                echo "EDITION=opensource" >> $GITHUB_ENV
+        with:
-            fi
+          name: cover
-        - name: docker compose up
+          path: source/_build/test/cover
-          if: env.EDITION == 'opensource'
+          if-no-files-found: warn
-          env:
+
-            MYSQL_TAG: 8
+  docker_ct:
-            REDIS_TAG: 6
+    needs: prepare
-            MONGO_TAG: 4
+    runs-on: ${{ matrix.runs-on }}
-            PGSQL_TAG: 13
+    strategy:
-            LDAP_TAG: 2.4.50
+      max-parallel: 12
-            GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      fail-fast: false
+      matrix:
+        app_name: ${{ fromJson(needs.prepare.outputs.docker_ct_apps) }}
+        runs-on:
+          - aws-amd64
+          - ubuntu-20.04
+        use-self-hosted:
+          - ${{ github.repository_owner == 'emqx' }}
+        exclude:
+          - runs-on: ubuntu-20.04
+            use-self-hosted: true
+          - runs-on: aws-amd64
+            use-self-hosted: false
+    steps:
+      - uses: AutoModality/action-clean@v1
+      - uses: actions/download-artifact@v3
+        with:
+          name: source
+          path: .
+      - name: unzip source code
+        run: unzip -q source.zip
+        # to avoid dirty self-hosted runners
+      - name: stop containers
         run: |
-            docker-compose \
+          docker rm -f $(docker ps -qa) || true
-                -f .ci/docker-compose-file/docker-compose.yaml \
+          docker network rm $(docker network ls -q) || true
-                -f .ci/docker-compose-file/docker-compose-ldap-tcp.yaml \
+          docker system prune --volumes -f
-                -f .ci/docker-compose-file/docker-compose-mongo-tcp.yaml \
-                -f .ci/docker-compose-file/docker-compose-mysql-tcp.yaml \
-                -f .ci/docker-compose-file/docker-compose-pgsql-tcp.yaml \
-                -f .ci/docker-compose-file/docker-compose-redis-single-tcp.yaml \
-                up -d --build
       - name: docker compose up
-          if: env.EDITION == 'enterprise'
+        working-directory: source
         env:
           MYSQL_TAG: 8
           REDIS_TAG: 6
@@ -88,16 +170,20 @@ jobs:
           KAFKA_TAG: 2.5.0
           PULSAR_TAG: 2.3.2
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          timeout-minutes: 20
+        timeout-minutes: 40
         run: |
-            docker-compose \
+          ulimit -n
+          docker_compose_files="\
               -f .ci/docker-compose-file/docker-compose.yaml \
-                -f .ci/docker-compose-file/docker-compose-ldap-tcp.yaml \
+              -f .ci/docker-compose-file/docker-compose-toxiproxy.yaml \
               -f .ci/docker-compose-file/docker-compose-mongo-tcp.yaml \
               -f .ci/docker-compose-file/docker-compose-mysql-tcp.yaml \
               -f .ci/docker-compose-file/docker-compose-pgsql-tcp.yaml \
-                -f .ci/docker-compose-file/docker-compose-redis-single-tcp.yaml \
+              -f .ci/docker-compose-file/docker-compose-redis-single-tcp.yaml"
+          ee_docker_compose_files="\
               -f .ci/docker-compose-file/docker-compose-enterprise.yaml \
+              -f .ci/docker-compose-file/docker-compose-mongo-replicaset-tcp.yaml \
+              -f .ci/docker-compose-file/docker-compose-mongo-sharded-tcp.yaml \
               -f .ci/docker-compose-file/docker-compose-enterprise-cassandra-tcp.yaml \
               -f .ci/docker-compose-file/docker-compose-enterprise-dynamodb-tcp.yaml \
               -f .ci/docker-compose-file/docker-compose-enterprise-influxdb-tcp.yaml \
@@ -107,40 +193,92 @@ jobs:
               -f .ci/docker-compose-file/docker-compose-enterprise-rabbit-tcp.yaml \
               -f .ci/docker-compose-file/docker-compose-enterprise-timescale-tcp.yaml \
               -f .ci/docker-compose-file/docker-compose-enterprise-mysql-client.yaml \
-                -f .ci/docker-compose-file/docker-compose-enterprise-pgsql-and-timescale-client.yaml \
+              -f .ci/docker-compose-file/docker-compose-enterprise-pgsql-and-timescale-client.yaml"
-                up -d --build
+          if [ -f EMQX_ENTERPRISE ]; then
+            docker_compose_files="${docker_compose_files} ${ee_docker_compose_files}"
+          fi
+          # only build ldap docker image when necessary, it takes 9 minutes
+          if [[ ${{ matrix.app_name }} == *ldap ]]; then
+            docker_compose_files="${docker_compose_files} -f .ci/docker-compose-file/docker-compose-ldap-tcp.yaml"
+          fi
+          docker-compose $docker_compose_files up -d --build
+          if [ -f EMQX_ENTERPRISE ]; then
             docker exec -i erlang bash -c "echo \"https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com\" > /root/.git-credentials && git config --global credential.helper store"
-            while [ $(docker ps -a --filter name=client --filter exited=0 | wc -l) \
+          fi
-                 != $(docker ps -a --filter name=client | wc -l) ]; do
+          docker exec -i erlang bash -c "git config --global --add safe.directory /emqx"
-              sleep 5
+          clients="$(docker ps -q --filter name=client)"
+          for client in ${clients}; do
+            docker ps -a --filter name=client
+            echo "waiting for docker ${client} to exit"
+            if ! timeout 60 docker wait "${client}"; then
+              docker-compose $docker_compose_files logs | tee docker-compose.log
+              exit 1
+            fi
           done
-        - name: run eunit
-          run: |
-            docker exec -i erlang bash -c "make eunit"
       - name: run common test
-          run: |
+        run: docker exec -i erlang bash -c "env CT_READABLE=true make ${{ matrix.app_name }}-ct-pipeline"
-            docker exec -i erlang bash -c "make ct"
-        - name: run cover
-          run: |
-            printenv > .env
-            docker exec -i erlang bash -c "make cover"
-            docker exec --env-file .env -i erlang bash -c "make coveralls"
       - name: cat rebar.crashdump
         if: failure()
+        working-directory: source
         run: if [ -f 'rebar3.crashdump' ];then cat 'rebar3.crashdump'; fi
-        - uses: actions/upload-artifact@v1
+      - name: set log file name
+        if: failure()
+        run: echo "LOGFILENAME=logs-$(echo ${{ matrix.app_name }} | tr '/' '_')" >> $GITHUB_ENV
+      - name: stop containers
+        run: |
+          docker rm -f $(docker ps -qa) || true
+          docker network rm $(docker network ls -q) || true
+          docker system prune --volumes -f
+      - uses: actions/upload-artifact@v3
         if: failure()
         with:
-            name: logs
+          name: ${{ env.LOGFILENAME }}
-            path: _build/test/logs
+          path: |
-        - uses: actions/upload-artifact@v1
+            docker-compose.log
+            source/_build/test/logs
+          if-no-files-found: warn
+      - uses: actions/upload-artifact@v3
         with:
           name: cover
-            path: _build/test/cover
+          path: source/_build/test/cover
+          if-no-files-found: warn
+
+  make_cover:
+    needs:
+      - eunit_and_proper
+      - fast_ct
+      - docker_ct
+    runs-on: aws-amd64
+    container: emqx/build-env:erl23.3.4.18-1-ubuntu20.04
+    steps:
+      - uses: AutoModality/action-clean@v1
+      - uses: actions/download-artifact@v3
+        with:
+          name: source
+          path: .
+      - name: unzip source code
+        run: unzip -q source.zip
+      - uses: actions/download-artifact@v3
+        name: download cover data
+        with:
+          name: cover
+          path: source/_build/test/cover
+      - name: make cover
+        working-directory: source
+        run: make cover
+      - name: send to coveralls
+        working-directory: source
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: make coveralls
+      - name: get coveralls logs
+        working-directory: source
+        if: failure()
+        run: cat rebar3.crashdump
 
   finish:
-      needs: run_common_test
+    needs: make_cover
-      runs-on: ubuntu-20.04
+    runs-on: aws-amd64
     steps:
       - name: Coveralls Finished
         env:

.gitignore

@@ -47,3 +47,16 @@ dist.zip
 scripts/git-token
 etc/*.seg
 _upgrade_base/
+erlang_ls.config
+.els_cache/
+# VSCode files
+.vs/
+.vscode/
+# Emacs Backup files
+*~
+# Emacs temporary files
+.#*
+*#
+# For direnv
+.envrc
+mix.lock

.tool-versions

@@ -1 +1 @@
-erlang 24.0.1-emqx-1
+erlang 23.3.4.9-3

CHANGES-4.3.md

@@ -0,0 +1,398 @@
+# EMQX 4.3 Changes
+
+Started tracking changes in CHANGE.md since EMQX v4.3.11
+
+NOTE: Keep prepending to the head of the file instead of the tail
+
+File format:
+
+- Use weight-2 heading for releases
+- One list item per change topic
+  Change log ends with a list of GitHub PRs
+
+## For 4.3.22 and later versions, please find details in `changes` dir
+
+## v4.3.21
+
+### Bug fixes
+
+- Deny POST an existing resource id using HTTP API with error 400 "Already Exists". [#9079](https://github.com/emqx/emqx/pull/9079)
+- Fix the issue that reseting rule metrics crashed under certain conditions. [#9079](https://github.com/emqx/emqx/pull/9079)
+
+### Enhancements
+
+- TLS listener memory usage optimization [#9005](https://github.com/emqx/emqx/pull/9005).
+  New config `listener.ssl.$NAME.hibernate_after` to hibernate TLS connection process after idling.
+  Hibernation can reduce RAM usage significantly, but may cost more CPU.
+  This configuration is by default disabled.
+  Our preliminary test shows a 50% of RAM usage decline when configured to '5s'.
+
+- TLS listener default buffer size to 4KB [#9007](https://github.com/emqx/emqx/pull/9007)
+  Eliminate uncertainty that the buffer size is set by OS default.
+
+- Disable authorization for `api/v4/emqx_prometheus` endpoint. [#8955](https://github.com/emqx/emqx/pull/8955)
+
+- Added a test to prevent a last will testament message to be
+  published when a client is denied connection. [#8894](https://github.com/emqx/emqx/pull/8894)
+
+- More rigorous checking of flapping to improve stability of the system. [#9045](https://github.com/emqx/emqx/pull/9045)
+
+- QoS1 and QoS2 messages in session's buffer are re-dispatched to other members in the group
+  when the session terminates [#9094](https://github.com/emqx/emqx/pull/9094).
+  Prior to this enhancement, one would have to set `broker.shared_dispatch_ack_enabled` to true
+  to prevent sessions from buffering messages, however this acknowledgement comes with a cost.
+
+- Prior to this fix, some of the time stamps were taken from the `os` module (system call),
+  while majority of other places are using `erlang` module (from Erlang virtual machine).
+  This inconsistent behaviour has caused some trouble for the Delayed Publish feature when OS time changes.
+  Now all time stamps are from `erlang` module. [#8908](https://github.com/emqx/emqx/pull/8908)
+
+### Bug fixes
+
+- Fix HTTP client library to handle SSL socket passive signal. [#9145](https://github.com/emqx/emqx/pull/9145)
+
+- Hide redis password in error logs [#9071](https://github.com/emqx/emqx/pull/9071)
+  More changes in redis client included in this release:
+  - Improve redis connection error logging [eredis #19](https://github.com/emqx/eredis/pull/19).
+    Also added support for eredis to accept an anonymous function as password instead of
+    passing around plaintext args which may get dumpped to crash logs (hard to predict where).
+    This change also added `format_status` callback for `gen_server` states which hold plaintext
+    password so the process termination log and `sys:get_status` will print '******' instead of
+    the password to console.
+  - Avoid pool name clashing [eredis_cluster #22](https://github.com/emqx/eredis_cluster/pull/22)
+    Same `format_status` callback is added here too for `gen_server`s which hold password in
+    their state.
+
+- Fix shared subscription message re-dispatches [#9094](https://github.com/emqx/emqx/pull/9094).
+  - When discarding QoS 2 inflight messages, there were excessive logs
+  - For wildcard deliveries, the re-dispatch used the wrong topic (the publishing topic,
+    but not the subscribing topic), caused messages to be lost when dispatching.
+
+- Fix shared subscription group member unsubscribe issue when 'sticky' strategy is used.
+  Prior to this fix, if a previously picked member unsubscribes from the group (without reconnect)
+  the message is still dispatched to it.
+  This issue only occurs when unsubscribe with the session kept.
+  Fixed in [#9119](https://github.com/emqx/emqx/pull/9119)
+
+- Fix shared subscription 'sticky' strategy when there is no local subscriptions at all.
+  Prior to this change, it may take a few rounds to randomly pick group members until a local subscriber
+  is hit (and then start sticking to it).
+  After this fix, it will start sticking to whichever randomly picked member even when it is a
+  subscriber from another node in the cluster.
+  Fixed in [#9122](https://github.com/emqx/emqx/pull/9122)
+
+- Fix rule engine fallback actions metrics reset. [#9125](https://github.com/emqx/emqx/pull/9125)
+
+## v4.3.20
+
+### Bug fixes
+
+- Fix rule-engine update behaviour which may initialize actions for disabled rules. [#8849](https://github.com/emqx/emqx/pull/8849)
+- Fix JWT plugin don't support non-integer timestamp claims. [#8862](https://github.com/emqx/emqx/pull/8862)
+- Fix a possible dead loop caused by shared subscriptions with `shared_dispatch_ack_enabled=true`. [#8918](https://github.com/emqx/emqx/pull/8918)
+- Fix dashboard binding IP address not working. [#8916](https://github.com/emqx/emqx/pull/8916)
+- Fix rule SQL topic matching to null values failed. [#8927](https://github.com/emqx/emqx/pull/8927)
+  The following SQL should not fail (crash) but return `{"r": false}`:
+  `SELECT topic =~ 't' as r FROM "$events/client_connected"`.
+  The topic is a null value as there's no such field in event `$events/client_connected`, so it
+  should return false if match it to a topic.
+
+## v4.3.19
+
+### Enhancements
+
+- Improve error message for LwM2M plugin when object ID is not valid. [#8654](https://github.com/emqx/emqx/pull/8654).
+- Add tzdata apk package to alpine docker image. [#8671](https://github.com/emqx/emqx/pull/8671)
+- Refine Rule Engine error log. RuleId will be logged when take action failed. [#8737](https://github.com/emqx/emqx/pull/8737)
+- Increases the latency interval for MQTT Bridge test connections to improve compatibility in high-latency environments. [#8745](https://github.com/emqx/emqx/pull/8745)
+- Close ExProto client process immediately if it's keepalive timeouted. [#8725](https://github.com/emqx/emqx/pull/8725)
+- Upgrade grpc-erl driver to 0.6.7 to support batch operation in sending stream. [#8725](https://github.com/emqx/emqx/pull/8725)
+- Improved jwt authentication module initialization process. [#8736](https://github.com/emqx/emqx/pull/8736)
+
+### Bug fixes
+
+- Fix rule SQL compare to null values always returns false. [#8743](https://github.com/emqx/emqx/pull/8743)
+  Before this change, the following SQL failed to match on the WHERE clause (`clientid != foo` returns false):
+  `SELECT 'some_var' as clientid FROM "t" WHERE clientid != foo`.
+  The `foo` variable is a null value, so `clientid != foo` should be evaluated as true.
+- Fix GET `/auth_clientid` and `/auth_username` counts. [#8655](https://github.com/emqx/emqx/pull/8655)
+- Add an idle timer for ExProto UDP client to avoid client leaking [#8628](https://github.com/emqx/emqx/pull/8628)
+- Fix ExHook can't be un-hooked if the grpc service stop first. [#8725](https://github.com/emqx/emqx/pull/8725)
+- Fix the problem that ExHook cannot continue hook chains execution for mismatched topics. [#8807](https://github.com/emqx/emqx/pull/8807)
+- Fix GET `/listeners/` crashes when listener is not ready. [#8752](https://github.com/emqx/emqx/pull/8752)
+- Fix repeated warning messages in bin/emqx [#8824](https://github.com/emqx/emqx/pull/8824)
+
+
+## v4.3.18
+
+### Enhancements
+
+- Upgrade Erlang/OTP from 23.2.7.2-emqx-3 to 23.3.4.9-3 [#8511](https://github.com/emqx/emqx/pull/8511)
+- Make possible to debug-print SSL handshake procedure by setting listener config `log_level=debug` [#8553](https://github.com/emqx/emqx/pull/8553)
+- Add option to perform GC on connection process after TLS/SSL handshake is performed. [#8649](https://github.com/emqx/emqx/pull/8649)
+  Expected to reduce around 35% memory consumption for each SSL connection. See [#8637](https://github.com/emqx/emqx/pull/8637) for more details.
+
+## v4.3.17
+
+### Bug fixes
+
+- Fixed issue where the dashboard APIs were being exposed under the
+  management listener. [#8411]
+
+- Fixed crash when shared persistent subscription [#8441]
+- Fixed issue in Lua hook that prevented messages from being
+  rejected [#8535]
+- Fix ExProto UDP client keepalive checking error.
+  This causes the clients to not expire as long as a new UDP packet arrives [#8575]
+
+### Enhancements
+
+- HTTP API(GET /rules/) support for pagination and fuzzy filtering. [#8450]
+- Add check_conf cli to check config format. [#8486]
+- Optimize performance of shared subscription
+
+## v4.3.16
+
+### Enhancements
+
+- Add the possibility of configuring the password for
+  password-protected private key files used for dashboard and
+  management HTTPS listeners. [#8129]
+- Add message republish supports using placeholder variables to specify QoS and Retain values. Set `${qos}` and `${flags.retain}` use the original QoS & Retain flag.
+- Add supports specifying the network interface address of the cluster listener & rcp call listener. Specify `0.0.0.0` use all network interfaces, or a particular network interface IP address.
+- ExHook supports to customize the socket parameters for gRPC client. [#8314]
+
+### Bug fixes
+
+- Avoid repeated writing `loaded_plugins` file if the plugin enable stauts has not changed [#8179]
+- Correctly tally `connack.auth_error` metrics when a client uses MQTT
+  3.1. [#8177]
+- Do not match ACL rules containing placeholders if there's no
+  information to fill them. [#8280]
+- Fixed issue in Lua hook that didn't prevent a topic from being
+  subscribed to. [#8288]
+- Ensuring that exhook dispatches the client events are sequential. [#8311]
+- Ensure start dashboard ok event if default_username is missing.
+- Fix key update from JWKS server by JWT auth. [#8337]
+- Better errors for JWT claim validations. [#8337]
+
+## v4.3.15
+
+### Enhancements
+
+* Refactored `bin/emqx` help messages.
+* Upgrade script refuses upgrade from incompatible versions. (e.g. hot upgrade from 4.3 to 4.4 will fail fast).
+* Made possible for EMQX to boot from a Linux directory which has white spaces in its path.
+* Add support for JWT authorization [#7596]
+  Now MQTT clients may be authorized with respect to a specific claim containing publish/subscribe topic whitelists.
+* Better randomisation of app screts (changed from timestamp seeded sha hash (uuid) to crypto:strong_rand_bytes)
+* Return a client_identifier_not_valid error when username is empty and username_as_clientid is set to true [#7862]
+* Add more rule engine date functions: format_date/3, format_date/4, date_to_unix_ts/3, date_to_unix_ts/4 [#7894]
+* Add proto_name and proto_ver fields for $event/client_disconnected event.
+* Mnesia auth/acl http api support multiple condition queries.
+* Inflight QoS1 Messages for shared topics are now redispatched to other alive subscribers upon chosen subscriber session termination.
+* Make auth metrics name more understandable.
+* Allow emqx_management http listener binding to specific interface [#8005]
+* Add rule-engine function float2str/2, user can specify the float output precision [#7991]
+
+### Bug fixes
+
+* List subscription topic (/api/v4/subscriptions), the result do not match with multiple conditions.
+* SSL closed error bug fixed for redis client.
+* Fix mqtt-sn client disconnected due to re-send a duplicated qos2 message
+* Rule-engine function hexstr2bin/1 support half byte [#7977]
+* Shared message delivery when all alive shared subs have full inflight [#7984]
+* Improved resilience against autocluster partitioning during cluster
+  startup. [#7876]
+  [ekka-158](https://github.com/emqx/ekka/pull/158)
+* Add regular expression check ^[0-9A-Za-z_\-]+$ for node name [#7979]
+* Fix `node_dump` variable sourcing. [#8026]
+* Fix heap size is growing too fast when trace large message.
+* Support customized timestamp format of the log messages.
+
+## v4.3.14
+
+### Enhancements
+
+* Add `RequestMeta` for exhook.proto in order to expose `cluster_name` of emqx in each gRPC request. [#7524]
+* Support customize emqx_exhook execution priority. [#7408]
+* add api: PUT /rules/{id}/reset_metrics.
+  This api reset the metrics of the rule engine of a rule, and reset the metrics of the action related to this rule. [#7474]
+* Enhanced rule engine error handling when json parsing error.
+* Add support for `RSA-PSK-AES256-GCM-SHA384`, `RSA-PSK-AES256-CBC-SHA384`,
+ `RSA-PSK-AES128-GCM-SHA256`, `RSA-PSK-AES128-CBC-SHA256` PSK ciphers, and remove `PSK-3DES-EDE-CBC-SHA`,
+ `PSK-RC4-SHA` from the default configuration. [#7427]
+* Diagnostic logging for mnesia `wait_for_table`
+  - prints check points of mnesia internal stats
+  - prints check points of per table loading stats
+  Help to locate the problem of long table loading time.
+* Add `local` strategy for Shared Subscription.
+  That will preferentially dispatch messages to a shared subscriber at the same
+  node. It will improves the efficiency of shared messages dispatching in certain
+  scenarios, especially when the emqx-bridge-mqtt plugin is configured as shared
+  subscription. [#7462]
+* Add some compression functions to rule-engine: gzip, gunzip, zip, unzip, zip_compress, zip_uncompress
+
+### Bug fixes
+
+* Prohibit empty topics in strict mode
+* Make sure ehttpc delete useless pool always succeed.
+* Update mongodb driver to fix potential process leak.
+* Fix a potential security issue #3155 with emqx-dashboard plugin.
+  In the earlier implementation, the Dashboard password is reset back to the
+  default value of emqx_dashboard.conf after the node left cluster.
+  Now we persist changed password to protect against reset. [#7518]
+* Silence grep/sed warnings in docker-entrypoint.sh. [#7520]
+* Generate `loaded_modules` and `loaded_plugins` files with default values when no such files exists. [#7520]
+* Fix the configuration `server_name_indication` set to disable does not take effect.
+* Fix backup files are not deleted and downloaded correctly when the API path has ISO8859-1 escape characters.
+
+## v4.3.13
+
+### Important changes
+
+* For docker image, /opt/emqx/etc has been removed from the VOLUME list,
+  this made it easier for the users to rebuild image on top with changed configs.
+* CentOS 7 Erlang runtime is rebuilt on OpenSSL-1.1.1n (previously on 1.0),
+  Prior to v4.3.13, EMQX pick certain cipher suites proposed by the clients,
+  but then fail to handshake resulting in a `malformed_handshake_data` exception.
+* CentOS 8 Erlang runtime is rebuilt on RockyLinux 8.
+  'centos8' will remain in the package name to keep it backward compatible.
+
+### Enhancements
+
+* CLI `emqx_ctl pem_cache clean` to force purge x509 certificate cache,
+  to force an immediate reload of all certificates after the files are updated on disk.
+* Refactor the ExProto so that anonymous clients can also be displayed on the dashboard [#6983]
+* Force shutdown of processes that cannot answer takeover event [#7026]
+* `topic` parameter in bridge configuration can have `${node}` substitution (just like in `clientid` parameter)
+* Add UTF-8 string validity check in `strict_mode` for MQTT packet.
+  When set to true, invalid UTF-8 strings will cause the client to be disconnected. i.e. client ID, topic name. [#7261]
+* Changed systemd service restart delay from 10 seconds to 60 seconds.
+* MQTT-SN gateway supports initiative to synchronize registered topics after session resumed. [#7300]
+* Add load control app for future development.
+* Change the precision of float to 17 digits after the decimal point when formatting a
+  float using payload templates of rule actions. The old precision is 10 digits before
+  this change. [#7336]
+* Return the cached resource status when querying a resource using HTTP APIs.
+  This is to avoid blocking the HTTP request if the resource is unavailable. [#7374]
+
+### Bug fixes
+
+* Fix the `{error,eexist}` error when do release upgrade again if last run failed. [#7121]
+* Fix case where publishing to a non-existent topic alias would crash the connection [#6979]
+* Fix HTTP-API 500 error on querying the lwm2m client list on the another node [#7009]
+* Fix the ExProto connection registry is not released after the client process abnormally exits [#6983]
+* Fix Server-KeepAlive wrongly applied on MQTT v3.0/v3.1 [#7085]
+* Fix Stomp client can not trigger `$event/client_connection` message [#7096]
+* Fix system memory false alarm at boot
+* Fix the MQTT-SN message replay when the topic is not registered to the client [#6970]
+* Fix rpc get node info maybe crash when other nodes is not ready.
+* Fix false alert level log “cannot_find_plugins” caused by duplicate plugin names in `loaded_plugins` files.
+* Prompt user how to change the dashboard's initial default password when emqx start.
+* Fix errno=13 'Permission denied' Cannot create FIFO boot error in Amazon Linux 2022 (el8 package)
+* Fix user or appid created, name only allow `^[A-Za-z]+[A-Za-z0-9-_]*$`
+* Fix subscribe http api crash by bad_qos `/mqtt/subscribe`,`/mqtt/subscribe_batch`.
+* Send DISCONNECT packet with reason code 0x98 if connection has been kicked [#7309]
+* Auto subscribe to an empty topic will be simply ignored now
+
+## v4.3.12
+### Important changes
+
+### Minor changes
+* Fix updating `emqx_auth_mnesia.conf` password and restarting the new password does not take effect [#6717]
+* Fix import data crash when emqx_auth_mnesia's record is not empty [#6717]
+* Fix `os_mon.sysmem_high_watermark` may not alert after reboot.
+* Enhancement: Log client status before killing it for holding the lock for too long.
+  [emqx-6959](https://github.com/emqx/emqx/pull/6959)
+  [ekka-144](https://github.com/emqx/ekka/pull/144)
+  [ekka-146](https://github.com/emqx/ekka/pull/146)
+
+## v4.3.11
+
+Important notes:
+
+- For Debian/Ubuntu users
+
+  We changed the package installed service from init.d to systemd.
+  The upgrade from init.d to systemd is verified, however it is
+  recommended to verify it before rolling out to production.
+  At least to ensure systemd is available in your system.
+
+- For Centos Users
+
+  RPM package now depends on `openssl11` which is NOT available
+  in certain centos distributions.
+  Please make sure the yum repo [epel-release](https://docs.fedoraproject.org/en-US/epel) is installed.
+
+### Important changes
+
+* Debian/Ubuntu package (deb) installed EMQX now runs on systemd [#6389]<br>
+  This is to take advantage of systemd's supervision functionality to ensure
+  EMQX service is restarted after crashes.
+
+### Minor changes
+
+* Clustering malfunction fixes [#6221, #6381]
+  Mostly changes made in [ekka](https://github.com/emqx/ekka/pull/134)<br>
+  From 0.8.1.4 to 0.8.1.6, fixes included intra-cluster RPC call timeouts,<br>
+  also fixed `ekka_locker` process crashed after killing a hanged lock owner.
+
+* Improved log message when TCP proxy is in use but proxy_protocol configuration is not turned on [#6416]<br>
+  "please check proxy_protocol config for specific listeners and zones" to hint a misconfiguration
+
+* Helm chart supports networking.k8s.io/v1 [#6368]
+
+* Fix session takeover race condition which may lead to message loss [#6396]
+
+* EMQX docker images are pushed to aws public ecr in an automated CI job [#6271]<br>
+  `docker pull public.ecr.aws/emqx/emqx:4.3.10`
+
+* Fix webhook URL path to allow rule-engine variable substitution [#6399]
+
+* Corrected RAM usage display [#6379]
+
+* Changed emqx_sn_registry table creation to runtime [#6357]<br>
+  This was a bug introduced in 4.3.3, in which the table is changed from ets to mnesia<br>
+  this will cause upgrade to fail when a later version node joins a 4.3.0-2 cluster<br>
+
+* Log level for normal termination changed from info to debug [#6358]
+
+* Added config `retainer.stop_publish_clear_msg` to enable/disable empty message retained message publish [#6343]<br>
+  In MQTT 3.1.1, it is unclear if a MQTT broker should publish the 'clear' (no payload) message<br>
+  to the subscribers, or just delete the retained message. So we have made it configurable
+
+* Fix mqtt bridge malfunction when remote host is unreachable (hangs the connection) [#6286, #6323]
+
+* System monitor now inspects `current_stacktrace` of suspicious process [#6290]<br>
+  `current_function` was not quite helpful
+
+* Changed default `max_topc_levels` config value to 128 [#6294, #6420]<br>
+  previously it has no limit (config value = 0), which can be a potential DoS threat
+
+* Collect only libcrypto and libtinfo so files for zip package [#6259]<br>
+  in 4.3.10 we tried to collect all so files, however glibc is not quite portable
+
+* Added openssl-1.1 to RPM dependency [#6239]
+
+* Http client duplicated header fix [#6195]
+
+* Fix `node_dump` issues when working with deb or rpm installation [#6209]
+
+* Pin Erlang/OTP 23.2.7.2-emqx-3 [#6246]<br>
+  4.3.10 is on 23.2.7.2-emqx-2, this bump is to fix an ECC signature name typo:
+  ecdsa_secp512r1_sha512 -> ecdsa_secp521r1_sha512
+
+* HTTP client performance improvement [#6474, #6414]<br>
+  The changes are mostly done in the dependency [repo](https://github.com/emqx/ehttpc).
+
+* For messages from gateways add message properties as MQTT message headers [#6142]<br>
+  e.g. messages from CoAP, LwM2M, Stomp, ExProto, when translated into MQTT message<br>
+  properties such as protocol name, protocol version, username (if any) peer-host<br>
+  etc. are filled as MQTT message headers.
+
+* Format the message id to hex strings in the log message [#6961]
+
+## v4.3.0~10
+
+Older version changes are not tracked here.

Makefile

@@ -3,11 +3,16 @@ REBAR_VERSION = 3.14.3-emqx-8
 REBAR = $(CURDIR)/rebar3
 BUILD = $(CURDIR)/build
 SCRIPTS = $(CURDIR)/scripts
+export EMQX_RELUP ?= true
+export EMQX_DEFAULT_BUILDER = emqx/build-env:erl23.3.4.9-3-alpine
+export EMQX_DEFAULT_RUNNER = alpine:3.12
 export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh)
-export EMQX_DESC ?= EMQ X
+export DOCKERFILE := deploy/docker/Dockerfile
-export EMQX_CE_DASHBOARD_VERSION ?= v4.3.1
 ifeq ($(OS),Windows_NT)
 	export REBAR_COLOR=none
+	FIND=/usr/bin/find
+else
+	FIND=find
 endif
 
 PROFILE ?= emqx
@@ -15,6 +20,8 @@ REL_PROFILES := emqx emqx-edge
 PKG_PROFILES := emqx-pkg emqx-edge-pkg
 PROFILES := $(REL_PROFILES) $(PKG_PROFILES) default
 
+CT_READABLE ?= true
+
 export REBAR_GIT_CLONE_OPTIONS += --depth=1
 
 .PHONY: default
@@ -51,11 +58,19 @@ APPS=$(shell $(CURDIR)/scripts/find-apps.sh)
 ## app/name-ct targets are intended for local tests hence cover is not enabled
 .PHONY: $(APPS:%=%-ct)
 define gen-app-ct-target
-$1-ct:
+$1-ct: $(REBAR)
-	$(REBAR) ct --name 'test@127.0.0.1' -v --suite $(shell $(CURDIR)/scripts/find-suites.sh $1)
+	$(REBAR) ct --name 'test@127.0.0.1' -v --readable $(CT_READABLE) --suite $(shell $(CURDIR)/scripts/find-suites.sh $1)
 endef
 $(foreach app,$(APPS),$(eval $(call gen-app-ct-target,$(app))))
 
+## app/name-ct-pipeline targets are used in pipeline -> make cover data for each app
+.PHONY: $(APPS:%=%-ct-pipeline)
+define gen-app-ct-target-pipeline
+$1-ct-pipeline: $(REBAR)
+	$(REBAR) ct --name 'test@127.0.0.1' -c -v --readable $(CT_READABLE) --cover_export_name $(PROFILE)-$(subst /,-,$1) --suite $(shell $(CURDIR)/scripts/find-suites.sh $1)
+endef
+$(foreach app,$(APPS),$(eval $(call gen-app-ct-target-pipeline,$(app))))
+
 ## apps/name-prop targets
 .PHONY: $(APPS:%=%-prop)
 define gen-app-prop-target
@@ -73,6 +88,7 @@ coveralls: $(REBAR)
 	@ENABLE_COVER_COMPILE=1 $(REBAR) as test coveralls send
 
 .PHONY: $(REL_PROFILES)
+
 $(REL_PROFILES:%=%): $(REBAR) get-dashboard
 	@$(REBAR) as $(@) do compile,release
 
@@ -85,16 +101,20 @@ $(REL_PROFILES:%=%): $(REBAR) get-dashboard
 clean: $(PROFILES:%=clean-%)
 $(PROFILES:%=clean-%):
 	@if [ -d _build/$(@:clean-%=%) ]; then \
+		rm -f rebar.lock; \
 		rm -rf _build/$(@:clean-%=%)/rel; \
-		find _build/$(@:clean-%=%) -name '*.beam' -o -name '*.so' -o -name '*.app' -o -name '*.appup' -o -name '*.o' -o -name '*.d' -type f | xargs rm -f; \
+		$(FIND) _build/$(@:clean-%=%) -name '*.beam' -o -name '*.so' -o -name '*.app' -o -name '*.appup' -o -name '*.o' -o -name '*.d' -type f | xargs rm -f; \
+		$(FIND) _build/$(@:clean-%=%) -type l -delete; \
 	fi
 
 .PHONY: clean-all
 clean-all:
 	@rm -rf _build
+	@rm -f rebar.lock
 
 .PHONY: deps-all
 deps-all: $(REBAR) $(PROFILES:%=deps-%)
+	@make clean # ensure clean at the end
 
 ## deps-<profile> is used in CI scripts to download deps and the
 ## share downloads between CI steps and/or copied into containers
@@ -102,10 +122,12 @@ deps-all: $(REBAR) $(PROFILES:%=deps-%)
 .PHONY: $(PROFILES:%=deps-%)
 $(PROFILES:%=deps-%): $(REBAR) get-dashboard
 	@$(REBAR) as $(@:deps-%=%) get-deps
+	@rm -f rebar.lock
 
 .PHONY: xref
-xref: $(REBAR)
+xref: $(REBAR) $(REL_PROFILES:%=%-rel)
 	@$(REBAR) as check xref
+	@scripts/xref-check.escript
 
 .PHONY: dialyzer
 dialyzer: $(REBAR)
@@ -118,10 +140,19 @@ COMMON_DEPS := $(REBAR) get-dashboard $(CONF_SEGS)
 $(REL_PROFILES:%=%-rel) $(PKG_PROFILES:%=%-rel): $(COMMON_DEPS)
 	@$(BUILD) $(subst -rel,,$(@)) rel
 
+## download relup base packages
+.PHONY: $(REL_PROFILES:%=%-relup-downloads)
+define download-relup-packages
+$1-relup-downloads:
+	@if [ "$${EMQX_RELUP}" = "true" ]; then $(CURDIR)/scripts/relup-base-packages.sh $1; fi
+endef
+ALL_ZIPS = $(REL_PROFILES)
+$(foreach zt,$(ALL_ZIPS),$(eval $(call download-relup-packages,$(zt))))
+
 ## relup target is to create relup instructions
 .PHONY: $(REL_PROFILES:%=%-relup)
 define gen-relup-target
-$1-relup: $(COMMON_DEPS)
+$1-relup: $1-relup-downloads $(COMMON_DEPS)
 	@$(BUILD) $1 relup
 endef
 ALL_ZIPS = $(REL_PROFILES)
@@ -144,11 +175,18 @@ $1: $1-rel
 endef
 $(foreach pt,$(PKG_PROFILES),$(eval $(call gen-pkg-target,$(pt))))
 
+## docker target is to create docker instructions
+.PHONY: $(REL_PROFILES:%=%-docker)
+define gen-docker-target
+$1-docker: $(COMMON_DEPS)
+	@$(BUILD) $1 docker
+endef
+ALL_ZIPS = $(REL_PROFILES)
+$(foreach zt,$(ALL_ZIPS),$(eval $(call gen-docker-target,$(zt))))
+
 .PHONY: run
 run: $(PROFILE) quickrun
 
 .PHONY: quickrun
 quickrun:
 	./_build/$(PROFILE)/rel/emqx/bin/emqx console
-
-include docker.mk

NOTICE

@@ -0,0 +1,5 @@
+EMQX, a highly scalable, highly available distributed MQTT messaging broker for IoT.
+Copyright (c) 2017-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+
+This product contains code developed at EMQ Technologies Co., Ltd.
+Visit https://www.emqx.come to learn more.

README-CN.md

@@ -1,32 +1,30 @@
-# EMQ X Broker
+# EMQX Broker
 
 [![GitHub Release](https://img.shields.io/github/release/emqx/emqx?color=brightgreen)](https://github.com/emqx/emqx/releases)
 [![Build Status](https://travis-ci.org/emqx/emqx.svg)](https://travis-ci.org/emqx/emqx)
 [![Coverage Status](https://coveralls.io/repos/github/emqx/emqx/badge.svg)](https://coveralls.io/github/emqx/emqx)
 [![Docker Pulls](https://img.shields.io/docker/pulls/emqx/emqx)](https://hub.docker.com/r/emqx/emqx)
-[![Slack Invite](<https://slack-invite.emqx.io/badge.svg>)](https://slack-invite.emqx.io)
+[![Slack](https://img.shields.io/badge/Slack-EMQ-39AE85?logo=slack)](https://slack-invite.emqx.io/)
 [![Twitter](https://img.shields.io/badge/Twitter-EMQ-1DA1F2?logo=twitter)](https://twitter.com/EMQTech)
 [![Community](https://img.shields.io/badge/Community-EMQ%20X-yellow)](https://askemq.com)
 [![YouTube](https://img.shields.io/badge/Subscribe-EMQ%20中文-FF0000?logo=youtube)](https://www.youtube.com/channel/UCir_r04HIsLjf2qqyZ4A8Cg)
 
-[![最棒的物联网 MQTT 开源团队期待您的加入](https://www.emqx.io/static/img/github_readme_cn_bg.png)](https://careers.emqx.cn/)
-
 [English](./README.md) | 简体中文 | [日本語](./README-JP.md) | [русский](./README-RU.md)
 
-*EMQ X* 是一款完全开源，高度可伸缩，高可用的分布式 MQTT 消息服务器，适用于 IoT、M2M 和移动应用程序，可处理千万级别的并发客户端。
+*EMQX* 是一款完全开源，高度可伸缩，高可用的分布式 MQTT 消息服务器，适用于 IoT、M2M 和移动应用程序，可处理千万级别的并发客户端。
 
-从 3.0 版本开始，*EMQ X* 完整支持 MQTT V5.0 协议规范，向下兼容 MQTT V3.1 和 V3.1.1，并支持 MQTT-SN、CoAP、LwM2M、WebSocket 和 STOMP 等通信协议。EMQ X 3.0 单集群可支持千万级别的 MQTT 并发连接。
+从 3.0 版本开始，*EMQX* 完整支持 MQTT V5.0 协议规范，向下兼容 MQTT V3.1 和 V3.1.1，并支持 MQTT-SN、CoAP、LwM2M、WebSocket 和 STOMP 等通信协议。EMQX 3.0 单集群可支持千万级别的 MQTT 并发连接。
 
-- 新功能的完整列表，请参阅 [EMQ X Release Notes](https://github.com/emqx/emqx/releases)。
+- 新功能的完整列表，请参阅 [EMQX Release Notes](https://github.com/emqx/emqx/releases)。
-- 获取更多信息，请访问 [EMQ X 官网](https://www.emqx.cn/)。
+- 获取更多信息，请访问 [EMQX 官网](https://www.emqx.cn/)。
 
 ## 安装
 
-*EMQ X* 是跨平台的，支持 Linux、Unix、macOS 以及 Windows。这意味着 *EMQ X* 可以部署在 x86_64 架构的服务器上，也可以部署在 Raspberry Pi 这样的 ARM 设备上。
+*EMQX* 是跨平台的，支持 Linux、Unix、macOS 以及 Windows。这意味着 *EMQX* 可以部署在 x86_64 架构的服务器上，也可以部署在 Raspberry Pi 这样的 ARM 设备上。
 
-Windows 上编译和运行 *EMQ X* 的详情参考：[Windows.md](./Windows.md)
+Windows 上编译和运行 *EMQX* 的详情参考：[Windows.md](./Windows.md)
 
-#### EMQ X Docker 镜像安装
+#### EMQX Docker 镜像安装
 
 ```
 docker run -d --name emqx -p 1883:1883 -p 8081:8081 -p 8083:8083 -p 8883:8883 -p 8084:8084 -p 18083:18083 emqx/emqx
@@ -34,14 +32,14 @@ docker run -d --name emqx -p 1883:1883 -p 8081:8081 -p 8083:8083 -p 8883:8883 -p
 
 #### 二进制软件包安装
 
-需从 [EMQ X 下载](https://www.emqx.cn/downloads) 页面获取相应操作系统的二进制软件包。
+需从 [EMQX 下载](https://www.emqx.cn/downloads) 页面获取相应操作系统的二进制软件包。
 
 - [单节点安装文档](https://docs.emqx.cn/broker/latest/getting-started/install.html)
 - [集群配置文档](https://docs.emqx.cn/broker/latest/advanced/cluster.html)
 
 ## 从源码构建
 
-3.0 版本开始，构建 *EMQ X* 需要 Erlang/OTP R21+。
+3.0 版本开始，构建 *EMQX* 需要 Erlang/OTP R21+。
 
 4.3 及以后的版本：
 
@@ -77,7 +75,7 @@ _build/emqx/rel/emqx/bin/emqx console
 ./bin/emqx stop
 ```
 
-*EMQ X* 启动，可以使用浏览器访问 http://localhost:18083 来查看 Dashboard。
+*EMQX* 启动，可以使用浏览器访问 http://localhost:18083 来查看 Dashboard。
 
 ## 测试
 
@@ -108,7 +106,7 @@ DIALYZER_ANALYSE_APP=emqx_lwm2m,emqx_auth_jwt,emqx_auth_ldap make dialyzer
 
 ### FAQ
 
-访问 [EMQ X FAQ](https://docs.emqx.cn/broker/latest/faq/faq.html) 以获取常见问题的帮助。
+访问 [EMQX FAQ](https://docs.emqx.cn/broker/latest/faq/faq.html) 以获取常见问题的帮助。
 
 ### 问答
 
@@ -117,7 +115,7 @@ DIALYZER_ANALYSE_APP=emqx_lwm2m,emqx_auth_jwt,emqx_auth_ldap make dialyzer
 
 ### 参与设计
 
-如果对 EMQ X 有改进建议，可以向[EIP](https://github.com/emqx/eip) 提交 PR 和 ISSUE
+如果对 EMQX 有改进建议，可以向[EIP](https://github.com/emqx/eip) 提交 PR 和 ISSUE
 
 ### 插件开发
 

README-JP.md

@@ -1,33 +1,31 @@
-# EMQ X Broker
+# EMQX Broker
 
 [![GitHub Release](https://img.shields.io/github/release/emqx/emqx?color=brightgreen)](https://github.com/emqx/emqx/releases)
 [![Build Status](https://travis-ci.org/emqx/emqx.svg)](https://travis-ci.org/emqx/emqx)
 [![Coverage Status](https://coveralls.io/repos/github/emqx/emqx/badge.svg)](https://coveralls.io/github/emqx/emqx)
 [![Docker Pulls](https://img.shields.io/docker/pulls/emqx/emqx)](https://hub.docker.com/r/emqx/emqx)
-[![Slack Invite](<https://slack-invite.emqx.io/badge.svg>)](https://slack-invite.emqx.io)
+[![Slack](https://img.shields.io/badge/Slack-EMQ-39AE85?logo=slack)](https://slack-invite.emqx.io/)
 [![Twitter](https://img.shields.io/badge/Twitter-EMQ-1DA1F2?logo=twitter)](https://twitter.com/EMQTech)
 [![YouTube](https://img.shields.io/badge/Subscribe-EMQ-FF0000?logo=youtube)](https://www.youtube.com/channel/UC5FjR77ErAxvZENEWzQaO5Q)
 
-[![The best IoT MQTT open source team looks forward to your joining](https://www.emqx.io/static/img/github_readme_en_bg.png)](https://www.emqx.io/careers)
-
 [English](./README.md) | [简体中文](./README-CN.md) | 日本語 | [русский](./README-RU.md)
 
-*EMQ X* は、高い拡張性と可用性をもつ、分散型のMQTTブローカーです。数千万のクライアントを同時に処理するIoT、M2M、モバイルアプリケーション向けです。
+*EMQX* は、高い拡張性と可用性をもつ、分散型のMQTTブローカーです。数千万のクライアントを同時に処理するIoT、M2M、モバイルアプリケーション向けです。
 
-version 3.0 以降、*EMQ X* は MQTT V5.0 の仕様を完全にサポートしており、MQTT V3.1およびV3.1.1とも下位互換性があります。
+version 3.0 以降、*EMQX* は MQTT V5.0 の仕様を完全にサポートしており、MQTT V3.1およびV3.1.1とも下位互換性があります。
 MQTT-SN、CoAP、LwM2M、WebSocket、STOMPなどの通信プロトコルをサポートしています。 MQTTの同時接続数は1つのクラスター上で1,000万以上にまでスケールできます。
 
-- 新機能の一覧については、[EMQ Xリリースノート](https://github.com/emqx/emqx/releases)を参照してください。
+- 新機能の一覧については、[EMQXリリースノート](https://github.com/emqx/emqx/releases)を参照してください。
-- 詳細はこちら[EMQ X公式ウェブサイト](https://www.emqx.io/)をご覧ください。
+- 詳細はこちら[EMQX公式ウェブサイト](https://www.emqx.io/)をご覧ください。
 
 ## インストール
 
-*EMQ X* はクロスプラットフォームで、Linux、Unix、macOS、Windowsをサポートしています。
+*EMQX* はクロスプラットフォームで、Linux、Unix、macOS、Windowsをサポートしています。
-そのため、x86_64アーキテクチャサーバー、またはRaspberryPiなどのARMデバイスに *EMQ X* をデプロイすることもできます。
+そのため、x86_64アーキテクチャサーバー、またはRaspberryPiなどのARMデバイスに *EMQX* をデプロイすることもできます。
 
-Windows上における *EMQ X* のビルドと実行については、[Windows.md](./Windows.md)をご参照ください。
+Windows上における *EMQX* のビルドと実行については、[Windows.md](./Windows.md)をご参照ください。
 
-#### Docker イメージによる EMQ X のインストール
+#### Docker イメージによる EMQX のインストール
 
 ```
 docker run -d --name emqx -p 1883:1883 -p 8083:8083 -p 8883:8883 -p 8084:8084 -p 18083:18083 emqx/emqx
@@ -35,14 +33,14 @@ docker run -d --name emqx -p 1883:1883 -p 8083:8083 -p 8883:8883 -p 8084:8084 -p
 
 #### バイナリパッケージによるインストール
 
-それぞれのOSに対応したバイナリソフトウェアパッケージは、[EMQ Xのダウンロード](https://www.emqx.io/downloads)ページから取得できます。
+それぞれのOSに対応したバイナリソフトウェアパッケージは、[EMQXのダウンロード](https://www.emqx.io/downloads)ページから取得できます。
 
 - [シングルノードインストール](https://docs.emqx.io/broker/latest/en/getting-started/installation.html)
 - [マルチノードインストール](https://docs.emqx.io/broker/latest/en/advanced/cluster.html)
 
 ## ソースからビルド
 
-version 3.0 以降の *EMQ X* をビルドするには Erlang/OTP R21+ が必要です。
+version 3.0 以降の *EMQX* をビルドするには Erlang/OTP R21+ が必要です。
 
 version 4.3 以降の場合：
 
@@ -71,7 +69,7 @@ emqx をソースコードからビルドした場合は、
 ./bin/emqx stop
 ```
 
-*EMQ X* の起動後、ブラウザで http://localhost:18083 にアクセスするとダッシュボードが表示されます。
+*EMQX* の起動後、ブラウザで http://localhost:18083 にアクセスするとダッシュボードが表示されます。
 
 ## テスト
 
@@ -102,7 +100,7 @@ DIALYZER_ANALYSE_APP=emqx_lwm2m,emqx_auth_jwt,emqx_auth_ldap make dialyzer
 
 ### FAQ
 
-よくある質問については、[EMQ X FAQ](https://docs.emqx.io/broker/latest/en/faq/faq.html)をご確認ください。
+よくある質問については、[EMQX FAQ](https://docs.emqx.io/broker/latest/en/faq/faq.html)をご確認ください。
 
 ### 質問する
 

README-RU.md

@@ -1,32 +1,30 @@
-# Брокер EMQ X
+# Брокер EMQX
 
 [![GitHub Release](https://img.shields.io/github/release/emqx/emqx?color=brightgreen)](https://github.com/emqx/emqx/releases)
 [![Build Status](https://travis-ci.org/emqx/emqx.svg)](https://travis-ci.org/emqx/emqx)
 [![Coverage Status](https://coveralls.io/repos/github/emqx/emqx/badge.svg?branch=master)](https://coveralls.io/github/emqx/emqx?branch=master)
 [![Docker Pulls](https://img.shields.io/docker/pulls/emqx/emqx)](https://hub.docker.com/r/emqx/emqx)
-[![Slack Invite](<https://slack-invite.emqx.io/badge.svg>)](https://slack-invite.emqx.io)
+[![Slack](https://img.shields.io/badge/Slack-EMQ-39AE85?logo=slack)](https://slack-invite.emqx.io/)
 [![Twitter](https://img.shields.io/badge/Follow-EMQ-1DA1F2?logo=twitter)](https://twitter.com/EMQTech)
 [![Community](https://img.shields.io/badge/Community-EMQ%20X-yellow?logo=github)](https://github.com/emqx/emqx/discussions)
 [![YouTube](https://img.shields.io/badge/Subscribe-EMQ-FF0000?logo=youtube)](https://www.youtube.com/channel/UC5FjR77ErAxvZENEWzQaO5Q)
 
-[![The best IoT MQTT open source team looks forward to your joining](https://www.emqx.io/static/img/github_readme_en_bg.png)](https://www.emqx.io/careers)
-
 [English](./README.md) | [简体中文](./README-CN.md) | [日本語](./README-JP.md) | русский
 
-*EMQ X* — это масштабируемый, высоко доступный, распределённый MQTT брокер с полностью открытым кодом для интернета вещей, межмашинного взаимодействия и мобильных приложений, который поддерживает миллионы одновременных подключений.
+*EMQX* — это масштабируемый, высоко доступный, распределённый MQTT брокер с полностью открытым кодом для интернета вещей, межмашинного взаимодействия и мобильных приложений, который поддерживает миллионы одновременных подключений.
 
-Начиная с релиза 3.0, брокер *EMQ X* полностью поддерживает протокол MQTT версии 5.0, и обратно совместим с версиями 3.1 и 3.1.1, а также протоколами MQTT-SN, CoAP, LwM2M, WebSocket и STOMP. Начиная с релиза 3.0, брокер *EMQ X* может масштабироваться до более чем 10 миллионов одновременных MQTT соединений на один кластер.
+Начиная с релиза 3.0, брокер *EMQX* полностью поддерживает протокол MQTT версии 5.0, и обратно совместим с версиями 3.1 и 3.1.1, а также протоколами MQTT-SN, CoAP, LwM2M, WebSocket и STOMP. Начиная с релиза 3.0, брокер *EMQX* может масштабироваться до более чем 10 миллионов одновременных MQTT соединений на один кластер.
 
-- Полный список возможностей доступен по ссылке: [EMQ X Release Notes](https://github.com/emqx/emqx/releases).
+- Полный список возможностей доступен по ссылке: [EMQX Release Notes](https://github.com/emqx/emqx/releases).
-- Более подробная информация доступна на нашем сайте: [EMQ X homepage](https://www.emqx.io).
+- Более подробная информация доступна на нашем сайте: [EMQX homepage](https://www.emqx.io).
 
 ## Установка
 
-Брокер *EMQ X* кросплатформенный, и поддерживает Linux, Unix, macOS и Windows. Он может работать на серверах с архитектурой x86_64 и устройствах на архитектуре ARM, таких как Raspberry Pi.
+Брокер *EMQX* кросплатформенный, и поддерживает Linux, Unix, macOS и Windows. Он может работать на серверах с архитектурой x86_64 и устройствах на архитектуре ARM, таких как Raspberry Pi.
 
 Более подробная информация о запуске на Windows по ссылке: [Windows.md](./Windows.md)
 
-#### Установка EMQ X с помощью Docker-образа
+#### Установка EMQX с помощью Docker-образа
 
 ```
 docker run -d --name emqx -p 1883:1883 -p 8081:8081 -p 8083:8083 -p 8883:8883 -p 8084:8084 -p 18083:18083 emqx/emqx
@@ -34,7 +32,7 @@ docker run -d --name emqx -p 1883:1883 -p 8081:8081 -p 8083:8083 -p 8883:8883 -p
 
 #### Установка бинарного пакета
 
-Сборки для различных операционных систем: [Загрузить EMQ X](https://www.emqx.io/downloads).
+Сборки для различных операционных систем: [Загрузить EMQX](https://www.emqx.io/downloads).
 
 - [Установка на одном сервере](https://docs.emqx.io/en/broker/latest/getting-started/install.html)
 - [Установка на кластере](https://docs.emqx.io/en/broker/latest/advanced/cluster.html)
@@ -111,7 +109,7 @@ DIALYZER_ANALYSE_APP=emqx_lwm2m,emqx_auth_jwt,emqx_auth_ldap make dialyzer
 
 ### FAQ
 
-Наиболее частые проблемы разобраны в [EMQ X FAQ](https://docs.emqx.io/en/broker/latest/faq/faq.html).
+Наиболее частые проблемы разобраны в [EMQX FAQ](https://docs.emqx.io/en/broker/latest/faq/faq.html).
 
 
 ### Вопросы

README.md

@@ -1,31 +1,29 @@
-# EMQ X Broker
+# EMQX Broker
 
 [![GitHub Release](https://img.shields.io/github/release/emqx/emqx?color=brightgreen)](https://github.com/emqx/emqx/releases)
 [![Build Status](https://travis-ci.org/emqx/emqx.svg)](https://travis-ci.org/emqx/emqx)
 [![Coverage Status](https://coveralls.io/repos/github/emqx/emqx/badge.svg?branch=master)](https://coveralls.io/github/emqx/emqx?branch=master)
 [![Docker Pulls](https://img.shields.io/docker/pulls/emqx/emqx)](https://hub.docker.com/r/emqx/emqx)
-[![Slack Invite](<https://slack-invite.emqx.io/badge.svg>)](https://slack-invite.emqx.io)
+[![Slack](https://img.shields.io/badge/Slack-EMQ-39AE85?logo=slack)](https://slack-invite.emqx.io/)
 [![Twitter](https://img.shields.io/badge/Follow-EMQ-1DA1F2?logo=twitter)](https://twitter.com/EMQTech)
 [![YouTube](https://img.shields.io/badge/Subscribe-EMQ-FF0000?logo=youtube)](https://www.youtube.com/channel/UC5FjR77ErAxvZENEWzQaO5Q)
 
-[![The best IoT MQTT open source team looks forward to your joining](https://www.emqx.io/static/img/github_readme_en_bg.png)](https://www.emqx.io/careers)
-
 English | [简体中文](./README-CN.md) | [日本語](./README-JP.md) | [русский](./README-RU.md)
 
-*EMQ X* broker is a fully open source, highly scalable, highly available distributed MQTT messaging broker for IoT, M2M and Mobile applications that can handle tens of millions of concurrent clients.
+*EMQX* broker is a fully open source, highly scalable, highly available distributed MQTT messaging broker for IoT, M2M and Mobile applications that can handle tens of millions of concurrent clients.
 
-Starting from 3.0 release, *EMQ X* broker fully supports MQTT V5.0 protocol specifications and backward compatible with MQTT V3.1 and V3.1.1,  as well as other communication protocols such as MQTT-SN, CoAP, LwM2M, WebSocket and STOMP. The 3.0 release of the *EMQ X* broker can scaled to 10+ million concurrent MQTT connections on one cluster.
+Starting from 3.0 release, *EMQX* broker fully supports MQTT V5.0 protocol specifications and backward compatible with MQTT V3.1 and V3.1.1,  as well as other communication protocols such as MQTT-SN, CoAP, LwM2M, WebSocket and STOMP. The 3.0 release of the *EMQX* broker can scaled to 10+ million concurrent MQTT connections on one cluster.
 
-- For full list of new features, please read [EMQ X Release Notes](https://github.com/emqx/emqx/releases).
+- For full list of new features, please read [EMQX Release Notes](https://github.com/emqx/emqx/releases).
-- For more information, please visit [EMQ X homepage](https://www.emqx.io).
+- For more information, please visit [EMQX homepage](https://www.emqx.io).
 
 ## Installation
 
-The *EMQ X* broker is cross-platform, which supports Linux, Unix, macOS and Windows. It means *EMQ X* can be deployed on x86_64 architecture servers and ARM devices like Raspberry Pi.
+The *EMQX* broker is cross-platform, which supports Linux, Unix, macOS and Windows. It means *EMQX* can be deployed on x86_64 architecture servers and ARM devices like Raspberry Pi.
 
-See more details for building and running *EMQ X* on Windows in [Windows.md](./Windows.md)
+See more details for building and running *EMQX* on Windows in [Windows.md](./Windows.md)
 
-#### Installing via EMQ X Docker Image
+#### Installing via EMQX Docker Image
 
 ```
 docker run -d --name emqx -p 1883:1883 -p 8081:8081 -p 8083:8083 -p 8883:8883 -p 8084:8084 -p 18083:18083 emqx/emqx
@@ -33,7 +31,7 @@ docker run -d --name emqx -p 1883:1883 -p 8081:8081 -p 8083:8083 -p 8883:8883 -p
 
 #### Installing via Binary Package
 
-Get the binary package of the corresponding OS from [EMQ X Download](https://www.emqx.io/downloads) page.
+Get the binary package of the corresponding OS from [EMQX Download](https://www.emqx.io/downloads) page.
 
 - [Single Node Install](https://docs.emqx.io/en/broker/latest/getting-started/install.html)
 - [Multi Node Install](https://docs.emqx.io/en/broker/latest/advanced/cluster.html)
@@ -41,7 +39,7 @@ Get the binary package of the corresponding OS from [EMQ X Download](https://www
 
 ## Build From Source
 
-The *EMQ X* broker requires Erlang/OTP R21+ to build since 3.0 release.
+The *EMQX* broker requires Erlang/OTP R21+ to build since 3.0 release.
 
 For 4.3 and later versions.
 
@@ -110,7 +108,7 @@ DIALYZER_ANALYSE_APP=emqx_lwm2m,emqx_auth_jwt,emqx_auth_ldap make dialyzer
 
 ### FAQ
 
-Visiting [EMQ X FAQ](https://docs.emqx.io/en/broker/latest/faq/faq.html) to get help of common problems.
+Visiting [EMQX FAQ](https://docs.emqx.io/en/broker/latest/faq/faq.html) to get help of common problems.
 
 
 ### Questions

Windows.md

@@ -1,4 +1,4 @@
-# Build and run EMQ X on Windows
+# Build and run EMQX on Windows
 
 NOTE: The instructions and examples are based on Windows 10.
 
@@ -6,7 +6,7 @@ NOTE: The instructions and examples are based on Windows 10.
 
 ### Visual studio for C/C++ compile and link
 
-EMQ X includes Erlang NIF (Native Implmented Function) components, implemented
+EMQX includes Erlang NIF (Native Implmented Function) components, implemented
 in C/C++. To compile and link C/C++ libraries, the easiest way is perhaps to
 install Visual Studio.
 
@@ -25,17 +25,17 @@ C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\Build
 
 Depending on your visual studio version and OS, the paths may differ.
 The first path is for rebar3 port compiler to find `cl.exe` and `link.exe`
-The second path is for Powershell or CMD to setup environment variables.
+The second path is for CMD to setup environment variables.
 
 ### Erlang/OTP
 
-Install Erlang/OTP 23.2 from https://www.erlang.org/downloads
+Install Erlang/OTP 23.3 from https://www.erlang.org/downloads
 You may need to edit the `Path` environment variable to allow running
-Erlang commands such as `erl` from powershell.
+Erlang commands such as `erl` from CMD.
 
-To validate Erlang installation in CMD or powershell:
+To validate Erlang installation in CMD :
 
-* Start (or restart) CMD or powershell
+* Start (or restart) CMD
 
 * Execute `erl` command to enter Erlang shell
 
@@ -51,7 +51,7 @@ Eshell V11.1.4  (abort with ^G)
 
 ### bash
 
-All EMQ X build/run scripts are either in `bash` or `escript`.
+All EMQX build/run scripts are either in `bash` or `escript`.
 `escript` is installed as a part of Erlang. To install a `bash`
 environment in Windows, there are quite a few options.
 
@@ -63,12 +63,12 @@ Cygwin is what we tested with.
   to `Path` list.
 
 * Validate installation.
-  Start (restart) CMD or powershell console and execute `which bash`, it should
+  Start (restart) CMD console and execute `which bash`, it should
   print out `/usr/bin/bash`
 
 ### Other tools
 
-Some of the unix world tools are required to build EMQ X.  Including:
+Some of the unix world tools are required to build EMQX.  Including:
 
 * git
 * curl
@@ -84,11 +84,11 @@ When using scoop:
 scoop install git curl make jq zip unzip
 ```
 
-## Build EMQ X source code
+## Build EMQX source code
 
 * Clone the repo: `git clone https://github.com/emqx/emqx.git`
 
-* Start CMD or Powershell
+* Start CMD
 
 * Execute `vcvarsall.bat x86_amd64` to load environment variables
 
@@ -112,11 +112,11 @@ scoop install git curl make jq zip unzip
   To fix it, Visual Studio's bin paths should be ordered prior to Cygwin's (or similar installation's)
   bin paths in `Path` environment variable.
 
-## Run EMQ X
+## Run EMQX
 
-To start EMQ X broker.
+To start EMQX broker.
 
-Execute `_build\emqx\rel\emqx>.\bin\emqx console` or `_build\emqx\rel\emqx>.\bin\emqx start` to start EMQ X.
+Execute `_build\emqx\rel\emqx>.\bin\emqx console` or `_build\emqx\rel\emqx>.\bin\emqx start` to start EMQX.
 
 Then execute `_build\emqx\rel\emqx>.\bin\emqx_ctl status` to check status.
 If everything works fine, it should print out

apps/emqx_auth_http/README.md

@@ -1,7 +1,7 @@
 emqx_auth_http
 ==============
 
-EMQ X HTTP Auth/ACL Plugin
+EMQX HTTP Auth/ACL Plugin
 
 Build
 -----
@@ -96,5 +96,5 @@ Apache License Version 2.0
 Author
 ------
 
-EMQ X Team.
+EMQX Team.
 

apps/emqx_auth_http/etc/emqx_auth_http.conf

@@ -42,18 +42,18 @@ auth.http.auth_req.params = clientid=%c,username=%u,password=%P
 ## Value: URL
 ##
 ## Examples: http://127.0.0.1:80/mqtt/superuser, https://[::1]:80/mqtt/superuser
-auth.http.super_req.url = http://127.0.0.1:80/mqtt/superuser
+# auth.http.super_req.url = http://127.0.0.1:80/mqtt/superuser
 
 ## HTTP Request Method for SuperUser Request
 ##
 ## Value: post | get
-auth.http.super_req.method = post
+# auth.http.super_req.method = post
 
 ## HTTP Request Headers for SuperUser Request, Content-Type header is configured by default.
 ## The possible values of the Content-Type header: application/x-www-form-urlencoded, application/json
 ##
 ## Examples: auth.http.super_req.headers.accept = */*
-auth.http.super_req.headers.content-type = application/x-www-form-urlencoded
+# auth.http.super_req.headers.content-type = application/x-www-form-urlencoded
 
 ## Parameters used to construct the request body or query string parameters
 ## When the request method is GET, these parameters will be converted into query string parameters
@@ -70,7 +70,7 @@ auth.http.super_req.headers.content-type = application/x-www-form-urlencoded
 ##  - %d: subject of client TLS cert
 ##
 ## Value: <K1>=<V1>,<K2>=<V2>,...
-auth.http.super_req.params = clientid=%c,username=%u
+# auth.http.super_req.params = clientid=%c,username=%u
 
 ## HTTP URL API path for ACL Request
 ## Comment out this config to disable ACL checks
@@ -136,6 +136,11 @@ auth.http.connect_timeout = 5s
 ## Value: Number
 auth.http.pool_size = 32
 
+## Whether to enable HTTP Pipelining
+##
+## See: https://en.wikipedia.org/wiki/HTTP_pipelining
+auth.http.enable_pipelining = 100
+
 ##------------------------------------------------------------------------------
 ## SSL options
 
@@ -163,7 +168,7 @@ auth.http.pool_size = 32
 
 ## If not specified, the server's names returned in server's certificate is validated against
 ## what's provided `auth.http.auth_req.url` config's host part.
-## Setting to 'disable' will make EMQ X ignore unmatched server names.
+## Setting to 'disable' will make EMQX ignore unmatched server names.
 ## If set with a host name, the server's names returned in server's certificate is validated
 ## against this value.
 ##

apps/emqx_auth_http/include/emqx_auth_http.hrl

@@ -1,23 +1,20 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
 
 -define(APP, emqx_auth_http).
 
--record(auth_metrics, {
+%% equals to the default value of ehttpc
-        success = 'client.auth.success',
+-define(DEFAULT_RETRY_TIMES, 2).
-        failure = 'client.auth.failure',
-        ignore = 'client.auth.ignore'
-    }).
-
--record(acl_metrics, {
-        allow = 'client.acl.allow',
-        deny = 'client.acl.deny',
-        ignore = 'client.acl.ignore'
-    }).
-
--define(METRICS(Type), tl(tuple_to_list(#Type{}))).
--define(METRICS(Type, K), #Type{}#Type.K).
-
--define(AUTH_METRICS, ?METRICS(auth_metrics)).
--define(AUTH_METRICS(K), ?METRICS(auth_metrics, K)).
-
--define(ACL_METRICS, ?METRICS(acl_metrics)).
--define(ACL_METRICS(K), ?METRICS(acl_metrics, K)).

apps/emqx_auth_http/priv/emqx_auth_http.schema

@@ -109,6 +109,30 @@ end}.
   {datatype, integer}
 ]}.
 
+{mapping, "auth.http.enable_pipelining", "emqx_auth_http.enable_pipelining", [
+  {default, "100"},
+  {datatype, string}
+]}.
+
+{translation, "emqx_auth_http.enable_pipelining", fun(Conf) ->
+  case cuttlefish:conf_get("auth.http.enable_pipelining", Conf, undefined) of
+    undefined -> 100;
+    Str ->
+      try
+        erlang:list_to_integer(Str)
+      catch _:_ ->
+        case erlang:list_to_atom(Str) of
+          true ->
+            100;
+          false ->
+            1;
+          _ ->
+            100
+        end
+      end
+  end
+end}.
+
 {mapping, "auth.http.ssl.cacertfile", "emqx_auth_http.cacertfile", [
   {datatype, string}
 ]}.

apps/emqx_auth_http/src/emqx_acl_http.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -24,38 +24,35 @@
 -logger_header("[ACL http]").
 
 -import(emqx_auth_http_cli,
-        [ request/6
+        [ request/7
         , feedvar/2
         ]).
 
 %% ACL callbacks
--export([ register_metrics/0
+-export([ check_acl/5
-        , check_acl/5
         , description/0
         ]).
 
--spec(register_metrics() -> ok).
-register_metrics() ->
-    lists:foreach(fun emqx_metrics:ensure/1, ?ACL_METRICS).
-
 %%--------------------------------------------------------------------
 %% ACL callbacks
 %%--------------------------------------------------------------------
 
-check_acl(ClientInfo, PubSub, Topic, AclResult, Params) ->
+check_acl(#{username := <<$$, _/binary>>}, _PubSub, _Topic, _AclResult, _Params) ->
-    return_with(fun inc_metrics/1,
-                do_check_acl(ClientInfo, PubSub, Topic, AclResult, Params)).
-
-do_check_acl(#{username := <<$$, _/binary>>}, _PubSub, _Topic, _AclResult, _Params) ->
     ok;
-do_check_acl(ClientInfo, PubSub, Topic, _AclResult, #{acl := ACLParams = #{path := Path}}) ->
+check_acl(ClientInfo, PubSub, Topic, _AclResult, #{acl := ACLParams = #{path := Path}}) ->
     ClientInfo1 = ClientInfo#{access => access(PubSub), topic => Topic},
+    Username = maps:get(username, ClientInfo1, undefined),
     case check_acl_request(ACLParams, ClientInfo1) of
         {ok, 200, <<"ignore">>} -> ok;
         {ok, 200, _Body} -> {stop, allow};
-        {ok, _Code, _Body}  -> {stop, deny};
+        {ok, Code, _Body} ->
+            ?LOG(warning, "Deny ~s to topic ~ts, username: ~ts, http response code: ~p",
+                 [PubSub, Topic, Username, Code]),
+            {stop, deny};
         {error, Error} ->
-            ?LOG(error, "Request ACL path ~s, error: ~p", [Path, Error]),
+            ?LOG(warning, "Deny ~s to topic ~ts, username: ~ts, due to request "
+                          "http server failure, path: ~p, error: ~0p",
+                          [PubSub, Topic, Username, Path, Error]),
             ok
     end.
 
@@ -65,23 +62,15 @@ description() -> "ACL with HTTP API".
 %% Internal functions
 %%--------------------------------------------------------------------
 
-inc_metrics(ok) ->
+check_acl_request(ACLParams =
-    emqx_metrics:inc(?ACL_METRICS(ignore));
+                  #{pool_name := PoolName,
-inc_metrics({stop, allow}) ->
-    emqx_metrics:inc(?ACL_METRICS(allow));
-inc_metrics({stop, deny}) ->
-    emqx_metrics:inc(?ACL_METRICS(deny)).
-
-return_with(Fun, Result) ->
-    Fun(Result), Result.
-
-check_acl_request(#{pool_name := PoolName,
                     path := Path,
                     method := Method,
                     headers := Headers,
                     params := Params,
                     timeout := Timeout}, ClientInfo) ->
-    request(PoolName, Method, Path, Headers, feedvar(Params, ClientInfo), Timeout).
+    Retry = maps:get(retry_times, ACLParams, ?DEFAULT_RETRY_TIMES),
+    request(PoolName, Method, Path, Headers, feedvar(Params, ClientInfo), Timeout, Retry).
 
 access(subscribe) -> 1;
 access(publish)   -> 2.

apps/emqx_auth_http/src/emqx_auth_http.app.src

@@ -1,6 +1,6 @@
 {application, emqx_auth_http,
  [{description, "EMQ X Authentication/ACL with HTTP API"},
-  {vsn, "4.3.0"}, % strict semver, bump manually!
+  {vsn, "4.3.9"}, % strict semver, bump manually!
   {modules, []},
   {registered, [emqx_auth_http_sup]},
   {applications, [kernel,stdlib,ehttpc]},

apps/emqx_auth_http/src/emqx_auth_http.appup.src

@@ -0,0 +1,71 @@
+%% -*- mode: erlang -*-
+%% Unless you know what you are doing, DO NOT edit manually!!
+{VSN,
+  [{"4.3.8",
+    [{load_module,emqx_auth_http,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_http,brutal_purge,soft_purge,[]}]},
+   {"4.3.7",
+    [{load_module,emqx_auth_http_cli,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_http,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_http,brutal_purge,soft_purge,[]}]},
+   {"4.3.6",
+    [{load_module,emqx_auth_http_cli,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_http,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_http,brutal_purge,soft_purge,[]}]},
+   {"4.3.5",
+    [{load_module,emqx_auth_http_cli,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_http,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_http_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_http,brutal_purge,soft_purge,[]}]},
+   {"4.3.4",
+    [{load_module,emqx_auth_http_cli,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_http,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_http_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_http,brutal_purge,soft_purge,[]}]},
+   {"4.3.3",
+    [{load_module,emqx_auth_http_cli,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_http_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_http,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_http,brutal_purge,soft_purge,[]}]},
+   {"4.3.2",
+    [{apply,{application,stop,[emqx_auth_http]}},
+     {load_module,emqx_auth_http_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_http,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_http,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_http_cli,brutal_purge,soft_purge,[]}]},
+   {<<"4.3.[0-1]">>,[{restart_application,emqx_auth_http}]},
+   {<<".*">>,[]}],
+  [{"4.3.8",
+    [{load_module,emqx_auth_http,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_http,brutal_purge,soft_purge,[]}]},
+   {"4.3.7",
+    [{load_module,emqx_auth_http_cli,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_http,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_http,brutal_purge,soft_purge,[]}]},
+   {"4.3.6",
+    [{load_module,emqx_auth_http_cli,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_http,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_http,brutal_purge,soft_purge,[]}]},
+   {"4.3.5",
+    [{load_module,emqx_auth_http_cli,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_http,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_http_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_http,brutal_purge,soft_purge,[]}]},
+   {"4.3.4",
+    [{load_module,emqx_auth_http_cli,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_http,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_http_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_http,brutal_purge,soft_purge,[]}]},
+   {"4.3.3",
+    [{load_module,emqx_auth_http_cli,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_http_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_http,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_http,brutal_purge,soft_purge,[]}]},
+   {"4.3.2",
+    [{apply,{application,stop,[emqx_auth_http]}},
+     {load_module,emqx_auth_http_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_http,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_http,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_http_cli,brutal_purge,soft_purge,[]}]},
+   {<<"4.3.[0-1]">>,[{restart_application,emqx_auth_http}]},
+   {<<".*">>,[]}]}.

apps/emqx_auth_http/src/emqx_auth_http.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -25,41 +25,36 @@
 -logger_header("[Auth http]").
 
 -import(emqx_auth_http_cli,
-        [ request/6
+        [ request/7
         , feedvar/2
         ]).
 
 %% Callbacks
--export([ register_metrics/0
+-export([ check/3
-        , check/3
         , description/0
         ]).
 
--spec(register_metrics() -> ok).
-register_metrics() ->
-    lists:foreach(fun emqx_metrics:ensure/1, ?AUTH_METRICS).
-
 check(ClientInfo, AuthResult, #{auth  := AuthParms = #{path := Path},
                                 super := SuperParams}) ->
+    Username = maps:get(username, ClientInfo, undefined),
     case authenticate(AuthParms, ClientInfo) of
         {ok, 200, <<"ignore">>} ->
-            emqx_metrics:inc(?AUTH_METRICS(ignore)), ok;
+            ok;
         {ok, 200, Body}  ->
-            emqx_metrics:inc(?AUTH_METRICS(success)),
             IsSuperuser = is_superuser(SuperParams, ClientInfo),
             {stop, AuthResult#{is_superuser => IsSuperuser,
                                 auth_result => success,
                                 anonymous   => false,
                                 mountpoint  => mountpoint(Body, ClientInfo)}};
         {ok, Code, _Body} ->
-            ?LOG(error, "Deny connection from path: ~s, response http code: ~p",
+            ?LOG(warning, "Deny connection from path: ~s, username: ~ts, http "
-                 [Path, Code]),
+                          "response code: ~p",
-            emqx_metrics:inc(?AUTH_METRICS(failure)),
+                          [Path, Username, Code]),
             {stop, AuthResult#{auth_result => http_to_connack_error(Code),
                                anonymous   => false}};
         {error, Error} ->
-            ?LOG(error, "Request auth path: ~s, error: ~p", [Path, Error]),
+            ?LOG_SENSITIVE(warning, "Deny connection from path: ~s, username: ~ts, due to "
-            emqx_metrics:inc(?AUTH_METRICS(failure)),
+                                    "request http-server failed: ~0p", [Path, Username, Error]),
             %%FIXME later: server_unavailable is not right.
             {stop, AuthResult#{auth_result => server_unavailable,
                                anonymous   => false}}
@@ -71,27 +66,34 @@ description() -> "Authentication by HTTP API".
 %% Requests
 %%--------------------------------------------------------------------
 
-authenticate(#{pool_name := PoolName,
+authenticate(AuthParams =
+             #{pool_name := PoolName,
                path := Path,
                method := Method,
                headers := Headers,
                params := Params,
                timeout := Timeout}, ClientInfo) ->
-   request(PoolName, Method, Path, Headers, feedvar(Params, ClientInfo), Timeout).
+    Retry = maps:get(retry_times, AuthParams, ?DEFAULT_RETRY_TIMES),
+    request(PoolName, Method, Path, Headers, feedvar(Params, ClientInfo), Timeout, Retry).
 
 -spec(is_superuser(maybe(map()), emqx_types:client()) -> boolean()).
 is_superuser(undefined, _ClientInfo) ->
     false;
-is_superuser(#{pool_name := PoolName,
+is_superuser(SuperParams =
+             #{pool_name := PoolName,
                path := Path,
                method := Method,
                headers := Headers,
                params := Params,
                timeout := Timeout}, ClientInfo) ->
-    case request(PoolName, Method, Path, Headers, feedvar(Params, ClientInfo), Timeout) of
+    Retry = maps:get(retry_times, SuperParams, ?DEFAULT_RETRY_TIMES),
-        {ok, 200, _Body}   -> true;
+    case request(PoolName, Method, Path, Headers, feedvar(Params, ClientInfo), Timeout, Retry) of
-        {ok, _Code, _Body} -> false;
+        {ok, 200, _Body} ->
-        {error, Error}     -> ?LOG(error, "Request superuser path ~s, error: ~p", [Path, Error]),
+            true;
+        {ok, _Code, _Body} ->
+            false;
+        {error, Error} ->
+            ?LOG_SENSITIVE(warning, "Request superuser path ~s, error: ~p", [Path, Error]),
             false
     end.
 

apps/emqx_auth_http/src/emqx_auth_http_app.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -50,14 +50,14 @@ translate_env(EnvName) ->
     case application:get_env(?APP, EnvName) of
         undefined -> ok;
         {ok, Req} ->
+            {ok, EnablePipelining} = application:get_env(?APP, enable_pipelining),
             {ok, PoolSize} = application:get_env(?APP, pool_size),
             {ok, ConnectTimeout} = application:get_env(?APP, connect_timeout),
             URL = proplists:get_value(url, Req),
             {ok, #{host := Host,
-                   path := Path0,
                    port := Port,
-                   scheme := Scheme}} = emqx_http_lib:uri_parse(URL),
+                   scheme := Scheme} = URIMap} = emqx_http_lib:uri_parse(URL),
-            Path = path(Path0),
+            Path = path(URIMap),
             MoreOpts = case Scheme of
                         http ->
                             [{transport_opts, emqx_misc:ipv6_probe([])}];
@@ -71,6 +71,7 @@ translate_env(EnvName) ->
                                      end,
                             SNI = case application:get_env(?APP, server_name_indication, undefined) of
                                     "disable" -> disable;
+                                    "" -> undefined;
                                     SNI0 -> SNI0
                                   end,
                             TLSOpts = lists:filter(
@@ -89,6 +90,7 @@ translate_env(EnvName) ->
                         end,
             PoolOpts = [{host, Host},
                         {port, Port},
+                        {enable_pipelining, EnablePipelining},
                         {pool_size, PoolSize},
                         {pool_type, random},
                         {connect_timeout, ConnectTimeout},
@@ -110,7 +112,6 @@ load_hooks() ->
     case application:get_env(?APP, auth_req) of
         undefined -> ok;
         {ok, AuthReq} ->
-            ok = emqx_auth_http:register_metrics(),
             PoolOpts = proplists:get_value(pool_opts, AuthReq),
             PoolName = proplists:get_value(pool_name, AuthReq),
             {ok, _} = ehttpc_sup:start_pool(PoolName, PoolOpts),
@@ -129,7 +130,6 @@ load_hooks() ->
     case application:get_env(?APP, acl_req) of
         undefined -> ok;
         {ok, ACLReq} ->
-            ok = emqx_acl_http:register_metrics(),
             PoolOpts2 = proplists:get_value(pool_opts, ACLReq),
             PoolName2 = proplists:get_value(pool_name, ACLReq),
             {ok, _} = ehttpc_sup:start_pool(PoolName2, PoolOpts2),
@@ -149,10 +149,13 @@ ensure_content_type_header(Method, Headers)
   when Method =:= post orelse Method =:= put ->
     Headers;
 ensure_content_type_header(_Method, Headers) ->
-    lists:keydelete("content-type", 1, Headers).
+    lists:keydelete(<<"content-type">>, 1, Headers).
 
-path("") ->
+path(#{path := "", 'query' := Query}) ->
+    "?" ++ Query;
+path(#{path := Path, 'query' := Query}) ->
+    Path ++ "?" ++ Query;
+path(#{path := ""}) ->
     "/";
-path(Path) ->
+path(#{path := Path}) ->
     Path.
-

apps/emqx_auth_http/src/emqx_auth_http_cli.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -19,6 +19,7 @@
 -include("emqx_auth_http.hrl").
 
 -export([ request/6
+        , request/7
         , feedvar/2
         , feedvar/3
         ]).
@@ -27,18 +28,21 @@
 %% HTTP Request
 %%--------------------------------------------------------------------
 
-request(PoolName, get, Path, Headers, Params, Timeout) ->
+request(PoolName, Method, Path, Headers, Params, Timeout) ->
-    NewPath = Path ++ "?" ++ binary_to_list(cow_qs:qs(bin_kw(Params))),
+    request(PoolName, Method, Path, Headers, Params, Timeout, ?DEFAULT_RETRY_TIMES).
-    reply(ehttpc:request(ehttpc_pool:pick_worker(PoolName), get, {NewPath, Headers}, Timeout));
 
-request(PoolName, post, Path, Headers, Params, Timeout) ->
+request(PoolName, get, Path, Headers, Params, Timeout, Retry) ->
-    Body = case proplists:get_value("content-type", Headers) of
+    NewPath = Path ++ "?" ++ binary_to_list(cow_qs:qs(bin_kw(Params))),
+    reply(ehttpc:request(PoolName, get, {NewPath, Headers}, Timeout, Retry));
+
+request(PoolName, post, Path, Headers, Params, Timeout, Retry) ->
+    Body = case proplists:get_value(<<"content-type">>, Headers) of
                "application/x-www-form-urlencoded" ->
                    cow_qs:qs(bin_kw(Params));
                "application/json" -> 
                    emqx_json:encode(bin_kw(Params))
            end,
-    reply(ehttpc:request(ehttpc_pool:pick_worker(PoolName), post, {Path, Headers, Body}, Timeout)).
+    reply(ehttpc:request(PoolName, post, {Path, Headers, Body}, Timeout, Retry)).
 
 reply({ok, StatusCode, _Headers}) ->
     {ok, StatusCode, <<>>};

apps/emqx_auth_http/src/emqx_auth_http_sup.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

apps/emqx_auth_http/test/emqx_auth_http_SUITE.erl

@@ -1,4 +1,4 @@
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

apps/emqx_auth_jwt/README.md

@@ -1,7 +1,7 @@
 
 # emqx-auth-jwt
 
-EMQ X JWT Authentication Plugin
+EMQX JWT Authentication Plugin
 
 Build
 -----
@@ -46,6 +46,11 @@ auth.jwt.verify_claims = off
 ##  - %u: username
 ##  - %c: clientid
 # auth.jwt.verify_claims.username = %u
+
+## Name of the claim containg ACL rules
+##
+## Value: String
+#auth.jwt.acl_claim_name = acl
 ```
 
 Load the Plugin
@@ -62,6 +67,33 @@ Example
 mosquitto_pub -t 'pub' -m 'hello' -i test -u test -P eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJuYW1lIjoiYm9iIiwiYWdlIjoyOX0.bIV_ZQ8D5nQi0LT8AVkpM4Pd6wmlbpR9S8nOLJAsA8o
 ```
 
+ACL
+---
+JWT may contain lists of topics allowed for subscribing/publishing (ACL rules):
+
+Payload example:
+```json
+{
+  "sub": "emqx",
+  "name": "John Doe",
+  "iat": 1516239022,
+  "exp": 1516239122,
+  "acl": {
+    "sub": [
+      "a/b",
+      "c/+",
+      "%u/%c"
+    ],
+    "pub": [
+      "a/b",
+      "c/+",
+      "%u/%c"
+    ]
+  }
+}
+```
+
+
 Algorithms
 ----------
 
@@ -87,4 +119,4 @@ Apache License Version 2.0
 Author
 ------
 
-EMQ X Team.
+EMQX Team.

apps/emqx_auth_jwt/etc/emqx_auth_jwt.conf

@@ -47,3 +47,8 @@ auth.jwt.verify_claims = off
 ## For example, to verify that the username in the JWT payload is the same
 ## as the client (MQTT protocol) username
 #auth.jwt.verify_claims.username = %u
+
+## Name of the claim containg ACL rules
+##
+## Value: String
+#auth.jwt.acl_claim_name = acl

apps/emqx_auth_jwt/priv/emqx_auth_jwt.schema

@@ -47,3 +47,12 @@
               end, [], cuttlefish_variable:filter_by_prefix("auth.jwt.verify_claims", Conf))
    end
 end}.
+
+{mapping, "auth.jwt.acl_claim_name", "emqx_auth_jwt.acl_claim_name", [
+  {default, "acl"},
+  {datatype, string}
+]}.
+
+{translation, "emqx_auth_jwt.acl_claim_name", fun(Conf) ->
+    list_to_binary(cuttlefish:conf_get("auth.jwt.acl_claim_name", Conf))
+end}.

apps/emqx_auth_jwt/src/emqx_auth_jwt.app.src

@@ -1,6 +1,6 @@
 {application, emqx_auth_jwt,
  [{description, "EMQ X Authentication with JWT"},
-  {vsn, "4.3.1"}, % strict semver, bump manually!
+  {vsn, "4.3.9"}, % strict semver, bump manually!
   {modules, []},
   {registered, [emqx_auth_jwt_sup]},
   {applications, [kernel,stdlib,jose]},

apps/emqx_auth_jwt/src/emqx_auth_jwt.appup.src

@@ -1,15 +1,21 @@
-%% -*-: erlang -*-
+%% -*- mode: erlang -*-
+%% Unless you know what you are doing, DO NOT edit manually!!
 {VSN,
- [
+  [{"4.3.8",[{load_module,emqx_auth_jwt,brutal_purge,soft_purge,[]}]},
-    {"4.3.0", [
+   {"4.3.7",
-      {load_module, emqx_auth_jwt_svr, brutal_purge, soft_purge, []}
+    [{load_module,emqx_auth_jwt_svr,brutal_purge,soft_purge,[]},
-    ]},
+     {load_module,emqx_auth_jwt,brutal_purge,soft_purge,[]}]},
-    {<<".*">>, []}
+   {<<"4\\.3\\.[3-6]">>,
- ],
+    [{load_module,emqx_auth_jwt,brutal_purge,soft_purge,[]},
- [
+     {load_module,emqx_auth_jwt_svr,brutal_purge,soft_purge,[]}]},
-    {"4.3.0", [
+   {<<"4\\.3\\.[0-2]">>,[{restart_application,emqx_auth_jwt}]},
-      {load_module, emqx_auth_jwt_svr, brutal_purge, soft_purge, []}
+   {<<".*">>,[]}],
-    ]},
+  [{"4.3.8",[{load_module,emqx_auth_jwt,brutal_purge,soft_purge,[]}]},
-    {<<".*">>, []}
+   {"4.3.7",
- ]
+    [{load_module,emqx_auth_jwt_svr,brutal_purge,soft_purge,[]},
-}.
+     {load_module,emqx_auth_jwt,brutal_purge,soft_purge,[]}]},
+   {<<"4\\.3\\.[3-6]">>,
+    [{load_module,emqx_auth_jwt,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_jwt_svr,brutal_purge,soft_purge,[]}]},
+   {<<"4\\.3\\.[0-2]">>,[{restart_application,emqx_auth_jwt}]},
+   {<<".*">>,[]}]}.

apps/emqx_auth_jwt/src/emqx_auth_jwt.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -18,69 +18,124 @@
 
 -include_lib("emqx/include/emqx.hrl").
 -include_lib("emqx/include/logger.hrl").
+-include_lib("snabbkaffe/include/snabbkaffe.hrl").
 
 -logger_header("[JWT]").
 
--export([ register_metrics/0
+-export([ check_auth/3
         , check/3
+        , check_acl/5
         , description/0
         ]).
 
--record(auth_metrics, {
+-export([string_to_number/1]).
-        success = 'client.auth.success',
-        failure = 'client.auth.failure',
-        ignore = 'client.auth.ignore'
-    }).
-
--define(METRICS(Type), tl(tuple_to_list(#Type{}))).
--define(METRICS(Type, K), #Type{}#Type.K).
-
--define(AUTH_METRICS, ?METRICS(auth_metrics)).
--define(AUTH_METRICS(K), ?METRICS(auth_metrics, K)).
-
--spec(register_metrics() -> ok).
-register_metrics() ->
-    lists:foreach(fun emqx_metrics:ensure/1, ?AUTH_METRICS).
 
 %%--------------------------------------------------------------------
 %% Authentication callbacks
 %%--------------------------------------------------------------------
 
-check(ClientInfo, AuthResult, #{pid := Pid,
+%% for compatibility with old versions
-                                from := From,
+check(ClientInfo, AuthResult, State) ->
-                                checklists := Checklists}) ->
+    ?MODULE:check_auth(ClientInfo, AuthResult, State).
+
+check_auth(ClientInfo, AuthResult, #{from := From, checklists := Checklists}) ->
     case maps:find(From, ClientInfo) of
         error ->
-            ok = emqx_metrics:inc(?AUTH_METRICS(ignore));
+            ok;
         {ok, undefined} ->
-            ok = emqx_metrics:inc(?AUTH_METRICS(ignore));
+            ok;
         {ok, Token} ->
-            case emqx_auth_jwt_svr:verify(Pid, Token) of
+            case emqx_auth_jwt_svr:verify(Token) of
                 {error, not_found} ->
-                    ok = emqx_metrics:inc(?AUTH_METRICS(ignore));
+                    ok;
                 {error, not_token} ->
-                    ok = emqx_metrics:inc(?AUTH_METRICS(ignore));
+                    ok;
                 {error, Reason} ->
-                    ok = emqx_metrics:inc(?AUTH_METRICS(failure)),
                     {stop, AuthResult#{auth_result => Reason, anonymous => false}};
                 {ok, Claims} ->
                     {stop, maps:merge(AuthResult, verify_claims(Checklists, Claims, ClientInfo))}
             end
     end.
 
+check_acl(ClientInfo = #{jwt_claims := Claims},
+          PubSub,
+          Topic,
+          _NoMatchAction,
+          #{acl_claim_name := AclClaimName}) ->
+    case Claims of
+        #{AclClaimName := Acl, <<"exp">> := Exp} ->
+            case is_expired(Exp) of
+                true ->
+                    ?DEBUG("acl_deny_due_to_jwt_expired", []),
+                    {stop, deny};
+                false ->
+                    verify_acl(ClientInfo, Acl, PubSub, Topic)
+            end;
+        #{AclClaimName := Acl} ->
+            verify_acl(ClientInfo, Acl, PubSub, Topic);
+        _ ->
+            ?DEBUG("no_acl_jwt_claim", []),
+            ignore
+    end;
+check_acl(_ClientInfo, _PubSub, _Topic, _NoMatchAction, _Env) ->
+    ?tp(debug, no_jwt_claim, #{}),
+    ignore.
+
+is_expired(Exp) when is_binary(Exp)  ->
+    case string_to_number(Exp) of
+        {ok, Val} ->
+            is_expired(Val);
+        _ ->
+            ?DEBUG("acl_deny_due_to_invalid_jwt_exp:~p", [Exp]),
+            true
+    end;
+is_expired(Exp) when is_number(Exp) ->
+    Now = erlang:system_time(second),
+    Now > Exp;
+is_expired(Exp) ->
+    ?DEBUG("acl_deny_due_to_invalid_jwt_exp:~p", [Exp]),
+    true.
+
 description() -> "Authentication with JWT".
 
+string_to_number(Bin) when is_binary(Bin) ->
+    string_to_number(Bin, fun erlang:binary_to_integer/1, fun erlang:binary_to_float/1);
+string_to_number(Str) when is_list(Str) ->
+    string_to_number(Str, fun erlang:list_to_integer/1, fun erlang:list_to_float/1);
+string_to_number(_) ->
+    false.
+
 %%------------------------------------------------------------------------------
 %% Verify Claims
 %%--------------------------------------------------------------------
 
+verify_acl(ClientInfo, Acl, PubSub, Topic) ->
+    Key = case PubSub of
+              subscribe -> <<"sub">>;
+              publish -> <<"pub">>
+          end,
+    Rules0 = lists:map(
+               fun(K) ->
+                       case maps:get(K, Acl, undefined) of
+                           R when is_list(R) -> R;
+                           _ -> []
+                       end
+               end, [<<"all">>, Key]),
+    Rules = lists:append(Rules0),
+    verify_acl(ClientInfo, Rules, Topic).
+
+verify_acl(_ClientInfo, [], _Topic) -> {stop, deny};
+verify_acl(ClientInfo, [AclTopic | AclTopics], Topic) ->
+    case match_topic(ClientInfo, AclTopic, Topic) of
+        true -> {stop, allow};
+        false -> verify_acl(ClientInfo, AclTopics, Topic)
+    end.
+
 verify_claims(Checklists, Claims, ClientInfo) ->
     case do_verify_claims(feedvar(Checklists, ClientInfo), Claims) of
         {error, Reason} ->
-            ok = emqx_metrics:inc(?AUTH_METRICS(failure)),
             #{auth_result => Reason, anonymous => false};
         ok ->
-            ok = emqx_metrics:inc(?AUTH_METRICS(success)),
             #{auth_result => success, anonymous => false, jwt_claims => Claims}
     end.
 
@@ -97,3 +152,20 @@ feedvar(Checklists, #{username := Username, clientid := ClientId}) ->
                  ({K, <<"%c">>}) -> {K, ClientId};
                  ({K, Expected}) -> {K, Expected}
               end, Checklists).
+
+match_topic(ClientInfo, AclTopic, Topic) ->
+    AclTopicWords = emqx_topic:words(AclTopic),
+    TopicWords = emqx_topic:words(Topic),
+    AclTopicRendered = emqx_access_rule:feed_var(ClientInfo, AclTopicWords),
+    emqx_topic:match(TopicWords, AclTopicRendered).
+
+string_to_number(Str, IntFun, FloatFun) ->
+    try
+        {ok, IntFun(Str)}
+    catch _:_ ->
+        try
+            {ok, FloatFun(Str)}
+        catch _:_ ->
+            false
+        end
+    end.

apps/emqx_auth_jwt/src/emqx_auth_jwt_app.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -31,16 +31,19 @@
 start(_Type, _Args) ->
     {ok, Sup} = supervisor:start_link({local, ?MODULE}, ?MODULE, []),
 
-    {ok, Pid} = start_auth_server(jwks_svr_options()),
+    {ok, _} = start_auth_server(jwks_svr_options()),
-    ok = emqx_auth_jwt:register_metrics(),
-    AuthEnv0 = auth_env(),
-    AuthEnv1 = AuthEnv0#{pid => Pid},
 
-    _ = emqx:hook('client.authenticate', {emqx_auth_jwt, check, [AuthEnv1]}),
+    AuthEnv = auth_env(),
-    {ok, Sup, AuthEnv1}.
+    _ = emqx:hook('client.authenticate', {emqx_auth_jwt, check_auth, [AuthEnv]}),
 
-stop(AuthEnv) ->
+    AclEnv = acl_env(),
-    emqx:unhook('client.authenticate', {emqx_auth_jwt, check, [AuthEnv]}).
+    _ = emqx:hook('client.check_acl', {emqx_auth_jwt, check_acl, [AclEnv]}),
+
+    {ok, Sup}.
+
+stop(_State) ->
+    emqx:unhook('client.authenticate', {emqx_auth_jwt, check_auth}),
+    emqx:unhook('client.check_acl', {emqx_auth_jwt, check_acl}).
 
 %%--------------------------------------------------------------------
 %% Dummy supervisor
@@ -69,6 +72,9 @@ auth_env() ->
      , checklists => Checklists
      }.
 
+acl_env() ->
+    #{acl_claim_name => env(acl_claim_name, <<"acl">>)}.
+
 jwks_svr_options() ->
     [{K, V} || {K, V}
              <- [{secret, env(secret, undefined)},

apps/emqx_auth_jwt/src/emqx_auth_jwt_svr.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -20,13 +20,14 @@
 
 -include_lib("emqx/include/logger.hrl").
 -include_lib("jose/include/jose_jwk.hrl").
+-include_lib("snabbkaffe/include/snabbkaffe.hrl").
 
 -logger_header("[JWT-SVR]").
 
 %% APIs
 -export([start_link/1]).
 
--export([verify/2]).
+-export([verify/1, trace/2]).
 
 %% gen_server callbacks
 -export([ init/1
@@ -44,8 +45,9 @@
                 | {interval, pos_integer()}.
 
 -define(INTERVAL, 300000).
+-define(TAB, ?MODULE).
 
--record(state, {static, remote, addr, tref, intv}).
+-record(state, {addr, tref, intv}).
 
 %%--------------------------------------------------------------------
 %% APIs
@@ -55,13 +57,13 @@
 start_link(Options) ->
     gen_server:start_link(?MODULE, [Options], []).
 
--spec verify(pid(), binary())
+-spec verify(binary())
     -> {error, term()}
      | {ok, Payload :: map()}.
-verify(S, JwsCompacted) when is_binary(JwsCompacted) ->
+verify(JwsCompacted) when is_binary(JwsCompacted) ->
     case catch jose_jws:peek(JwsCompacted) of
         {'EXIT', _} -> {error, not_token};
-        _ -> gen_server:call(S, {verify, JwsCompacted})
+        _ -> do_verify(JwsCompacted)
     end.
 
 %%--------------------------------------------------------------------
@@ -70,43 +72,25 @@ verify(S, JwsCompacted) when is_binary(JwsCompacted) ->
 
 init([Options]) ->
     ok = jose:json_module(jiffy),
-    {Static, Remote} = do_init_jwks(Options),
+    _ = ets:new(?TAB, [set, protected, named_table]),
+    Static = do_init_jwks(Options),
+    to_request_jwks(Options),
+    true = ets:insert(?TAB, [{static, Static}, {remote, undefined}]),
     Intv = proplists:get_value(interval, Options, ?INTERVAL),
     {ok, reset_timer(
            #state{
-              static = Static,
-              remote = Remote,
               addr = proplists:get_value(jwks_addr, Options),
               intv = Intv})}.
 
 %% @private
 do_init_jwks(Options) ->
-    K2J = fun(K, F) ->
+    OctJwk = key2jwt_value(secret,
-              case proplists:get_value(K, Options) of
+                           fun(V) ->
-                  undefined -> undefined;
-                  V ->
-                     try F(V) of
-                         {error, Reason} ->
-                             ?LOG(warning, "Build ~p JWK ~p failed: {error, ~p}~n",
-                                  [K, V, Reason]),
-                             undefined;
-                         J -> J
-                     catch T:R:_ ->
-                         ?LOG(warning, "Build ~p JWK ~p failed: {~p, ~p}~n",
-                              [K, V, T, R]),
-                         undefined
-                     end
-              end
-          end,
-    OctJwk = K2J(secret, fun(V) ->
                                    jose_jwk:from_oct(list_to_binary(V))
-                         end),
+                           end,
-    PemJwk = K2J(pubkey, fun jose_jwk:from_pem_file/1),
+                           Options),
-    Remote = K2J(jwks_addr, fun request_jwks/1),
+    PemJwk = key2jwt_value(pubkey, fun jose_jwk:from_pem_file/1, Options),
-    {[J ||J <- [OctJwk, PemJwk], J /= undefined], Remote}.
+    [J ||J <- [OctJwk, PemJwk], J /= undefined].
-
-handle_call({verify, JwsCompacted}, _From, State) ->
-    handle_verify(JwsCompacted, State);
 
 handle_call(_Req, _From, State) ->
     {reply, ok, State}.
@@ -116,12 +100,18 @@ handle_cast(_Msg, State) ->
 
 handle_info({timeout, _TRef, refresh}, State = #state{addr = Addr}) ->
     NState = try
-                 State#state{remote = request_jwks(Addr)}
+                 true = ets:insert(?TAB, {remote, request_jwks(Addr)}),
+                 State
              catch _:_ ->
                  State
              end,
     {noreply, reset_timer(NState)};
 
+handle_info({request_jwks, Options}, State) ->
+    Remote = key2jwt_value(jwks_addr, fun request_jwks/1, Options),
+    true = ets:insert(?TAB, {remote, Remote}),
+    {noreply, State};
+
 handle_info(_Info, State) ->
     {noreply, State}.
 
@@ -136,24 +126,10 @@ code_change(_OldVsn, State, _Extra) ->
 %% Internal funcs
 %%--------------------------------------------------------------------
 
-handle_verify(JwsCompacted,
+keys(Type) ->
-              State = #state{static = Static, remote = Remote}) ->
+    case ets:lookup(?TAB, Type) of
-    try
+        [{_, Keys}] -> Keys;
-        Jwks = case emqx_json:decode(jose_jws:peek_protected(JwsCompacted), [return_maps]) of
+        [] -> []
-                   #{<<"kid">> := Kid} when Remote /= undefined ->
-                       [J || J <- Remote, maps:get(<<"kid">>, J#jose_jwk.fields, undefined) =:= Kid];
-                   _ -> Static
-               end,
-        case Jwks of
-            [] -> {reply, {error, not_found}, State};
-            _ ->
-                {reply, do_verify(JwsCompacted, Jwks), State}
-        end
-    catch
-        Class : Reason : Stk ->
-            ?LOG(error, "Handle JWK crashed: ~p, ~p, stacktrace: ~p~n",
-                        [Class, Reason, Stk]),
-            {reply, {error, invalid_signature}, State}
     end.
 
 request_jwks(Addr) ->
@@ -163,9 +139,12 @@ request_jwks(Addr) ->
         {ok, {_Code, _Headers, Body}} ->
             try
                 JwkSet = jose_jwk:from(emqx_json:decode(Body, [return_maps])),
-                {_, Jwks} = JwkSet#jose_jwk.keys, Jwks
+                {_, Jwks} = JwkSet#jose_jwk.keys,
+                ?tp(debug, emqx_auth_jwt_svr_jwks_updated, #{jwks => Jwks, pid => self()}),
+                Jwks
             catch _:_ ->
-                ?LOG(error, "Invalid jwks server response: ~p~n", [Body]),
+                ?MODULE:trace(jwks_server_reesponse, Body),
+                ?LOG(error, "Invalid jwks server response, body is not logged for security reasons, trace it if inspection is required", []),
                 error(badarg)
             end
     end.
@@ -181,6 +160,26 @@ cancel_timer(State = #state{tref = TRef}) ->
     _ = erlang:cancel_timer(TRef),
     State#state{tref = undefined}.
 
+do_verify(JwsCompacted) ->
+    try
+        Remote = keys(remote),
+        Jwks = case emqx_json:decode(jose_jws:peek_protected(JwsCompacted), [return_maps]) of
+                   #{<<"kid">> := Kid} when Remote /= undefined ->
+                       [J || J <- Remote, maps:get(<<"kid">>, J#jose_jwk.fields, undefined) =:= Kid];
+                   _ -> keys(static)
+               end,
+        case Jwks of
+            [] -> {error, not_found};
+            _ ->
+                do_verify(JwsCompacted, Jwks)
+        end
+    catch
+        Class : Reason : Stk ->
+            ?LOG_SENSITIVE(error, "verify JWK crashed: ~p, ~p, stacktrace: ~p~n",
+                        [Class, Reason, Stk]),
+            {error, invalid_signature}
+    end.
+
 do_verify(_JwsCompated, []) ->
     {error, invalid_signature};
 do_verify(JwsCompacted, [Jwk|More]) ->
@@ -190,7 +189,11 @@ do_verify(JwsCompacted, [Jwk|More]) ->
             case check_claims(Claims) of
                 {false, <<"exp">>} ->
                     {error, {invalid_signature, expired}};
-                NClaims ->
+                {false, <<"iat">>} ->
+                    {error, {invalid_signature, issued_in_future}};
+                {false, <<"nbf">>} ->
+                    {error, {invalid_signature, not_valid_yet}};
+                {true, NClaims} ->
                     {ok, NClaims}
             end;
         {false, _, _} ->
@@ -198,27 +201,64 @@ do_verify(JwsCompacted, [Jwk|More]) ->
     end.
 
 check_claims(Claims) ->
-    Now = os:system_time(seconds),
+    Now = erlang:system_time(seconds),
-    Checker = [{<<"exp">>, fun(ExpireTime) ->
+    Checker = [{<<"exp">>, with_num_value(
-                               Now < ExpireTime
+                             fun(ExpireTime) -> Now < ExpireTime end)},
-                           end},
+               {<<"iat">>, with_num_value(
-               {<<"iat">>, fun(IssueAt) ->
+                             fun(IssueAt) -> IssueAt =< Now end)},
-                               IssueAt =< Now
+               {<<"nbf">>, with_num_value(
-                           end},
+                             fun(NotBefore) -> NotBefore =< Now end)}
-               {<<"nbf">>, fun(NotBefore) ->
-                               NotBefore =< Now
-                           end}
               ],
     do_check_claim(Checker, Claims).
 
-do_check_claim([], Claims) ->
+with_num_value(Fun) ->
-    Claims;
+    fun(Value) ->
-do_check_claim([{K, F}|More], Claims) ->
+            case Value of
-    case maps:take(K, Claims) of
+                Num when is_number(Num) -> Fun(Num);
-        error -> do_check_claim(More, Claims);
+                Bin when is_binary(Bin) ->
-        {V, NClaims} ->
+                    case emqx_auth_jwt:string_to_number(Bin) of
-            case F(V) of
+                        {ok, Num} -> Fun(Num);
-                true -> do_check_claim(More, NClaims);
+                        _ -> false
-                _ -> {false, K}
+                    end;
+                Str when is_list(Str) ->
+                    case emqx_auth_jwt:string_to_number(Str) of
+                        {ok, Num} -> Fun(Num);
+                        _ -> false
+                    end
             end
     end.
+
+do_check_claim([], Claims) ->
+    {true, Claims};
+do_check_claim([{K, F}|More], Claims) ->
+    case Claims of
+        #{K := V} ->
+            case F(V) of
+                true -> do_check_claim(More, Claims);
+                _ -> {false, K}
+            end;
+        _ ->
+            do_check_claim(More, Claims)
+    end.
+
+to_request_jwks(Options) ->
+    erlang:send(self(), {request_jwks, Options}).
+
+key2jwt_value(Key, Func, Options) ->
+    case proplists:get_value(Key, Options) of
+        undefined -> undefined;
+        V ->
+            try Func(V) of
+                {error, Reason} ->
+                    ?LOG_SENSITIVE(warning, "Build ~p JWK ~p failed: {error, ~p}~n",
+                         [Key, V, Reason]),
+                    undefined;
+                J -> J
+            catch T:R ->
+                    ?LOG_SENSITIVE(warning, "Build ~p JWK ~p failed: {~p, ~p}~n",
+                         [Key, V, T, R]),
+                    undefined
+            end
+    end.
+
+trace(_Tag, _Data) -> ok.

apps/emqx_auth_jwt/test/emqx_auth_jwt_SUITE.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -19,29 +19,20 @@
 -compile(export_all).
 -compile(nowarn_export_all).
 
--include_lib("emqx/include/emqx.hrl").
+-include_lib("emqx/include/emqx_mqtt.hrl").
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("common_test/include/ct.hrl").
+-include_lib("snabbkaffe/include/snabbkaffe.hrl").
 
--define(APP, emqx_auth_jwt).
+all() -> emqx_ct:all(?MODULE).
 
-all() ->
+init_per_testcase(TestCase, Config) ->
-    [{group, emqx_auth_jwt}].
+    ?MODULE:TestCase(init, Config),
-
-groups() ->
-    [{emqx_auth_jwt, [sequence], [ t_check_auth
-                                 , t_check_claims
-                                 , t_check_claims_clientid
-                                 , t_check_claims_username
-                                 , t_check_claims_kid_in_header
-                                 ]}
-    ].
-
-init_per_suite(Config) ->
     emqx_ct_helpers:start_apps([emqx_auth_jwt], fun set_special_configs/1),
     Config.
 
-end_per_suite(_Config) ->
+end_per_testcase(TestCase, Config) ->
+    try ?MODULE:TestCase('end', Config) catch _:_ -> ok end,
     emqx_ct_helpers:stop_apps([emqx_auth_jwt]).
 
 set_special_configs(emqx) ->
@@ -78,11 +69,13 @@ sign(Payload, Alg, Key) ->
 %% Testcases
 %%------------------------------------------------------------------------------
 
-t_check_auth(_) ->
+t_check_auth(init, _Config) ->
+    application:unset_env(emqx_auth_jwt, verify_claims).
+t_check_auth(_Config) ->
     Plain = #{clientid => <<"client1">>, username => <<"plain">>, zone => external},
     Jwt = sign([{clientid, <<"client1">>},
                 {username, <<"plain">>},
-                {exp, os:system_time(seconds) + 3}], <<"HS256">>, <<"emqxsecret">>),
+                {exp, erlang:system_time(seconds) + 2}], <<"HS256">>, <<"emqxsecret">>),
     ct:pal("Jwt: ~p~n", [Jwt]),
 
     Result0 = emqx_access_control:authenticate(Plain#{password => Jwt}),
@@ -91,7 +84,7 @@ t_check_auth(_) ->
 
     ct:sleep(3100),
     Result1 = emqx_access_control:authenticate(Plain#{password => Jwt}),
-    ct:pal("Auth result after 1000ms: ~p~n", [Result1]),
+    ct:pal("Auth result after 3100ms: ~p~n", [Result1]),
     ?assertMatch({error, _}, Result1),
 
     Jwt_Error = sign([{client_id, <<"client1">>},
@@ -102,15 +95,121 @@ t_check_auth(_) ->
     ?assertEqual({error, invalid_signature}, Result2),
     ?assertMatch({error, _}, emqx_access_control:authenticate(Plain#{password => <<"asd">>})).
 
-t_check_claims(_) ->
+t_check_nbf(init, _Config) ->
-    application:set_env(emqx_auth_jwt, verify_claims, [{sub, <<"value">>}]),
+    application:unset_env(emqx_auth_jwt, verify_claims).
-    application:stop(emqx_auth_jwt), application:start(emqx_auth_jwt),
+t_check_nbf(_Config) ->
+    Plain = #{clientid => <<"client1">>, username => <<"plain">>, zone => external},
+    Jwt = sign([{clientid, <<"client1">>},
+                {username, <<"plain">>},
+                {nbf, erlang:system_time(seconds) + 3}], <<"HS256">>, <<"emqxsecret">>),
+    ct:pal("Jwt: ~p~n", [Jwt]),
 
+    Result0 = emqx_access_control:authenticate(Plain#{password => Jwt}),
+    ct:pal("Auth result: ~p~n", [Result0]),
+    ?assertEqual({error, {invalid_signature, not_valid_yet}}, Result0).
+
+t_check_iat(init, _Config) ->
+    application:unset_env(emqx_auth_jwt, verify_claims).
+t_check_iat(_Config) ->
+    Plain = #{clientid => <<"client1">>, username => <<"plain">>, zone => external},
+    Jwt = sign([{clientid, <<"client1">>},
+                {username, <<"plain">>},
+                {iat, erlang:system_time(seconds) + 3}], <<"HS256">>, <<"emqxsecret">>),
+    ct:pal("Jwt: ~p~n", [Jwt]),
+
+    Result0 = emqx_access_control:authenticate(Plain#{password => Jwt}),
+    ct:pal("Auth result: ~p~n", [Result0]),
+    ?assertEqual({error, {invalid_signature, issued_in_future}}, Result0).
+
+t_check_auth_invalid_exp(init, _Config) ->
+    application:unset_env(emqx_auth_jwt, verify_claims).
+t_check_auth_invalid_exp(_Config) ->
+    Plain = #{clientid => <<"client1">>, username => <<"plain">>, zone => external},
+    Jwt0 = sign([{clientid, <<"client1">>},
+                 {username, <<"plain">>},
+                 {exp, [{foo, bar}]}], <<"HS256">>, <<"emqxsecret">>),
+    ct:pal("Jwt: ~p~n", [Jwt0]),
+
+    Result0 = emqx_access_control:authenticate(Plain#{password => Jwt0}),
+    ct:pal("Auth result: ~p~n", [Result0]),
+    ?assertMatch({error, _}, Result0),
+
+    Jwt1 = sign([{clientid, <<"client1">>},
+                 {username, <<"plain">>},
+                 {exp, <<"foobar">>}], <<"HS256">>, <<"emqxsecret">>),
+    ct:pal("Jwt: ~p~n", [Jwt1]),
+
+    Result1 = emqx_access_control:authenticate(Plain#{password => Jwt1}),
+    ct:pal("Auth result: ~p~n", [Result1]),
+    ?assertMatch({error, _}, Result1).
+
+t_check_auth_str_exp(init, _Config) ->
+    application:unset_env(emqx_auth_jwt, verify_claims).
+t_check_auth_str_exp(_Config) ->
+    Plain = #{clientid => <<"client1">>, username => <<"plain">>, zone => external},
+    Exp = integer_to_binary(erlang:system_time(seconds) + 3),
+
+    Jwt0 = sign([{clientid, <<"client1">>},
+                 {username, <<"plain">>},
+                 {exp, Exp}], <<"HS256">>, <<"emqxsecret">>),
+    ct:pal("Jwt: ~p~n", [Jwt0]),
+
+    Result0 = emqx_access_control:authenticate(Plain#{password => Jwt0}),
+    ct:pal("Auth result: ~p~n", [Result0]),
+    ?assertMatch({ok, #{auth_result := success, jwt_claims := _}}, Result0),
+
+    Jwt1 = sign([{clientid, <<"client1">>},
+                 {username, <<"plain">>},
+                 {exp, <<"0">>}], <<"HS256">>, <<"emqxsecret">>),
+    ct:pal("Jwt: ~p~n", [Jwt1]),
+
+    Result1 = emqx_access_control:authenticate(Plain#{password => Jwt1}),
+    ct:pal("Auth result: ~p~n", [Result1]),
+    ?assertMatch({error, _}, Result1),
+
+    Exp2 = float_to_binary(erlang:system_time(seconds) + 3.5),
+
+    Jwt2 = sign([{clientid, <<"client1">>},
+                 {username, <<"plain">>},
+                 {exp, Exp2}], <<"HS256">>, <<"emqxsecret">>),
+    ct:pal("Jwt: ~p~n", [Jwt2]),
+
+    Result2 = emqx_access_control:authenticate(Plain#{password => Jwt2}),
+    ct:pal("Auth result: ~p~n", [Result2]),
+    ?assertMatch({ok, #{auth_result := success, jwt_claims := _}}, Result2).
+
+t_check_auth_float_exp(init, _Config) ->
+    application:unset_env(emqx_auth_jwt, verify_claims).
+t_check_auth_float_exp(_Config) ->
+    Plain = #{clientid => <<"client1">>, username => <<"plain">>, zone => external},
+    Exp = erlang:system_time(seconds) + 3.5,
+
+    Jwt0 = sign([{clientid, <<"client1">>},
+                 {username, <<"plain">>},
+                 {exp, Exp}], <<"HS256">>, <<"emqxsecret">>),
+    ct:pal("Jwt: ~p~n", [Jwt0]),
+
+    Result0 = emqx_access_control:authenticate(Plain#{password => Jwt0}),
+    ct:pal("Auth result: ~p~n", [Result0]),
+    ?assertMatch({ok, #{auth_result := success, jwt_claims := _}}, Result0),
+
+    Jwt1 = sign([{clientid, <<"client1">>},
+                 {username, <<"plain">>},
+                 {exp, 1.5}], <<"HS256">>, <<"emqxsecret">>),
+    ct:pal("Jwt: ~p~n", [Jwt1]),
+
+    Result1 = emqx_access_control:authenticate(Plain#{password => Jwt1}),
+    ct:pal("Auth result: ~p~n", [Result1]),
+    ?assertMatch({error, _}, Result1).
+
+t_check_claims(init, _Config) ->
+    application:set_env(emqx_auth_jwt, verify_claims, [{sub, <<"value">>}]).
+t_check_claims(_Config) ->
     Plain = #{clientid => <<"client1">>, username => <<"plain">>, zone => external},
     Jwt = sign([{client_id, <<"client1">>},
                 {username, <<"plain">>},
                 {sub, value},
-                {exp, os:system_time(seconds) + 3}], <<"HS256">>, <<"emqxsecret">>),
+                {exp, erlang:system_time(seconds) + 3}], <<"HS256">>, <<"emqxsecret">>),
     Result0 = emqx_access_control:authenticate(Plain#{password => Jwt}),
     ct:pal("Auth result: ~p~n", [Result0]),
     ?assertMatch({ok, #{auth_result := success, jwt_claims := _}}, Result0),
@@ -120,13 +219,13 @@ t_check_claims(_) ->
     ct:pal("Auth result for the invalid jwt: ~p~n", [Result2]),
     ?assertEqual({error, invalid_signature}, Result2).
 
-t_check_claims_clientid(_) ->
+t_check_claims_clientid(init, _Config) ->
-    application:set_env(emqx_auth_jwt, verify_claims, [{clientid, <<"%c">>}]),
+    application:set_env(emqx_auth_jwt, verify_claims, [{clientid, <<"%c">>}]).
-    application:stop(emqx_auth_jwt), application:start(emqx_auth_jwt),
+t_check_claims_clientid(_Config) ->
     Plain = #{clientid => <<"client23">>, username => <<"plain">>, zone => external},
     Jwt = sign([{clientid, <<"client23">>},
                 {username, <<"plain">>},
-                {exp, os:system_time(seconds) + 3}], <<"HS256">>, <<"emqxsecret">>),
+                {exp, erlang:system_time(seconds) + 3}], <<"HS256">>, <<"emqxsecret">>),
     Result0 = emqx_access_control:authenticate(Plain#{password => Jwt}),
     ct:pal("Auth result: ~p~n", [Result0]),
     ?assertMatch({ok, #{auth_result := success, jwt_claims := _}}, Result0),
@@ -136,14 +235,13 @@ t_check_claims_clientid(_) ->
     ct:pal("Auth result for the invalid jwt: ~p~n", [Result2]),
     ?assertEqual({error, invalid_signature}, Result2).
 
-t_check_claims_username(_) ->
+t_check_claims_username(init, _Config) ->
-    application:set_env(emqx_auth_jwt, verify_claims, [{username, <<"%u">>}]),
+    application:set_env(emqx_auth_jwt, verify_claims, [{username, <<"%u">>}]).
-    application:stop(emqx_auth_jwt), application:start(emqx_auth_jwt),
+t_check_claims_username(_Config) ->
-
     Plain = #{clientid => <<"client23">>, username => <<"plain">>, zone => external},
     Jwt = sign([{client_id, <<"client23">>},
                 {username, <<"plain">>},
-                {exp, os:system_time(seconds) + 3}], <<"HS256">>, <<"emqxsecret">>),
+                {exp, erlang:system_time(seconds) + 3}], <<"HS256">>, <<"emqxsecret">>),
     Result0 = emqx_access_control:authenticate(Plain#{password => Jwt}),
     ct:pal("Auth result: ~p~n", [Result0]),
     ?assertMatch({ok, #{auth_result := success, jwt_claims := _}}, Result0),
@@ -153,14 +251,264 @@ t_check_claims_username(_) ->
     ct:pal("Auth result for the invalid jwt: ~p~n", [Result3]),
     ?assertEqual({error, invalid_signature}, Result3).
 
-t_check_claims_kid_in_header(_) ->
+t_check_claims_kid_in_header(init, _Config) ->
-    application:set_env(emqx_auth_jwt, verify_claims, []),
+    application:set_env(emqx_auth_jwt, verify_claims, []).
+t_check_claims_kid_in_header(_Config) ->
     Plain = #{clientid => <<"client23">>, username => <<"plain">>, zone => external},
     Jwt = sign([{clientid, <<"client23">>},
                 {username, <<"plain">>},
-                {exp, os:system_time(seconds) + 3}],
+                {exp, erlang:system_time(seconds) + 3}],
                #{<<"alg">> => <<"HS256">>,
                  <<"kid">> => <<"a_kid_str">>}, <<"emqxsecret">>),
     Result0 = emqx_access_control:authenticate(Plain#{password => Jwt}),
     ct:pal("Auth result: ~p~n", [Result0]),
     ?assertMatch({ok, #{auth_result := success, jwt_claims := _}}, Result0).
+
+t_keys_update(init, _Config) ->
+    ok = meck:new(httpc, [passthrough, no_history]),
+    ok = meck:expect(
+           httpc,
+           request,
+           fun(get, _, _, _) ->
+                   {ok,
+                    {200,
+                     [],
+                     jiffy:encode(#{<<"keys">> => []})}}
+           end),
+
+    application:set_env(emqx_auth_jwt, verify_claims, []),
+    application:set_env(emqx_auth_jwt, refresh_interval, 100),
+    application:set_env(emqx_auth_jwt, jwks, "http://localhost:4001/keys.json").
+t_keys_update(_Config) ->
+    ?check_trace(
+       snabbkaffe:block_until(
+         ?match_n_events(2, #{?snk_kind := emqx_auth_jwt_svr_jwks_updated}),
+         _Timeout    = infinity,
+         _BackInTIme = 0),
+       fun(_, Trace) ->
+               ?assertMatch([#{pid := Pid}, #{pid := Pid} |  _],
+                            ?of_kind(emqx_auth_jwt_svr_jwks_updated, Trace))
+       end).
+
+t_check_jwt_acl(init, _Config) ->
+    application:set_env(emqx_auth_jwt, verify_claims, [{sub, <<"value">>}]).
+t_check_jwt_acl(_Config) ->
+    Jwt = sign([{client_id, <<"client1">>},
+                {username, <<"plain">>},
+                {sub, value},
+                {acl, [{sub, [<<"a/b">>]},
+                       {pub, [<<"c/d">>]},
+                       {all, [<<"all">>]}]},
+                {exp, erlang:system_time(seconds) + 10}],
+               <<"HS256">>,
+               <<"emqxsecret">>),
+
+    {ok, C} = emqtt:start_link(
+                [{clean_start, true},
+                 {proto_ver, v5},
+                 {client_id, <<"client1">>},
+                 {password, Jwt}]),
+    {ok, _} = emqtt:connect(C),
+
+    ?assertMatch(
+       {ok, #{}, [0]},
+       emqtt:subscribe(C, <<"a/b">>, 0)),
+
+    ?assertMatch(
+       ok,
+       emqtt:publish(C, <<"c/d">>, <<"hi">>, 0)),
+
+    ?assertMatch(
+       {ok, #{}, [?RC_NOT_AUTHORIZED]},
+       emqtt:subscribe(C, <<"c/d">>, 0)),
+
+    ok = emqtt:publish(C, <<"a/b">>, <<"hi">>, 0),
+
+    receive
+        {publish, #{topic := <<"a/b">>}} ->
+            ?assert(false, "Publish to `a/b` should not be allowed")
+    after 100 -> ok
+    end,
+
+    %% can pub/sub to all rules
+    ?assertMatch(
+      {ok, #{}, [0]},
+      emqtt:subscribe(C, <<"all">>, 0)),
+
+    ?assertMatch(
+      ok,
+      emqtt:publish(C, <<"all">>, <<"hi">>, 0)),
+    receive
+        {publish, #{topic := <<"all">>}} -> ok
+    after 2000 ->
+              ?assert(false, "Publish to `all` should be allowed")
+    end,
+    ok = emqtt:disconnect(C).
+
+t_check_jwt_acl_no_recs(init, _Config) ->
+    application:set_env(emqx_auth_jwt, verify_claims, [{sub, <<"value">>}]).
+t_check_jwt_acl_no_recs(_Config) ->
+    Jwt = sign([{client_id, <<"client1">>},
+                {username, <<"plain">>},
+                {sub, value},
+                {acl, []},
+                {exp, erlang:system_time(seconds) + 10}],
+               <<"HS256">>,
+               <<"emqxsecret">>),
+
+    {ok, C} = emqtt:start_link(
+                [{clean_start, true},
+                 {proto_ver, v5},
+                 {client_id, <<"client1">>},
+                 {password, Jwt}]),
+    {ok, _} = emqtt:connect(C),
+
+    ?assertMatch(
+       {ok, #{}, [?RC_NOT_AUTHORIZED]},
+       emqtt:subscribe(C, <<"a/b">>, 0)),
+
+    ok = emqtt:disconnect(C).
+
+t_check_jwt_acl_no_acl_claim(init, _Config) ->
+    application:set_env(emqx_auth_jwt, verify_claims, [{sub, <<"value">>}]).
+t_check_jwt_acl_no_acl_claim(_Config) ->
+    Jwt = sign([{client_id, <<"client1">>},
+                {username, <<"plain">>},
+                {sub, value},
+                {exp, erlang:system_time(seconds) + 10}],
+               <<"HS256">>,
+               <<"emqxsecret">>),
+
+    {ok, C} = emqtt:start_link(
+                [{clean_start, true},
+                 {proto_ver, v5},
+                 {client_id, <<"client1">>},
+                 {password, Jwt}]),
+    {ok, _} = emqtt:connect(C),
+
+    ?assertMatch(
+       {ok, #{}, [?RC_NOT_AUTHORIZED]},
+       emqtt:subscribe(C, <<"a/b">>, 0)),
+
+    ok = emqtt:disconnect(C).
+
+t_check_jwt_acl_no_jwt_claims_helper(_ClientInfo, _LastAuthResult) ->
+    {stop, #{auth_result => success, anonymous => false}}.
+t_check_jwt_acl_no_jwt_claims(init, _Config) ->
+    ok;
+t_check_jwt_acl_no_jwt_claims('end', _Config) ->
+    ok = emqx_hooks:del(
+            'client.authenticate',
+            {?MODULE, t_check_jwt_acl_no_jwt_claims_helper, []}
+          ).
+t_check_jwt_acl_no_jwt_claims(_Config) ->
+    %% bypass the jwt authentication checking
+    ok = emqx_hooks:add(
+            'client.authenticate',
+            {?MODULE, t_check_jwt_acl_no_jwt_claims_helper, []},
+            _Priority = 99999
+          ),
+
+    {ok, C} = emqtt:start_link(
+                [{clean_start, true},
+                 {proto_ver, v5},
+                 {client_id, <<"client1">>},
+                 {username, <<"client1">>},
+                 {password, <<"password">>}]),
+    {ok, _} = emqtt:connect(C),
+
+    ok = snabbkaffe:start_trace(),
+
+    ?assertMatch(
+       {ok, #{}, [?RC_NOT_AUTHORIZED]},
+       emqtt:subscribe(C, <<"a/b">>, 0)),
+
+    {ok, _} = ?block_until(#{?snk_kind := no_jwt_claim}, 1000),
+    Trace = snabbkaffe:collect_trace(),
+    ?assertEqual(1, length(?of_kind(no_jwt_claim, Trace))),
+
+    snabbkaffe:stop(),
+    ok = emqtt:disconnect(C).
+
+t_check_jwt_acl_expire(init, _Config) ->
+    application:set_env(emqx_auth_jwt, verify_claims, [{sub, <<"value">>}]).
+t_check_jwt_acl_expire(_Config) ->
+    Jwt = sign([{client_id, <<"client1">>},
+                {username, <<"plain">>},
+                {sub, value},
+                {acl, [{sub, [<<"a/b">>]}]},
+                {exp, erlang:system_time(seconds) + 1}],
+               <<"HS256">>,
+               <<"emqxsecret">>),
+
+    {ok, C} = emqtt:start_link(
+                [{clean_start, true},
+                 {proto_ver, v5},
+                 {client_id, <<"client1">>},
+                 {password, Jwt}]),
+    {ok, _} = emqtt:connect(C),
+
+    ?assertMatch(
+       {ok, #{}, [0]},
+       emqtt:subscribe(C, <<"a/b">>, 0)),
+
+    ?assertMatch(
+       {ok, #{}, [0]},
+       emqtt:unsubscribe(C, <<"a/b">>)),
+
+    timer:sleep(2000),
+
+    ?assertMatch(
+       {ok, #{}, [?RC_NOT_AUTHORIZED]},
+       emqtt:subscribe(C, <<"a/b">>, 0)),
+
+    Default = emqx_zone:get_env(external, acl_nomatch, deny),
+    emqx_zone:set_env(external, acl_nomatch, allow),
+    try
+        ?assertMatch(
+           {ok, #{}, [?RC_NOT_AUTHORIZED]},
+           emqtt:subscribe(C, <<"a/b">>, 0))
+    after
+        emqx_zone:set_env(external, acl_nomatch, Default)
+    end,
+
+    ok = emqtt:disconnect(C).
+
+t_check_jwt_acl_no_exp(init, _Config) ->
+    application:set_env(emqx_auth_jwt, verify_claims, [{sub, <<"value">>}]).
+t_check_jwt_acl_no_exp(_Config) ->
+    Jwt = sign([{client_id, <<"client1">>},
+                {username, <<"plain">>},
+                {sub, value},
+                {acl, [{sub, [<<"a/b">>]}]}],
+               <<"HS256">>,
+               <<"emqxsecret">>),
+
+    {ok, C} = emqtt:start_link(
+                [{clean_start, true},
+                 {proto_ver, v5},
+                 {client_id, <<"client1">>},
+                 {password, Jwt}]),
+    {ok, _} = emqtt:connect(C),
+
+    ?assertMatch(
+       {ok, #{}, [0]},
+       emqtt:subscribe(C, <<"a/b">>, 0)),
+
+    ok = emqtt:disconnect(C).
+
+t_check_compatibility(init, _Config) -> ok.
+t_check_compatibility(_Config) ->
+
+    %% We literary want emqx_auth_jwt:check call emqx_auth_jwt:check_auth, so check with meck
+
+    ok = meck:new(emqx_auth_jwt, [passthrough, no_history]),
+    ok = meck:expect(emqx_auth_jwt, check_auth, fun(a, b, c) -> ok end),
+
+    ?assertEqual(
+       ok,
+       emqx_auth_jwt:check(a, b, c)
+      ),
+
+    meck:validate(emqx_auth_jwt),
+    meck:unload(emqx_auth_jwt).

apps/emqx_auth_ldap/README.md

@@ -1,7 +1,7 @@
 emqx_auth_ldap
 ==============
 
-EMQ X LDAP Authentication Plugin
+EMQX LDAP Authentication Plugin
 
 Build
 -----
@@ -92,5 +92,5 @@ Apache License Version 2.0
 Author
 ------
 
-EMQ X Team.
+EMQX Team.
 

apps/emqx_auth_ldap/include/emqx_auth_ldap.hrl

@@ -1,23 +1 @@
-
 -define(APP, emqx_auth_ldap).
-
--record(auth_metrics, {
-        success = 'client.auth.success',
-        failure = 'client.auth.failure',
-        ignore = 'client.auth.ignore'
-    }).
-
--record(acl_metrics, {
-        allow = 'client.acl.allow',
-        deny = 'client.acl.deny',
-        ignore = 'client.acl.ignore'
-    }).
-
--define(METRICS(Type), tl(tuple_to_list(#Type{}))).
--define(METRICS(Type, K), #Type{}#Type.K).
-
--define(AUTH_METRICS, ?METRICS(auth_metrics)).
--define(AUTH_METRICS(K), ?METRICS(auth_metrics, K)).
-
--define(ACL_METRICS, ?METRICS(acl_metrics)).
--define(ACL_METRICS(K), ?METRICS(acl_metrics, K)).

apps/emqx_auth_ldap/priv/emqx_auth_ldap.schema

@@ -73,6 +73,7 @@
                  {verify, cuttlefish:conf_get("auth.ldap.ssl.verify", Conf, undefined)},
                  {server_name_indication, case cuttlefish:conf_get("auth.ldap.ssl.server_name_indication", Conf, undefined) of
                                             "disable" -> disable;
+                                            "" -> undefined;
                                             SNI -> SNI
                                           end}]
               end,

apps/emqx_auth_ldap/src/emqx_acl_ldap.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -22,24 +22,15 @@
 -include_lib("eldap/include/eldap.hrl").
 -include_lib("emqx/include/logger.hrl").
 
--export([ register_metrics/0
+-export([ check_acl/5
-        , check_acl/5
         , description/0
         ]).
 
--import(proplists, [get_value/2]).
-
--import(emqx_auth_ldap_cli, [search/4]).
-
--spec(register_metrics() -> ok).
-register_metrics() ->
-    lists:foreach(fun emqx_metrics:ensure/1, ?ACL_METRICS).
-
 check_acl(ClientInfo, PubSub, Topic, NoMatchAction, State) ->
     case do_check_acl(ClientInfo, PubSub, Topic, NoMatchAction, State) of
-        ok -> emqx_metrics:inc(?ACL_METRICS(ignore)), ok;
+        ok -> ok;
-        {stop, allow} -> emqx_metrics:inc(?ACL_METRICS(allow)), {stop, allow};
+        {stop, allow} -> {stop, allow};
-        {stop, deny} -> emqx_metrics:inc(?ACL_METRICS(deny)), {stop, deny}
+        {stop, deny} -> {stop, deny}
     end.
 
 do_check_acl(#{username := <<$$, _/binary>>}, _PubSub, _Topic, _NoMatchAction, _State) ->
@@ -70,14 +61,14 @@ do_check_acl(#{username := Username}, PubSub, Topic, _NoMatchAction,
 
     BaseDN = emqx_auth_ldap:replace_vars(CustomBaseDN, ReplaceRules),
 
-    case search(Pool, BaseDN, Filter, [Attribute, Attribute1]) of
+    case emqx_auth_ldap_cli:search(Pool, BaseDN, Filter, [Attribute, Attribute1]) of
         {error, noSuchObject} ->
             ok;
         {ok, #eldap_search_result{entries = []}} ->
             ok;
         {ok, #eldap_search_result{entries = [Entry]}} ->
-            Topics = get_value(Attribute, Entry#eldap_entry.attributes)
+            Topics = proplists:get_value(Attribute, Entry#eldap_entry.attributes, [])
-                ++ get_value(Attribute1, Entry#eldap_entry.attributes),
+                ++ proplists:get_value(Attribute1, Entry#eldap_entry.attributes, []),
             match(Topic, Topics);
         Error ->
             ?LOG(error, "[LDAP] search error:~p", [Error]),
@@ -95,4 +86,3 @@ match(Topic, [Filter | Topics]) ->
 
 description() ->
     "ACL with LDAP".
-

apps/emqx_auth_ldap/src/emqx_auth_ldap.app.src

@@ -1,6 +1,6 @@
 {application, emqx_auth_ldap,
  [{description, "EMQ X Authentication/ACL with LDAP"},
-  {vsn, "4.3.0"}, % strict semver, bump manually!
+  {vsn, "4.3.6"}, % strict semver, bump manually!
   {modules, []},
   {registered, [emqx_auth_ldap_sup]},
   {applications, [kernel,stdlib,eldap2,ecpool]},

apps/emqx_auth_ldap/src/emqx_auth_ldap.appup.src

@@ -0,0 +1,39 @@
+%% -*- mode: erlang -*-
+%% Unless you know what you are doing, DO NOT edit manually!!
+{VSN,
+  [{"4.3.5",
+    [{load_module,emqx_auth_ldap,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_ldap_cli,brutal_purge,soft_purge,[]}]},
+   {<<"4\\.3\\.[3-4]">>,
+    [{load_module,emqx_auth_ldap_cli,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_ldap_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_ldap,brutal_purge,soft_purge,[]}]},
+   {"4.3.2",
+    [{load_module,emqx_auth_ldap_cli,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_ldap_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_ldap,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_ldap,brutal_purge,soft_purge,[]}]},
+   {<<"4\\.3\\.[0-1]">>,
+    [{load_module,emqx_auth_ldap_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_ldap,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_ldap,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_ldap_cli,brutal_purge,soft_purge,[]}]},
+   {<<".*">>,[]}],
+  [{"4.3.5",
+    [{load_module,emqx_auth_ldap,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_ldap_cli,brutal_purge,soft_purge,[]}]},
+   {<<"4\\.3\\.[3-4]">>,
+    [{load_module,emqx_auth_ldap_cli,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_ldap_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_ldap,brutal_purge,soft_purge,[]}]},
+   {"4.3.2",
+    [{load_module,emqx_auth_ldap_cli,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_ldap_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_ldap,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_ldap,brutal_purge,soft_purge,[]}]},
+   {<<"4\\.3\\.[0-1]">>,
+    [{load_module,emqx_auth_ldap_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_ldap,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_ldap,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_ldap_cli,brutal_purge,soft_purge,[]}]},
+   {<<".*">>,[]}]}.

apps/emqx_auth_ldap/src/emqx_auth_ldap.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -26,17 +26,12 @@
 
 -import(emqx_auth_ldap_cli, [search/3]).
 
--export([ register_metrics/0
+-export([ check/3
-        , check/3
         , description/0
         , prepare_filter/4
         , replace_vars/2
         ]).
 
--spec(register_metrics() -> ok).
-register_metrics() ->
-    lists:foreach(fun emqx_metrics:ensure/1, ?AUTH_METRICS).
-
 check(ClientInfo = #{username := Username, password := Password}, AuthResult,
       State = #{password_attr := PasswdAttr, bind_as_user := BindAsUserRequired, pool := Pool}) ->
     CheckResult =
@@ -63,13 +58,11 @@ check(ClientInfo = #{username := Username, password := Password}, AuthResult,
         end,
     case CheckResult of
         ok ->
-            ok = emqx_metrics:inc(?AUTH_METRICS(success)),
             {stop, AuthResult#{auth_result => success, anonymous => false}};
         {error, not_found} ->
-            emqx_metrics:inc(?AUTH_METRICS(ignore));
+            ok;
         {error, ResultCode} ->
-            ok = emqx_metrics:inc(?AUTH_METRICS(failure)),
+            ?LOG_SENSITIVE(error, "[LDAP] Auth from ldap failed: ~p", [ResultCode]),
-            ?LOG(error, "[LDAP] Auth from ldap failed: ~p", [ResultCode]),
             {stop, AuthResult#{auth_result => ResultCode, anonymous => false}}
     end.
 

apps/emqx_auth_ldap/src/emqx_auth_ldap_app.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -49,12 +49,10 @@ stop(_State) ->
     ok.
 
 load_auth_hook(DeviceDn) ->
-    ok = emqx_auth_ldap:register_metrics(),
     Params = maps:from_list(DeviceDn),
     emqx:hook('client.authenticate', fun emqx_auth_ldap:check/3, [Params#{pool => ?APP}]).
 
 load_acl_hook(DeviceDn) ->
-    ok = emqx_acl_ldap:register_metrics(),
     Params = maps:from_list(DeviceDn),
     emqx:hook('client.check_acl', fun emqx_acl_ldap:check_acl/5 , [Params#{pool => ?APP}]).
 

apps/emqx_auth_ldap/src/emqx_auth_ldap_cli.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -54,30 +54,30 @@ connect(Opts) ->
                        false ->
                            [{port, Port}, {timeout, Timeout}]
                    end,
-    ?LOG(debug, "[LDAP] Connecting to OpenLDAP server: ~p, Opts:~p ...", [Servers, LdapOpts]),
+    ?LOG_SENSITIVE(debug, "[LDAP] Connecting to OpenLDAP server: ~p, Opts:~p ...", [Servers, LdapOpts]),
 
     case eldap2:open(Servers, LdapOpts) of
         {ok, LDAP} ->
             try eldap2:simple_bind(LDAP, BindDn, BindPassword) of
                 ok -> {ok, LDAP};
                 {error, Error} ->
-                    ?LOG(error, "[LDAP] Can't authenticated to OpenLDAP server: ~p", [Error]),
+                    ?LOG_SENSITIVE(error, "[LDAP] Can't authenticated to OpenLDAP server: ~p", [Error]),
                     {error, Error}
             catch
                 error:Reason ->
-                    ?LOG(error, "[LDAP] Can't authenticated to OpenLDAP server: ~p", [Reason]),
+                    ?LOG_SENSITIVE(error, "[LDAP] Can't authenticated to OpenLDAP server: ~p", [Reason]),
                     {error, Reason}
             end;
         {error, Reason} ->
-            ?LOG(error, "[LDAP] Can't connect to OpenLDAP server: ~p", [Reason]),
+            ?LOG_SENSITIVE(error, "[LDAP] Can't connect to OpenLDAP server: ~p", [Reason]),
             {error, Reason}
     end.
 
 search(Pool, Base, Filter) ->
     ecpool:with_client(Pool,
         fun(C) ->
-                case application:get_env(?APP, bind_as_user) of
+                case application:get_env(?APP, bind_as_user, false) of
-                    {ok, true} ->
+                    true ->
                         {ok, Opts} = application:get_env(?APP, ldap),
                         BindDn       = get_value(bind_dn, Opts),
                         BindPassword = get_value(bind_password, Opts),
@@ -91,7 +91,7 @@ search(Pool, Base, Filter) ->
                         catch
                             error:Reason -> {error, Reason}
                         end;
-                    {ok, false} ->
+                    false ->
                         eldap2:search(C, [{base, Base},
                                           {filter, Filter},
                                           {deref, eldap2:derefFindingBaseObj()}])
@@ -101,8 +101,8 @@ search(Pool, Base, Filter) ->
 search(Pool, Base, Filter, Attributes) ->
     ecpool:with_client(Pool,
         fun(C) ->
-                case application:get_env(?APP, bind_as_user) of
+                case application:get_env(?APP, bind_as_user, false) of
-                    {ok, true} ->
+                    true ->
                         {ok, Opts} = application:get_env(?APP, ldap),
                         BindDn       = get_value(bind_dn, Opts),
                         BindPassword = get_value(bind_password, Opts),
@@ -117,7 +117,7 @@ search(Pool, Base, Filter, Attributes) ->
                         catch
                             error:Reason -> {error, Reason}
                         end;
-                    {ok, false} ->
+                    false ->
                         eldap2:search(C, [{base, Base},
                                           {filter, Filter},
                                           {attributes, Attributes},
@@ -147,4 +147,3 @@ init_args(ENVS) ->
            match_objectclass => ObjectClass,
            username_attr => UidAttr,
            password_attr => PasswdAttr}}.
-

apps/emqx_auth_ldap/src/emqx_auth_ldap_sup.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.

apps/emqx_auth_ldap/test/emqx_auth_ldap_SUITE.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -48,7 +48,9 @@ init_per_group(GrpName, Cfg) ->
     Cfg.
 
 end_per_group(_GrpName, _Cfg) ->
-    emqx_ct_helpers:stop_apps([emqx_auth_ldap]).
+    emqx_ct_helpers:stop_apps([emqx_auth_ldap]),
+    %% clear the application envs to avoid cross-suite testcase failure
+    application:unload(emqx_auth_ldap).
 
 %%--------------------------------------------------------------------
 %% Cases

apps/emqx_auth_ldap/test/emqx_auth_ldap_bind_as_user_SUITE.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -40,7 +40,9 @@ init_per_suite(Config) ->
     Config.
 
 end_per_suite(_Config) ->
-    emqx_ct_helpers:stop_apps([emqx_auth_ldap]).
+    emqx_ct_helpers:stop_apps([emqx_auth_ldap]),
+    %% clear the application envs to avoid cross-suite testcase failure
+    application:unload(emqx_auth_ldap).
 
 check_auth(_) ->
     MqttUser1 = #{clientid => <<"mqttuser1">>,

apps/emqx_auth_mnesia/include/emqx_auth_mnesia.hrl

@@ -1,38 +1,49 @@
 -define(APP, emqx_auth_mnesia).
 
--type(login():: {clientid, binary()}
+-type(login() :: {clientid, binary()}
               | {username, binary()}).
 
+-type(acl_target() :: login() | all).
+
+-type(acl_target_type() :: clientid | username | all).
+
+-type(access():: allow | deny).
+-type(action():: pub | sub).
+-type(legacy_action():: action() | pubsub).
+-type(created_at():: integer()).
+
 -record(emqx_user, {
+          login,
+          password,
+          created_at
+        }).
+
+-type(emqx_user() :: #emqx_user{
           login :: login(),
           password :: binary(),
-          created_at :: integer()
+          created_at :: created_at()
         }).
 
--record(emqx_acl, {
+-define(ACL_TABLE, emqx_acl).
-          filter:: {login() | all, emqx_topic:topic()},
+
-          action :: pub | sub | pubsub,
+-define(MIGRATION_MARK_KEY, emqx_acl2_migration_started).
-          access :: allow | deny,
+
-          created_at :: integer()
+-record(?ACL_TABLE, {
+          filter :: {acl_target(), emqx_topic:topic()} | ?MIGRATION_MARK_KEY,
+          action :: legacy_action(),
+          access :: access(),
+          created_at :: created_at()
          }).
 
--record(auth_metrics, {
+-define(MIGRATION_MARK_RECORD, #?ACL_TABLE{filter = ?MIGRATION_MARK_KEY, action = pub, access = deny, created_at = 0}).
-        success = 'client.auth.success',
+
-        failure = 'client.auth.failure',
+-type(rule() :: {access(), action(), emqx_topic:topic(), created_at()}).
-        ignore = 'client.auth.ignore'
+
+-define(ACL_TABLE2, emqx_acl2).
+
+-record(?ACL_TABLE2, {
+          who :: acl_target(),
+          rules :: [ rule() ]
          }).
 
--record(acl_metrics, {
+-type(acl_record() :: {acl_target(), emqx_topic:topic(), action(), access(), created_at()}).
-        allow = 'client.acl.allow',
-        deny = 'client.acl.deny',
-        ignore = 'client.acl.ignore'
-    }).
-
--define(METRICS(Type), tl(tuple_to_list(#Type{}))).
--define(METRICS(Type, K), #Type{}#Type.K).
-
--define(AUTH_METRICS, ?METRICS(auth_metrics)).
--define(AUTH_METRICS(K), ?METRICS(auth_metrics, K)).
-
--define(ACL_METRICS, ?METRICS(acl_metrics)).
--define(ACL_METRICS(K), ?METRICS(acl_metrics, K)).

apps/emqx_auth_mnesia/src/emqx_acl_mnesia.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -18,51 +18,35 @@
 
 -include("emqx_auth_mnesia.hrl").
 
--include_lib("stdlib/include/ms_transform.hrl").
-
--define(TABLE, emqx_acl).
-
 %% ACL Callbacks
 -export([ init/0
-        , register_metrics/0
         , check_acl/5
         , description/0
        ]).
 
 init() ->
-    ok = ekka_mnesia:create_table(emqx_acl, [
+    ok = emqx_acl_mnesia_db:create_table(),
-            {type, bag},
+    ok = emqx_acl_mnesia_db:create_table2().
-            {disc_copies, [node()]},
-            {attributes, record_info(fields, emqx_acl)},
-            {storage_properties, [{ets, [{read_concurrency, true}]}]}]),
-    ok = ekka_mnesia:copy_table(emqx_acl, disc_copies).
-
--spec(register_metrics() -> ok).
-register_metrics() ->
-    lists:foreach(fun emqx_metrics:ensure/1, ?ACL_METRICS).
 
 check_acl(ClientInfo = #{ clientid := Clientid }, PubSub, Topic, _NoMatchAction, _Params) ->
     Username = maps:get(username, ClientInfo, undefined),
 
     Acls = case Username of
                undefined ->
-                   emqx_acl_mnesia_cli:lookup_acl({clientid, Clientid}) ++
+                   emqx_acl_mnesia_db:lookup_acl({clientid, Clientid}) ++
-                   emqx_acl_mnesia_cli:lookup_acl(all);
+                   emqx_acl_mnesia_db:lookup_acl(all);
                _ ->
-                   emqx_acl_mnesia_cli:lookup_acl({clientid, Clientid}) ++
+                   emqx_acl_mnesia_db:lookup_acl({clientid, Clientid}) ++
-                   emqx_acl_mnesia_cli:lookup_acl({username, Username}) ++
+                   emqx_acl_mnesia_db:lookup_acl({username, Username}) ++
-                   emqx_acl_mnesia_cli:lookup_acl(all)
+                   emqx_acl_mnesia_db:lookup_acl(all)
            end,
 
     case match(ClientInfo, PubSub, Topic, Acls) of
         allow ->
-            emqx_metrics:inc(?ACL_METRICS(allow)),
             {stop, allow};
         deny ->
-            emqx_metrics:inc(?ACL_METRICS(deny)),
             {stop, deny};
         _ ->
-            emqx_metrics:inc(?ACL_METRICS(ignore)),
             ok
     end.
 
@@ -83,7 +67,6 @@ match(ClientInfo, PubSub, Topic, [ {_, ACLTopic, Action, Access, _} | Acls]) ->
 match_topic(ClientInfo, Topic, ACLTopic) when is_binary(Topic) ->
     emqx_topic:match(Topic, feed_var(ClientInfo, ACLTopic)).
 
-match_actions(_, pubsub) -> true;
 match_actions(subscribe, sub) -> true;
 match_actions(publish, pub) -> true;
 match_actions(_, _) -> false.

apps/emqx_auth_mnesia/src/emqx_acl_mnesia_api.erl

@@ -1,5 +1,5 @@
 %c%--------------------------------------------------------------------
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -16,8 +16,6 @@
 
 -module(emqx_acl_mnesia_api).
 
--include("emqx_auth_mnesia.hrl").
-
 -include_lib("stdlib/include/ms_transform.hrl").
 
 -import(proplists, [ get_value/2
@@ -98,27 +96,29 @@
         , delete/2
         ]).
 
+-define(CLIENTID_SCHEMA, [{<<"clientid">>, binary}, {<<"_like_clientid">>, binary}] ++ ?COMMON_SCHEMA).
+-define(USERNAME_SCHEMA, [{<<"username">>, binary}, {<<"_like_username">>, binary}] ++ ?COMMON_SCHEMA).
+-define(COMMON_SCHEMA, [{<<"topic">>, binary}, {<<"action">>, atom}, {<<"access">>, atom}]).
+
 list_clientid(_Bindings, Params) ->
-    MatchSpec = ets:fun2ms(
+    {_, Params1 = {_Qs, _Fuzzy}} = emqx_mgmt_api:params2qs(Params, ?CLIENTID_SCHEMA),
-                  fun({emqx_acl, {{clientid, Clientid}, Topic}, Action, Access, CreatedAt}) -> {{clientid,Clientid}, Topic, Action,Access, CreatedAt} end),
+    Table = emqx_acl_mnesia_db:login_acl_table(clientid, Params1),
-    return({ok, emqx_auth_mnesia_api:paginate(emqx_acl, MatchSpec, Params, fun emqx_acl_mnesia_cli:comparing/2, fun format/1)}).
+    return({ok, paginate_qh(Table, count(Table), Params, fun emqx_acl_mnesia_db:comparing/2, fun format/1)}).
 
 list_username(_Bindings, Params) ->
-    MatchSpec = ets:fun2ms(
+    {_, Params1 = {_Qs, _Fuzzy}} = emqx_mgmt_api:params2qs(Params, ?USERNAME_SCHEMA),
-                  fun({emqx_acl, {{username, Username}, Topic}, Action, Access, CreatedAt}) -> {{username, Username}, Topic, Action,Access, CreatedAt} end),
+    Table = emqx_acl_mnesia_db:login_acl_table(username, Params1),
-    return({ok, emqx_auth_mnesia_api:paginate(emqx_acl, MatchSpec, Params, fun emqx_acl_mnesia_cli:comparing/2, fun format/1)}).
+    return({ok, paginate_qh(Table, count(Table), Params, fun emqx_acl_mnesia_db:comparing/2, fun format/1)}).
 
 list_all(_Bindings, Params) ->
-    MatchSpec = ets:fun2ms(
+    {_, Params1 = {_Qs, _Fuzzy}} = emqx_mgmt_api:params2qs(Params, ?COMMON_SCHEMA),
-                  fun({emqx_acl, {all, Topic}, Action, Access, CreatedAt}) -> {all, Topic, Action,Access, CreatedAt}end
+    Table = emqx_acl_mnesia_db:login_acl_table(all, Params1),
-                 ),
+    return({ok, paginate_qh(Table, count(Table), Params, fun emqx_acl_mnesia_db:comparing/2, fun format/1)}).
-    return({ok, emqx_auth_mnesia_api:paginate(emqx_acl, MatchSpec, Params, fun emqx_acl_mnesia_cli:comparing/2, fun format/1)}).
-
 
 lookup(#{clientid := Clientid}, _Params) ->
-    return({ok, format(emqx_acl_mnesia_cli:lookup_acl({clientid, urldecode(Clientid)}))});
+    return({ok, format(emqx_acl_mnesia_db:lookup_acl({clientid, urldecode(Clientid)}))});
 lookup(#{username := Username}, _Params) ->
-    return({ok, format(emqx_acl_mnesia_cli:lookup_acl({username, urldecode(Username)}))}).
+    return({ok, format(emqx_acl_mnesia_db:lookup_acl({username, urldecode(Username)}))}).
 
 add(_Bindings, Params) ->
     [ P | _] = Params,
@@ -144,15 +144,15 @@ do_add(Params) ->
     Username = get_value(<<"username">>, Params, undefined),
     Login = case {Clientid, Username} of
                 {undefined, undefined} -> all;
-                {_, undefined} -> {clientid, urldecode(Clientid)};
+                {_, undefined} -> {clientid, Clientid};
-                {undefined, _} -> {username, urldecode(Username)}
+                {undefined, _} -> {username, Username}
             end,
-    Topic = urldecode(get_value(<<"topic">>, Params)),
+    Topic = get_value(<<"topic">>, Params),
-    Action = urldecode(get_value(<<"action">>, Params)),
+    Action = get_value(<<"action">>, Params),
-    Access = urldecode(get_value(<<"access">>, Params)),
+    Access = get_value(<<"access">>, Params),
     Re = case validate([login, topic, action, access], [Login, Topic, Action, Access]) of
         ok ->
-            emqx_acl_mnesia_cli:add_acl(Login, Topic, erlang:binary_to_atom(Action, utf8), erlang:binary_to_atom(Access, utf8));
+            emqx_acl_mnesia_db:add_acl(Login, Topic, erlang:binary_to_atom(Action, utf8), erlang:binary_to_atom(Access, utf8));
         Err -> Err
     end,
     maps:merge(#{topic => Topic,
@@ -165,15 +165,23 @@ do_add(Params) ->
                    end).
 
 delete(#{clientid := Clientid, topic := Topic}, _) ->
-    return(emqx_acl_mnesia_cli:remove_acl({clientid, urldecode(Clientid)}, urldecode(Topic)));
+    return(emqx_acl_mnesia_db:remove_acl({clientid, urldecode(Clientid)}, urldecode(Topic)));
 delete(#{username := Username, topic := Topic}, _) ->
-    return(emqx_acl_mnesia_cli:remove_acl({username, urldecode(Username)}, urldecode(Topic)));
+    return(emqx_acl_mnesia_db:remove_acl({username, urldecode(Username)}, urldecode(Topic)));
 delete(#{topic := Topic}, _) ->
-    return(emqx_acl_mnesia_cli:remove_acl(all, urldecode(Topic))).
+    return(emqx_acl_mnesia_db:remove_acl(all, urldecode(Topic))).
 
 %%------------------------------------------------------------------------------
 %% Interval Funcs
 %%------------------------------------------------------------------------------
+
+count(QH) ->
+    Count = qlc:fold(fun(_, Sum) -> Sum + 1 end, 0, QH),
+    case is_integer(Count) of
+        true -> Count;
+        false -> 0
+    end.
+
 format({{clientid, Clientid}, Topic, Action, Access, _CreatedAt}) ->
     #{clientid => Clientid, topic => Topic, action => Action, access => Access};
 format({{username, Username}, Topic, Action, Access, _CreatedAt}) ->
@@ -224,3 +232,27 @@ format_msg(Message) when is_tuple(Message) ->
 
 urldecode(S) ->
     emqx_http_lib:uri_decode(S).
+
+paginate_qh(Qh, Count, Params, ComparingFun, RowFun) ->
+    Page = page(Params),
+    Limit = limit(Params),
+    Cursor = qlc:cursor(Qh),
+    case Page > 1 of
+        true  ->
+            _ = qlc:next_answers(Cursor, (Page - 1) * Limit),
+            ok;
+        false -> ok
+    end,
+    Rows = qlc:next_answers(Cursor, Limit),
+    qlc:delete_cursor(Cursor),
+    #{meta  => #{page => Page, limit => Limit, count => Count},
+        data  => [RowFun(Row) || Row <- lists:sort(ComparingFun, Rows)]}.
+
+page(Params) ->
+    binary_to_integer(proplists:get_value(<<"_page">>, Params, <<"1">>)).
+
+limit(Params) ->
+    case proplists:get_value(<<"_limit">>, Params) of
+        undefined -> 50;
+        Size      -> binary_to_integer(Size)
+    end.

apps/emqx_auth_mnesia/src/emqx_acl_mnesia_cli.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -16,110 +16,28 @@
 
 -module(emqx_acl_mnesia_cli).
 
--include("emqx_auth_mnesia.hrl").
--include_lib("emqx/include/logger.hrl").
--include_lib("stdlib/include/ms_transform.hrl").
--define(TABLE, emqx_acl).
-
-%% Acl APIs
--export([ add_acl/4
-        , lookup_acl/1
-        , all_acls/0
-        , all_acls/1
-        , remove_acl/2
-        ]).
-
 -export([cli/1]).
--export([comparing/2]).
-%%--------------------------------------------------------------------
-%% Acl API
-%%--------------------------------------------------------------------
-
-%% @doc Add Acls
--spec(add_acl(login() | all, emqx_topic:topic(), pub | sub | pubsub, allow | deny) ->
-        ok | {error, any()}).
-add_acl(Login, Topic, Action, Access) ->
-    Filter = {Login, Topic},
-    Acl = #?TABLE{
-             filter = Filter,
-             action = Action,
-             access = Access,
-             created_at = erlang:system_time(millisecond)
-            },
-    ret(mnesia:transaction(
-          fun() ->
-                  OldRecords = mnesia:wread({?TABLE, Filter}),
-                  case Action of
-                      pubsub ->
-                          update_permission(pub, Acl, OldRecords),
-                          update_permission(sub, Acl, OldRecords);
-                      _ ->
-                          update_permission(Action, Acl, OldRecords)
-                  end
-          end)).
-
-%% @doc Lookup acl by login
--spec(lookup_acl(login() | all) -> list()).
-lookup_acl(undefined) -> [];
-lookup_acl(Login) ->
-    MatchSpec = ets:fun2ms(fun({?TABLE, {Filter, ACLTopic}, Action, Access, CreatedAt})
-                                 when Filter =:= Login ->
-                                   {Filter, ACLTopic, Action, Access, CreatedAt}
-                           end),
-    lists:sort(fun comparing/2, ets:select(?TABLE, MatchSpec)).
-
-%% @doc Remove acl
--spec(remove_acl(login() | all, emqx_topic:topic()) -> ok | {error, any()}).
-remove_acl(Login, Topic) ->
-    ret(mnesia:transaction(fun mnesia:delete/1, [{?TABLE, {Login, Topic}}])).
-
-%% @doc All logins
--spec(all_acls() -> list()).
-all_acls() ->
-    all_acls(clientid) ++
-    all_acls(username) ++
-    all_acls(all).
-
-all_acls(clientid) ->
-    MatchSpec = ets:fun2ms(
-                  fun({?TABLE, {{clientid, Clientid}, Topic}, Action, Access, CreatedAt}) ->
-                          {{clientid, Clientid}, Topic, Action, Access, CreatedAt}
-                  end),
-    lists:sort(fun comparing/2, ets:select(?TABLE, MatchSpec));
-all_acls(username) ->
-    MatchSpec = ets:fun2ms(
-                  fun({?TABLE, {{username, Username}, Topic}, Action, Access, CreatedAt}) ->
-                          {{username, Username}, Topic, Action, Access, CreatedAt}
-                  end),
-    lists:sort(fun comparing/2, ets:select(?TABLE, MatchSpec));
-all_acls(all) ->
-    MatchSpec = ets:fun2ms(
-                  fun({?TABLE, {all, Topic}, Action, Access, CreatedAt}) ->
-                          {all, Topic, Action, Access, CreatedAt}
-                  end
-                 ),
-    lists:sort(fun comparing/2, ets:select(?TABLE, MatchSpec)).
 
 %%--------------------------------------------------------------------
 %% ACL Cli
 %%--------------------------------------------------------------------
 
 cli(["list"]) ->
-    [print_acl(Acl) || Acl <- all_acls()];
+    [print_acl(Acl) || Acl <- emqx_acl_mnesia_db:all_acls()];
 
 cli(["list", "clientid"]) ->
-    [print_acl(Acl) || Acl <- all_acls(clientid)];
+    [print_acl(Acl) || Acl <- emqx_acl_mnesia_db:all_acls(clientid)];
 
 cli(["list", "username"]) ->
-    [print_acl(Acl) || Acl <- all_acls(username)];
+    [print_acl(Acl) || Acl <- emqx_acl_mnesia_db:all_acls(username)];
 
 cli(["list", "_all"]) ->
-    [print_acl(Acl) || Acl <- all_acls(all)];
+    [print_acl(Acl) || Acl <- emqx_acl_mnesia_db:all_acls(all)];
 
 cli(["add", "clientid", Clientid, Topic, Action, Access]) ->
     case validate(action, Action) andalso validate(access, Access) of
         true ->
-            case add_acl(
+            case emqx_acl_mnesia_db:add_acl(
                    {clientid, iolist_to_binary(Clientid)},
                    iolist_to_binary(Topic),
                    list_to_existing_atom(Action),
@@ -135,7 +53,7 @@ cli(["add", "clientid", Clientid, Topic, Action, Access]) ->
 cli(["add", "username", Username, Topic, Action, Access]) ->
     case validate(action, Action) andalso validate(access, Access) of
         true ->
-            case add_acl(
+            case emqx_acl_mnesia_db:add_acl(
                    {username, iolist_to_binary(Username)},
                    iolist_to_binary(Topic),
                    list_to_existing_atom(Action),
@@ -151,7 +69,7 @@ cli(["add", "username", Username, Topic, Action, Access]) ->
 cli(["add", "_all", Topic, Action, Access]) ->
     case validate(action, Action) andalso validate(access, Access) of
         true ->
-            case add_acl(
+            case emqx_acl_mnesia_db:add_acl(
                    all,
                    iolist_to_binary(Topic),
                    list_to_existing_atom(Action),
@@ -165,16 +83,16 @@ cli(["add", "_all", Topic, Action, Access]) ->
     end;
 
 cli(["show", "clientid", Clientid]) ->
-    [print_acl(Acl) || Acl <- lookup_acl({clientid, iolist_to_binary(Clientid)})];
+    [print_acl(Acl) || Acl <- emqx_acl_mnesia_db:lookup_acl({clientid, iolist_to_binary(Clientid)})];
 
 cli(["show", "username", Username]) ->
-    [print_acl(Acl) || Acl <- lookup_acl({username, iolist_to_binary(Username)})];
+    [print_acl(Acl) || Acl <- emqx_acl_mnesia_db:lookup_acl({username, iolist_to_binary(Username)})];
 
 cli(["del", "clientid", Clientid, Topic])->
     cli(["delete", "clientid", Clientid, Topic]);
 
 cli(["delete", "clientid", Clientid, Topic])->
-    case remove_acl({clientid, iolist_to_binary(Clientid)}, iolist_to_binary(Topic)) of
+    case emqx_acl_mnesia_db:remove_acl({clientid, iolist_to_binary(Clientid)}, iolist_to_binary(Topic)) of
          ok -> emqx_ctl:print("ok~n");
         {error, Reason} -> emqx_ctl:print("Error: ~p~n", [Reason])
     end;
@@ -183,7 +101,7 @@ cli(["del", "username", Username, Topic])->
     cli(["delete", "username", Username, Topic]);
 
 cli(["delete", "username", Username, Topic])->
-    case remove_acl({username, iolist_to_binary(Username)}, iolist_to_binary(Topic)) of
+    case emqx_acl_mnesia_db:remove_acl({username, iolist_to_binary(Username)}, iolist_to_binary(Topic)) of
          ok -> emqx_ctl:print("ok~n");
         {error, Reason} -> emqx_ctl:print("Error: ~p~n", [Reason])
     end;
@@ -192,7 +110,7 @@ cli(["del", "_all", Topic])->
     cli(["delete", "_all", Topic]);
 
 cli(["delete", "_all", Topic])->
-    case remove_acl(all, iolist_to_binary(Topic)) of
+    case emqx_acl_mnesia_db:remove_acl(all, iolist_to_binary(Topic)) of
          ok -> emqx_ctl:print("ok~n");
         {error, Reason} -> emqx_ctl:print("Error: ~p~n", [Reason])
     end;
@@ -201,10 +119,11 @@ cli(_) ->
     emqx_ctl:usage([ {"acl list clientid", "List clientid acls"}
                    , {"acl list username", "List username acls"}
                    , {"acl list _all", "List $all acls"}
+                   , {"acl list ", "List all acls"}
                    , {"acl show clientid <Clientid>", "Lookup clientid acl detail"}
                    , {"acl show username <Username>", "Lookup username acl detail"}
-                   , {"acl aad clientid <Clientid> <Topic> <Action> <Access>", "Add clientid acl"}
+                   , {"acl add clientid <Clientid> <Topic> <Action> <Access>", "Add clientid acl"}
-                   , {"acl add Username <Username> <Topic> <Action> <Access>", "Add username acl"}
+                   , {"acl add username <Username> <Topic> <Action> <Access>", "Add username acl"}
                    , {"acl add _all <Topic> <Action> <Access>", "Add $all acl"}
                    , {"acl delete clientid <Clientid> <Topic>", "Delete clientid acl"}
                    , {"acl delete username <Username> <Topic>", "Delete username acl"}
@@ -215,13 +134,6 @@ cli(_) ->
 %% Internal functions
 %%--------------------------------------------------------------------
 
-comparing({_, _, _, _, CreatedAt1},
-          {_, _, _, _, CreatedAt2}) ->
-    CreatedAt1 >= CreatedAt2.
-
-ret({atomic, ok})     -> ok;
-ret({aborted, Error}) -> {error, Error}.
-
 validate(action, "pub") -> true;
 validate(action, "sub") -> true;
 validate(action, "pubsub") -> true;
@@ -244,27 +156,3 @@ print_acl({all, Topic, Action, Access, _}) ->
         "Acl($all topic = ~p action = ~p access = ~p)~n",
         [Topic, Action, Access]
      ).
-
-update_permission(Action, Acl0, OldRecords) ->
-    Acl = Acl0 #?TABLE{action = Action},
-    maybe_delete_shadowed_records(Action, OldRecords),
-    mnesia:write(Acl).
-
-maybe_delete_shadowed_records(_, []) ->
-    ok;
-maybe_delete_shadowed_records(Action1, [Rec = #emqx_acl{action = Action2} | Rest]) ->
-    if Action1 =:= Action2 ->
-            ok = mnesia:delete_object(Rec);
-       Action2 =:= pubsub ->
-            %% Perform migration from the old data format on the
-            %% fly. This is needed only for the enterprise version,
-            %% delete this branch on 5.0
-            mnesia:delete_object(Rec),
-            mnesia:write(Rec#?TABLE{action = other_action(Action1)});
-       true ->
-            ok
-    end,
-    maybe_delete_shadowed_records(Action1, Rest).
-
-other_action(pub) -> sub;
-other_action(sub) -> pub.

apps/emqx_auth_mnesia/src/emqx_acl_mnesia_db.erl

@@ -0,0 +1,388 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_acl_mnesia_db).
+
+-include("emqx_auth_mnesia.hrl").
+-include_lib("stdlib/include/ms_transform.hrl").
+-include_lib("stdlib/include/qlc.hrl").
+
+%% ACL APIs
+-export([ create_table/0
+        , create_table2/0
+        ]).
+
+-export([ add_acl/4
+        , lookup_acl/1
+        , all_acls_export/0
+        , all_acls/0
+        , all_acls/1
+        , remove_acl/2
+        , merge_acl_records/3
+        , login_acl_table/1
+        , login_acl_table/2
+        , is_migration_started/0
+        ]).
+
+-export([comparing/2]).
+
+%%--------------------------------------------------------------------
+%% ACL API
+%%--------------------------------------------------------------------
+
+%% @doc Create table `emqx_acl` of old format rules
+-spec(create_table() -> ok).
+create_table() ->
+    ok = ekka_mnesia:create_table(?ACL_TABLE, [
+        {type, bag},
+        {disc_copies, [node()]},
+        {attributes, record_info(fields, ?ACL_TABLE)},
+        {storage_properties, [{ets, [{read_concurrency, true}]}]}]),
+    ok = ekka_mnesia:copy_table(?ACL_TABLE, disc_copies).
+
+%% @doc Create table `emqx_acl2` of new format rules
+-spec(create_table2() -> ok).
+create_table2() ->
+    ok = ekka_mnesia:create_table(?ACL_TABLE2, [
+            {type, ordered_set},
+            {disc_copies, [node()]},
+            {attributes, record_info(fields, ?ACL_TABLE2)},
+            {storage_properties, [{ets, [{read_concurrency, true}]}]}]),
+    ok = ekka_mnesia:copy_table(?ACL_TABLE2, disc_copies).
+
+%% @doc Add Acls
+-spec(add_acl(acl_target(), emqx_topic:topic(), legacy_action(), access()) ->
+        ok | {error, any()}).
+add_acl(Login, Topic, Action, Access) ->
+    ret(mnesia:transaction(fun() ->
+                              case is_migration_started() of
+                                  true -> add_acl_new(Login, Topic, Action, Access);
+                                  false -> add_acl_old(Login, Topic, Action, Access)
+                              end
+                           end)).
+
+%% @doc Lookup acl by login
+-spec(lookup_acl(acl_target()) -> list(acl_record())).
+lookup_acl(undefined) -> [];
+lookup_acl(Login) ->
+    % After migration to ?ACL_TABLE2, ?ACL_TABLE never has any rules. This lookup should be removed later.
+    MatchSpec = ets:fun2ms(fun(#?ACL_TABLE{filter = {Filter, _}} = Rec)
+                                 when Filter =:= Login -> Rec
+                           end),
+    OldRecs = ets:select(?ACL_TABLE, MatchSpec),
+
+    NewAcls = ets:lookup(?ACL_TABLE2, Login),
+    MergedAcl = merge_acl_records(Login, OldRecs, NewAcls),
+    lists:sort(fun comparing/2, acl_to_list(MergedAcl)).
+
+%% @doc Remove ACL
+-spec remove_acl(acl_target(), emqx_topic:topic()) -> ok | {error, any()}.
+remove_acl(Login, Topic) ->
+    ret(mnesia:transaction(fun() ->
+                              mnesia:delete({?ACL_TABLE, {Login, Topic}}),
+                              case mnesia:wread({?ACL_TABLE2, Login}) of
+                                  [] -> ok;
+                                  [#?ACL_TABLE2{rules = Rules} = Acl] ->
+                                      case delete_topic_rules(Topic, Rules) of
+                                          [] -> mnesia:delete({?ACL_TABLE2, Login});
+                                          [_ | _] = RemainingRules ->
+                                              mnesia:write(Acl#?ACL_TABLE2{rules = RemainingRules})
+                                      end
+                              end
+                           end)).
+
+%% @doc All ACL rules
+-spec(all_acls() -> list(acl_record())).
+all_acls() ->
+    all_acls(username) ++
+    all_acls(clientid) ++
+    all_acls(all).
+
+%% @doc All ACL rules of specified type
+-spec(all_acls(acl_target_type()) -> list(acl_record())).
+all_acls(AclTargetType) ->
+    lists:sort(fun comparing/2, qlc:eval(login_acl_table(AclTargetType))).
+
+%% @doc All ACL rules fetched transactionally
+-spec(all_acls_export() -> list(acl_record())).
+all_acls_export() ->
+    AclTargetTypes = [username, clientid, all],
+    MatchSpecNew = lists:flatmap(fun login_match_spec_new/1, AclTargetTypes),
+    MatchSpecOld = lists:flatmap(fun login_match_spec_old/1, AclTargetTypes),
+
+    {atomic, Records} = mnesia:transaction(
+        fun() ->
+            QH = acl_table(MatchSpecNew, MatchSpecOld, {#{}, #{}}, fun mnesia:table/2, fun lookup_mnesia/2),
+            qlc:eval(QH)
+        end),
+    Records.
+
+%% @doc QLC table of logins matching spec
+-spec(login_acl_table(acl_target_type()) -> qlc:query_handle()).
+login_acl_table(AclTargetType) ->
+    login_acl_table(AclTargetType, {[], []}).
+
+login_acl_table(AclTargetType, {Qs, Fuzzy}) ->
+    ToMap = fun({Type, Symbol, Val}, Acc) -> Acc#{{Type, Symbol} => Val} end,
+    Qs1 = lists:foldl(ToMap, #{}, Qs),
+    Fuzzy1 = lists:foldl(ToMap, #{}, Fuzzy),
+    MatchSpecNew = login_match_spec_new(AclTargetType, Qs1),
+    MatchSpecOld = login_match_spec_old(AclTargetType, Qs1),
+    acl_table(MatchSpecNew, MatchSpecOld, {Qs1, Fuzzy1}, fun ets:table/2, fun lookup_ets/2).
+
+%% @doc Combine old `emqx_acl` ACL records with a new `emqx_acl2` ACL record for a given login
+-spec(merge_acl_records(acl_target(), [#?ACL_TABLE{}], [#?ACL_TABLE2{}]) -> #?ACL_TABLE2{}).
+merge_acl_records(Login, OldRecs, Acls) ->
+    OldRules = old_recs_to_rules(OldRecs),
+    NewRules = case Acls of
+        [] -> [];
+        [#?ACL_TABLE2{rules = Rules}] -> Rules
+    end,
+    #?ACL_TABLE2{who = Login, rules = merge_rules(NewRules, OldRules)}.
+
+%% @doc Checks if background migration of ACL rules from `emqx_acl` to `emqx_acl2` format started.
+%% Should be run in transaction
+-spec(is_migration_started() -> boolean()).
+is_migration_started() ->
+    case mnesia:read({?ACL_TABLE, ?MIGRATION_MARK_KEY}) of
+        [?MIGRATION_MARK_RECORD | _] -> true;
+        [] -> false
+    end.
+
+%%--------------------------------------------------------------------
+%% Internal functions
+%%--------------------------------------------------------------------
+
+add_acl_new(Login, Topic, Action, Access) ->
+    Rule = {Access, Action, Topic, erlang:system_time(millisecond)},
+    Rules = normalize_rule(Rule),
+    OldAcl = mnesia:wread({?ACL_TABLE2, Login}),
+    NewAcl = case OldAcl of
+        [#?ACL_TABLE2{rules = OldRules} = Acl] ->
+            Acl#?ACL_TABLE2{rules = merge_rules(Rules, OldRules)};
+        [] ->
+            #?ACL_TABLE2{who = Login, rules = Rules}
+    end,
+    mnesia:write(NewAcl).
+
+add_acl_old(Login, Topic, Action, Access) ->
+    Filter = {Login, Topic},
+    Acl = #?ACL_TABLE{
+             filter = Filter,
+             action = Action,
+             access = Access,
+             created_at = erlang:system_time(millisecond)
+            },
+    OldRecords = mnesia:wread({?ACL_TABLE, Filter}),
+    case Action of
+        pubsub ->
+            update_permission(pub, Acl, OldRecords),
+            update_permission(sub, Acl, OldRecords);
+        _ ->
+            update_permission(Action, Acl, OldRecords)
+    end.
+
+old_recs_to_rules(OldRecs) ->
+    lists:flatmap(fun old_rec_to_rules/1, OldRecs).
+
+old_rec_to_rules(#?ACL_TABLE{filter = {_, Topic}, action = Action, access = Access, created_at = CreatedAt}) ->
+    normalize_rule({Access, Action, Topic, CreatedAt}).
+
+normalize_rule({Access, pubsub, Topic, CreatedAt}) ->
+    [{Access, pub, Topic, CreatedAt}, {Access, sub, Topic, CreatedAt}];
+normalize_rule({Access, Action, Topic, CreatedAt}) ->
+    [{Access, Action, Topic, CreatedAt}].
+
+merge_rules([], OldRules) -> OldRules;
+merge_rules([NewRule | RestNewRules], OldRules) ->
+    merge_rules(RestNewRules, merge_rule(NewRule, OldRules)).
+
+merge_rule({_, Action, Topic, _ } = NewRule, OldRules) ->
+    [NewRule | lists:filter(
+        fun({_, OldAction, OldTopic, _}) ->
+            {Action, Topic} =/= {OldAction, OldTopic}
+        end, OldRules)].
+
+acl_to_list(#?ACL_TABLE2{who = Login, rules = Rules}) ->
+    [{Login, Topic, Action, Access, CreatedAt} || {Access, Action, Topic, CreatedAt} <- Rules].
+
+delete_topic_rules(Topic, Rules) ->
+    [Rule || {_, _, T, _} = Rule <- Rules, T =/= Topic].
+
+comparing({_, _, _, _, CreatedAt} = Rec1,
+          {_, _, _, _, CreatedAt} = Rec2) ->
+        Rec1 >= Rec2;
+
+comparing({_, _, _, _, CreatedAt1},
+          {_, _, _, _, CreatedAt2}) ->
+    CreatedAt1 >= CreatedAt2.
+
+login_match_spec_old(Type) -> login_match_spec_old(Type, #{}).
+
+login_match_spec_old(all, _) ->
+    ets:fun2ms(fun(#?ACL_TABLE{filter = {all, _}} = Record) ->
+                Record
+               end);
+
+login_match_spec_old(Type, Params) when (Type =:= username) orelse (Type =:= clientid) ->
+    case maps:get({Type, '=:='}, Params, undefined) of
+        undefined ->
+            ets:fun2ms(fun(#?ACL_TABLE{filter = {{RType, _}, _}} = Rec) when RType =:= Type -> Rec end);
+        Val ->
+            ets:fun2ms(fun(#?ACL_TABLE{filter = {{RType, RVal}, _}} = Rec)
+                when RType =:= Type andalso RVal =:= Val -> Rec end)
+    end.
+
+login_match_spec_new(Type) -> login_match_spec_new(Type, #{}).
+
+login_match_spec_new(all, _) ->
+    ets:fun2ms(fun(#?ACL_TABLE2{who = all} = Record) ->
+                Record
+               end);
+
+login_match_spec_new(Type, Params) when (Type =:= username) orelse (Type =:= clientid) ->
+    case maps:get({Type, '=:='}, Params, undefined) of
+        undefined ->
+            ets:fun2ms(fun(#?ACL_TABLE2{who = {RType, _}} = Rec) when RType =:= Type -> Rec end);
+        Val ->
+            ets:fun2ms(fun(#?ACL_TABLE2{who = {RType, RVal}} = Rec)
+                when RType =:= Type andalso RVal =:= Val  -> Rec end)
+    end.
+
+acl_table(MatchSpecNew, MatchSpecOld, Params, TableFun, LookupFun) ->
+    TraverseFun =
+        fun() ->
+           CursorNew =
+               qlc:cursor(
+                   TableFun(?ACL_TABLE2, [{traverse, {select, MatchSpecNew}}])),
+           CursorOld =
+               qlc:cursor(
+                   TableFun(?ACL_TABLE, [{traverse, {select, MatchSpecOld}}])),
+           traverse_new(CursorNew, CursorOld, Params, #{}, LookupFun)
+        end,
+
+    qlc:table(TraverseFun, []).
+
+
+% These are traverse funs for qlc table created by `acl_table/4`.
+% Traversing consumes memory: it collects logins present in `?ACL_TABLE` and
+% at the same time having rules in `?ACL_TABLE2`.
+% Such records appear if ACLs are inserted before migration started.
+% After migration, number of such logins is zero, so traversing starts working in
+% constant memory.
+
+traverse_new(CursorNew, CursorOld, Params, FoundKeys, LookupFun) ->
+    Acls = qlc:next_answers(CursorNew, 1),
+    case Acls of
+        [] ->
+            qlc:delete_cursor(CursorNew),
+            traverse_old(CursorOld, Params, FoundKeys);
+        [#?ACL_TABLE2{who = Login, rules = Rules} = Acl] ->
+            Keys = lists:usort([{Login, Topic} || {_, _, Topic, _} <- Rules]),
+            OldRecs = lists:flatmap(fun(Key) -> LookupFun(?ACL_TABLE, Key) end, Keys),
+            MergedAcl = merge_acl_records(Login, OldRecs, [Acl]),
+            NewFoundKeys =
+                lists:foldl(fun(#?ACL_TABLE{filter = Key}, Found) -> maps:put(Key, true, Found) end,
+                            FoundKeys,
+                            OldRecs),
+            case acl_to_list(MergedAcl) of
+                [] ->
+                    traverse_new(CursorNew, CursorOld, Params, NewFoundKeys, LookupFun);
+                List ->
+                    filter_params(List, Params) ++
+                    fun() -> traverse_new(CursorNew, CursorOld, Params, NewFoundKeys, LookupFun) end
+            end
+    end.
+
+filter_params(List, {Qs, Fuzzy}) ->
+    case maps:size(Qs) =:= 0 andalso maps:size(Fuzzy) =:= 0 of
+        false ->
+            Topic = maps:get({topic, '=:='}, Qs, undefined),
+            Action = maps:get({action, '=:='}, Qs, undefined),
+            Access = maps:get({access, '=:='}, Qs, undefined),
+            lists:filter(fun({Target, Topic0, Action0, Access0, _CreatedAt}) ->
+                CheckList = [{Topic, Topic0}, {Action, Action0}, {Access, Access0}],
+                case lists:all(fun is_match/1, CheckList) of
+                    true ->
+                        case Target of
+                            {Type, Login} ->
+                                case maps:get({Type, 'like'}, Fuzzy, <<>>) of
+                                    <<>> -> true;
+                                    LikeSchema -> binary:match(Login, LikeSchema) =/= nomatch
+                                end;
+                            all -> true
+                        end;
+                    false -> false
+                end
+                         end, List);
+        true -> List
+    end.
+
+is_match({Schema, Val}) ->
+    Schema =:= undefined orelse Schema =:= Val.
+
+traverse_old(CursorOld, Params, FoundKeys) ->
+    OldAcls = qlc:next_answers(CursorOld),
+    case OldAcls of
+        [] ->
+            qlc:delete_cursor(CursorOld),
+            [];
+        _ ->
+            Records = [{Login, Topic, Action, Access, CreatedAt}
+                || #?ACL_TABLE{filter = {Login, Topic}, action = LegacyAction, access = Access, created_at = CreatedAt} <- OldAcls,
+                {_, Action, _, _} <- normalize_rule({Access, LegacyAction, Topic, CreatedAt}),
+                not maps:is_key({Login, Topic}, FoundKeys)
+            ],
+            case Records of
+                [] -> traverse_old(CursorOld, Params, FoundKeys);
+                List ->
+                    filter_params(List, Params)
+                    ++ fun() -> traverse_old(CursorOld, Params, FoundKeys) end
+            end
+    end.
+
+lookup_mnesia(Tab, Key) ->
+    mnesia:read({Tab, Key}).
+
+lookup_ets(Tab, Key) ->
+    ets:lookup(Tab, Key).
+
+update_permission(Action, Acl0, OldRecords) ->
+    Acl = Acl0 #?ACL_TABLE{action = Action},
+    maybe_delete_shadowed_records(Action, OldRecords),
+    mnesia:write(Acl).
+
+maybe_delete_shadowed_records(_, []) ->
+    ok;
+maybe_delete_shadowed_records(Action1, [Rec = #emqx_acl{action = Action2} | Rest]) ->
+    if Action1 =:= Action2 ->
+            ok = mnesia:delete_object(Rec);
+       Action2 =:= pubsub ->
+            %% Perform migration from the old data format on the
+            %% fly. This is needed only for the enterprise version,
+            %% delete this branch on 5.0
+            mnesia:delete_object(Rec),
+            mnesia:write(Rec#?ACL_TABLE{action = other_action(Action1)});
+       true ->
+            ok
+    end,
+    maybe_delete_shadowed_records(Action1, Rest).
+
+other_action(pub) -> sub;
+other_action(sub) -> pub.
+
+ret({atomic, ok})     -> ok;
+ret({aborted, Error}) -> {error, Error}.

apps/emqx_auth_mnesia/src/emqx_acl_mnesia_migrator.erl

@@ -0,0 +1,215 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_acl_mnesia_migrator).
+
+-include("emqx_auth_mnesia.hrl").
+-include_lib("emqx/include/logger.hrl").
+-include_lib("snabbkaffe/include/snabbkaffe.hrl").
+
+-behaviour(gen_statem).
+
+-define(CHECK_ALL_NODES_INTERVAL, 60000).
+
+-type(migration_delay_reason() :: old_nodes | bad_nodes).
+
+-export([
+    callback_mode/0,
+    init/1
+]).
+
+-export([
+    waiting_all_nodes/3,
+    checking_old_table/3,
+    migrating/3
+]).
+
+-export([
+    start_link/0,
+    start_link/1,
+    start_supervised/0,
+    stop_supervised/0,
+    migrate_records/0,
+    is_migrating_on_node/1,
+    is_old_table_migrated/0
+]).
+
+%%--------------------------------------------------------------------
+%% External interface
+%%--------------------------------------------------------------------
+
+start_link() ->
+    start_link(?MODULE).
+
+start_link(Name) when is_atom(Name) ->
+    start_link(#{
+        name => Name
+    });
+
+start_link(#{name := Name} = Opts) ->
+    gen_statem:start_link({local, Name}, ?MODULE, Opts, []).
+
+start_supervised() ->
+    try
+        {ok, _} = supervisor:restart_child(emqx_auth_mnesia_sup, ?MODULE),
+        ok
+    catch
+        exit:{noproc, _} -> ok
+    end.
+
+stop_supervised() ->
+    try
+        ok = supervisor:terminate_child(emqx_auth_mnesia_sup, ?MODULE),
+        ok = supervisor:delete_child(emqx_auth_mnesia_sup, ?MODULE)
+    catch
+        exit:{noproc, _} -> ok
+    end.
+
+%%--------------------------------------------------------------------
+%% gen_statem callbacks
+%%--------------------------------------------------------------------
+
+callback_mode() -> state_functions.
+
+init(Opts) ->
+    ok = emqx_acl_mnesia_db:create_table(),
+    ok = emqx_acl_mnesia_db:create_table2(),
+    Name = maps:get(name, Opts, ?MODULE),
+    CheckNodesInterval = maps:get(check_nodes_interval, Opts, ?CHECK_ALL_NODES_INTERVAL),
+    GetNodes = maps:get(get_nodes, Opts, fun all_nodes/0),
+    Data =
+        #{name => Name,
+          check_nodes_interval => CheckNodesInterval,
+          get_nodes => GetNodes},
+    {ok, waiting_all_nodes, Data, [{state_timeout, 0, check_nodes}]}.
+
+%%--------------------------------------------------------------------
+%% state callbacks
+%%--------------------------------------------------------------------
+
+waiting_all_nodes(state_timeout, check_nodes, Data) ->
+    #{name := Name, check_nodes_interval := CheckNodesInterval, get_nodes := GetNodes} = Data,
+    case is_all_nodes_migrating(Name, GetNodes()) of
+        true ->
+            ?tp(info, emqx_acl_mnesia_migrator_check_old_table, #{}),
+            {next_state, checking_old_table, Data, [{next_event, internal, check_old_table}]};
+        {false, Reason, Nodes} ->
+            ?tp(info,
+                emqx_acl_mnesia_migrator_bad_nodes_delay,
+                #{delay => CheckNodesInterval,
+                  reason => Reason,
+                  name => Name,
+                  nodes => Nodes}),
+            {keep_state_and_data, [{state_timeout, CheckNodesInterval, check_nodes}]}
+    end.
+
+checking_old_table(internal, check_old_table, Data) ->
+    case is_old_table_migrated() of
+        true ->
+            ?tp(info, emqx_acl_mnesia_migrator_finish, #{}),
+            {next_state, finished, Data, [{hibernate, true}]};
+        false ->
+            ?tp(info, emqx_acl_mnesia_migrator_start_migration, #{}),
+            {next_state, migrating, Data, [{next_event, internal, start_migration}]}
+    end.
+
+migrating(internal, start_migration, Data) ->
+    ok = migrate_records(),
+    {next_state, checking_old_table, Data, [{next_event, internal, check_old_table}]}.
+
+%% @doc Returns `true` if migration is started in the local node, otherwise crash.
+-spec(is_migrating_on_node(atom()) -> true).
+is_migrating_on_node(Name) ->
+    true = is_pid(erlang:whereis(Name)).
+
+%% @doc Run migration of records
+-spec(migrate_records() -> ok).
+migrate_records() ->
+    ok = add_migration_mark(),
+    Key = peek_record(),
+    do_migrate_records(Key).
+
+%% @doc Run migration of records
+-spec(is_all_nodes_migrating(atom(), list(node())) -> true | {false, migration_delay_reason(), list(node())}).
+is_all_nodes_migrating(Name, Nodes) ->
+    case rpc:multicall(Nodes, ?MODULE, is_migrating_on_node, [Name]) of
+        {Results, []} ->
+            OldNodes = [ Node || {Node, Result} <- lists:zip(Nodes, Results), Result =/= true ],
+            case OldNodes of
+                [] -> true;
+                _ -> {false, old_nodes, OldNodes}
+            end;
+        {_, [_BadNode | _] = BadNodes} ->
+            {false, bad_nodes, BadNodes}
+    end.
+
+%%--------------------------------------------------------------------
+%% Internal functions
+%%--------------------------------------------------------------------
+
+all_nodes() ->
+    ekka_mnesia:cluster_nodes(all).
+
+is_old_table_migrated() ->
+    Result =
+        mnesia:transaction(fun() ->
+                              case mnesia:first(?ACL_TABLE) of
+                                  ?MIGRATION_MARK_KEY ->
+                                      case mnesia:next(?ACL_TABLE, ?MIGRATION_MARK_KEY) of
+                                          '$end_of_table' -> true;
+                                          _OtherKey -> false
+                                      end;
+                                  '$end_of_table' -> false;
+                                  _OtherKey -> false
+                              end
+                           end),
+    case Result of
+        {atomic, true} ->
+            true;
+        _ ->
+            false
+    end.
+
+add_migration_mark() ->
+    {atomic, ok} = mnesia:transaction(fun() -> mnesia:write(?MIGRATION_MARK_RECORD) end),
+    ok.
+
+peek_record() ->
+    Key = mnesia:dirty_first(?ACL_TABLE),
+    case Key of
+        ?MIGRATION_MARK_KEY ->
+            mnesia:dirty_next(?ACL_TABLE, Key);
+        _ -> Key
+    end.
+
+do_migrate_records('$end_of_table') -> ok;
+do_migrate_records({_Login, _Topic} = Key) ->
+    ?tp(emqx_acl_mnesia_migrator_record_selected, #{key => Key}),
+    _ = mnesia:transaction(fun migrate_one_record/1, [Key]),
+    do_migrate_records(peek_record()).
+
+migrate_one_record({Login, _Topic} = Key) ->
+    case mnesia:wread({?ACL_TABLE, Key}) of
+        [] ->
+            ?tp(emqx_acl_mnesia_migrator_record_missed, #{key => Key}),
+            record_missing;
+        OldRecs ->
+            Acls = mnesia:wread({?ACL_TABLE2, Login}),
+            UpdatedAcl = emqx_acl_mnesia_db:merge_acl_records(Login, OldRecs, Acls),
+            ok = mnesia:write(UpdatedAcl),
+            ok = mnesia:delete({?ACL_TABLE, Key}),
+            ?tp(emqx_acl_mnesia_migrator_record_migrated, #{key => Key})
+    end.

apps/emqx_auth_mnesia/src/emqx_auth_mnesia.app.src

@@ -1,6 +1,6 @@
 {application, emqx_auth_mnesia,
  [{description, "EMQ X Authentication with Mnesia"},
-  {vsn, "4.3.0"}, % strict semver, bump manually
+  {vsn, "4.3.10"}, % strict semver, bump manually
   {modules, []},
   {registered, []},
   {applications, [kernel,stdlib,mnesia]},

apps/emqx_auth_mnesia/src/emqx_auth_mnesia.appup.src

@@ -0,0 +1,71 @@
+%% -*- mode: erlang -*-
+%% Unless you know what you are doing, DO NOT edit manually!!
+{VSN,
+  [{"4.3.9",[{load_module,emqx_acl_mnesia_cli,brutal_purge,soft_purge,[]}]},
+   {"4.3.8",[{load_module,emqx_acl_mnesia_cli,brutal_purge,soft_purge,[]}]},
+   {"4.3.7",
+    [{load_module,emqx_auth_mnesia_api,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_mnesia_cli,brutal_purge,soft_purge,[]}]},
+   {<<"4\\.3\\.[5-6]">>,
+    [{load_module,emqx_auth_mnesia_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_mnesia,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_mnesia_api,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_mnesia_db,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_mnesia_cli,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_mnesia_api,brutal_purge,soft_purge,[]}]},
+   {<<"4\\.3\\.[0-3]">>,
+    [{load_module,emqx_auth_mnesia_cli,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_mnesia,brutal_purge,soft_purge,[]},
+     {add_module,emqx_acl_mnesia_db},
+     {add_module,emqx_acl_mnesia_migrator,[emqx_acl_mnesia_db]},
+     {update,emqx_auth_mnesia_sup,supervisor},
+     {apply,{emqx_acl_mnesia_migrator,start_supervised,[]}},
+     {load_module,emqx_auth_mnesia_api,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_mnesia,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_mnesia_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_mnesia_api,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_mnesia_cli,brutal_purge,soft_purge,[]}]},
+   {"4.3.4",
+    [{load_module,emqx_auth_mnesia_api,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_mnesia,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_mnesia_db,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_mnesia_api,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_mnesia_cli,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_mnesia,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_mnesia_cli,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_mnesia_app,brutal_purge,soft_purge,[]}]},
+   {<<".*">>,[]}],
+  [{"4.3.9",[{load_module,emqx_acl_mnesia_cli,brutal_purge,soft_purge,[]}]},
+   {"4.3.8",[{load_module,emqx_acl_mnesia_cli,brutal_purge,soft_purge,[]}]},
+   {"4.3.7",
+    [{load_module,emqx_auth_mnesia_api,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_mnesia_cli,brutal_purge,soft_purge,[]}]},
+   {<<"4\\.3\\.[5-6]">>,
+    [{load_module,emqx_auth_mnesia_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_mnesia,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_mnesia_api,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_mnesia_db,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_mnesia_cli,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_mnesia_api,brutal_purge,soft_purge,[]}]},
+   {<<"4\\.3\\.[0-3]">>,
+    [{load_module,emqx_auth_mnesia_cli,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_mnesia,brutal_purge,soft_purge,[]},
+     {apply,{emqx_acl_mnesia_migrator,stop_supervised,[]}},
+     {update,emqx_auth_mnesia_sup,supervisor},
+     {load_module,emqx_acl_mnesia_cli,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_mnesia_api,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_mnesia_api,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_mnesia,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_mnesia_app,brutal_purge,soft_purge,[]},
+     {delete_module,emqx_acl_mnesia_migrator},
+     {delete_module,emqx_acl_mnesia_db}]},
+   {"4.3.4",
+    [{load_module,emqx_auth_mnesia_api,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_mnesia_api,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_mnesia,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_mnesia_cli,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_mnesia_db,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_mnesia,brutal_purge,soft_purge,[]},
+     {load_module,emqx_acl_mnesia_cli,brutal_purge,soft_purge,[]},
+     {load_module,emqx_auth_mnesia_app,brutal_purge,soft_purge,[]}]},
+   {<<".*">>,[]}]}.

apps/emqx_auth_mnesia/src/emqx_auth_mnesia.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -27,29 +27,31 @@
 -define(TABLE, emqx_user).
 %% Auth callbacks
 -export([ init/1
-        , register_metrics/0
         , check/3
         , description/0
         ]).
 
+-export([ match_password/3
+        , hash_type/0
+        ]).
+
 init(#{clientid_list := ClientidList, username_list := UsernameList}) ->
     ok = ekka_mnesia:create_table(?TABLE, [
             {disc_copies, [node()]},
             {attributes, record_info(fields, emqx_user)},
             {storage_properties, [{ets, [{read_concurrency, true}]}]}]),
-    _ = [ add_default_user({{clientid, iolist_to_binary(Clientid)}, iolist_to_binary(Password)})
+    lists:foreach(fun({Clientid, Password}) ->
-      || {Clientid, Password} <- ClientidList],
+        emqx_auth_mnesia_cli:add_default_user(clientid, iolist_to_binary(Clientid), iolist_to_binary(Password))
-    _ = [ add_default_user({{username, iolist_to_binary(Username)}, iolist_to_binary(Password)})
+    end, ClientidList),
-      || {Username, Password} <- UsernameList],
+
+    lists:foreach(fun({Username, Password}) ->
+        emqx_auth_mnesia_cli:add_default_user(username, iolist_to_binary(Username), iolist_to_binary(Password))
+    end, UsernameList),
+
     ok = ekka_mnesia:copy_table(?TABLE, disc_copies).
 
-%% @private
+hash_type() ->
-add_default_user({Login, Password}) when is_tuple(Login) ->
+    application:get_env(emqx_auth_mnesia, password_hash, sha256).
-    emqx_auth_mnesia_cli:add_user(Login, Password).
-
--spec(register_metrics() -> ok).
-register_metrics() ->
-    lists:foreach(fun emqx_metrics:ensure/1, ?AUTH_METRICS).
 
 check(ClientInfo = #{ clientid := Clientid
                     , password := NPassword
@@ -60,16 +62,14 @@ check(ClientInfo = #{ clientid := Clientid
                            end),
     case ets:select(?TABLE, MatchSpec) of
         [] ->
-            emqx_metrics:inc(?AUTH_METRICS(ignore)),
             ok;
         List ->
             case match_password(NPassword, HashType, List)  of
                 false ->
-                    ?LOG(error, "[Mnesia] Auth from mnesia failed: ~p", [ClientInfo]),
+                    Info = maps:without([password], ClientInfo),
-                    emqx_metrics:inc(?AUTH_METRICS(failure)),
+                    ?LOG(info, "[Mnesia] Auth from mnesia failed: ~p", [Info]),
                     {stop, AuthResult#{anonymous => false, auth_result => password_error}};
                 _ ->
-                    emqx_metrics:inc(?AUTH_METRICS(success)),
                     {stop, AuthResult#{anonymous => false, auth_result => success}}
             end
     end.

apps/emqx_auth_mnesia/src/emqx_auth_mnesia_api.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -18,18 +18,20 @@
 
 -include_lib("stdlib/include/qlc.hrl").
 -include_lib("stdlib/include/ms_transform.hrl").
+-include("emqx_auth_mnesia.hrl").
 
 -define(TABLE, emqx_user).
 
 -import(proplists, [get_value/2]).
 -import(minirest,  [return/1]).
--export([paginate/5]).
 
 -export([ list_clientid/2
         , lookup_clientid/2
         , add_clientid/2
         , update_clientid/2
         , delete_clientid/2
+        , query_clientid/3
+        , query_username/3
         ]).
 
 -rest_api(#{name   => list_clientid,
@@ -109,13 +111,32 @@
             descr  => "Delete username in the cluster"
            }).
 
+-define(CLIENTID_SCHEMA, {?TABLE,
+    [
+        {<<"clientid">>, binary},
+        {<<"_like_clientid">>, binary}
+    ]}).
+
+-define(USERNAME_SCHEMA, {?TABLE,
+    [
+        {<<"username">>, binary},
+        {<<"_like_username">>, binary}
+    ]}).
+
+-define(query_clientid, {?MODULE, query_clientid}).
+-define(query_username, {?MODULE, query_username}).
+
 %%------------------------------------------------------------------------------
 %% Auth Clientid Api
 %%------------------------------------------------------------------------------
 
 list_clientid(_Bindings, Params) ->
-    MatchSpec = ets:fun2ms(fun({?TABLE, {clientid, Clientid}, Password, CreatedAt}) -> {?TABLE, {clientid, Clientid}, Password, CreatedAt} end),
+    SortFun = fun(#{created_at := C1}, #{created_at := C2}) -> C1 > C2 end,
-    return({ok, paginate(?TABLE, MatchSpec, Params, fun emqx_auth_mnesia_cli:comparing/2, fun({?TABLE, {clientid, X}, _, _}) -> #{clientid => X} end)}).
+    CountFun = fun() ->
+        MatchSpec = [{{?TABLE, {clientid, '_'}, '_', '_'}, [], [true]}],
+        ets:select_count(?TABLE, MatchSpec)
+               end,
+    return({ok, emqx_mgmt_api:node_query(node(), Params, ?CLIENTID_SCHEMA, ?query_clientid, SortFun, CountFun)}).
 
 lookup_clientid(#{clientid := Clientid}, _Params) ->
     return({ok, format(emqx_auth_mnesia_cli:lookup_user({clientid, urldecode(Clientid)}))}).
@@ -133,15 +154,15 @@ add_clientid(_Bindings, Params) ->
     end.
 
 do_add_clientid([ Params | ParamsN ], ReList ) ->
-    Clientid = urldecode(get_value(<<"clientid">>, Params)),
+    Clientid = get_value(<<"clientid">>, Params),
     do_add_clientid(ParamsN, [{Clientid, format_msg(do_add_clientid(Params))} | ReList]);
 
 do_add_clientid([], ReList) ->
     {ok, ReList}.
 
 do_add_clientid(Params) ->
-    Clientid = urldecode(get_value(<<"clientid">>, Params)),
+    Clientid = get_value(<<"clientid">>, Params),
-    Password = urldecode(get_value(<<"password">>, Params)),
+    Password = get_value(<<"password">>, Params),
     Login = {clientid, Clientid},
     case validate([login, password], [Login, Password]) of
         ok ->
@@ -152,7 +173,7 @@ do_add_clientid(Params) ->
 update_clientid(#{clientid := Clientid}, Params) ->
     Password = get_value(<<"password">>, Params),
     case validate([password], [Password]) of
-        ok -> return(emqx_auth_mnesia_cli:update_user({clientid, urldecode(Clientid)}, urldecode(Password)));
+        ok -> return(emqx_auth_mnesia_cli:update_user({clientid, urldecode(Clientid)}, Password));
         Err -> return(Err)
     end.
 
@@ -164,8 +185,12 @@ delete_clientid(#{clientid := Clientid}, _) ->
 %%------------------------------------------------------------------------------
 
 list_username(_Bindings, Params) ->
-    MatchSpec = ets:fun2ms(fun({?TABLE, {username, Username}, Password, CreatedAt}) -> {?TABLE, {username, Username}, Password, CreatedAt} end),
+    SortFun = fun(#{created_at := C1}, #{created_at := C2}) -> C1 > C2 end,
-    return({ok, paginate(?TABLE, MatchSpec, Params, fun emqx_auth_mnesia_cli:comparing/2, fun({?TABLE, {username, X}, _, _}) -> #{username => X} end)}).
+    CountFun = fun() ->
+        MatchSpec = [{{?TABLE, {username, '_'}, '_', '_'}, [], [true]}],
+        ets:select_count(?TABLE, MatchSpec)
+               end,
+    return({ok, emqx_mgmt_api:node_query(node(), Params, ?USERNAME_SCHEMA, ?query_username, SortFun, CountFun)}).
 
 lookup_username(#{username := Username}, _Params) ->
     return({ok, format(emqx_auth_mnesia_cli:lookup_user({username, urldecode(Username)}))}).
@@ -182,15 +207,15 @@ add_username(_Bindings, Params) ->
     end.
 
 do_add_username([ Params | ParamsN ], ReList ) ->
-    Username = urldecode(get_value(<<"username">>, Params)),
+    Username = get_value(<<"username">>, Params),
     do_add_username(ParamsN, [{Username, format_msg(do_add_username(Params))} | ReList]);
 
 do_add_username([], ReList) ->
     {ok, ReList}.
 
 do_add_username(Params) ->
-    Username = urldecode(get_value(<<"username">>, Params)),
+    Username = get_value(<<"username">>, Params),
-    Password = urldecode(get_value(<<"password">>, Params)),
+    Password = get_value(<<"password">>, Params),
     Login = {username, Username},
     case validate([login, password], [Login, Password]) of
         ok ->
@@ -201,7 +226,7 @@ do_add_username(Params) ->
 update_username(#{username := Username}, Params) ->
     Password = get_value(<<"password">>, Params),
     case validate([password], [Password]) of
-        ok -> return(emqx_auth_mnesia_cli:update_user({username, urldecode(Username)}, urldecode(Password)));
+        ok -> return(emqx_auth_mnesia_cli:update_user({username, urldecode(Username)}, Password));
         Err -> return(Err)
     end.
 
@@ -211,68 +236,52 @@ delete_username(#{username := Username}, _) ->
 %%------------------------------------------------------------------------------
 %% Paging Query
 %%------------------------------------------------------------------------------
+query_clientid(Qs, Start, Limit) -> query(clientid, Qs, Start, Limit).
+query_username(Qs, Start, Limit) -> query(username, Qs, Start, Limit).
 
-paginate(Tables, MatchSpec, Params, ComparingFun, RowFun) ->
+query(Type, {Qs, []}, Start, Limit) ->
-    Qh = query_handle(Tables, MatchSpec),
+    Ms = qs2ms(Type, Qs),
-    Count = count(Tables, MatchSpec),
+    emqx_mgmt_api:select_table(?TABLE, Ms, Start, Limit, fun format/1);
-    Page = page(Params),
-    Limit = limit(Params),
-    Cursor = qlc:cursor(Qh),
-    case Page > 1 of
-        true  ->
-            _ = qlc:next_answers(Cursor, (Page - 1) * Limit),
-            ok;
-        false -> ok
-    end,
-    Rows = qlc:next_answers(Cursor, Limit),
-    qlc:delete_cursor(Cursor),
-    #{meta  => #{page => Page, limit => Limit, count => Count},
-      data  => [RowFun(Row) || Row <- lists:sort(ComparingFun, Rows)]}.
 
-query_handle(Table, MatchSpec) when is_atom(Table) ->
+query(Type, {Qs, Fuzzy}, Start, Limit) ->
-    Options = {traverse, {select, MatchSpec}},
+    Ms = qs2ms(Type, Qs),
-    qlc:q([R|| R <- ets:table(Table, Options)]);
+    MatchFun = match_fun(Ms, Fuzzy),
-query_handle([Table], MatchSpec) when is_atom(Table) ->
+    emqx_mgmt_api:traverse_table(?TABLE, MatchFun, Start, Limit, fun format/1).
-    Options = {traverse, {select, MatchSpec}},
-    qlc:q([R|| R <- ets:table(Table, Options)]);
-query_handle(Tables, MatchSpec) ->
-    Options = {traverse, {select, MatchSpec}},
-    qlc:append([qlc:q([E || E <- ets:table(T, Options)]) || T <- Tables]).
 
-count(Table, MatchSpec) when is_atom(Table) ->
+-spec qs2ms(clientid | username, list()) -> ets:match_spec().
-    [{MatchPattern, Where, _Re}] = MatchSpec,
+qs2ms(Type, Qs) ->
-    NMatchSpec = [{MatchPattern, Where, [true]}],
+    Init = #?TABLE{login = {Type, '_'}, password  = '_', created_at = '_'},
-    ets:select_count(Table, NMatchSpec);
+    MatchHead = lists:foldl(fun(Q, Acc) ->  match_ms(Q, Acc) end, Init, Qs),
-count([Table], MatchSpec) when is_atom(Table) ->
+    [{MatchHead, [], ['$_']}].
-    [{MatchPattern, Where, _Re}] = MatchSpec,
-    NMatchSpec = [{MatchPattern, Where, [true]}],
-    ets:select_count(Table, NMatchSpec);
-count(Tables, MatchSpec) ->
-    lists:sum([count(T, MatchSpec) || T <- Tables]).
 
-page(Params) ->
+match_ms({Type, '=:=', Value}, MatchHead) -> MatchHead#?TABLE{login = {Type, Value}};
-    binary_to_integer(proplists:get_value(<<"_page">>, Params, <<"1">>)).
+match_ms(_, MatchHead) -> MatchHead.
 
-limit(Params) ->
+match_fun(Ms, Fuzzy) ->
-    case proplists:get_value(<<"_limit">>, Params) of
+    MsC = ets:match_spec_compile(Ms),
-        undefined -> 10;
+    fun(Rows) ->
-        Size      -> binary_to_integer(Size)
+        Ls = ets:match_spec_run(Rows, MsC),
+        lists:filter(fun(E) -> run_fuzzy_match(E, Fuzzy) end, Ls)
     end.
 
+run_fuzzy_match(_, []) -> true;
+run_fuzzy_match(E = #?TABLE{login = {Key, Str}}, [{Key, like, SubStr}|Fuzzy]) ->
+    binary:match(Str, SubStr) /= nomatch andalso run_fuzzy_match(E, Fuzzy);
+run_fuzzy_match(_E, [{_Key, like, _SubStr}| _Fuzzy]) -> false.
+
 %%------------------------------------------------------------------------------
 %% Interval Funcs
 %%------------------------------------------------------------------------------
 
-format([{?TABLE, {clientid, ClientId}, Password, _InterTime}]) ->
+format([{?TABLE, {clientid, ClientId}, _Password, CreatedAt}]) ->
-    #{clientid => ClientId,
+    #{clientid => ClientId, created_at => CreatedAt};
-      password => Password};
 
-format([{?TABLE, {username, Username}, Password, _InterTime}]) ->
+format([{?TABLE, {username, Username}, _Password, CreatedAt}]) ->
-    #{username => Username,
+    #{username => Username, created_at => CreatedAt};
-      password => Password};
 
 format([]) ->
-    #{}.
+    #{};
+format(User) -> format([User]).
 
 validate([], []) ->
     ok;

apps/emqx_auth_mnesia/src/emqx_auth_mnesia_app.erl

@@ -1,5 +1,5 @@
 %%--------------------------------------------------------------------
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%% Copyright (c) 2020-2022 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%
 %% Licensed under the Apache License, Version 2.0 (the "License");
 %% you may not use this file except in compliance with the License.
@@ -56,13 +56,9 @@ load_auth_hook() ->
     ClientidList = application:get_env(?APP, clientid_list, []),
     UsernameList = application:get_env(?APP, username_list, []),
     ok = emqx_auth_mnesia:init(#{clientid_list => ClientidList, username_list => UsernameList}),
-    ok = emqx_auth_mnesia:register_metrics(),
+    Params = #{hash_type => emqx_auth_mnesia:hash_type()},
-    Params = #{
-            hash_type => application:get_env(emqx_auth_mnesia, password_hash, sha256)
-            },
     emqx:hook('client.authenticate', fun emqx_auth_mnesia:check/3, [Params]).
 
 load_acl_hook() ->
     ok = emqx_acl_mnesia:init(),
-    ok = emqx_acl_mnesia:register_metrics(),
     emqx:hook('client.check_acl', fun emqx_acl_mnesia:check_acl/5, [#{}]).