chore: merge upstream/master release-50

.ci/docker-compose-file/.env

@@ -6,5 +6,6 @@ LDAP_TAG=2.4.50
 INFLUXDB_TAG=2.5.0
 TDENGINE_TAG=3.0.2.4
 DYNAMO_TAG=1.21.0
+CASSANDRA_TAG=3.11.6
 
 TARGET=emqx/emqx

.ci/docker-compose-file/cassandra/Dockerfile

@@ -0,0 +1,4 @@
+ARG CASSANDRA_TAG=3.11.6
+FROM cassandra:${CASSANDRA_TAG}
+COPY cassandra.yaml /etc/cassandra/cassandra.yaml
+CMD ["cassandra", "-f"]

.ci/docker-compose-file/certs/README.md

@@ -0,0 +1,23 @@
+Certificate and Key files for testing
+
+## Cassandra (v3.x)
+
+### How to convert server PEM to JKS Format
+
+1. Convert server.crt and server.key to server.p12
+
+```bash
+openssl pkcs12 -export -in server.crt -inkey server.key -out server.p12 -name "certificate"
+```
+
+2. Convert server.p12 to server.jks
+
+```bash
+keytool -importkeystore -srckeystore server.p12 -srcstoretype pkcs12 -destkeystore server.jks
+```
+
+### How to convert CA PEM certificate to truststore.jks
+
+```
+keytool -import -file ca.pem -keystore truststore.jks
+```

.ci/docker-compose-file/certs/client.key

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEAzs74tdftT7xGMGXQSoX/nnFkFAOjNtEVOI3bChzR+w6Xwo8Z
+OUiOuOjynKvsJeltdmc0L+cbHZh7j+aHuAqVYxavqaqhFneF0f03t17qju9AixoV
+JXgNT3ru56aZFa6Ov6NhfZfRirGnbNrg2RhuNeYZ4TYLH7iMR36exNFP83glXwXM
+inMd1tsHL7xHLf3KjCbkusA5ncFWcpIUtpuWVn9aAE402dN7BJWfAbkQ4Y3VToR1
+P/T+W6WBldv0i2WlNbfiuAzuapA3EzJwoyTrG2Qyz7EtXM8XZdOZ6oJmW4s7c4V/
+FBT5knNtmXTt78xBBlIPFas5BAJIeV4eADx9MwIDAQABAoIBAQCZTvcynpJuxIxn
+vmItjK5U/4wIBjZNIawQk6BoG7tR2JyJ/1jcjTw4OX/4wr450JRz7MfUJweD5hDb
+OTMtLLNXlG6+YR4vsIUEiSlvhy5srVH0jG5Wq2t6mxBVq7vaRd/OkshnuU79+Pq7
+iHqclS7GSACxYkXWyxE6wtPh5aTWP8joK/LvYFiOqKPilUnLZ4hBhmL7CRUCZ0ZA
+QGNyEhlmiAL+LNKW2RLXPBxlKX21X78ahUQmkkTM0lBK9x6hm4dD3SpLqmZyQQ9M
+UfiMbU6XOYlDva/USZzrvTDlRf9uCG9QOsZzngP1aIy8Cq3QHECOeMIPO9WQLMll
+SyY+SpyJAoGBAP4fhnbDpQC6ekd9TNoU9GE/FNNNGKLh82GDgnGcWU/oIzv8GlaR
+rkEHTb6aRoPpjTxWIjJpScs9kycC+7N3oNo9rub4s5UvllI+EgQ95+j/5fnZx6gO
+la8ousLy1hTYu9C0nTWdTV3YtfC0l0opn7Friv5QafNmhSn74DqrH0BHAoGBANBV
+/NhBDAH1PHzYA+XuNLYTLv56Q4osmoen17nPnFNWb1TtWblzb0yWp86GGDFcs8CZ
+eH0mXCRUzGMSWtOHe4CbIm2brAYXuL2t6+DZ1A22gsnW5avNrosZRS7eN7BE7DDj
+5cp9+Es9UWnArzJU7jSWwAtA6o47WHfHU/pqRB21AoGAGx6eKPqEF2nPNuXmV7e4
+xNAIluw5XtiiMpvoRdubpG1vpS0oWmi9oe73mwm30MgR7Ih8qciWuXvewmENH3/6
+yI+gpMGR2K/1aN166rz4jOMSVfGp3wN/cev00m0774mZsZI03M3mvccs031ST/XV
+Nwf1E2Ldi747I9nfeiNc+G0CgYEAslFHD1ntiyd6VGkYPQ978nPM/2dqs7OluILC
+tHmslfAfbpOQ/ph9JRK2IqDHyEhOWoWBiazxpO8n2Yx2TSNjZBpkh2h8/uIC7+cT
+Q+tuAya6H0ReZISx5sEEZC8zfx4fA2Gs53qWsN+U9W1FB1GGaWC2k2tG1+KXwD3N
+9UJLdxkCgYBB96dsfT7nXmy0JLUz0rQ4umBje6H5uvuaevWdVMEptHB+O7+6CAse
+OVwqlFLQ4QC7s4/P9FQwfr/0uMRInB1aC043Haa1LbiRcRIlSuBDUezK5xidUbz+
+uB/ABkwwEuqW3Ns1+QieJyyfoNYKZ2v0RtYxBuieKOpUCm3oNFZRWg==
+-----END RSA PRIVATE KEY-----

.ci/docker-compose-file/certs/client.pem

@@ -0,0 +1,25 @@
+-----BEGIN CERTIFICATE-----
+MIIEMjCCAhoCFCOrAvLNRztbFFcN0zrCQXoj73cHMA0GCSqGSIb3DQEBCwUAMDQx
+EjAQBgNVBAoMCUVNUVggVGVzdDEeMBwGA1UEAwwVQ2VydGlmaWNhdGUgQXV0aG9y
+aXR5MB4XDTIzMDMxNzA5MzgzMVoXDTMzMDMxNDA5MzgzMVowdzELMAkGA1UEBhMC
+U0UxEjAQBgNVBAgMCVN0b2NraG9sbTESMBAGA1UEBwwJU3RvY2tob2xtMRIwEAYD
+VQQKDAlNeU9yZ05hbWUxGDAWBgNVBAsMD015U2VydmljZUNsaWVudDESMBAGA1UE
+AwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAzs74
+tdftT7xGMGXQSoX/nnFkFAOjNtEVOI3bChzR+w6Xwo8ZOUiOuOjynKvsJeltdmc0
+L+cbHZh7j+aHuAqVYxavqaqhFneF0f03t17qju9AixoVJXgNT3ru56aZFa6Ov6Nh
+fZfRirGnbNrg2RhuNeYZ4TYLH7iMR36exNFP83glXwXMinMd1tsHL7xHLf3KjCbk
+usA5ncFWcpIUtpuWVn9aAE402dN7BJWfAbkQ4Y3VToR1P/T+W6WBldv0i2WlNbfi
+uAzuapA3EzJwoyTrG2Qyz7EtXM8XZdOZ6oJmW4s7c4V/FBT5knNtmXTt78xBBlIP
+Fas5BAJIeV4eADx9MwIDAQABMA0GCSqGSIb3DQEBCwUAA4ICAQBHgfJgMjTgWZXG
+eyzIVxaqzWTLxrT7zPy09Mw4qsAl1TfWg9/r8nuskq4bjBQuKm0k9H0HQXz//eFC
+Qn85qTHyAmZok6c4ljO2P+kTIl3nkKk5zudmeCTy3W9YBdyWvDXQ/GhbywIfO+1Y
+fYA82I5rXVg4c9fUVTNczUFyDNcZzoJoqCS8jwFDtNR0N/fptJN14j8pnYvNV+4c
+hZ+pcnhSoz7dD8WjyYCc/QCajJdTyb15i072HxuGmhwltjnwIE/2xfeXCCeUTzsJ
+8h4/ABRu9VEqjqDQHepXIflYuVhU38SL0f4ly7neMXmytAbXwGLVM+ME81HG60Bw
+8hkfSwKBbEkhUmD6+V1bdUz14I6HjWJt/INtFU+O+MYZbIFt4ep9GKLV3nk97CyL
+fwDv5b4WXdC68iWMZqSrADAXr+VG3DgHqpNItj0XmhY6ihmt5tA3Z6IZJj45TShA
+vRqTCx3Hf6EO3zf4KCrzaPSSSfVLnGKftA/6oz3bl8EK2e2M44lOspRk4l9k+iBR
+sfHPmpiWY0hIiFtd3LD/uGDSBcGkKjU/fLvJZXJpVXwmT9pmK9LzkAPOK1rr97e9
+esHqwe1bo3z7IdeREZ0wdxqGL3BNpm4f1NaIzV/stX+vScau0AyFYXzumjeBIpKa
+Gt0A+dZnUfWG6qn5NiRENXxFQSppaA==
+-----END CERTIFICATE-----

.ci/docker-compose-file/docker-compose-cassandra.yaml

@@ -0,0 +1,32 @@
+version: '3.9'
+
+services:
+  cassandra_server:
+    container_name: cassandra
+    build:
+      context: ./cassandra
+      args:
+        CASSANDRA_TAG: ${CASSANDRA_TAG}
+    image: emqx-cassandra
+    restart: always
+    environment:
+      CASSANDRA_BROADCAST_ADDRESS: "1.2.3.4"
+      CASSANDRA_RPC_ADDRESS: "0.0.0.0"
+      HEAP_NEWSIZE: "128M"
+      MAX_HEAP_SIZE: "2048M"
+    volumes:
+      - ./certs:/certs
+    #ports:
+    #  - "9042:9042"
+    #  - "9142:9142"
+    command:
+      - /bin/bash
+      - -c
+      - |
+        /opt/cassandra/bin/cassandra -f -R > /cassandra.log &
+        /opt/cassandra/bin/cqlsh -u cassandra -p cassandra -e "CREATE KEYSPACE mqtt WITH REPLICATION = { 'class':'SimpleStrategy','replication_factor':1};"
+        while [[ $$? -ne 0 ]];do sleep 5; /opt/cassandra/bin/cqlsh -u cassandra -p cassandra -e "CREATE KEYSPACE mqtt WITH REPLICATION = { 'class':'SimpleStrategy','replication_factor':1};"; done
+        /opt/cassandra/bin/cqlsh -u cassandra -p cassandra -e "describe keyspaces;"
+        tail -f /cassandra.log
+    networks:
+      - emqx_bridge

.ci/docker-compose-file/docker-compose-kafka.yaml

@@ -18,7 +18,7 @@ services:
       - /tmp/emqx-ci/emqx-shared-secret:/var/lib/secret
   kdc:
     hostname: kdc.emqx.net
-    image:  ghcr.io/emqx/emqx-builder/5.0-28:1.13.4-24.3.4.2-2-ubuntu20.04
+    image:  ghcr.io/emqx/emqx-builder/5.0-33:1.13.4-24.3.4.2-3-ubuntu20.04
     container_name: kdc.emqx.net
     expose:
       - 88 # kdc

.ci/docker-compose-file/docker-compose-rocketmq.yaml

@@ -0,0 +1,34 @@
+version: '3.9'
+
+services:
+  mqnamesrv:
+    image: apache/rocketmq:4.9.4
+    container_name: rocketmq_namesrv
+#    ports:
+#      - 9876:9876
+    volumes:
+      - ./rocketmq/logs:/opt/logs
+      - ./rocketmq/store:/opt/store
+    command: ./mqnamesrv 
+    networks:
+      - emqx_bridge
+
+  mqbroker:
+    image: apache/rocketmq:4.9.4 
+    container_name: rocketmq_broker
+#    ports:
+#      - 10909:10909
+#      - 10911:10911
+    volumes:
+      - ./rocketmq/logs:/opt/logs
+      - ./rocketmq/store:/opt/store
+      - ./rocketmq/conf/broker.conf:/etc/rocketmq/broker.conf
+    environment:
+        NAMESRV_ADDR: "rocketmq_namesrv:9876"
+        JAVA_OPTS: " -Duser.home=/opt"
+        JAVA_OPT_EXT: "-server -Xms1024m -Xmx1024m -Xmn1024m"
+    command: ./mqbroker -c /etc/rocketmq/broker.conf
+    depends_on:
+      - mqnamesrv
+    networks:
+      - emqx_bridge

.ci/docker-compose-file/docker-compose-toxiproxy.yaml

@@ -22,6 +22,9 @@ services:
       - 15433:5433
       - 16041:6041
       - 18000:8000
+      - 19876:9876
+      - 19042:9042
+      - 19142:9142
     command:
       - "-host=0.0.0.0"
       - "-config=/config/toxiproxy.json"

.ci/docker-compose-file/docker-compose.yaml

@@ -3,7 +3,7 @@ version: '3.9'
 services:
   erlang:
     container_name: erlang
-    image: ${DOCKER_CT_RUNNER_IMAGE:-ghcr.io/emqx/emqx-builder/5.0-28:1.13.4-24.3.4.2-2-ubuntu20.04}
+    image: ${DOCKER_CT_RUNNER_IMAGE:-ghcr.io/emqx/emqx-builder/5.0-33:1.13.4-24.3.4.2-3-ubuntu20.04}
     env_file:
       - conf.env
     environment:

.ci/docker-compose-file/rocketmq/conf/broker.conf

@@ -0,0 +1,22 @@
+brokerClusterName=DefaultCluster
+brokerName=broker-a
+brokerId=0
+
+brokerIP1=rocketmq_broker
+
+defaultTopicQueueNums=4
+autoCreateTopicEnable=true
+autoCreateSubscriptionGroup=true
+
+listenPort=10911
+deleteWhen=04
+
+fileReservedTime=120
+mapedFileSizeCommitLog=1073741824
+mapedFileSizeConsumeQueue=300000
+diskMaxUsedSpaceRatio=100
+maxMessageSize=65536
+
+brokerRole=ASYNC_MASTER
+
+flushDiskType=ASYNC_FLUSH

.ci/docker-compose-file/scripts/run-emqx.sh

@@ -29,7 +29,7 @@ esac
 is_node_up() {
   local node="$1"
   docker exec -i "$node" \
-         bash -c "emqx eval-erl \"['emqx@node1.emqx.io','emqx@node2.emqx.io'] = maps:get(running_nodes, ekka_cluster:info()).\"" > /dev/null 2>&1
+         bash -c "emqx eval \"['emqx@node1.emqx.io','emqx@node2.emqx.io'] = maps:get(running_nodes, ekka_cluster:info()).\"" > /dev/null 2>&1
 }
 
 is_node_listening() {

.ci/docker-compose-file/toxiproxy.json

@@ -77,5 +77,23 @@
     "listen": "0.0.0.0:9295",
     "upstream": "kafka-1.emqx.net:9295",
     "enabled": true
+  },
+  {
+    "name": "rocketmq",
+    "listen": "0.0.0.0:9876",
+    "upstream": "rocketmq_namesrv:9876",
+    "enabled": true
+  },
+  {
+    "name": "cassa_tcp",
+    "listen": "0.0.0.0:9042",
+    "upstream": "cassandra:9042",
+    "enabled": true
+  },
+  {
+    "name": "cassa_tls",
+    "listen": "0.0.0.0:9142",
+    "upstream": "cassandra:9142",
+    "enabled": true
   }
 ]

.github/CODEOWNERS

@@ -22,5 +22,8 @@
 ## CI
 /deploy/  @emqx/emqx-review-board @Rory-Z
 
+## @Meggielqk owns all files in any i18n directory anywhere in the project
+/i18n/ @Meggielqk
+
 ## no owner for changelogs, anyone can approve
 /changes

.github/pull_request_template.md

@@ -1,11 +1,16 @@
 Fixes <issue-or-jira-number>
 
+<!-- Make sure to target release-50 branch if this PR is intended to fix the issues for the release candidate. -->
+
+## Summary
+copilot:summary
+
 ## PR Checklist
 Please convert it to a draft if any of the following conditions are not met. Reviewers may skip over until all the items are checked:
 
 - [ ] Added tests for the changes
 - [ ] Changed lines covered in coverage report
-- [ ] Change log has been added to `changes/{ce,ee}/(feat|perf|fix)-<PR-id>.en.md` and `.zh.md` files
+- [ ] Change log has been added to `changes/{ce,ee}/(feat|perf|fix)-<PR-id>.en.md` files
 - [ ] For internal contributor: there is a jira ticket to track this change
 - [ ] If there should be document changes, a PR to emqx-docs.git is sent, or a jira ticket is created to follow up
 - [ ] Schema changes are backward compatible

.github/workflows/build_and_push_docker_images.yaml

@@ -25,7 +25,7 @@ jobs:
   prepare:
     runs-on: ubuntu-22.04
     # prepare source with any OTP version, no need for a matrix
-    container: "ghcr.io/emqx/emqx-builder/5.0-32:1.13.4-24.3.4.2-2-ubuntu22.04"
+    container: "ghcr.io/emqx/emqx-builder/5.0-33:1.13.4-24.3.4.2-3-ubuntu22.04"
 
     outputs:
       PROFILE: ${{ steps.get_profile.outputs.PROFILE }}
@@ -121,9 +121,9 @@ jobs:
         # NOTE: 'otp' and 'elixir' are to configure emqx-builder image
         #       only support latest otp and elixir, not a matrix
         builder:
-          - 5.0-32 # update to latest
+          - 5.0-33 # update to latest
         otp:
-          - 24.3.4.2-2 # switch to 25 once ready to release 5.1
+          - 24.3.4.2-3 # switch to 25 once ready to release 5.1
         elixir:
           - 'no_elixir'
           - '1.13.4' # update to latest

.github/workflows/build_packages.yaml

@@ -24,7 +24,7 @@ jobs:
   prepare:
     runs-on: ubuntu-22.04
     if: (github.repository_owner == 'emqx' && github.event_name == 'schedule') || github.event_name != 'schedule'
-    container: ghcr.io/emqx/emqx-builder/5.0-32:1.13.4-24.3.4.2-2-ubuntu22.04
+    container: ghcr.io/emqx/emqx-builder/5.0-33:1.13.4-24.3.4.2-3-ubuntu22.04
     outputs:
       BUILD_PROFILE: ${{ steps.get_profile.outputs.BUILD_PROFILE }}
       IS_EXACT_TAG: ${{ steps.get_profile.outputs.IS_EXACT_TAG }}
@@ -151,7 +151,7 @@ jobs:
         profile:
           - ${{ needs.prepare.outputs.BUILD_PROFILE }}
         otp:
-          - 24.3.4.2-2
+          - 24.3.4.2-3
         os:
           - macos-11
           - macos-12
@@ -203,7 +203,7 @@ jobs:
         profile:
           - ${{ needs.prepare.outputs.BUILD_PROFILE }}
         otp:
-          - 24.3.4.2-2
+          - 24.3.4.2-3
         arch:
           - amd64
           - arm64
@@ -221,7 +221,7 @@ jobs:
           - aws-arm64
           - ubuntu-22.04
         builder:
-          - 5.0-32
+          - 5.0-33
         elixir:
           - 1.13.4
         exclude:
@@ -231,19 +231,19 @@ jobs:
           build_machine: aws-arm64
         include:
           - profile: emqx
-            otp: 25.1.2-2
+            otp: 25.1.2-3
             arch: amd64
             os: ubuntu22.04
             build_machine: ubuntu-22.04
-            builder: 5.0-32
+            builder: 5.0-33
             elixir: 1.13.4
             release_with: elixir
           - profile: emqx
-            otp: 25.1.2-2
+            otp: 25.1.2-3
             arch: amd64
             os: amzn2
             build_machine: ubuntu-22.04
-            builder: 5.0-32
+            builder: 5.0-33
             elixir: 1.13.4
             release_with: elixir
 

.github/workflows/build_slim_packages.yaml

@@ -30,12 +30,12 @@ jobs:
       fail-fast: false
       matrix:
         profile:
-          - ["emqx", "24.3.4.2-2", "el7", "erlang"]
+          - ["emqx", "24.3.4.2-3", "el7", "erlang"]
-          - ["emqx", "25.1.2-2", "ubuntu22.04", "elixir"]
+          - ["emqx", "25.1.2-3", "ubuntu22.04", "elixir"]
-          - ["emqx-enterprise", "24.3.4.2-2", "amzn2", "erlang"]
+          - ["emqx-enterprise", "24.3.4.2-3", "amzn2", "erlang"]
-          - ["emqx-enterprise", "25.1.2-2", "ubuntu20.04", "erlang"]
+          - ["emqx-enterprise", "25.1.2-3", "ubuntu20.04", "erlang"]
         builder:
-          - 5.0-32
+          - 5.0-33
         elixir:
           - '1.13.4'
 
@@ -132,7 +132,7 @@ jobs:
         - emqx
         - emqx-enterprise
         otp:
-        - 24.3.4.2-2
+        - 24.3.4.2-3
         os:
         - macos-11
         - macos-12-arm64
@@ -165,19 +165,21 @@ jobs:
       fail-fast: false
       matrix:
         profile:
-          - emqx
+          - ["emqx", "5.0.16"]
-          - emqx-enterprise
+          - ["emqx-enterprise", "5.0.1"]
 
     steps:
     - uses: actions/checkout@v3
     - name: prepare
       run: |
-        EMQX_NAME=${{ matrix.profile }}
+        EMQX_NAME=${{ matrix.profile[0] }}
         PKG_VSN=${PKG_VSN:-$(./pkg-vsn.sh $EMQX_NAME)}
         EMQX_IMAGE_TAG=emqx/$EMQX_NAME:test
+        EMQX_IMAGE_OLD_VERSION_TAG=emqx/$EMQX_NAME:${{ matrix.profile[1] }}
         echo "EMQX_NAME=$EMQX_NAME" >> $GITHUB_ENV
         echo "PKG_VSN=$PKG_VSN" >> $GITHUB_ENV
         echo "EMQX_IMAGE_TAG=$EMQX_IMAGE_TAG" >> $GITHUB_ENV
+        echo "EMQX_IMAGE_OLD_VERSION_TAG=$EMQX_IMAGE_OLD_VERSION_TAG" >> $GITHUB_ENV
     - uses: docker/setup-buildx-action@v2
     - name: build and export to Docker
       uses: docker/build-push-action@v4
@@ -192,14 +194,24 @@ jobs:
       run: |
         CID=$(docker run -d --rm -P $EMQX_IMAGE_TAG)
         HTTP_PORT=$(docker inspect --format='{{(index (index .NetworkSettings.Ports "18083/tcp") 0).HostPort}}' $CID)
+        export EMQX_SMOKE_TEST_CHECK_HIDDEN_FIELDS='yes'
         ./scripts/test/emqx-smoke-test.sh localhost $HTTP_PORT
         docker stop $CID
+    - name: test two nodes cluster with proto_dist=inet_tls in docker
+      run: |
+        ./scripts/test/start-two-nodes-in-docker.sh -P $EMQX_IMAGE_TAG $EMQX_IMAGE_OLD_VERSION_TAG
+        HTTP_PORT=$(docker inspect --format='{{(index (index .NetworkSettings.Ports "18083/tcp") 0).HostPort}}' haproxy)
+        # versions before 5.0.22 have hidden fields included in the API spec
+        export EMQX_SMOKE_TEST_CHECK_HIDDEN_FIELDS='no'
+        ./scripts/test/emqx-smoke-test.sh localhost $HTTP_PORT
+        # cleanup
+        ./scripts/test/start-two-nodes-in-docker.sh -c
     - name: export docker image
       run: |
         docker save $EMQX_IMAGE_TAG | gzip > $EMQX_NAME-$PKG_VSN.tar.gz
     - uses: actions/upload-artifact@v3
       with:
-        name: "${{ matrix.profile }}-docker"
+        name: "${{ matrix.profile[0] }}-docker"
         path: "${{ env.EMQX_NAME }}-${{ env.PKG_VSN }}.tar.gz"
 
   spellcheck:

.github/workflows/check_deps_integrity.yaml

@@ -6,7 +6,7 @@ on:
 jobs:
   check_deps_integrity:
     runs-on: ubuntu-22.04
-    container: ghcr.io/emqx/emqx-builder/5.0-32:1.13.4-25.1.2-2-ubuntu22.04
+    container: ghcr.io/emqx/emqx-builder/5.0-33:1.13.4-25.1.2-3-ubuntu22.04
 
     steps:
       - uses: actions/checkout@v3

.github/workflows/code_style_check.yaml

@@ -5,7 +5,7 @@ on: [pull_request]
 jobs:
   code_style_check:
     runs-on: ubuntu-22.04
-    container: "ghcr.io/emqx/emqx-builder/5.0-32:1.13.4-25.1.2-2-ubuntu22.04"
+    container: "ghcr.io/emqx/emqx-builder/5.0-33:1.13.4-25.1.2-3-ubuntu22.04"
     steps:
       - uses: actions/checkout@v3
         with:

.github/workflows/elixir_apps_check.yaml

@@ -9,7 +9,7 @@ jobs:
   elixir_apps_check:
     runs-on: ubuntu-22.04
     # just use the latest builder
-    container: "ghcr.io/emqx/emqx-builder/5.0-32:1.13.4-25.1.2-2-ubuntu22.04"
+    container: "ghcr.io/emqx/emqx-builder/5.0-33:1.13.4-25.1.2-3-ubuntu22.04"
 
     strategy:
       fail-fast: false

.github/workflows/elixir_deps_check.yaml

@@ -8,7 +8,7 @@ on:
 jobs:
   elixir_deps_check:
     runs-on: ubuntu-22.04
-    container: ghcr.io/emqx/emqx-builder/5.0-32:1.13.4-25.1.2-2-ubuntu22.04
+    container: ghcr.io/emqx/emqx-builder/5.0-33:1.13.4-25.1.2-3-ubuntu22.04
 
     steps:
       - name: Checkout

.github/workflows/elixir_release.yml

@@ -17,7 +17,7 @@ jobs:
         profile:
           - emqx
           - emqx-enterprise
-    container: ghcr.io/emqx/emqx-builder/5.0-32:1.13.4-25.1.2-2-ubuntu22.04
+    container: ghcr.io/emqx/emqx-builder/5.0-33:1.13.4-25.1.2-3-ubuntu22.04
     steps:
       - name: Checkout
         uses: actions/checkout@v3

.github/workflows/geen_master.yaml

@@ -0,0 +1,26 @@
+---
+
+name: Keep master green
+
+on:
+  schedule:
+    # run hourly
+    - cron: "0 * * * *"
+  workflow_dispatch:
+
+jobs:
+  rerun-failed-jobs:
+    runs-on: ubuntu-22.04
+    if: github.repository_owner == 'emqx'
+    permissions:
+      checks: read
+      actions: write
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: run script
+        shell: bash
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          python3 scripts/rerun-failed-checks.py

.github/workflows/release.yaml

@@ -54,7 +54,7 @@ jobs:
           OUTPUT_DIR=${{ steps.profile.outputs.s3dir }}
           aws s3 cp --recursive s3://$BUCKET/$OUTPUT_DIR/${{ github.ref_name }} packages
           cd packages
-          DEFAULT_BEAM_PLATFORM='otp24.3.4.2-2'
+          DEFAULT_BEAM_PLATFORM='otp24.3.4.2-3'
           # all packages including full-name and default-name are uploaded to s3
           # but we only upload default-name packages (and elixir) as github artifacts
           # so we rename (overwrite) non-default packages before uploading

.github/workflows/run_emqx_app_tests.yaml

@@ -12,10 +12,10 @@ jobs:
     strategy:
       matrix:
         builder:
-          - 5.0-32
+          - 5.0-33
         otp:
-          - 24.3.4.2-2
+          - 24.3.4.2-3
-          - 25.1.2-2
+          - 25.1.2-3
         # no need to use more than 1 version of Elixir, since tests
         # run using only Erlang code.  This is needed just to specify
         # the base image.

.github/workflows/run_fvt_tests.yaml

@@ -17,7 +17,7 @@ jobs:
   prepare:
     runs-on: ubuntu-22.04
     # prepare source with any OTP version, no need for a matrix
-    container: ghcr.io/emqx/emqx-builder/5.0-32:1.13.4-24.3.4.2-2-debian11
+    container: ghcr.io/emqx/emqx-builder/5.0-33:1.13.4-24.3.4.2-3-debian11
 
     steps:
       - uses: actions/checkout@v3
@@ -50,9 +50,9 @@ jobs:
         os:
           - ["debian11", "debian:11-slim"]
         builder:
-          - 5.0-32
+          - 5.0-33
         otp:
-          - 24.3.4.2-2
+          - 24.3.4.2-3
         elixir:
           - 1.13.4
         arch:
@@ -123,9 +123,9 @@ jobs:
         os:
         - ["debian11", "debian:11-slim"]
         builder:
-        - 5.0-32
+        - 5.0-33
         otp:
-        - 24.3.4.2-2
+        - 24.3.4.2-3
         elixir:
         - 1.13.4
         arch:

.github/workflows/run_relup_tests.yaml

@@ -15,7 +15,7 @@ concurrency:
 jobs:
   relup_test_plan:
     runs-on: ubuntu-22.04
-    container: "ghcr.io/emqx/emqx-builder/5.0-32:1.13.4-24.3.4.2-2-ubuntu22.04"
+    container: "ghcr.io/emqx/emqx-builder/5.0-33:1.13.4-24.3.4.2-3-ubuntu22.04"
     outputs:
       CUR_EE_VSN: ${{ steps.find-versions.outputs.CUR_EE_VSN }}
       OLD_VERSIONS: ${{ steps.find-versions.outputs.OLD_VERSIONS }}

.github/workflows/run_test_cases.yaml

@@ -31,13 +31,13 @@ jobs:
           MATRIX="$(echo "${APPS}" | jq -c '
             [
               (.[] | select(.profile == "emqx") | . + {
-                builder: "5.0-32",
+                builder: "5.0-33",
-                otp: "25.1.2-2",
+                otp: "25.1.2-3",
                 elixir: "1.13.4"
               }),
               (.[] | select(.profile == "emqx-enterprise") | . + {
-                builder: "5.0-32",
+                builder: "5.0-33",
-                otp: ["24.3.4.2-2", "25.1.2-2"][],
+                otp: ["24.3.4.2-3", "25.1.2-3"][],
                 elixir: "1.13.4"
               })
             ]
@@ -230,12 +230,12 @@ jobs:
       - ct
       - ct_docker
     runs-on: ubuntu-22.04
-    container: "ghcr.io/emqx/emqx-builder/5.0-32:1.13.4-24.3.4.2-2-ubuntu22.04"
+    container: "ghcr.io/emqx/emqx-builder/5.0-33:1.13.4-24.3.4.2-3-ubuntu22.04"
     steps:
       - uses: AutoModality/action-clean@v1
       - uses: actions/download-artifact@v3
         with:
-          name: source-emqx-enterprise-24.3.4.2-2
+          name: source-emqx-enterprise-24.3.4.2-3
           path: .
       - name: unzip source code
         run: unzip -q source.zip

.tool-versions

@@ -1,2 +1,2 @@
-erlang 24.3.4.2-2
+erlang 24.3.4.2-3
 elixir 1.13.4-otp-24

Makefile

@@ -82,7 +82,7 @@ ct: $(REBAR) merge-config
 static_checks:
 	@$(REBAR) as check do xref, dialyzer
 	@if [ "$${PROFILE}" = 'emqx-enterprise' ]; then $(REBAR) ct --suite apps/emqx/test/emqx_static_checks --readable $(CT_READABLE); fi
-	@if [ "$${PROFILE}" = 'emqx-enterprise' ]; then ./scripts/check-i18n-style.sh; fi
+	./scripts/check-i18n-style.sh
 
 APPS=$(shell $(SCRIPTS)/find-apps.sh)
 
@@ -152,6 +152,7 @@ $(PROFILES:%=clean-%):
 .PHONY: clean-all
 clean-all:
 	@rm -f rebar.lock
+	@rm -rf deps
 	@rm -rf _build
 
 .PHONY: deps-all

README-CN.md

@@ -11,9 +11,6 @@
 [![YouTube](https://img.shields.io/badge/Subscribe-EMQ%20中文-FF0000?logo=youtube)](https://www.youtube.com/channel/UCir_r04HIsLjf2qqyZ4A8Cg)
 
 
-
-[English](./README.md) | 简体中文 | [русский](./README-RU.md)
-
 EMQX 是一款全球下载量超千万的大规模分布式物联网 MQTT 服务器，单集群支持 1 亿物联网设备连接，消息分发时延低于 1 毫秒。为高可靠、高性能的物联网实时数据移动、处理和集成提供动力，助力企业构建关键业务的 IoT 平台与应用。
 
 EMQX 自 2013 年在 GitHub 发布开源版本以来，获得了来自 50 多个国家和地区的 20000 余家企业用户的广泛认可，累计连接物联网关键设备超过 1 亿台。

README-RU.md

@@ -9,7 +9,6 @@
 [![Twitter](https://img.shields.io/badge/Follow-EMQ-1DA1F2?logo=twitter)](https://twitter.com/EMQTech)
 [![YouTube](https://img.shields.io/badge/Subscribe-EMQ-FF0000?logo=youtube)](https://www.youtube.com/channel/UC5FjR77ErAxvZENEWzQaO5Q)
 
-[English](./README.md) | [简体中文](./README-CN.md) | русский
 
 *EMQX* — это самый масштабируемый и популярный высокопроизводительный MQTT брокер с полностью открытым кодом для интернета вещей, межмашинного взаимодействия и мобильных приложений. EMQX может поддерживать более чем 100 миллионов одновременных соединенией на одном кластере с задержкой в 1 миллисекунду, а также принимать и обрабабывать миллионы MQTT сообщений в секунду.
 

README.md

@@ -10,9 +10,6 @@
 [![YouTube](https://img.shields.io/badge/Subscribe-EMQ-FF0000?logo=youtube)](https://www.youtube.com/channel/UC5FjR77ErAxvZENEWzQaO5Q)
 
 
-
-English | [简体中文](./README-CN.md) | [русский](./README-RU.md)
-
 EMQX is the world's most scalable open-source MQTT broker with a high performance that connects 100M+ IoT devices in 1 cluster, while maintaining 1M message per second throughput and sub-millisecond latency.
 
 EMQX supports multiple open standard protocols like MQTT, HTTP, QUIC, and WebSocket. It’s 100% compliant with MQTT 5.0 and 3.x standard, and secures bi-directional communication with MQTT over TLS/SSL and various authentication mechanisms.
@@ -25,7 +22,7 @@ For more information, please visit [EMQX homepage](https://www.emqx.io/).
 
 ## Get Started
 
-#### EMQX Cloud
+#### Run EMQX in the Cloud
 
 The simplest way to set up EMQX is to create a managed deployment with EMQX Cloud. You can [try EMQX Cloud for free](https://www.emqx.com/en/signup?utm_source=github.com&utm_medium=referral&utm_campaign=emqx-readme-to-cloud&continue=https://cloud-intl.emqx.com/console/deployments/0?oper=new), no credit card required.
 
@@ -62,6 +59,7 @@ For more organised improvement proposals, you can send pull requests to [EIP](ht
 ## Get Involved
 
 - Follow [@EMQTech on Twitter](https://twitter.com/EMQTech).
+- Join our [Slack](https://slack-invite.emqx.io/).
 - If you have a specific question, check out our [discussion forums](https://github.com/emqx/emqx/discussions).
 - For general discussions, join us on the [official Discord](https://discord.gg/xYGf3fQnES) team.
 - Keep updated on [EMQX YouTube](https://www.youtube.com/channel/UC5FjR77ErAxvZENEWzQaO5Q) by subscribing.

apps/emqx/rebar.config

@@ -26,10 +26,10 @@
     {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}},
     {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}},
     {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}},
-    {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.4"}}},
+    {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.6"}}},
-    {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.14.5"}}},
+    {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.14.6"}}},
     {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}},
-    {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.37.2"}}},
+    {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.38.0"}}},
     {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.2"}}},
     {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}},
     {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}},
@@ -59,4 +59,12 @@
     {statistics, true}
 ]}.
 
-{project_plugins, [erlfmt]}.
+{project_plugins, [
+    {erlfmt, [
+        {files, [
+            "{src,include,test}/*.{hrl,erl,app.src}",
+            "rebar.config",
+            "rebar.config.script"
+        ]}
+    ]}
+]}.

apps/emqx/rebar.config.script

@@ -24,7 +24,7 @@ IsQuicSupp = fun() ->
 end,
 
 Bcrypt = {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {tag, "0.6.0"}}},
-Quicer = {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.113"}}}.
+Quicer = {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.114"}}}.
 
 Dialyzer = fun(Config) ->
     {dialyzer, OldDialyzerConfig} = lists:keyfind(dialyzer, 1, Config),
@@ -37,7 +37,7 @@ Dialyzer = fun(Config) ->
         Config,
         {dialyzer, NewDialyzerConfig}
     )
-           end.
+end.
 
 ExtraDeps = fun(C) ->
     {deps, Deps0} = lists:keyfind(deps, 1, C),

apps/emqx/src/emqx.app.src

@@ -3,7 +3,7 @@
     {id, "emqx"},
     {description, "EMQX Core"},
     % strict semver, bump manually!
-    {vsn, "5.0.21"},
+    {vsn, "5.0.22"},
     {modules, []},
     {registered, []},
     {applications, [

apps/emqx/src/emqx_app.erl

@@ -72,9 +72,13 @@ set_init_config_load_done() ->
 get_init_config_load_done() ->
     application:get_env(emqx, init_config_load_done, false).
 
+%% @doc Set the transaction id from which this node should start applying after boot.
+%% The transaction ID is received from the core node which we just copied the latest
+%% config from.
 set_init_tnx_id(TnxId) ->
     application:set_env(emqx, cluster_rpc_init_tnx_id, TnxId).
 
+%% @doc Get the transaction id from which this node should start applying after boot.
 get_init_tnx_id() ->
     application:get_env(emqx, cluster_rpc_init_tnx_id, -1).
 

apps/emqx/src/emqx_channel.erl

@@ -276,7 +276,9 @@ init(
     ),
     {NClientInfo, NConnInfo} = take_ws_cookie(ClientInfo, ConnInfo),
     #channel{
-        conninfo = NConnInfo,
+        %% We remove the peercert because it duplicates to what's stored in the socket,
+        %% Saving a copy here causes unnecessary wast of memory (about 1KB per connection).
+        conninfo = maps:put(peercert, undefined, NConnInfo),
         clientinfo = NClientInfo,
         topic_aliases = #{
             inbound => #{},
@@ -2128,17 +2130,23 @@ publish_will_msg(
     ClientInfo = #{mountpoint := MountPoint},
     Msg = #message{topic = Topic}
 ) ->
-    case emqx_access_control:authorize(ClientInfo, publish, Topic) of
+    PublishingDisallowed = emqx_access_control:authorize(ClientInfo, publish, Topic) =/= allow,
-        allow ->
+    ClientBanned = emqx_banned:check(ClientInfo),
-            NMsg = emqx_mountpoint:mount(MountPoint, Msg),
+    case PublishingDisallowed orelse ClientBanned of
-            _ = emqx_broker:publish(NMsg),
+        true ->
-            ok;
-        deny ->
             ?tp(
                 warning,
                 last_will_testament_publish_denied,
-                #{topic => Topic}
+                #{
+                    topic => Topic,
+                    client_banned => ClientBanned,
+                    publishing_disallowed => PublishingDisallowed
+                }
             ),
+            ok;
+        false ->
+            NMsg = emqx_mountpoint:mount(MountPoint, Msg),
+            _ = emqx_broker:publish(NMsg),
             ok
     end.
 

apps/emqx/src/emqx_cm.erl

@@ -465,23 +465,23 @@ request_stepdown(Action, ConnMod, Pid) ->
         catch
             % emqx_ws_connection: call
             _:noproc ->
-                ok = ?tp(debug, "session_already_gone", #{pid => Pid, action => Action}),
+                ok = ?tp(debug, "session_already_gone", #{stale_pid => Pid, action => Action}),
                 {error, noproc};
             % emqx_connection: gen_server:call
             _:{noproc, _} ->
-                ok = ?tp(debug, "session_already_gone", #{pid => Pid, action => Action}),
+                ok = ?tp(debug, "session_already_gone", #{stale_pid => Pid, action => Action}),
                 {error, noproc};
             _:{shutdown, _} ->
-                ok = ?tp(debug, "session_already_shutdown", #{pid => Pid, action => Action}),
+                ok = ?tp(debug, "session_already_shutdown", #{stale_pid => Pid, action => Action}),
                 {error, noproc};
             _:{{shutdown, _}, _} ->
-                ok = ?tp(debug, "session_already_shutdown", #{pid => Pid, action => Action}),
+                ok = ?tp(debug, "session_already_shutdown", #{stale_pid => Pid, action => Action}),
                 {error, noproc};
             _:{timeout, {gen_server, call, _}} ->
                 ?tp(
                     warning,
                     "session_stepdown_request_timeout",
-                    #{pid => Pid, action => Action, stale_channel => stale_channel_info(Pid)}
+                    #{stale_pid => Pid, action => Action, stale_channel => stale_channel_info(Pid)}
                 ),
                 ok = force_kill(Pid),
                 {error, timeout};
@@ -490,7 +490,7 @@ request_stepdown(Action, ConnMod, Pid) ->
                     error,
                     "session_stepdown_request_exception",
                     #{
-                        pid => Pid,
+                        stale_pid => Pid,
                         action => Action,
                         reason => Error,
                         stacktrace => St,
@@ -671,7 +671,7 @@ handle_cast(Msg, State) ->
     {noreply, State}.
 
 handle_info({'DOWN', _MRef, process, Pid, _Reason}, State = #{chan_pmon := PMon}) ->
-    ?tp(emqx_cm_process_down, #{pid => Pid, reason => _Reason}),
+    ?tp(emqx_cm_process_down, #{stale_pid => Pid, reason => _Reason}),
     ChanPids = [Pid | emqx_misc:drain_down(?BATCH_SIZE)],
     {Items, PMon1} = emqx_pmon:erase_all(ChanPids, PMon),
     lists:foreach(fun mark_channel_disconnected/1, ChanPids),

apps/emqx/src/emqx_crl_cache.erl

@@ -0,0 +1,314 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%
+%% @doc EMQX CRL cache.
+%%--------------------------------------------------------------------
+
+-module(emqx_crl_cache).
+
+%% API
+-export([
+    start_link/0,
+    start_link/1,
+    register_der_crls/2,
+    refresh/1,
+    evict/1
+]).
+
+%% gen_server callbacks
+-export([
+    init/1,
+    handle_call/3,
+    handle_cast/2,
+    handle_info/2
+]).
+
+%% internal exports
+-export([http_get/2]).
+
+-behaviour(gen_server).
+
+-include("logger.hrl").
+-include_lib("snabbkaffe/include/snabbkaffe.hrl").
+
+-define(HTTP_TIMEOUT, timer:seconds(15)).
+-define(RETRY_TIMEOUT, 5_000).
+-ifdef(TEST).
+-define(MIN_REFRESH_PERIOD, timer:seconds(5)).
+-else.
+-define(MIN_REFRESH_PERIOD, timer:minutes(1)).
+-endif.
+-define(DEFAULT_REFRESH_INTERVAL, timer:minutes(15)).
+-define(DEFAULT_CACHE_CAPACITY, 100).
+
+-record(state, {
+    refresh_timers = #{} :: #{binary() => timer:tref()},
+    refresh_interval = timer:minutes(15) :: timer:time(),
+    http_timeout = ?HTTP_TIMEOUT :: timer:time(),
+    %% keeps track of URLs by insertion time
+    insertion_times = gb_trees:empty() :: gb_trees:tree(timer:time(), url()),
+    %% the set of cached URLs, for testing if an URL is already
+    %% registered.
+    cached_urls = sets:new([{version, 2}]) :: sets:set(url()),
+    cache_capacity = 100 :: pos_integer(),
+    %% for future use
+    extra = #{} :: map()
+}).
+-type url() :: uri_string:uri_string().
+-type state() :: #state{}.
+
+%%--------------------------------------------------------------------
+%% API
+%%--------------------------------------------------------------------
+
+start_link() ->
+    Config = gather_config(),
+    start_link(Config).
+
+start_link(Config = #{cache_capacity := _, refresh_interval := _, http_timeout := _}) ->
+    gen_server:start_link({local, ?MODULE}, ?MODULE, Config, []).
+
+-spec refresh(url()) -> ok.
+refresh(URL) ->
+    gen_server:cast(?MODULE, {refresh, URL}).
+
+-spec evict(url()) -> ok.
+evict(URL) ->
+    gen_server:cast(?MODULE, {evict, URL}).
+
+%% Adds CRLs in DER format to the cache and register them for periodic
+%% refresh.
+-spec register_der_crls(url(), [public_key:der_encoded()]) -> ok.
+register_der_crls(URL, CRLs) when is_list(CRLs) ->
+    gen_server:cast(?MODULE, {register_der_crls, URL, CRLs}).
+
+%%--------------------------------------------------------------------
+%% gen_server behaviour
+%%--------------------------------------------------------------------
+
+init(Config) ->
+    #{
+        cache_capacity := CacheCapacity,
+        refresh_interval := RefreshIntervalMS,
+        http_timeout := HTTPTimeoutMS
+    } = Config,
+    State = #state{
+        cache_capacity = CacheCapacity,
+        refresh_interval = RefreshIntervalMS,
+        http_timeout = HTTPTimeoutMS
+    },
+    {ok, State}.
+
+handle_call(Call, _From, State) ->
+    {reply, {error, {bad_call, Call}}, State}.
+
+handle_cast({evict, URL}, State0 = #state{refresh_timers = RefreshTimers0}) ->
+    emqx_ssl_crl_cache:delete(URL),
+    MTimer = maps:get(URL, RefreshTimers0, undefined),
+    emqx_misc:cancel_timer(MTimer),
+    RefreshTimers = maps:without([URL], RefreshTimers0),
+    State = State0#state{refresh_timers = RefreshTimers},
+    ?tp(
+        crl_cache_evict,
+        #{url => URL}
+    ),
+    {noreply, State};
+handle_cast({register_der_crls, URL, CRLs}, State0) ->
+    handle_register_der_crls(State0, URL, CRLs);
+handle_cast({refresh, URL}, State0) ->
+    case do_http_fetch_and_cache(URL, State0#state.http_timeout) of
+        {error, Error} ->
+            ?tp(crl_refresh_failure, #{error => Error, url => URL}),
+            ?SLOG(error, #{
+                msg => "failed_to_fetch_crl_response",
+                url => URL,
+                error => Error
+            }),
+            {noreply, ensure_timer(URL, State0, ?RETRY_TIMEOUT)};
+        {ok, _CRLs} ->
+            ?SLOG(debug, #{
+                msg => "fetched_crl_response",
+                url => URL
+            }),
+            {noreply, ensure_timer(URL, State0)}
+    end;
+handle_cast(_Cast, State) ->
+    {noreply, State}.
+
+handle_info(
+    {timeout, TRef, {refresh, URL}},
+    State = #state{
+        refresh_timers = RefreshTimers,
+        http_timeout = HTTPTimeoutMS
+    }
+) ->
+    case maps:get(URL, RefreshTimers, undefined) of
+        TRef ->
+            ?tp(debug, crl_refresh_timer, #{url => URL}),
+            case do_http_fetch_and_cache(URL, HTTPTimeoutMS) of
+                {error, Error} ->
+                    ?SLOG(error, #{
+                        msg => "failed_to_fetch_crl_response",
+                        url => URL,
+                        error => Error
+                    }),
+                    {noreply, ensure_timer(URL, State, ?RETRY_TIMEOUT)};
+                {ok, _CRLs} ->
+                    ?tp(debug, crl_refresh_timer_done, #{url => URL}),
+                    {noreply, ensure_timer(URL, State)}
+            end;
+        _ ->
+            {noreply, State}
+    end;
+handle_info(_Info, State) ->
+    {noreply, State}.
+
+%%--------------------------------------------------------------------
+%% internal functions
+%%--------------------------------------------------------------------
+
+http_get(URL, HTTPTimeout) ->
+    httpc:request(
+        get,
+        {URL, [{"connection", "close"}]},
+        [{timeout, HTTPTimeout}],
+        [{body_format, binary}]
+    ).
+
+do_http_fetch_and_cache(URL, HTTPTimeoutMS) ->
+    ?tp(crl_http_fetch, #{crl_url => URL}),
+    Resp = ?MODULE:http_get(URL, HTTPTimeoutMS),
+    case Resp of
+        {ok, {{_, 200, _}, _, Body}} ->
+            case parse_crls(Body) of
+                error ->
+                    {error, invalid_crl};
+                CRLs ->
+                    %% Note: must ensure it's a string and not a
+                    %% binary because that's what the ssl manager uses
+                    %% when doing lookups.
+                    emqx_ssl_crl_cache:insert(to_string(URL), {der, CRLs}),
+                    ?tp(crl_cache_insert, #{url => URL, crls => CRLs}),
+                    {ok, CRLs}
+            end;
+        {ok, {{_, Code, _}, _, Body}} ->
+            {error, {bad_response, #{code => Code, body => Body}}};
+        {error, Error} ->
+            {error, {http_error, Error}}
+    end.
+
+parse_crls(Bin) ->
+    try
+        [CRL || {'CertificateList', CRL, not_encrypted} <- public_key:pem_decode(Bin)]
+    catch
+        _:_ ->
+            error
+    end.
+
+ensure_timer(URL, State = #state{refresh_interval = Timeout}) ->
+    ensure_timer(URL, State, Timeout).
+
+ensure_timer(URL, State = #state{refresh_timers = RefreshTimers0}, Timeout) ->
+    ?tp(crl_cache_ensure_timer, #{url => URL, timeout => Timeout}),
+    MTimer = maps:get(URL, RefreshTimers0, undefined),
+    emqx_misc:cancel_timer(MTimer),
+    RefreshTimers = RefreshTimers0#{
+        URL => emqx_misc:start_timer(
+            Timeout,
+            {refresh, URL}
+        )
+    },
+    State#state{refresh_timers = RefreshTimers}.
+
+-spec gather_config() ->
+    #{
+        cache_capacity := pos_integer(),
+        refresh_interval := timer:time(),
+        http_timeout := timer:time()
+    }.
+gather_config() ->
+    %% TODO: add a config handler to refresh the config when those
+    %% globals change?
+    CacheCapacity = emqx_config:get([crl_cache, capacity], ?DEFAULT_CACHE_CAPACITY),
+    RefreshIntervalMS0 = emqx_config:get([crl_cache, refresh_interval], ?DEFAULT_REFRESH_INTERVAL),
+    MinimumRefreshInverval = ?MIN_REFRESH_PERIOD,
+    RefreshIntervalMS = max(RefreshIntervalMS0, MinimumRefreshInverval),
+    HTTPTimeoutMS = emqx_config:get([crl_cache, http_timeout], ?HTTP_TIMEOUT),
+    #{
+        cache_capacity => CacheCapacity,
+        refresh_interval => RefreshIntervalMS,
+        http_timeout => HTTPTimeoutMS
+    }.
+
+-spec handle_register_der_crls(state(), url(), [public_key:der_encoded()]) -> {noreply, state()}.
+handle_register_der_crls(State0, URL0, CRLs) ->
+    #state{cached_urls = CachedURLs0} = State0,
+    URL = to_string(URL0),
+    case sets:is_element(URL, CachedURLs0) of
+        true ->
+            {noreply, State0};
+        false ->
+            emqx_ssl_crl_cache:insert(URL, {der, CRLs}),
+            ?tp(debug, new_crl_url_inserted, #{url => URL}),
+            State1 = do_register_url(State0, URL),
+            State2 = handle_cache_overflow(State1),
+            State = ensure_timer(URL, State2),
+            {noreply, State}
+    end.
+
+-spec do_register_url(state(), url()) -> state().
+do_register_url(State0, URL) ->
+    #state{
+        cached_urls = CachedURLs0,
+        insertion_times = InsertionTimes0
+    } = State0,
+    Now = erlang:monotonic_time(),
+    CachedURLs = sets:add_element(URL, CachedURLs0),
+    InsertionTimes = gb_trees:enter(Now, URL, InsertionTimes0),
+    State0#state{
+        cached_urls = CachedURLs,
+        insertion_times = InsertionTimes
+    }.
+
+-spec handle_cache_overflow(state()) -> state().
+handle_cache_overflow(State0) ->
+    #state{
+        cached_urls = CachedURLs0,
+        insertion_times = InsertionTimes0,
+        cache_capacity = CacheCapacity,
+        refresh_timers = RefreshTimers0
+    } = State0,
+    case sets:size(CachedURLs0) > CacheCapacity of
+        false ->
+            State0;
+        true ->
+            {_Time, OldestURL, InsertionTimes} = gb_trees:take_smallest(InsertionTimes0),
+            emqx_ssl_crl_cache:delete(OldestURL),
+            MTimer = maps:get(OldestURL, RefreshTimers0, undefined),
+            emqx_misc:cancel_timer(MTimer),
+            RefreshTimers = maps:remove(OldestURL, RefreshTimers0),
+            CachedURLs = sets:del_element(OldestURL, CachedURLs0),
+            ?tp(debug, crl_cache_overflow, #{oldest_url => OldestURL}),
+            State0#state{
+                insertion_times = InsertionTimes,
+                cached_urls = CachedURLs,
+                refresh_timers = RefreshTimers
+            }
+    end.
+
+to_string(B) when is_binary(B) ->
+    binary_to_list(B);
+to_string(L) when is_list(L) ->
+    L.

apps/emqx/src/emqx_kernel_sup.erl

@@ -36,7 +36,8 @@ init([]) ->
             child_spec(emqx_stats, worker),
             child_spec(emqx_metrics, worker),
             child_spec(emqx_authn_authz_metrics_sup, supervisor),
-            child_spec(emqx_ocsp_cache, worker)
+            child_spec(emqx_ocsp_cache, worker),
+            child_spec(emqx_crl_cache, worker)
         ]
     }}.
 

apps/emqx/src/emqx_listeners.erl

@@ -388,7 +388,11 @@ do_start_listener(quic, ListenerName, #{bind := Bind} = Opts) ->
                 ] ++
                     case maps:get(cacertfile, SSLOpts, undefined) of
                         undefined -> [];
-                        CaCertFile -> [{cacertfile, binary_to_list(CaCertFile)}]
+                        CaCertFile -> [{cacertfile, str(CaCertFile)}]
+                    end ++
+                    case maps:get(password, SSLOpts, undefined) of
+                        undefined -> [];
+                        Password -> [{password, str(Password)}]
                     end ++
                     optional_quic_listener_opts(Opts),
             ConnectionOpts = #{
@@ -487,7 +491,8 @@ esockd_opts(ListenerId, Type, Opts0) ->
             tcp ->
                 Opts3#{tcp_options => tcp_opts(Opts0)};
             ssl ->
-                OptsWithSNI = inject_sni_fun(ListenerId, Opts0),
+                OptsWithCRL = inject_crl_config(Opts0),
+                OptsWithSNI = inject_sni_fun(ListenerId, OptsWithCRL),
                 SSLOpts = ssl_opts(OptsWithSNI),
                 Opts3#{ssl_options => SSLOpts, tcp_options => tcp_opts(Opts0)}
         end
@@ -794,3 +799,17 @@ inject_sni_fun(ListenerId, Conf = #{ssl_options := #{ocsp := #{enable_ocsp_stapl
     emqx_ocsp_cache:inject_sni_fun(ListenerId, Conf);
 inject_sni_fun(_ListenerId, Conf) ->
     Conf.
+
+inject_crl_config(
+    Conf = #{ssl_options := #{enable_crl_check := true} = SSLOpts}
+) ->
+    HTTPTimeout = emqx_config:get([crl_cache, http_timeout], timer:seconds(15)),
+    Conf#{
+        ssl_options := SSLOpts#{
+            %% `crl_check => true' doesn't work
+            crl_check => peer,
+            crl_cache => {emqx_ssl_crl_cache, {internal, [{http, HTTPTimeout}]}}
+        }
+    };
+inject_crl_config(Conf) ->
+    Conf.

apps/emqx/src/emqx_misc.erl

@@ -545,10 +545,23 @@ readable_error_msg(Error) ->
                 {ok, Msg} ->
                     Msg;
                 false ->
-                    iolist_to_binary(io_lib:format("~0p", [Error]))
+                    to_hr_error(Error)
             end
     end.
 
+to_hr_error(nxdomain) ->
+    <<"Could not resolve host">>;
+to_hr_error(econnrefused) ->
+    <<"Connection refused">>;
+to_hr_error({unauthorized_client, _}) ->
+    <<"Unauthorized client">>;
+to_hr_error({not_authorized, _}) ->
+    <<"Not authorized">>;
+to_hr_error({malformed_username_or_password, _}) ->
+    <<"Bad username or password">>;
+to_hr_error(Error) ->
+    iolist_to_binary(io_lib:format("~0p", [Error])).
+
 try_to_existing_atom(Convert, Data, Encoding) ->
     try Convert(Data, Encoding) of
         Atom ->

apps/emqx/src/emqx_schema.erl

@@ -44,6 +44,7 @@
 -type port_number() :: 1..65536.
 -type server_parse_option() :: #{default_port => port_number(), no_port => boolean()}.
 -type url() :: binary().
+-type json_binary() :: binary().
 
 -typerefl_from_string({duration/0, emqx_schema, to_duration}).
 -typerefl_from_string({duration_s/0, emqx_schema, to_duration_s}).
@@ -58,6 +59,7 @@
 -typerefl_from_string({cipher/0, emqx_schema, to_erl_cipher_suite}).
 -typerefl_from_string({comma_separated_atoms/0, emqx_schema, to_comma_separated_atoms}).
 -typerefl_from_string({url/0, emqx_schema, to_url}).
+-typerefl_from_string({json_binary/0, emqx_schema, to_json_binary}).
 
 -export([
     validate_heap_size/1,
@@ -84,7 +86,8 @@
     to_ip_port/1,
     to_erl_cipher_suite/1,
     to_comma_separated_atoms/1,
-    to_url/1
+    to_url/1,
+    to_json_binary/1
 ]).
 
 -export([
@@ -112,7 +115,8 @@
     ip_port/0,
     cipher/0,
     comma_separated_atoms/0,
-    url/0
+    url/0,
+    json_binary/0
 ]).
 
 -export([namespace/0, roots/0, roots/1, fields/1, desc/1, tags/0]).
@@ -226,6 +230,11 @@ roots(low) ->
             sc(
                 ref("trace"),
                 #{}
+            )},
+        {"crl_cache",
+            sc(
+                ref("crl_cache"),
+                #{importance => ?IMPORTANCE_HIDDEN}
             )}
     ].
 
@@ -794,6 +803,37 @@ fields("listeners") ->
                 }
             )}
     ];
+fields("crl_cache") ->
+    %% Note: we make the refresh interval and HTTP timeout global (not
+    %% per-listener) because multiple SSL listeners might point to the
+    %% same URL.  If they had diverging timeout options, it would be
+    %% confusing.
+    [
+        {"refresh_interval",
+            sc(
+                duration(),
+                #{
+                    default => <<"15m">>,
+                    desc => ?DESC("crl_cache_refresh_interval")
+                }
+            )},
+        {"http_timeout",
+            sc(
+                duration(),
+                #{
+                    default => <<"15s">>,
+                    desc => ?DESC("crl_cache_refresh_http_timeout")
+                }
+            )},
+        {"capacity",
+            sc(
+                pos_integer(),
+                #{
+                    default => 100,
+                    desc => ?DESC("crl_cache_capacity")
+                }
+            )}
+    ];
 fields("mqtt_tcp_listener") ->
     mqtt_listener(1883) ++
         [
@@ -1456,7 +1496,7 @@ fields("broker") ->
         {"perf",
             sc(
                 ref("broker_perf"),
-                #{}
+                #{importance => ?IMPORTANCE_HIDDEN}
             )},
         {"shared_subscription_group",
             sc(
@@ -1844,7 +1884,9 @@ mqtt_listener(Bind) ->
                         default => <<"3s">>
                     }
                 )},
-            {?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME, authentication(listener)}
+            {?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME, (authentication(listener))#{
+                importance => ?IMPORTANCE_HIDDEN
+            }}
         ].
 
 base_listener(Bind) ->
@@ -2065,6 +2107,8 @@ desc("shared_subscription_group") ->
     "Per group dispatch strategy for shared subscription";
 desc("ocsp") ->
     "Per listener OCSP Stapling configuration.";
+desc("crl_cache") ->
+    "Global CRL cache options.";
 desc(_) ->
     undefined.
 
@@ -2261,16 +2305,25 @@ server_ssl_opts_schema(Defaults, IsRanchListener) ->
                         #{
                             required => false,
                             %% TODO: remove after e5.0.2
-                            hidden => true,
+                            importance => ?IMPORTANCE_HIDDEN,
                             validator => fun ocsp_inner_validator/1
                         }
+                    )},
+                {"enable_crl_check",
+                    sc(
+                        boolean(),
+                        #{
+                            default => false,
+                            desc => ?DESC("server_ssl_opts_schema_enable_crl_check")
+                        }
                     )}
             ]
         ].
 
 mqtt_ssl_listener_ssl_options_validator(Conf) ->
     Checks = [
-        fun ocsp_outer_validator/1
+        fun ocsp_outer_validator/1,
+        fun crl_outer_validator/1
     ],
     case emqx_misc:pipeline(Checks, Conf, not_used) of
         {ok, _, _} ->
@@ -2305,6 +2358,18 @@ ocsp_inner_validator(#{<<"enable_ocsp_stapling">> := true} = Conf) ->
     ),
     ok.
 
+crl_outer_validator(
+    #{<<"enable_crl_check">> := true} = SSLOpts
+) ->
+    case maps:get(<<"verify">>, SSLOpts) of
+        verify_peer ->
+            ok;
+        _ ->
+            {error, "verify must be verify_peer when CRL check is enabled"}
+    end;
+crl_outer_validator(_SSLOpts) ->
+    ok.
+
 %% @doc Make schema for SSL client.
 -spec client_ssl_opts_schema(map()) -> hocon_schema:field_schema().
 client_ssl_opts_schema(Defaults) ->
@@ -2515,6 +2580,14 @@ to_url(Str) ->
             Error
     end.
 
+to_json_binary(Str) ->
+    case emqx_json:safe_decode(Str) of
+        {ok, _} ->
+            {ok, iolist_to_binary(Str)};
+        Error ->
+            Error
+    end.
+
 to_bar_separated_list(Str) ->
     {ok, string:tokens(Str, "| ")}.
 
@@ -2938,7 +3011,7 @@ quic_feature_toggle(Desc) ->
         typerefl:alias("boolean", typerefl:union([true, false, 0, 1])),
         #{
             desc => Desc,
-            hidden => true,
+            importance => ?IMPORTANCE_HIDDEN,
             required => false,
             converter => fun
                 (true) -> 1;
@@ -2953,7 +3026,7 @@ quic_lowlevel_settings_uint(Low, High, Desc) ->
         range(Low, High),
         #{
             required => false,
-            hidden => true,
+            importance => ?IMPORTANCE_HIDDEN,
             desc => Desc
         }
     ).
@@ -2964,9 +3037,9 @@ is_quic_ssl_opts(Name) ->
         "cacertfile",
         "certfile",
         "keyfile",
-        "verify"
+        "verify",
+        "password"
         %% Followings are planned
-        %% , "password"
         %% , "hibernate_after"
         %% , "fail_if_no_peer_cert"
         %% , "handshake_timeout"

apps/emqx/src/emqx_ssl_crl_cache.erl

@@ -0,0 +1,237 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2015-2022. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%
+%% %CopyrightEnd%
+
+%%--------------------------------------------------------------------
+%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+%----------------------------------------------------------------------
+% Based on `otp/lib/ssl/src/ssl_crl_cache.erl'
+%----------------------------------------------------------------------
+
+%----------------------------------------------------------------------
+%% Purpose: Simple default CRL cache
+%%----------------------------------------------------------------------
+
+-module(emqx_ssl_crl_cache).
+
+-include_lib("ssl/src/ssl_internal.hrl").
+-include_lib("public_key/include/public_key.hrl").
+
+-behaviour(ssl_crl_cache_api).
+
+-export_type([crl_src/0, uri/0]).
+-type crl_src() :: {file, file:filename()} | {der, public_key:der_encoded()}.
+-type uri() :: uri_string:uri_string().
+
+-export([lookup/3, select/2, fresh_crl/2]).
+-export([insert/1, insert/2, delete/1]).
+
+%% Allow usage of OTP certificate record fields (camelCase).
+-elvis([
+    {elvis_style, atom_naming_convention, #{
+        regex => "^([a-z][a-z0-9]*_?)([a-zA-Z0-9]*_?)*$",
+        enclosed_atoms => ".*"
+    }}
+]).
+
+%%====================================================================
+%% Cache callback API
+%%====================================================================
+
+lookup(
+    #'DistributionPoint'{distributionPoint = {fullName, Names}},
+    _Issuer,
+    CRLDbInfo
+) ->
+    get_crls(Names, CRLDbInfo);
+lookup(_, _, _) ->
+    not_available.
+
+select(GenNames, CRLDbHandle) when is_list(GenNames) ->
+    lists:flatmap(
+        fun
+            ({directoryName, Issuer}) ->
+                select(Issuer, CRLDbHandle);
+            (_) ->
+                []
+        end,
+        GenNames
+    );
+select(Issuer, {{_Cache, Mapping}, _}) ->
+    case ssl_pkix_db:lookup(Issuer, Mapping) of
+        undefined ->
+            [];
+        CRLs ->
+            CRLs
+    end.
+
+fresh_crl(#'DistributionPoint'{distributionPoint = {fullName, Names}}, CRL) ->
+    case get_crls(Names, undefined) of
+        not_available ->
+            CRL;
+        NewCRL ->
+            NewCRL
+    end.
+
+%%====================================================================
+%% API
+%%====================================================================
+
+insert(CRLs) ->
+    insert(?NO_DIST_POINT, CRLs).
+
+insert(URI, {file, File}) when is_list(URI) ->
+    case file:read_file(File) of
+        {ok, PemBin} ->
+            PemEntries = public_key:pem_decode(PemBin),
+            CRLs = [
+                CRL
+             || {'CertificateList', CRL, not_encrypted} <-
+                    PemEntries
+            ],
+            do_insert(URI, CRLs);
+        Error ->
+            Error
+    end;
+insert(URI, {der, CRLs}) ->
+    do_insert(URI, CRLs).
+
+delete({file, File}) ->
+    case file:read_file(File) of
+        {ok, PemBin} ->
+            PemEntries = public_key:pem_decode(PemBin),
+            CRLs = [
+                CRL
+             || {'CertificateList', CRL, not_encrypted} <-
+                    PemEntries
+            ],
+            ssl_manager:delete_crls({?NO_DIST_POINT, CRLs});
+        Error ->
+            Error
+    end;
+delete({der, CRLs}) ->
+    ssl_manager:delete_crls({?NO_DIST_POINT, CRLs});
+delete(URI) ->
+    case uri_string:normalize(URI, [return_map]) of
+        #{scheme := "http", path := Path} ->
+            ssl_manager:delete_crls(string:trim(Path, leading, "/"));
+        _ ->
+            {error, {only_http_distribution_points_supported, URI}}
+    end.
+
+%%--------------------------------------------------------------------
+%%% Internal functions
+%%--------------------------------------------------------------------
+do_insert(URI, CRLs) ->
+    case uri_string:normalize(URI, [return_map]) of
+        #{scheme := "http", path := Path} ->
+            ssl_manager:insert_crls(string:trim(Path, leading, "/"), CRLs);
+        _ ->
+            {error, {only_http_distribution_points_supported, URI}}
+    end.
+
+get_crls([], _) ->
+    not_available;
+get_crls(
+    [{uniformResourceIdentifier, "http" ++ _ = URL} | Rest],
+    CRLDbInfo
+) ->
+    case cache_lookup(URL, CRLDbInfo) of
+        [] ->
+            handle_http(URL, Rest, CRLDbInfo);
+        CRLs ->
+            CRLs
+    end;
+get_crls([_ | Rest], CRLDbInfo) ->
+    %% unsupported CRL location
+    get_crls(Rest, CRLDbInfo).
+
+http_lookup(URL, Rest, CRLDbInfo, Timeout) ->
+    case application:ensure_started(inets) of
+        ok ->
+            http_get(URL, Rest, CRLDbInfo, Timeout);
+        _ ->
+            get_crls(Rest, CRLDbInfo)
+    end.
+
+http_get(URL, Rest, CRLDbInfo, Timeout) ->
+    case emqx_crl_cache:http_get(URL, Timeout) of
+        {ok, {_Status, _Headers, Body}} ->
+            case Body of
+                <<"-----BEGIN", _/binary>> ->
+                    Pem = public_key:pem_decode(Body),
+                    CRLs = lists:filtermap(
+                        fun
+                            ({'CertificateList', CRL, not_encrypted}) ->
+                                {true, CRL};
+                            (_) ->
+                                false
+                        end,
+                        Pem
+                    ),
+                    emqx_crl_cache:register_der_crls(URL, CRLs),
+                    CRLs;
+                _ ->
+                    try public_key:der_decode('CertificateList', Body) of
+                        _ ->
+                            CRLs = [Body],
+                            emqx_crl_cache:register_der_crls(URL, CRLs),
+                            CRLs
+                    catch
+                        _:_ ->
+                            get_crls(Rest, CRLDbInfo)
+                    end
+            end;
+        {error, _Reason} ->
+            get_crls(Rest, CRLDbInfo)
+    end.
+
+cache_lookup(_, undefined) ->
+    [];
+cache_lookup(URL, {{Cache, _}, _}) ->
+    #{path := Path} = uri_string:normalize(URL, [return_map]),
+    case ssl_pkix_db:lookup(string:trim(Path, leading, "/"), Cache) of
+        undefined ->
+            [];
+        [CRLs] ->
+            CRLs
+    end.
+
+handle_http(URI, Rest, {_, [{http, Timeout}]} = CRLDbInfo) ->
+    CRLs = http_lookup(URI, Rest, CRLDbInfo, Timeout),
+    %% Uncomment to improve performance, but need to
+    %% implement cache limit and or cleaning to prevent
+    %% DoS attack possibilities
+    %%insert(URI, {der, CRLs}),
+    CRLs;
+handle_http(_, Rest, CRLDbInfo) ->
+    get_crls(Rest, CRLDbInfo).

apps/emqx/test/emqx_client_SUITE.erl

@@ -390,4 +390,10 @@ tls_certcn_as_clientid(TLSVsn, RequiredTLSVsn) ->
     {ok, _} = emqtt:connect(Client),
     #{clientinfo := #{clientid := CN}} = emqx_cm:get_chan_info(CN),
     confirm_tls_version(Client, RequiredTLSVsn),
+    %% verify that the peercert won't be stored in the conninfo
+    [ChannPid] = emqx_cm:lookup_channels(CN),
+    SysState = sys:get_state(ChannPid),
+    ChannelRecord = lists:keyfind(channel, 1, tuple_to_list(SysState)),
+    ConnInfo = lists:nth(2, tuple_to_list(ChannelRecord)),
+    ?assertMatch(#{peercert := undefined}, ConnInfo),
     emqtt:disconnect(Client).

apps/emqx/test/emqx_common_test_helpers.erl

@@ -16,7 +16,7 @@
 
 -module(emqx_common_test_helpers).
 
--include("emqx_authentication.hrl").
+-include_lib("emqx/include/emqx_authentication.hrl").
 
 -type special_config_handler() :: fun().
 
@@ -85,6 +85,13 @@
     reset_proxy/2
 ]).
 
+%% TLS certs API
+-export([
+    gen_ca/2,
+    gen_host_cert/3,
+    gen_host_cert/4
+]).
+
 -define(CERTS_PATH(CertName), filename:join(["etc", "certs", CertName])).
 
 -define(MQTT_SSL_CLIENT_CERTS, [
@@ -202,7 +209,6 @@ start_apps(Apps, SpecAppConfig, Opts) when is_function(SpecAppConfig) ->
     %% Because, minirest, ekka etc.. application will scan these modules
     lists:foreach(fun load/1, [emqx | Apps]),
     ok = start_ekka(),
-    mnesia:clear_table(emqx_admin),
     ok = emqx_ratelimiter_SUITE:load_conf(),
     lists:foreach(fun(App) -> start_app(App, SpecAppConfig, Opts) end, [emqx | Apps]).
 
@@ -262,12 +268,13 @@ app_schema(App) ->
     end.
 
 mustache_vars(App, Opts) ->
-    ExtraMustacheVars = maps:get(extra_mustache_vars, Opts, []),
+    ExtraMustacheVars = maps:get(extra_mustache_vars, Opts, #{}),
-    [
+    Defaults = #{
-        {platform_data_dir, app_path(App, "data")},
+        platform_data_dir => app_path(App, "data"),
-        {platform_etc_dir, app_path(App, "etc")},
+        platform_etc_dir => app_path(App, "etc"),
-        {platform_log_dir, app_path(App, "log")}
+        platform_log_dir => app_path(App, "log")
-    ] ++ ExtraMustacheVars.
+    },
+    maps:merge(Defaults, ExtraMustacheVars).
 
 render_config_file(ConfigFile, Vars0) ->
     Temp =
@@ -275,7 +282,7 @@ render_config_file(ConfigFile, Vars0) ->
             {ok, T} -> T;
             {error, Reason} -> error({failed_to_read_config_template, ConfigFile, Reason})
         end,
-    Vars = [{atom_to_list(N), iolist_to_binary(V)} || {N, V} <- Vars0],
+    Vars = [{atom_to_list(N), iolist_to_binary(V)} || {N, V} <- maps:to_list(Vars0)],
     Targ = bbmustache:render(Temp, Vars),
     NewName = ConfigFile ++ ".rendered",
     ok = file:write_file(NewName, Targ),
@@ -299,6 +306,7 @@ generate_config(SchemaModule, ConfigFile) when is_atom(SchemaModule) ->
 -spec stop_apps(list()) -> ok.
 stop_apps(Apps) ->
     [application:stop(App) || App <- Apps ++ [emqx, ekka, mria, mnesia]],
+    ok = mria_mnesia:delete_schema(),
     %% to avoid inter-suite flakiness
     application:unset_env(emqx, init_config_load_done),
     persistent_term:erase(?EMQX_AUTHENTICATION_SCHEMA_MODULE_PT_KEY),
@@ -561,6 +569,7 @@ ensure_quic_listener(Name, UdpPort, ExtraSettings) ->
         mountpoint => <<>>,
         zone => default
     },
+
     Conf2 = maps:merge(Conf, ExtraSettings),
     emqx_config:put([listeners, quic, Name], Conf2),
     case emqx_listeners:start_listener(emqx_listeners:listener_id(quic, Name)) of
@@ -651,6 +660,7 @@ start_slave(Name, Opts) when is_list(Opts) ->
 start_slave(Name, Opts) when is_map(Opts) ->
     SlaveMod = maps:get(peer_mod, Opts, ct_slave),
     Node = node_name(Name),
+    put_peer_mod(Node, SlaveMod),
     DoStart =
         fun() ->
             case SlaveMod of
@@ -660,13 +670,14 @@ start_slave(Name, Opts) when is_map(Opts) ->
                         [
                             {kill_if_fail, true},
                             {monitor_master, true},
-                            {init_timeout, 10000},
+                            {init_timeout, 20_000},
-                            {startup_timeout, 10000},
+                            {startup_timeout, 20_000},
                             {erl_flags, erl_flags()}
                         ]
                     );
                 slave ->
-                    slave:start_link(host(), Name, ebin_path())
+                    Env = " -env HOCON_ENV_OVERRIDE_PREFIX EMQX_",
+                    slave:start_link(host(), Name, ebin_path() ++ Env)
             end
         end,
     case DoStart() of
@@ -678,7 +689,6 @@ start_slave(Name, Opts) when is_map(Opts) ->
             throw(Other)
     end,
     pong = net_adm:ping(Node),
-    put_peer_mod(Node, SlaveMod),
     setup_node(Node, Opts),
     ok = snabbkaffe:forward_trace(Node),
     Node.
@@ -723,7 +733,7 @@ setup_node(Node, Opts) when is_map(Opts) ->
     ConfigureGenRpc = maps:get(configure_gen_rpc, Opts, true),
     LoadSchema = maps:get(load_schema, Opts, true),
     SchemaMod = maps:get(schema_mod, Opts, emqx_schema),
-    LoadApps = maps:get(load_apps, Opts, [gen_rpc, emqx, ekka, mria] ++ Apps),
+    LoadApps = maps:get(load_apps, Opts, Apps),
     Env = maps:get(env, Opts, []),
     Conf = maps:get(conf, Opts, []),
     ListenerPorts = maps:get(listener_ports, Opts, [
@@ -740,13 +750,28 @@ setup_node(Node, Opts) when is_map(Opts) ->
     %% `emqx_conf' app and correctly catch up the config.
     StartAutocluster = maps:get(start_autocluster, Opts, false),
 
+    ct:pal(
+        "setting up node ~p:\n  ~p",
+        [
+            Node,
+            #{
+                start_autocluster => StartAutocluster,
+                load_apps => LoadApps,
+                apps => Apps,
+                env => Env,
+                start_apps => StartApps
+            }
+        ]
+    ),
+
     %% Load env before doing anything to avoid overriding
-    lists:foreach(fun(App) -> rpc:call(Node, ?MODULE, load, [App]) end, LoadApps),
+    [ok = erpc:call(Node, ?MODULE, load, [App]) || App <- [gen_rpc, ekka, mria, emqx | LoadApps]],
+
     %% Ensure a clean mnesia directory for each run to avoid
     %% inter-test flakiness.
     MnesiaDataDir = filename:join([
         PrivDataDir,
-        node(),
+        Node,
         integer_to_list(erlang:unique_integer()),
         "mnesia"
     ]),
@@ -763,10 +788,7 @@ setup_node(Node, Opts) when is_map(Opts) ->
         end,
 
     %% Setting env before starting any applications
-    [
+    set_envs(Node, Env),
-        ok = rpc:call(Node, application, set_env, [Application, Key, Value])
-     || {Application, Key, Value} <- Env
-    ],
 
     %% Here we start the apps
     EnvHandlerForRpc =
@@ -784,8 +806,9 @@ setup_node(Node, Opts) when is_map(Opts) ->
                         node(),
                         integer_to_list(erlang:unique_integer())
                     ]),
+                    Cookie = atom_to_list(erlang:get_cookie()),
                     os:putenv("EMQX_NODE__DATA_DIR", NodeDataDir),
-                    os:putenv("EMQX_NODE__COOKIE", atom_to_list(erlang:get_cookie())),
+                    os:putenv("EMQX_NODE__COOKIE", Cookie),
                     emqx_config:init_load(SchemaMod),
                     os:unsetenv("EMQX_NODE__DATA_DIR"),
                     os:unsetenv("EMQX_NODE__COOKIE"),
@@ -816,7 +839,15 @@ setup_node(Node, Opts) when is_map(Opts) ->
             ok;
         _ ->
             StartAutocluster andalso
-                (ok = rpc:call(Node, emqx_machine_boot, start_autocluster, [])),
+                begin
+                    %% Note: we need to re-set the env because
+                    %% starting the apps apparently make some of them
+                    %% to be lost...  This is particularly useful for
+                    %% setting extra apps to be restarted after
+                    %% joining.
+                    set_envs(Node, Env),
+                    ok = erpc:call(Node, emqx_machine_boot, start_autocluster, [])
+                end,
             case rpc:call(Node, ekka, join, [JoinTo]) of
                 ok ->
                     ok;
@@ -873,6 +904,14 @@ merge_opts(Opts1, Opts2) ->
         Opts2
     ).
 
+set_envs(Node, Env) ->
+    lists:foreach(
+        fun({Application, Key, Value}) ->
+            ok = rpc:call(Node, application, set_env, [Application, Key, Value])
+        end,
+        Env
+    ).
+
 erl_flags() ->
     %% One core and redirecting logs to master
     "+S 1:1 -master " ++ atom_to_list(node()) ++ " " ++ ebin_path().
@@ -1073,6 +1112,104 @@ latency_up_proxy(off, Name, ProxyHost, ProxyPort) ->
     ).
 
 %%-------------------------------------------------------------------------------
+%% TLS certs
+%%-------------------------------------------------------------------------------
+gen_ca(Path, Name) ->
+    %% Generate ca.pem and ca.key which will be used to generate certs
+    %% for hosts server and clients
+    ECKeyFile = filename(Path, "~s-ec.key", [Name]),
+    filelib:ensure_dir(ECKeyFile),
+    os:cmd("openssl ecparam -name secp256r1 > " ++ ECKeyFile),
+    Cmd = lists:flatten(
+        io_lib:format(
+            "openssl req -new -x509 -nodes "
+            "-newkey ec:~s "
+            "-keyout ~s -out ~s -days 3650 "
+            "-subj \"/C=SE/O=Internet Widgits Pty Ltd CA\"",
+            [
+                ECKeyFile,
+                ca_key_name(Path, Name),
+                ca_cert_name(Path, Name)
+            ]
+        )
+    ),
+    os:cmd(Cmd).
+
+ca_cert_name(Path, Name) ->
+    filename(Path, "~s.pem", [Name]).
+ca_key_name(Path, Name) ->
+    filename(Path, "~s.key", [Name]).
+
+gen_host_cert(H, CaName, Path) ->
+    gen_host_cert(H, CaName, Path, #{}).
+
+gen_host_cert(H, CaName, Path, Opts) ->
+    ECKeyFile = filename(Path, "~s-ec.key", [CaName]),
+    CN = str(H),
+    HKey = filename(Path, "~s.key", [H]),
+    HCSR = filename(Path, "~s.csr", [H]),
+    HPEM = filename(Path, "~s.pem", [H]),
+    HEXT = filename(Path, "~s.extfile", [H]),
+    PasswordArg =
+        case maps:get(password, Opts, undefined) of
+            undefined ->
+                " -nodes ";
+            Password ->
+                io_lib:format(" -passout pass:'~s' ", [Password])
+        end,
+    CSR_Cmd =
+        lists:flatten(
+            io_lib:format(
+                "openssl req -new ~s -newkey ec:~s "
+                "-keyout ~s -out ~s "
+                "-addext \"subjectAltName=DNS:~s\" "
+                "-addext keyUsage=digitalSignature,keyAgreement "
+                "-subj \"/C=SE/O=Internet Widgits Pty Ltd/CN=~s\"",
+                [PasswordArg, ECKeyFile, HKey, HCSR, CN, CN]
+            )
+        ),
+    create_file(
+        HEXT,
+        "keyUsage=digitalSignature,keyAgreement\n"
+        "subjectAltName=DNS:~s\n",
+        [CN]
+    ),
+    CERT_Cmd =
+        lists:flatten(
+            io_lib:format(
+                "openssl x509 -req "
+                "-extfile ~s "
+                "-in ~s -CA ~s -CAkey ~s -CAcreateserial "
+                "-out ~s -days 500",
+                [
+                    HEXT,
+                    HCSR,
+                    ca_cert_name(Path, CaName),
+                    ca_key_name(Path, CaName),
+                    HPEM
+                ]
+            )
+        ),
+    ct:pal(os:cmd(CSR_Cmd)),
+    ct:pal(os:cmd(CERT_Cmd)),
+    file:delete(HEXT).
+
+filename(Path, F, A) ->
+    filename:join(Path, str(io_lib:format(F, A))).
+
+str(Arg) ->
+    binary_to_list(iolist_to_binary(Arg)).
+
+create_file(Filename, Fmt, Args) ->
+    filelib:ensure_dir(Filename),
+    {ok, F} = file:open(Filename, [write]),
+    try
+        io:format(F, Fmt, Args)
+    after
+        file:close(F)
+    end,
+    ok.
+%%-------------------------------------------------------------------------------
 %% Testcase teardown utilities
 %%-------------------------------------------------------------------------------
 

apps/emqx/test/emqx_crl_cache_SUITE_data/ca-chain.cert.pem

@@ -0,0 +1,68 @@
+-----BEGIN CERTIFICATE-----
+MIIF+zCCA+OgAwIBAgICEAAwDQYJKoZIhvcNAQELBQAwbzELMAkGA1UEBhMCU0Ux
+EjAQBgNVBAgMCVN0b2NraG9sbTESMBAGA1UEBwwJU3RvY2tob2xtMRIwEAYDVQQK
+DAlNeU9yZ05hbWUxETAPBgNVBAsMCE15Um9vdENBMREwDwYDVQQDDAhNeVJvb3RD
+QTAeFw0yMzAxMTIxMzA4MTZaFw0zMzAxMDkxMzA4MTZaMGsxCzAJBgNVBAYTAlNF
+MRIwEAYDVQQIDAlTdG9ja2hvbG0xEjAQBgNVBAoMCU15T3JnTmFtZTEZMBcGA1UE
+CwwQTXlJbnRlcm1lZGlhdGVDQTEZMBcGA1UEAwwQTXlJbnRlcm1lZGlhdGVDQTCC
+AiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBALQG7dMeU/y9HDNHzhydR0bm
+wN9UGplqJOJPwqJRaZZcrn9umgJ9SU2il2ceEVxMDwzBWCRKJO5/H9A9k13SqsXM
+2c2c9xXfIF1kb820lCm1Uow5hZ/auDjxliNk9kNJDigCRi3QoIs/dVeWzFsgEC2l
+gxRqauN2eNFb6/yXY788YALHBsCRV2NFOFXxtPsvLXpD9Q/8EqYsSMuLARRdHVNU
+ryaEF5lhShpcuz0TlIuTy2TiuXJUtJ+p7a4Z7friZ6JsrmQWsVQBj44F8TJRHWzW
+C7vm9c+dzEX9eqbr5iPL+L4ctMW9Lz6ePcYfIXne6CElusRUf8G+xM1uwovF9bpV
++9IqY7tAu9G1iY9iNtJgNNDKOCcOGKcZCx6Cg1XYOEKReNnUMazvYeqRrrjV5WQ0
+vOcD5zcBRNTXCddCLa7U0guXP9mQrfuk4NTH1Bt77JieTJ8cfDXHwtaKf6aGbmZP
+wl1Xi/GuXNUP/xeog78RKyFwBmjt2JKwvWzMpfmH4mEkG9moh2alva+aEz6LIJuP
+16g6s0Q6c793/OvUtpNcewHw4Vjn39LD9o6VLp854G4n8dVpUWSbWS+sXD1ZE69H
+g/sMNMyq+09ufkbewY8xoCm/rQ1pqDZAVMWsstJEaYu7b/eb7R+RGOj1YECCV/Yp
+EZPdDotbSNRkIi2d/a1NAgMBAAGjgaQwgaEwHQYDVR0OBBYEFExwhjsVUom6tQ+S
+qq6xMUETvnPzMB8GA1UdIwQYMBaAFD90kfU5pc5l48THu0Ayj9SNpHuhMBIGA1Ud
+EwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgGGMDsGA1UdHwQ0MDIwMKAuoCyG
+Kmh0dHA6Ly9sb2NhbGhvc3Q6OTg3OC9pbnRlcm1lZGlhdGUuY3JsLnBlbTANBgkq
+hkiG9w0BAQsFAAOCAgEAK6NgdWQYtPNKQNBGjsgtgqTRh+k30iqSO6Y3yE1KGABO
+EuQdVqkC2qUIbCB0M0qoV0ab50KNLfU6cbshggW4LDpcMpoQpI05fukNh1jm3ZuZ
+0xsB7vlmlsv00tpqmfIl/zykPDynHKOmFh/hJP/KetMy4+wDv4/+xP31UdEj5XvG
+HvMtuqOS23A+H6WPU7ol7KzKBnU2zz/xekvPbUD3JqV+ynP5bgbIZHAndd0o9T8e
+NFX23Us4cTenU2/ZlOq694bRzGaK+n3Ksz995Nbtzv5fbUgqmf7Mcq4iHGRVtV11
+MRyBrsXZp2vbF63c4hrf2Zd6SWRoaDKRhP2DMhajpH9zZASSTlfejg/ZRO2s+Clh
+YrSTkeMAdnRt6i/q4QRcOTCfsX75RFM5v67njvTXsSaSTnAwaPi78tRtf+WSh0EP
+VVPzy++BszBVlJ1VAf7soWZHCjZxZ8ZPqVTy5okoHwWQ09WmYe8GfulDh1oj0wbK
+3FjN7bODWHJN+bFf5aQfK+tumYKoPG8RXL6QxpEzjFWjxhIMJHHMKfDWnAV1o1+7
+/1/aDzq7MzEYBbrgQR7oE5ZHtyqhCf9LUgw0Kr7/8QWuNAdeDCJzjXRROU0hJczp
+dOyfRlLbHmLLmGOnROlx6LsGNQ17zuz6SPi7ei8/ylhykawDOAGkM1+xFakmQhM=
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIIFzzCCA7egAwIBAgIUYjc7hD7/UJ0/VPADfNfp/WpOwRowDQYJKoZIhvcNAQEL
+BQAwbzELMAkGA1UEBhMCU0UxEjAQBgNVBAgMCVN0b2NraG9sbTESMBAGA1UEBwwJ
+U3RvY2tob2xtMRIwEAYDVQQKDAlNeU9yZ05hbWUxETAPBgNVBAsMCE15Um9vdENB
+MREwDwYDVQQDDAhNeVJvb3RDQTAeFw0yMzAxMTIxMzA4MTRaFw00MzAxMDcxMzA4
+MTRaMG8xCzAJBgNVBAYTAlNFMRIwEAYDVQQIDAlTdG9ja2hvbG0xEjAQBgNVBAcM
+CVN0b2NraG9sbTESMBAGA1UECgwJTXlPcmdOYW1lMREwDwYDVQQLDAhNeVJvb3RD
+QTERMA8GA1UEAwwITXlSb290Q0EwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIK
+AoICAQCnBwSOYVJw47IoMHMXTVDtOYvUt3rqsurEhFcB4O8xmf2mmwr6m7s8A5Ft
+AvAehg1GvnXT3t/KiyU7BK+acTwcErGyZwS2wvdB0lpHWSpOn/u5y+4ZETvQefcj
+ZTdDOM9VN5nutpitgNb+1yL8sqSexfVbY7DnYYvFjOVBYoP/SGvM9jVjCad+0WL3
+FhuD+L8QAxzCieX3n9UMymlFwINQuEc+TDjuNcEqt+0J5EgS1fwzxb2RCVL0TNv4
+9a71hFGCNRj20AeZm99hbdufm7+0AFO7ocV5q43rLrWFUoBzqKPYIjga/cv/UdWZ
+c5RLRXw3JDSrCqkf/mOlaEhNPlmWRF9MSus5Da3wuwgGCaVzmrf30rWR5aHHcscG
+e+AOgJ4HayvBUQeb6ZlRXc0YlACiLToMKxuyxDyUcDfVEXpUIsDILF8dkiVQxEU3
+j9g6qjXiqPVdNiwpqXfBKObj8vNCzORnoHYs8cCgib3RgDVWeqkDmlSwlZE7CvQh
+U4Loj4l7813xxzYEKkVaT1JdXPWu42CG/b4Y/+f4V+3rkJkYzUwndX6kZNksIBai
+phmtvKt+CTdP1eAbT+C9AWWF3PT31+BIhuT0u9tR8BVSkXdQB8dG4M/AAJcTo640
+0mdYYOXT153gEKHJuUBm750ZTy+r6NjNvpw8VrMAakJwHqnIdQIDAQABo2MwYTAd
+BgNVHQ4EFgQUP3SR9TmlzmXjxMe7QDKP1I2ke6EwHwYDVR0jBBgwFoAUP3SR9Tml
+zmXjxMe7QDKP1I2ke6EwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAYYw
+DQYJKoZIhvcNAQELBQADggIBAFMFv4C+I0+xOAb9v6G/IOpfPBZ1ez31EXKJJBra
+lulP4nRHQMeb310JS8BIeQ3dl+7+PkSxPABZSwc3jkxdSMvhc+Z4MQtTgos+Qsjs
+gH7sTqwWeeQ0lHYxWmkXijrh5OPRZwTKzYQlkcn85BCUXl2KDuNEdiqPbDTao+lc
+lA0/UAvC6NCyFKq/jqf4CmW5Kx6yG1v1LaE+IXn7cbIXj+DaehocVXi0wsXqj03Q
+DDUHuLHZP+LBsg4e91/0Jy2ekNRTYJifSqr+9ufHl0ZX1pFDZyf396IgZ5CQZ0PJ
+nRxZHlCfsxWxmxxdy3FQSE6YwXhdTjjoAa1ApZcKkkt1beJa6/oRLze/ux5x+5q+
+4QczufHd6rjoKBi6BM3FgFQ8As5iNohHXlMHd/xITo1Go3CWw2j9TGH5vzksOElK
+B0mcwwt2zwNEjvfytc+tI5jcfGN3tiT5fVHS8hw9dWKevypLL+55Ua9G8ZgDHasT
+XFRJHgmnbyFcaAe26D2dSKmhC9u2mHBH+MaI8dj3e7wNBfpxNgp41aFIk+QTmiFW
+VXFED6DHQ/Mxq93ACalHdYg18PlIYClbT6Pf2xXBnn33YPhn5xzoTZ+cDH/RpaQp
+s0UUTSJT1UTXgtXPnZWQfvKlMjJEIiVFiLEC0sgZRlWuZDRAY0CdZJJxvQp59lqu
+cbTm
+-----END CERTIFICATE-----

apps/emqx/test/emqx_crl_cache_SUITE_data/client-no-dist-points.cert.pem

@@ -0,0 +1,32 @@
+-----BEGIN CERTIFICATE-----
+MIIFdTCCA12gAwIBAgICEAUwDQYJKoZIhvcNAQELBQAwazELMAkGA1UEBhMCU0Ux
+EjAQBgNVBAgMCVN0b2NraG9sbTESMBAGA1UECgwJTXlPcmdOYW1lMRkwFwYDVQQL
+DBBNeUludGVybWVkaWF0ZUNBMRkwFwYDVQQDDBBNeUludGVybWVkaWF0ZUNBMB4X
+DTIzMDExODEyMzY1NloXDTMzMDQyNTEyMzY1NlowgYQxCzAJBgNVBAYTAlNFMRIw
+EAYDVQQIDAlTdG9ja2hvbG0xEjAQBgNVBAcMCVN0b2NraG9sbTESMBAGA1UECgwJ
+TXlPcmdOYW1lMRkwFwYDVQQLDBBNeUludGVybWVkaWF0ZUNBMR4wHAYDVQQDDBVj
+bGllbnQtbm8tZGlzdC1wb2ludHMwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK
+AoIBAQCYQqNF7o20tEwyXphDgtwkZ628baYzQoCmmaufR+5SPQWdTN+GFeApv0dP
+4y/ncZV24rgButMo73e4+wPsILwSGhaVIU0mMaCmexyC4W6INBkQsVB5FAd/YM0O
+gdxS6A42h9HZTaAJ+4ftgFdOOHiP3lwicXeIYykAE7Y5ikxlnHgi8p1PTLowN4Q+
+AjuXChRzmU16cUEAevZKkTVf7VCcK66aJsxBsxfykkGHhc6qLqmlMt6Te6DPCi/R
+KP/kARTDWNEkp6qtpvzByYFYAKPSZxPuryajAC3RLuGNkVSB+PZ6NnZW6ASeTdra
+Lwuiwsi5XPBeFb0147naQOBzSGG/AgMBAAGjggEHMIIBAzAJBgNVHRMEAjAAMBEG
+CWCGSAGG+EIBAQQEAwIFoDBBBglghkgBhvhCAQ0ENBYyT3BlblNTTCBHZW5lcmF0
+ZWQgQ2xpZW50IENlcnRpZmljYXRlIChubyBDUkwgaW5mbykwHQYDVR0OBBYEFBiV
+sjDe46MixvftT/wej1mxGuN7MB8GA1UdIwQYMBaAFExwhjsVUom6tQ+Sqq6xMUET
+vnPzMA4GA1UdDwEB/wQEAwIF4DAdBgNVHSUEFjAUBggrBgEFBQcDAgYIKwYBBQUH
+AwQwMQYIKwYBBQUHAQEEJTAjMCEGCCsGAQUFBzABhhVodHRwOi8vbG9jYWxob3N0
+Ojk4NzcwDQYJKoZIhvcNAQELBQADggIBAKBEnKYVLFtZb3MI0oMJkrWBssVCq5ja
+OYomZ61I13QLEeyPevTSWAcWFQ4zQDF/SWBsXjsrC+JIEjx2xac6XCpxcx3jDUgo
+46u/hx2rT8tMKa60hW0V1Dk6w8ZHiCe94BlFLsWFKnn6dVzoJd2u3vgUaleh3uxF
+hug8XY+wmHd36rO0kVe3DrsqdIdOfhMiJLDxU0cBA79vI5kCvqB8DIwCWtOzkA82
+EPl3Iws5NPhuFAR9u0xOQu0akzmSJFcEGLZ4qfatHD/tZGRduyFvMKy5iIeMzuEs
+2etm01tfLHqgKGOKp5LjPm7Aoac/GeVoTvctGF+wayvOuYE7inlGZToz3kQMMzHZ
+ZGBBgOhXbR2y74QoFv6DUqmmTRbGfiLYyErA5r881ntgciQi02xrGjoAFntvKb+H
+HNB22Qprz16OmdC9dJKF2RhO6Cketdhv65wFWw6xlhRMCWYPY3CI8tWkxS4A4yit
+RZQZg3yaeHXMaCAu5HxuqAQXKGjz+7w7N6diwbT7o7CfKk8iHUrGfkQ5nCS0GZ1r
+lU1vgKtdzVvJ6HmBrCRcdNqh/L/wdIltwI/52j+TKRtELM1qHuLAYmhcRBW+2wuH
+ewaNA9KEgEk6JC+iR8uOBi0ZLkMIm47j+ZLJRJVUfgkVEEFjyiYSFfpwwcgT+/Aw
+EczVZOdUEbDM
+-----END CERTIFICATE-----

apps/emqx/test/emqx_crl_cache_SUITE_data/client-no-dist-points.key.pem

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCYQqNF7o20tEwy
+XphDgtwkZ628baYzQoCmmaufR+5SPQWdTN+GFeApv0dP4y/ncZV24rgButMo73e4
++wPsILwSGhaVIU0mMaCmexyC4W6INBkQsVB5FAd/YM0OgdxS6A42h9HZTaAJ+4ft
+gFdOOHiP3lwicXeIYykAE7Y5ikxlnHgi8p1PTLowN4Q+AjuXChRzmU16cUEAevZK
+kTVf7VCcK66aJsxBsxfykkGHhc6qLqmlMt6Te6DPCi/RKP/kARTDWNEkp6qtpvzB
+yYFYAKPSZxPuryajAC3RLuGNkVSB+PZ6NnZW6ASeTdraLwuiwsi5XPBeFb0147na
+QOBzSGG/AgMBAAECggEACSMuozq+vFJ5pCgzIRIQXgruzTkTWU4rZFQijYuGjN7m
+oFsFqwlTC45UHEI5FL2nR5wxiMEKfRFp8Or3gEsyni98nXSDKcCesH8A5gXbWUcv
+HeZWOv3tuUI47B709vDAMZuTB2R2L0MuFB24n5QaACBLDTIcB05UHpIQRIG9NffH
+MhxqFB2kuakp67VekYGZkBCNkqfL3VQZIGRpQC8SvpnRXELqZgI4MyJgvkK6myWj
+Vtpwm8YiOQoJHJx4raoVfS2NWTsCwL0M0aXMMtmM2QfMP/xB9OifxnmDDBs7Tie8
+0Wri845xLTCYthaU8B06rhoQdKXoqKmQMoF2doPm8QKBgQDN+0E0PtPkyxIho8pV
+CsQnmif91EQQqWxOdkHbE96lT0UKu6ziBSbB4ClRHYil5c8p7INxRpj7pruOY3Kw
+MAcacIMMBNhLBJL4R0hr/pwr18WOZxCIMcLHTaCfbVqL71TKp4/6C+GexZfaYJ46
+IZEpLU5RPmD4f9MPIDDm6KcPxwKBgQC9O9TOor93g+A4sU54CGOqvVDrdi5TnGF8
+YdimvUsT20gl2WGX5vq3OohzZi7U8FuxKHWpbgh2efqGLcFsRNFZ/T0ZXX4DDafN
+Gzyu/DMVuFO4ccgFJNnl45w3/yFG40kL6yS8kss/iEYu550/uOZ1FjH+kJ0vjV6G
+JD8q0PgOSQKBgG2i9cLcSia2nBEBwFlhoKS/ndeyWwRPWZGtykHUoqZ0ufgLiurG
++SkqqnM9eBVta8YR2Ki7fgQ8bApPDqWO+sjs6CPGlGXhqmSydG7fF7sSX1n7q8YC
+Tn2M6RjSuOZQ3l37sFvUZSQAYmJfGPkyErTLI6uEu1KpnuqnJMBTR1DTAoGAIGQn
+bx9oirqmHM4s0lsNRGKXgVZ/Y4x3G2VcQl5QhZuZY/ErxWaiL87zIF2zUnu6Fj8I
+tPHCvRTwDxux6ih1dWPlm3vnX/psaK1q28ELtYIRwpanWEoQiktFqEghmBK7pDCh
+3y15YOygptK6lfe+avhboml6nnMiZO+7aEbQzxECgYALuUM4fo1dQYmYuZIqZoFJ
+TXGyzMkNGs61SMiD6mW6XgXj5h5T8Q0MdpmHkwsm+z9A/1of5cxkE6d8HCCz+dt5
+tnY7OC0gYB1+gDld8MZgFgP6k0qklreLVhzEz11TbMldifa1EE4VjUDG/NeAEtbq
+GbLaw0NhGJtRCgL9Bc7i7g==
+-----END PRIVATE KEY-----

apps/emqx/test/emqx_crl_cache_SUITE_data/client-revoked.cert.pem

@@ -0,0 +1,32 @@
+-----BEGIN CERTIFICATE-----
+MIIFnDCCA4SgAwIBAgICEAIwDQYJKoZIhvcNAQELBQAwazELMAkGA1UEBhMCU0Ux
+EjAQBgNVBAgMCVN0b2NraG9sbTESMBAGA1UECgwJTXlPcmdOYW1lMRkwFwYDVQQL
+DBBNeUludGVybWVkaWF0ZUNBMRkwFwYDVQQDDBBNeUludGVybWVkaWF0ZUNBMB4X
+DTIzMDExMjEzMDgxNloXDTMzMDQxOTEzMDgxNlowfTELMAkGA1UEBhMCU0UxEjAQ
+BgNVBAgMCVN0b2NraG9sbTESMBAGA1UEBwwJU3RvY2tob2xtMRIwEAYDVQQKDAlN
+eU9yZ05hbWUxGTAXBgNVBAsMEE15SW50ZXJtZWRpYXRlQ0ExFzAVBgNVBAMMDmNs
+aWVudC1yZXZva2VkMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAs+R6
+PDtIxVlUoLYbDBbaVcxgoLjnWcvqL8wSqyWuqi/Y3cjuNYCziR9nR5dWajtkBjzJ
+HyhgAr6gBVSRt4RRmDXoOcprK3GcpowAr65UAmC4hdH0af6FdKjKCnFw67byUg52
+f7ueXZ6t/XuuKxlU/f2rjXVwmmnlhBi5EHDkXxvfgWXJekDfsPbW9j0kaCUWCpfj
+rzGbfkXqrPkslO41PYlCbPxoiRItJjindFjcQySYvRq7A2uYMGsrxv4n3rzo5NGt
+goBmnGj61ii9WOdopcFxKirhIB9zrxC4x0opRfIaF/n1ZXk6NOnaDxu1LTZ18wfC
+ZB979ge6pleeKoPf7QIDAQABo4IBNjCCATIwCQYDVR0TBAIwADARBglghkgBhvhC
+AQEEBAMCBaAwMwYJYIZIAYb4QgENBCYWJE9wZW5TU0wgR2VuZXJhdGVkIENsaWVu
+dCBDZXJ0aWZpY2F0ZTAdBgNVHQ4EFgQUQeItXr3nc6CZ++G9UCoq1YlQ9oowHwYD
+VR0jBBgwFoAUTHCGOxVSibq1D5KqrrExQRO+c/MwDgYDVR0PAQH/BAQDAgXgMB0G
+A1UdJQQWMBQGCCsGAQUFBwMCBggrBgEFBQcDBDA7BgNVHR8ENDAyMDCgLqAshipo
+dHRwOi8vbG9jYWxob3N0Ojk4NzgvaW50ZXJtZWRpYXRlLmNybC5wZW0wMQYIKwYB
+BQUHAQEEJTAjMCEGCCsGAQUFBzABhhVodHRwOi8vbG9jYWxob3N0Ojk4NzcwDQYJ
+KoZIhvcNAQELBQADggIBAIFuhokODd54/1B2JiNyG6FMq/2z8B+UquC2iw3p2pyM
+g/Jz4Ouvg6gGwUwmykEua06FRCxx5vJ5ahdhXvKst/zH/0qmYTFNMhNsDy76J/Ot
+Ss+VwQ8ddpEG3EIUI9BQxB3xL7z7kRQzploQjakNcDWtDt1BmN05Iy2vz4lnYJky
+Kss6ya9jEkNibHekhxJuchJ0fVGlVe74MO7RNDFG7+O3tMlxu0zH/LpW093V7BI2
+snXNAwQBizvWTrDKWLDu5JsX8KKkrmDtFTs9gegnxDCOYdtG5GbbMq+H1SjWUJPV
+wiXTF8/eE02s4Jzm7ZAxre4bRt/hAg7xTGmDQ1Hn+LzLn18I9LaW5ZWqSwwpgv+g
+Z/jiLO9DJ/y525Cl7DLCpSFoDTWlQXouKhcgALcVay/cXCsZ3oFZCustburLiJi/
+zgBeEk1gVpwljriJLeZifyfWtJx6yfgB/h6fid8XLsGRD+Yc8Tzs8J1LIgi+j4ZT
+UzKX3B85Kht/dr43UDMtWOF3edkOMaJu7rcg5tTsK+LIyHtXvebKPVvvA9f27Dz/
+4gmhAwwqS87Xv3FMVhZ03DNOJ6XAF+T6OTEqwYs+iK56IMSl1Jy+bCzo0j5jZVbl
+XFwGxUHzM7pfM6PDx657oUxG1QwM/fIWA18F+kY/yigXxq6pYMeAiQsPanOThgHp
+-----END CERTIFICATE-----

apps/emqx/test/emqx_crl_cache_SUITE_data/client-revoked.key.pem

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCz5Ho8O0jFWVSg
+thsMFtpVzGCguOdZy+ovzBKrJa6qL9jdyO41gLOJH2dHl1ZqO2QGPMkfKGACvqAF
+VJG3hFGYNeg5ymsrcZymjACvrlQCYLiF0fRp/oV0qMoKcXDrtvJSDnZ/u55dnq39
+e64rGVT9/auNdXCaaeWEGLkQcORfG9+BZcl6QN+w9tb2PSRoJRYKl+OvMZt+Reqs
++SyU7jU9iUJs/GiJEi0mOKd0WNxDJJi9GrsDa5gwayvG/ifevOjk0a2CgGacaPrW
+KL1Y52ilwXEqKuEgH3OvELjHSilF8hoX+fVleTo06doPG7UtNnXzB8JkH3v2B7qm
+V54qg9/tAgMBAAECggEAAml+HRgjZ+gEezot3yngSBW7NvR7v6e9DmKDXpGdB7Go
+DANBdGyzG5PU9/AGy9pbgzzl6nnJXcgOD7w8TvRifrK8WCgHa1f05IPMj458GGMR
+HlQ8HX647eFEgkLWo4Z6tdB1VM2geDtkNFmn8nJ+wgAYgIdSWPOyDOUi+B43ZbIN
+eaLWkP2fiX9tcJp41cytW+ng2YIm4s90Nt4FJPNBNzOrhVm35jciId02MmEjCEnr
+0YbK9uoMDC2YLg8vhRcjtsUHV2rREkwEAQj8nCWvWWheIwk943d6OicGAD/yebpV
+PTjtlZlpIbrovfvuMcoTxJg3WS8LTg/+cNWAX5a3eQKBgQDcRY7nVSJusYyN0Bij
+YWc9H47wU+YucaGT25xKe26w1pl6s4fmr1Sc3NcaN2iyUv4BuAvaQzymHe4g9deU
+D9Ws/NCQ9EjHJJsklNyn2KCgkSp7oPKhPwyl64XfPdV2gr5AD6MILf7Rkyib5sSf
+1WK8i25KatT7M4mCtrBVJYHNpQKBgQDREjwPIaQBPXouVpnHhSwRHfKD0B1a2koq
+4VE6Fnf3ogkiGfV9kqXwIfPHL0tfotFraM3FFmld8RcxhKUPr4oj+K9KTxmMD9lm
+9Hal0ANXYmHs5a1iHyoNmTpBGHALWLT9fCoeg+EIYabi2+P1c7cDIdUPkEzo4GmI
+nCIpv7hGqQKBgEFUC+8GK+EinWoN1tDV+ZWCP5V9fJ43q1E7592bQBgIfZqLlnnP
+dEvVn6Ix3sZMoPMHj9Ra7qjh5Zc28ooCLEBS9tSW7uLJM44k7FCHihQ1GaFy+aLj
+HTA0aw7rutycKCq9uH+bjKDBgWDDj3tMAS2kOMCvcJ1UCquO3TtTlWzVAoGBAIDN
+8yJ/X0NEVNnnkKZTbWq+QILk3LD0e20fk6Nt5Es0ENxpkczjZEglIsM8Z/trnAnI
+b71UqWWu+tMPHYIka77tn1DwmpSnzxCW2+Ib3XMgsaP5fHBPMuFd3X3tSFo1NIxW
+yrwyE5nOT7rELhUyTTYoydLk2/09BMedKY7/BtDBAoGAXeX1pX74K1i/uWyYKwYZ
+sskRueSo9whDJuZWgNiUovArr57eA+oA+bKdFpiE419348bkFF8jNoGFQ6MXMedD
+LqHAYIj+ZPIC4+rObHqO5EaIyblgutwx3citkQp7HXDBxojnOKA9mKQXj1vxCaL1
+/1fFNJQCzEqwnKwnhI2MJ28=
+-----END PRIVATE KEY-----

apps/emqx/test/emqx_crl_cache_SUITE_data/client.cert.pem

@@ -0,0 +1,32 @@
+-----BEGIN CERTIFICATE-----
+MIIFljCCA36gAwIBAgICEAEwDQYJKoZIhvcNAQELBQAwazELMAkGA1UEBhMCU0Ux
+EjAQBgNVBAgMCVN0b2NraG9sbTESMBAGA1UECgwJTXlPcmdOYW1lMRkwFwYDVQQL
+DBBNeUludGVybWVkaWF0ZUNBMRkwFwYDVQQDDBBNeUludGVybWVkaWF0ZUNBMB4X
+DTIzMDExMjEzMDgxNloXDTMzMDQxOTEzMDgxNlowdzELMAkGA1UEBhMCU0UxEjAQ
+BgNVBAgMCVN0b2NraG9sbTESMBAGA1UEBwwJU3RvY2tob2xtMRIwEAYDVQQKDAlN
+eU9yZ05hbWUxGTAXBgNVBAsMEE15SW50ZXJtZWRpYXRlQ0ExETAPBgNVBAMMCE15
+Q2xpZW50MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvGuAShewEo8V
+/+aWVO/MuUt92m8K0Ut4nC2gOvpjMjf8mhSSf6KfnxPklsFwP4fdyPOjOiXwCsf3
+1QO5fjVr8to3iGTHhEyZpzRcRqmw1eYJC7iDh3BqtYLAT30R+Kq6Mk+f4tXB5Lp/
+2jXgdi0wshWagCPgJO3CtiwGyE8XSa+Q6EBYwzgh3NFbgYdJma4x+S86Y/5WfmXP
+zF//UipsFp4gFUqwGuj6kJrN9NnA1xCiuOxCyN4JuFNMfM/tkeh26jAp0OHhJGsT
+s3YiUm9Dpt7Rs7o0so9ov9K+hgDFuQw9HZW3WIJI99M5a9QZ4ZEQqKpABtYBl/Nb
+VPXcr+T3fQIDAQABo4IBNjCCATIwCQYDVR0TBAIwADARBglghkgBhvhCAQEEBAMC
+BaAwMwYJYIZIAYb4QgENBCYWJE9wZW5TU0wgR2VuZXJhdGVkIENsaWVudCBDZXJ0
+aWZpY2F0ZTAdBgNVHQ4EFgQUOIChBA5aZB0dPWEtALfMIfSopIIwHwYDVR0jBBgw
+FoAUTHCGOxVSibq1D5KqrrExQRO+c/MwDgYDVR0PAQH/BAQDAgXgMB0GA1UdJQQW
+MBQGCCsGAQUFBwMCBggrBgEFBQcDBDA7BgNVHR8ENDAyMDCgLqAshipodHRwOi8v
+bG9jYWxob3N0Ojk4NzgvaW50ZXJtZWRpYXRlLmNybC5wZW0wMQYIKwYBBQUHAQEE
+JTAjMCEGCCsGAQUFBzABhhVodHRwOi8vbG9jYWxob3N0Ojk4NzcwDQYJKoZIhvcN
+AQELBQADggIBAE0qTL5WIWcxRPU9oTrzJ+oxMTp1JZ7oQdS+ZekLkQ8mP7T6C/Ew
+6YftjvkopnHUvn842+PTRXSoEtlFiTccmA60eMAai2tn5asxWBsLIRC9FH3LzOgV
+/jgyY7HXuh8XyDBCDD+Sj9QityO+accTHijYAbHPAVBwmZU8nO5D/HsxLjRrCfQf
+qf4OQpX3l1ryOi19lqoRXRGwcoZ95dqq3YgTMlLiEqmerQZSR6iSPELw3bcwnAV1
+hoYYzeKps3xhwszCTz2+WaSsUO2sQlcFEsZ9oHex/02UiM4a8W6hGFJl5eojErxH
+7MqaSyhwwyX6yt8c75RlNcUThv+4+TLkUTbTnWgC9sFjYfd5KSfAdIMp3jYzw3zw
+XEMTX5FaLaOCAfUDttPzn+oNezWZ2UyFTQXQE2CazpRdJoDd04qVg9WLpQxLYRP7
+xSFEHulOPccdAYF2C45yNtJAZyWKfGaAZIxrgEXbMkcdDMlYphpRwpjS8SIBNZ31
+KFE8BczKrg2qO0ywIjanPaRgrFVmeSvBKeU/YLQVx6fZMgOk6vtidLGZLyDXy0Ff
+yaZSoj+on++RDz1IXb96Y8scuNlfcYI8QeoNjwiLtf80BV8SRJiG4e/jTvMf/z9L
+kWrnDWvx4xkUmxFg4TK42dkNp7sEYBTlVVq9fjKE92ha7FGZRqsxOLNQ
+-----END CERTIFICATE-----

apps/emqx/test/emqx_crl_cache_SUITE_data/client.key.pem

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQC8a4BKF7ASjxX/
+5pZU78y5S33abwrRS3icLaA6+mMyN/yaFJJ/op+fE+SWwXA/h93I86M6JfAKx/fV
+A7l+NWvy2jeIZMeETJmnNFxGqbDV5gkLuIOHcGq1gsBPfRH4qroyT5/i1cHkun/a
+NeB2LTCyFZqAI+Ak7cK2LAbITxdJr5DoQFjDOCHc0VuBh0mZrjH5Lzpj/lZ+Zc/M
+X/9SKmwWniAVSrAa6PqQms302cDXEKK47ELI3gm4U0x8z+2R6HbqMCnQ4eEkaxOz
+diJSb0Om3tGzujSyj2i/0r6GAMW5DD0dlbdYgkj30zlr1BnhkRCoqkAG1gGX81tU
+9dyv5Pd9AgMBAAECggEAAifx6dZKIeNkQ8OaNp5V2IKIPSqBOV4/h/xKMkUZXisV
+eDmTCf8du0PR7hfLqrt9xYsGDv+6FQ1/8K231l8qR0tP/6CTl/0ynM4qqEAGeFXN
+3h2LvM4liFbdjImechrcwcnVaNKg/DogT5zHUYSMtB/rokaG0VBO3IX/+SGz0aXi
+LOLAx6SPaLOVX9GYUCiigTSEDwaQA+F3F6J2fR4u8PrXo+OQUqxjQ/fGXWp+4IfA
+6djlpvzO2849/WPB1tL20iLXJlL2OL0UgQNtbKWTjexMe+wgCR5BzCwTyPsQvMwX
+YOQrTOwgF3b6O+gLks5wSRT0ivq1sKgzA534+X4M+wKBgQDirPTLlrYobOO8KUpV
+LOJU8x9leiRNU9CZWrW/mOw/BXGXikqNWvgL595vvADsjYciuRxSqEE7lClB8Pp9
+20TMlES9orx7gdoQJCodpNV1BuBJhE9YtUiXzWAj+7m3D9LsXM1ewW/2A7Vvopj3
+4zKY7uHAFlo3nXwLOfChG5/i9wKBgQDUy5fPFa58xmn7Elb6x4vmUDHg6P4pf75E
+XHRQvNA8I7DTrpqfcsF1N4WuJ3Lm//RSpw7bnyqP20GoEfGHu/iCUPf29B7CuXhO
+vvD+I8uPdn8EcKUBWV+V0xNQN/gCe0TzrEjAkZcO2Lq0j93R8HVl3BbowxgRvQV9
+GmxQG/boKwKBgFeV8uSzsGEAaiKrZbBxrmaappgEUQCcES8gULfes/JJ/TFL2zCx
+ZMTc7CMKZuUAbqXpFtuNbd9CiYqUPYXh8ryF0eXgeqnSa9ruzmMz7NLSPFnLyQkC
+yzD0x2BABOuKLrrrxOMHJWbO2g1vq2GlJUjYjNw3BtcUf/iqg6MM1IPTAoGAWYWJ
+SSqS7JVAcsrFYt1eIrdsNHVwr565OeM3X9v/Mr3FH1jeXeQWNSz1hU29Ticx7y+u
+1YBBlKGmHoHl/bd7lb9ggjkzU7JZRa+YjSIb+i/cwc5t7IJf7xUMk/vnz4tyd5zs
+Qm89gJZ2/Y1kwXSKvx53WNbyokvGKlpaZN1O418CgYACliGux77pe4bWeXSFFd9N
+50ipxDLVghw1c5AiZn25GR5YHJZaV4R0wmFcHdZvogLKi0jDMPvU69PaiT8eX/A1
+COkxv7jY1vtKlEtb+gugMjMN8wvb2va4kyFamjqnleiZlBSqIF/Y17wBoMvaWgZ0
+bEPCN//ts5hBwgb1TwGrrg==
+-----END PRIVATE KEY-----

apps/emqx/test/emqx_crl_cache_SUITE_data/client1.cert.pem

@@ -0,0 +1,32 @@
+-----BEGIN CERTIFICATE-----
+MIIFljCCA36gAwIBAgICEAowDQYJKoZIhvcNAQELBQAwazELMAkGA1UEBhMCU0Ux
+EjAQBgNVBAgMCVN0b2NraG9sbTESMBAGA1UECgwJTXlPcmdOYW1lMRkwFwYDVQQL
+DBBNeUludGVybWVkaWF0ZUNBMRkwFwYDVQQDDBBNeUludGVybWVkaWF0ZUNBMB4X
+DTIzMDMxNjIwMjAzMloXDTMzMDYyMTIwMjAzMlowdjELMAkGA1UEBhMCU0UxEjAQ
+BgNVBAgMCVN0b2NraG9sbTESMBAGA1UEBwwJU3RvY2tob2xtMRIwEAYDVQQKDAlN
+eU9yZ05hbWUxGTAXBgNVBAsMEE15SW50ZXJtZWRpYXRlQ0ExEDAOBgNVBAMMB0Ns
+aWVudDEwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDcDhlEvUIYc9uA
+ocOBXt5thKrovs+8V0Eus/WrHMTKBk0Kw4X+7HBaRBoZj2sZpYfN63lVaO75kW4I
+uJuorGj5PAXYWJj+4uAsCc95xAN/liCuHJnxE5togWVt8W+z0Zll98RIpiCohqiE
+FLDL4X6FREL07GLgQZ/BFORvAwU+Gog05AFh43iZDnJl8MmrG2HBSRXtSZ6vQj9A
+NrOSqz5eK4YIHEEsgwTWQmhtNwu3Y+GzrAPWCA4TeYrSRwIrnGh20fOWXkAMldS4
+eRXmBztEsXMGqbe6oYO1QPYOlmoGO8EaaDPJ2sFIuM0zn98Alq3kCnRhM5Bi9RpJ
+7IpudIopAgMBAAGjggE3MIIBMzAJBgNVHRMEAjAAMBEGCWCGSAGG+EIBAQQEAwIF
+oDAzBglghkgBhvhCAQ0EJhYkT3BlblNTTCBHZW5lcmF0ZWQgQ2xpZW50IENlcnRp
+ZmljYXRlMB0GA1UdDgQWBBQoIuXq3wG6JEzAEj9wPe7am0OVgjAfBgNVHSMEGDAW
+gBRMcIY7FVKJurUPkqqusTFBE75z8zAOBgNVHQ8BAf8EBAMCBeAwHQYDVR0lBBYw
+FAYIKwYBBQUHAwIGCCsGAQUFBwMEMDwGA1UdHwQ1MDMwMaAvoC2GK2h0dHA6Ly9s
+b2NhbGhvc3Q6OTg3OC9pbnRlcm1lZGlhdGUxLmNybC5wZW0wMQYIKwYBBQUHAQEE
+JTAjMCEGCCsGAQUFBzABhhVodHRwOi8vbG9jYWxob3N0Ojk4NzcwDQYJKoZIhvcN
+AQELBQADggIBAHqKYcwkm3ODPD7Mqxq3bsswSXregWfc8tqfIBc5FZg2F+IzhxcJ
+kINB0lmcNdLALK6ka0sDs1Nrj1KB96NcHUqE+WY/qPS1Yksr34yFatb1ddlKQ9HK
+VRrIsi0ZfjBpHpvoQ0GsLeyRKm7iN/Fm5H9u8rw6RBu0Oe/l20FVSQIDzldYw51L
+uV/E9No8ZhdQ2Dffujs8madI7b7I1NMXS+Z1pZ+gYrz6O60tDEprE+rYuYWypURr
+fK+DnLLl+KQ+eekTPynw7LRpFzI/1cOMmd4BRnsBHCbCObfNp7WPasemZOEXGIlZ
+CQwZS62DYOJE4u4Nz5pSF+JgXfr6X/Im6Y1SV900xVHfoL0GpFDI9k+0Y5ncHfSH
++V9HlRWB3zqQF+yla32XOpBbER0vFDH52gp8/o1ZGg7rr6KrP4QKxnqywNLiAPDX
+txaAykZhON7uG8j+Lbjx5Ik91NRn9Fd5NH/vtT33a4uig2TP9EWd7EPcD2z8ONuD
+yiK3S37XAnmSKKX4HcCpEb+LedtqQo/+sqWyWXkpKdpkUSozvcYS4J/ob3z9N2IE
+qIH5I+Mty1I4EB4W89Pem8DHNq86Lt0Ea6TBtPTV8NwR5aG2vvLzb5lNdpANXYcp
+nGr57mTWaHnQh+yqgy66J++k+WokWkAkwE989AvUfNoQ+Jr6cTH8nKo2
+-----END CERTIFICATE-----

apps/emqx/test/emqx_crl_cache_SUITE_data/client1.key.pem

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDcDhlEvUIYc9uA
+ocOBXt5thKrovs+8V0Eus/WrHMTKBk0Kw4X+7HBaRBoZj2sZpYfN63lVaO75kW4I
+uJuorGj5PAXYWJj+4uAsCc95xAN/liCuHJnxE5togWVt8W+z0Zll98RIpiCohqiE
+FLDL4X6FREL07GLgQZ/BFORvAwU+Gog05AFh43iZDnJl8MmrG2HBSRXtSZ6vQj9A
+NrOSqz5eK4YIHEEsgwTWQmhtNwu3Y+GzrAPWCA4TeYrSRwIrnGh20fOWXkAMldS4
+eRXmBztEsXMGqbe6oYO1QPYOlmoGO8EaaDPJ2sFIuM0zn98Alq3kCnRhM5Bi9RpJ
+7IpudIopAgMBAAECggEARcly2gnrXDXh9vlWN0EO6UyZpxZcay6AzX7k+k81WZyF
+8lPvutjhCL9wR4rkPE3ys6tp31xX7W3hp4JkWynSYLhYYjQ20R7CWTUDR2qScXP7
+CTyo1XuSXaIruKJI+o4OR/g7l46X7NpHtxuYtg/dQAZV9bbB5LzrHSCzEUGz9+17
+jV//cBgLBiMdlbdLuQoGt4NQpBkNrauBVFq7Nq648uKkICmUo3Bzn/dfn3ehB+Zc
++580S+tawYd224j19tFQmd5oK8tfjqKuHenNGjp/gsRoY86N7qAtc3VIQ0yjE6ez
+tgREo/ftCb8kGfwRJOAQIeeDamBv+FWNT6QzcOtbwQKBgQDzWhY9BUgI8JVzjYg0
+oWfU90On81BtckKsEo//8MTlgwOD2PnUF0hTZF3RcSPYT+HybouTqHT8EOLBAzqy
+1+koH06MnAc/Y2ipaAe2fGaVH3SuXAsV/b8VcWWl4Qx7tYJDhE7sKmdl3/+jHZ7A
+hZQzgOQnxxCANBo3pwF9KboDbwKBgQDnfglSpgYdGzFpWp1hZnPl2RKIfX/4M2z2
+s+hVN1tp+1VySPrBRUC3J6hKPQUzzeUzPICclHOnO+kP7jAos/rlJ9VcNdAQTbTL
+7Ds9Em1KJTBphE038YbW3e93rydQpukXh50wRD9RI/3F3A/1rKhab92DXZZr6Wqu
+JouhNV8f5wKBgQCLQ3XQi/Iyc4QDse5NuETUgoCsX7kaOTZghOr1nFMByV08mfI2
+5vAUES8DigzqYKS8eXjVEqWIDx3FOVThPmCG/ouUOkKHixs9P3SSgVSvaGX81l3d
+wu4UlmWGbWkYbsJSYyhLTOUJTwxby7qrEIbEhrGK9gfCZo7OZHucpkF2bwKBgFhl
+1qWK5JbExY+XnLWO6/7/b4ZTdkSPTrK+bJ/t7aiA41Yq7CZVjarjJ+6BcrUfkMCK
+AArK3Yck55C/wgApCkvrdBwsKHGxWrLsWIqvuLAxl1UTwnD0eCsgwMsRRZAUzLnB
+fZLq3MrdVZDywd1suzUdtpbta/11OtmZuoQq31JNAoGAIzmevuPaUZRqnjDpLXAm
+Bo11q6CunhG5qZX4wifeZ9Fp5AaQu97F36igZ5/eDxFkDCrFRq6teMiNjRQZSLA3
+5tMBkq6BtN2Ozzm/6D135c4BF14ODHqAMPUy4sXbw5PS/kRFs4fKAH/+LcAOPgyI
+N/jJIY1LfM7PzrG2NdyscMU=
+-----END PRIVATE KEY-----

apps/emqx/test/emqx_crl_cache_SUITE_data/client2.cert.pem

@@ -0,0 +1,32 @@
+-----BEGIN CERTIFICATE-----
+MIIFljCCA36gAwIBAgICEAswDQYJKoZIhvcNAQELBQAwazELMAkGA1UEBhMCU0Ux
+EjAQBgNVBAgMCVN0b2NraG9sbTESMBAGA1UECgwJTXlPcmdOYW1lMRkwFwYDVQQL
+DBBNeUludGVybWVkaWF0ZUNBMRkwFwYDVQQDDBBNeUludGVybWVkaWF0ZUNBMB4X
+DTIzMDMxNjIwMjAzMloXDTMzMDYyMTIwMjAzMlowdjELMAkGA1UEBhMCU0UxEjAQ
+BgNVBAgMCVN0b2NraG9sbTESMBAGA1UEBwwJU3RvY2tob2xtMRIwEAYDVQQKDAlN
+eU9yZ05hbWUxGTAXBgNVBAsMEE15SW50ZXJtZWRpYXRlQ0ExEDAOBgNVBAMMB0Ns
+aWVudDIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDFLcCjzNhfY6Sk
+2nSdrB/6UPPeTCCH5NBBVLvu1hdlqLS4qEdq8+EjyMDZTtspRtYPkBfjpOrlBWUO
+lKyxw2mZOjZ8iWvd4sJaAI/6KZl5X0Rdsg1RjzW03kUdLx9XJCyrYY0YFrT1dgJo
+Ih56jk2SJX7wrz0NCJ05VPIdpaOF6CcziA+YhdVHcE6xyHagsYI0JdDWxFZrl9zT
+LyhaDgBUN/yUQBnxKzxs8TMT4YVSi73ouh5IP9Xvs52hd6HO8ZGVr+YthQZKo95p
+OlwFF+AQWxdDIKoPYUPFo8XMOXvOeQ9iUJarxrYSrelLXtGkaGLBolAvqo/YKE7j
+rcJWjRGHAgMBAAGjggE3MIIBMzAJBgNVHRMEAjAAMBEGCWCGSAGG+EIBAQQEAwIF
+oDAzBglghkgBhvhCAQ0EJhYkT3BlblNTTCBHZW5lcmF0ZWQgQ2xpZW50IENlcnRp
+ZmljYXRlMB0GA1UdDgQWBBTOo9YSgx1h5k/imP7nOfRfzQrRxjAfBgNVHSMEGDAW
+gBRMcIY7FVKJurUPkqqusTFBE75z8zAOBgNVHQ8BAf8EBAMCBeAwHQYDVR0lBBYw
+FAYIKwYBBQUHAwIGCCsGAQUFBwMEMDwGA1UdHwQ1MDMwMaAvoC2GK2h0dHA6Ly9s
+b2NhbGhvc3Q6OTg3OC9pbnRlcm1lZGlhdGUyLmNybC5wZW0wMQYIKwYBBQUHAQEE
+JTAjMCEGCCsGAQUFBzABhhVodHRwOi8vbG9jYWxob3N0Ojk4NzcwDQYJKoZIhvcN
+AQELBQADggIBAFo91lLqjPY67Wmj2yWxZuTTuUwXdXXUQxL6sEUUnfkECvRhNyBA
+eCHkfVopNbXZ5tdLfsUvXF0ulaC76GCK/P7gHOG9D/RJX/85VzhuJcqa4dsEEifg
+IiKIG7viYxSA6HFXuyzHvwNco3FqTBHbY46lKf1lWRVLhiAtcwcyPP34/RWcPfQi
+6NZfLyitu5U7Z9XVN5wCp8sg0ayaO5Ib2ejIYuBCUddV1gV//tSDf+rKCgtAbm/X
+K64Bf3GdaX3h6EhoqMZ+Z2f4XpKSXTabsWAU44xdVxisI82eo+NwT8KleE65GpOv
+nPvr/dLq5fQ6VtHbRL3wWqhzB1VKVCtd8a6RE2k8HVWflU3qgwJ+woF19ed921eq
+OZxc+KzjsGFyW1D2fPdgoZFmePadSstIME7qtCNEi7D3im01/1KKzE2m/nosrHeW
+ePjY2YrXu0w47re/N2kBJL2xRbj+fAjBsfNn9RhvQsWheXG6mgg8w1ac6y72ZA2W
+72pWoDkgXQMX5XBBj/zMnmwtrX9zTILFjNGFuWMPYgBRI0xOf2FoqqZ67cQ2yTW/
+1T/6Mp0FSh4cIo/ENiNSdvlt3BIo84EyOm3iHHy28Iv5SiFjF0pkwtXlYYvjM3+R
+BeWqlPsVCZXcVC1rPVDzfWZE219yghldY4I3QPJ7dlmszi8eI0HtzhTK
+-----END CERTIFICATE-----

apps/emqx/test/emqx_crl_cache_SUITE_data/client2.key.pem

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDFLcCjzNhfY6Sk
+2nSdrB/6UPPeTCCH5NBBVLvu1hdlqLS4qEdq8+EjyMDZTtspRtYPkBfjpOrlBWUO
+lKyxw2mZOjZ8iWvd4sJaAI/6KZl5X0Rdsg1RjzW03kUdLx9XJCyrYY0YFrT1dgJo
+Ih56jk2SJX7wrz0NCJ05VPIdpaOF6CcziA+YhdVHcE6xyHagsYI0JdDWxFZrl9zT
+LyhaDgBUN/yUQBnxKzxs8TMT4YVSi73ouh5IP9Xvs52hd6HO8ZGVr+YthQZKo95p
+OlwFF+AQWxdDIKoPYUPFo8XMOXvOeQ9iUJarxrYSrelLXtGkaGLBolAvqo/YKE7j
+rcJWjRGHAgMBAAECggEABJYUCcyJcnbagytBxfnaNQUuAp8AIypFG3kipq0l5Stk
+gGaTJq5F4OTGS4ofRsqeu07IgBSAfqJcJH8toPkDQqfvs6ftO1Mso2UzakMOcP51
+Ywxd91Kjm+LKOyHkHGDirPGnutUg/YpLLrrMvTk/bJHDZCM4i/WP1WTREVFjUgl7
+4L6Y53x2Lk5shJJhv0MzTGaoZzQcW0EbhNH1AI6MBv5/CN5m/7/+HCPlHSNKnozl
+o3PXD6l0XNfOY2Hi6MgS/Vd70s3VmDT9UCJNsDjdFpKNHmI7vr9FScOLN8EwbqWe
+maFa0TPknmPDmVjEGMtgGlJWL7Sm0MpNW+WsEXcDPQKBgQDv3sp0nVML9pxdzX/w
+rGebFaZaMYDWmV9w0V1uXYh4ZkpFmqrWkq/QSTGpwIP/x8WH9FBDUZqspLpPBNgG
+ft1XhuY34y3hoCxOyRhQcR/1dY+lgCzuN4G4MG3seq/cAXhrmkPljma/iO8KzoRK
+Pa+uaKFGHy1vWY2AmOhT20zr4wKBgQDScA3478TFHg9THlSFzqpzvVn5eAvmmrCQ
+RMYIZKFWPortqzeJHdA5ShVF1XBBar1yNMid7E7FXqi/P8Oh+E6Nuc7JxyVIJWlV
+mcBE1ceTKdZn7A0nuQIaU6hcn7xz/UHmxGur1ZcNQm3diFJ2CPn11lzZlkSZLSCN
+V86nndA9DQKBgQCWsUxXPo7xsRhDBdseg/ECyPMdLoRWTTxcT+t2bmRR31FBsQ0q
+iDTTkWgV0NAcXJCH/MB/ykB1vXceNVjRm9nKJwFyktI8MLglNsiDoM4HErgPrRqM
+/WoNIL+uFNVuTa4tS1jkWjXKlmg2Tc9mJKK92xWWS/frQENZSraKF/eXKQKBgGR9
+ni6CUTTQZgELOtGrHzql8ZFwAj7dH/PE48yeQW0t8KoOWTbhRc4V0pLGmhSjJFSl
+YCgJ8JPP4EVz7bgrG1gSou04bFVHiEWYZnh4nhVopTp7Psz5TEfGK2AP5658Ajxx
+D/m+xaNPVae0sawsHTGIbE57s8ZyBll41Pa2JfsBAoGBANtS7SOehkflSdry0eAZ
+50Ec3CmY+fArC044hQCmXxol5SiTUnXf/OIQH8y+RZUjF4F3PbbrFNLm/J6wuUPw
+XUIb4gAL75srakL8DXqyTYfO02aNrFEHhXzMs+GoAnRkFy16IAAFwpjbYSPanfed
+PfHCTWz6Y0pGdh1hUJAFV/3v
+-----END PRIVATE KEY-----

apps/emqx/test/emqx_crl_cache_SUITE_data/client3.cert.pem

@@ -0,0 +1,32 @@
+-----BEGIN CERTIFICATE-----
+MIIFljCCA36gAwIBAgICEAwwDQYJKoZIhvcNAQELBQAwazELMAkGA1UEBhMCU0Ux
+EjAQBgNVBAgMCVN0b2NraG9sbTESMBAGA1UECgwJTXlPcmdOYW1lMRkwFwYDVQQL
+DBBNeUludGVybWVkaWF0ZUNBMRkwFwYDVQQDDBBNeUludGVybWVkaWF0ZUNBMB4X
+DTIzMDMxNjIwMjAzMloXDTMzMDYyMTIwMjAzMlowdjELMAkGA1UEBhMCU0UxEjAQ
+BgNVBAgMCVN0b2NraG9sbTESMBAGA1UEBwwJU3RvY2tob2xtMRIwEAYDVQQKDAlN
+eU9yZ05hbWUxGTAXBgNVBAsMEE15SW50ZXJtZWRpYXRlQ0ExEDAOBgNVBAMMB0Ns
+aWVudDMwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDEOZ6fYNjZDNXX
+eOyapHMOMeNeYM3b7vsWXAbiJIt4utVrTS0A+/G640t/U0g8F9jbKgbjEEPtgPJ7
+GltjLWObfqDWKSO2D9/ei2+NauqgiN/HX+dQnSKHob0McXBXvLfrA4tn4braKrbg
+p1fZB8bAECuT/bUhVBqWlzrUwDMpqjMJWDab48ixezb2gnc/ePE6wq/d3ecDb0/k
+cYWQ0LX4JiQBgaTGhwczyoGfL1z2vx5kJqptK+r0Hc2jNCn6kFvoZUCYjCWgWNxZ
+sQk7fObQQkUb/XQyqRaKJBWDyqsNcuK2gOg3LGeolAlgtMiEqGhHv77XdJnJug/w
+3OiHpP/7AgMBAAGjggE3MIIBMzAJBgNVHRMEAjAAMBEGCWCGSAGG+EIBAQQEAwIF
+oDAzBglghkgBhvhCAQ0EJhYkT3BlblNTTCBHZW5lcmF0ZWQgQ2xpZW50IENlcnRp
+ZmljYXRlMB0GA1UdDgQWBBRxZFdIkSg6zDZCakXmIest5a6dBzAfBgNVHSMEGDAW
+gBRMcIY7FVKJurUPkqqusTFBE75z8zAOBgNVHQ8BAf8EBAMCBeAwHQYDVR0lBBYw
+FAYIKwYBBQUHAwIGCCsGAQUFBwMEMDwGA1UdHwQ1MDMwMaAvoC2GK2h0dHA6Ly9s
+b2NhbGhvc3Q6OTg3OC9pbnRlcm1lZGlhdGUzLmNybC5wZW0wMQYIKwYBBQUHAQEE
+JTAjMCEGCCsGAQUFBzABhhVodHRwOi8vbG9jYWxob3N0Ojk4NzcwDQYJKoZIhvcN
+AQELBQADggIBAEntkhiPpQtModUF/ffnxruq+cqopPhIdMXhMD8gtU5e4e7o3EHX
+lfZKIbxyw56v6dFPrl4TuHBiBudqIvBCsPtllWKixWvg6FV3CrEeTcg4shUIaJcD
+pqv1qHLwS4pue6oau/lb8jv1GuzuBXoMFQwlmiOXO7xXqXjV2GdmkFJCDdB/0BW1
+VHvh0DXgotaxITWKhCpSNB7F7LSvegRwZIAN6JXrLDpue7tgqLqBB1EzpmS6ALbn
+uZDdikOs/tGAFB3un/3Gl7jEPL8UGOoSj/H9PUT5AFHrHJDH72+QSXu09agz8RWJ
+V939njYFCAxQ8Jt2mOK8BJQDJgPtLfIIb1iYicQV13Eypt8uIUYvp0i0Wq8WxPbq
+rOEvQYpcGUsreS5XqZ7y68hgq6ePiR18Fnc3GyTV5o6qT3W7IOvPArTzNV5fFCwM
+lx8xSEm+ebJrJPphp6Uc/h8evohvAN8R/Z7FSo9OL6V+F3ywPqWTXaqiIiRc9PS0
+0vxsYZ96EeZY5HzjN6LzHxmkv4KYM5I1qmXlviQlaU+sotp3tzegADlM4K78nUFh
+HuXamecEcS73eAgjk+FGqJ9E25B0TLlQMcP6tCKdaUIGn6ZsF5wT87GzqT99wL/5
+foHCYIkyG7ZmAQmoaKBd4q6xqVOUHovmsPza69FuSrsBxoRR39PtAnrY
+-----END CERTIFICATE-----

apps/emqx/test/emqx_crl_cache_SUITE_data/client3.key.pem

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDEOZ6fYNjZDNXX
+eOyapHMOMeNeYM3b7vsWXAbiJIt4utVrTS0A+/G640t/U0g8F9jbKgbjEEPtgPJ7
+GltjLWObfqDWKSO2D9/ei2+NauqgiN/HX+dQnSKHob0McXBXvLfrA4tn4braKrbg
+p1fZB8bAECuT/bUhVBqWlzrUwDMpqjMJWDab48ixezb2gnc/ePE6wq/d3ecDb0/k
+cYWQ0LX4JiQBgaTGhwczyoGfL1z2vx5kJqptK+r0Hc2jNCn6kFvoZUCYjCWgWNxZ
+sQk7fObQQkUb/XQyqRaKJBWDyqsNcuK2gOg3LGeolAlgtMiEqGhHv77XdJnJug/w
+3OiHpP/7AgMBAAECggEADSe89sig5E63SKAlFXcGw0H2XgqIzDP/TGMnqPvNoYhX
+eSXUgxDhBptpB9e9a4RaKwaFxxPjlSXEdYFX9O22YSN1RMMl6Q8Zl9g3edhcDR6W
+b7Qbx2x8qj6Rjibnlh8JiFPiaDjN2wUeSDBss/9D98NkKiJ9Ue2YCYmJAOA3B3w9
+2t4Co5+3YrxkdzkvibTQCUSEwHFeB1Nim21126fknMPxyrf+AezRBRc8JNAHqzWb
+4QEeMnmIJDOzc3Oh7+P85tNyejOeRm9T7X3EQ0jKXgLYe+HUzXclBQ66b9x9Nc9b
+tNn6XkMlLlsQ3f149Th6PtHksH3hM+GF8bMuCp9yxQKBgQDGk0PYPkLqTD8jHjJW
+s8wBNhozigZPGaynxdTsD7L6UtDdOl1sSW/jFOj9UIs1duBce9dP1IjFc0jY+Kin
+lMLv3qCtk5ZjxRglOoLipR9hdClcM69rDoRZdoQK8KYa+QHcOTSazIp3fnw4gWSX
+nscelMfd1rtVP0dOGTuqE/73/QKBgQD8+F5WAi2IOVPHnBxAAOP+6XTs9Ntn1sDi
+L5wNgm+QA28aJJ4KRAwdXIc3IFPlHxZI77c2K1L9dKDu9X4UcyZIZYDvGVLuOOt5
+twaRaGuJW03cjbgOWC7rGyfzfZ49YlCZi2YuxERclBkbqgWD9hfa8twUfKNguF2Y
+AyiOhohtVwKBgQCJB8zUp7pzhqQ3LrpcHHzWBSi1kjTiVvxPVnSlZfwDRCz/zSv0
+8wRz9tUFIZS/E0ama4tcenTblL+bgpSX+E9BSiclQOiR9su/vQ3fK0Vpccis6LnP
+rdflCKT8C68Eg/slppBHloijBzTfpWLuQlJ0JwV5b5ocrKsfGMiUiHH1XQKBgQDg
+RnakfEPP7TtY0g+9ssxwOJxAZImM0zmojpsk4wpzvIeovuQap9+xvFHoztFyZhBE
+07oz3U8zhE4V7TI9gSVktBEOaf47U914yIqbKd+FJJywODkBBq96I1ZVKn67X0mk
+B5GtTrZo+agU/bTsHKdjp0L1KtdSLcJUviAb1Cxp+wKBgDrGqS01CCgxSUwMaZe4
+8HFWp/oMSyVDG9lTSC3uP/VL76zNFI55T3X06Q87hDN3gCJGUOmHzDZ/oCOgM4/S
+SU55M4lXeSEdFe84tMXJKOv5JXTkulzBYzATJ5J8DeS/4YZxMKyPDLXX8wgwmU+l
+i6Imd3qCPhh5eI3z9eSNDX+6
+-----END PRIVATE KEY-----

apps/emqx/test/emqx_crl_cache_SUITE_data/crl.pem

@@ -0,0 +1,20 @@
+-----BEGIN X509 CRL-----
+MIIDPDCCASQCAQEwDQYJKoZIhvcNAQELBQAwazELMAkGA1UEBhMCU0UxEjAQBgNV
+BAgMCVN0b2NraG9sbTESMBAGA1UECgwJTXlPcmdOYW1lMRkwFwYDVQQLDBBNeUlu
+dGVybWVkaWF0ZUNBMRkwFwYDVQQDDBBNeUludGVybWVkaWF0ZUNBFw0yMjA3MjAy
+MDIzNTNaFw0zMjEwMjUyMDIzNTNaMBUwEwICEAIXDTIyMDYxMzEyNDIwNVqgbjBs
+MB8GA1UdIwQYMBaAFCuv1TkzC1fSgTfzE1m1u5pRCJsVMDwGA1UdHAQ1MDOgLqAs
+hipodHRwOi8vbG9jYWxob3N0Ojk4NzgvaW50ZXJtZWRpYXRlLmNybC5wZW2EAf8w
+CwYDVR0UBAQCAhADMA0GCSqGSIb3DQEBCwUAA4ICAQBbWdqRFsIrG6coL6ln1RL+
+uhgW9l3XMmjNlyiYHHNzOgnlBok9xu9UdaVCOKC6GEthWSzSlBY1AZugje57DQQd
+RkIJol9am94lKMTjF/qhzFLiSjho8fwZGDGyES5YeZXkLqNMOf6m/drKaI3iofWf
+l63qU9jY8dnSrVDkwgCguUL2FTx60v5H9NPxSctQ3VDxDvDj0sTAcHFknQcZbfvY
+ZWpOYNS0FAJlQPVK9wUoDxI0LhrWDq5h/T1jcGO34fPT8RUA5HRtFVUevqSuOLWx
+WTfTx5oDeMZPJTvHWUcg4yMElHty4tEvtkFxLSYbZqj7qTU+mi/LAN3UKBH/gBEN
+y2OsJvFhVRgHf+zPYegf3WzBSoeaXNAJZ4UnRo34P9AL3Mrh+4OOUP++oYRKjWno
+pYtAmTrIwEYoLyisEhhZ6aD92f/Op3dIYsxwhHt0n0lKrbTmUfiJUAe7kUZ4PMn4
+Gg/OHlbEDaDxW1dCymjyRGl+3/8kjy7bkYUXCf7w6JBeL2Hw2dFp1Gh13NRjre93
+PYlSOvI6QNisYGscfuYPwefXogVrNjf/ttCksMa51tUk+ylw7ZMZqQjcPPSzmwKc
+5CqpnQHfolvRuN0xIVZiAn5V6/MdHm7ocrXxOkzWQyaoNODTq4js8h8eYXgAkt1w
+p1PTEFBucGud7uBDE6Ub6A==
+-----END X509 CRL-----

apps/emqx/test/emqx_crl_cache_SUITE_data/emqx.conf

@@ -0,0 +1,12 @@
+crl_cache.refresh_interval = {{ refresh_interval }}
+crl_cache.http_timeout = 17s
+crl_cache.capacity = {{ cache_capacity }}
+listeners.ssl.default {
+  ssl_options {
+    keyfile = "{{ test_data_dir }}/server.key.pem"
+    certfile = "{{ test_data_dir }}/server.cert.pem"
+    cacertfile = "{{ test_data_dir }}/ca-chain.cert.pem"
+    verify = verify_peer
+    enable_crl_check = true
+  }
+}

apps/emqx/test/emqx_crl_cache_SUITE_data/emqx_crl_cache_http_server.erl

@@ -0,0 +1,67 @@
+-module(emqx_crl_cache_http_server).
+
+-behaviour(gen_server).
+-compile([nowarn_export_all, export_all]).
+
+set_crl(CRLPem) ->
+    ets:insert(?MODULE, {crl, CRLPem}).
+
+%%--------------------------------------------------------------------
+%% `gen_server' APIs
+%%--------------------------------------------------------------------
+
+start_link(Parent, BasePort, CRLPem, Opts) ->
+    process_flag(trap_exit, true),
+    stop_http(),
+    timer:sleep(100),
+    gen_server:start_link(?MODULE, {Parent, BasePort, CRLPem, Opts}, []).
+
+init({Parent, BasePort, CRLPem, Opts}) ->
+    Tab = ets:new(?MODULE, [named_table, ordered_set, public]),
+    ets:insert(Tab, {crl, CRLPem}),
+    ok = start_http(Parent, [{port, BasePort} | Opts]),
+    Parent ! {self(), ready},
+    {ok, #{parent => Parent}}.
+
+handle_call(_Request, _From, State) ->
+    {reply, ignored, State}.
+
+handle_cast(_Msg, State) ->
+    {noreply, State}.
+
+terminate(_Reason, _State) ->
+    stop_http().
+
+stop(Pid) ->
+    ok = gen_server:stop(Pid).
+
+%%--------------------------------------------------------------------
+%% Callbacks
+%%--------------------------------------------------------------------
+
+start_http(Parent, Opts) ->
+    {ok, _Pid1} = cowboy:start_clear(http, Opts, #{
+        env => #{dispatch => compile_router(Parent)}
+    }),
+    ok.
+
+stop_http() ->
+    cowboy:stop_listener(http),
+    ok.
+
+compile_router(Parent) ->
+    {ok, _} = application:ensure_all_started(cowboy),
+    cowboy_router:compile([
+        {'_', [{'_', ?MODULE, #{parent => Parent}}]}
+    ]).
+
+init(Req, #{parent := Parent} = State) ->
+    %% assert
+    <<"GET">> = cowboy_req:method(Req),
+    [{crl, CRLPem}] = ets:lookup(?MODULE, crl),
+    Parent ! {http_get, iolist_to_binary(cowboy_req:uri(Req))},
+    Reply = reply(Req, CRLPem),
+    {ok, Reply, State}.
+
+reply(Req, CRLPem) ->
+    cowboy_req:reply(200, #{<<"content-type">> => <<"text/plain">>}, CRLPem, Req).

apps/emqx/test/emqx_crl_cache_SUITE_data/emqx_just_verify.conf

@@ -0,0 +1,12 @@
+node.name = test@127.0.0.1
+node.cookie = emqxsecretcookie
+node.data_dir = "{{ test_priv_dir }}"
+listeners.ssl.default {
+  ssl_options {
+    keyfile = "{{ test_data_dir }}/server.key.pem"
+    certfile = "{{ test_data_dir }}/server.cert.pem"
+    cacertfile = "{{ test_data_dir }}/ca-chain.cert.pem"
+    verify = verify_peer
+    enable_crl_check = false
+  }
+}

apps/emqx/test/emqx_crl_cache_SUITE_data/intermediate-not-revoked.crl.pem

@@ -0,0 +1,19 @@
+-----BEGIN X509 CRL-----
+MIIDJTCCAQ0CAQEwDQYJKoZIhvcNAQELBQAwazELMAkGA1UEBhMCU0UxEjAQBgNV
+BAgMCVN0b2NraG9sbTESMBAGA1UECgwJTXlPcmdOYW1lMRkwFwYDVQQLDBBNeUlu
+dGVybWVkaWF0ZUNBMRkwFwYDVQQDDBBNeUludGVybWVkaWF0ZUNBFw0yMzAxMTIx
+MzA4MTZaFw0zMzAxMDkxMzA4MTZaoG4wbDAfBgNVHSMEGDAWgBRMcIY7FVKJurUP
+kqqusTFBE75z8zA8BgNVHRwENTAzoC6gLIYqaHR0cDovL2xvY2FsaG9zdDo5ODc4
+L2ludGVybWVkaWF0ZS5jcmwucGVthAH/MAsGA1UdFAQEAgIQADANBgkqhkiG9w0B
+AQsFAAOCAgEAJGOZuqZL4m7zUaRyBrxeT6Tqo+XKz7HeD5zvO4BTNX+0E0CRyki4
+HhIGbxjv2NKWoaUv0HYbGAiZdO4TaPu3w3tm4+pGEDBclBj2KTdbB+4Hlzv956gD
+KXZ//ziNwx1SCoxxkxB+TALxReN0shE7Mof9GlB5HPskhLorZgg/pmgJtIykEpsq
+QAjJo4aq+f2/L+9dzRM205fVFegtsHvgEVNKz6iK6skt+kDhj/ks9BKsnfCDIGr+
+XnPYwS9yDnnhFdoJ40AQQDtomxggAjfgcSnqtHCxZwKJohuztbSWUgD/4yxzlrwP
+Dk1cT/Ajjjqb2dXVOfTLK1VB2168uuouArxZ7KYbXwBjHduYWGGkA6FfkNJO/jpF
+SL9qhX3oxcRF3hDhWigN1ZRD7NpDKwVal3Y9tmvO5bWhb5VF+3qv0HGeSGp6V0dp
+sjwhIj+78bkUrcXxrivACLAXgSTGonx1uXD+T4P4NCt148dgRAbgd8sUXK5FcgU2
+cdBl8Kv2ZUjEaod5gUzDtf22VGSoO9lHvfHdpG9o2H3wC7s4tyLTidNrduIguJff
+IIgc44Y252iV0sOmZ5S0jjTRiF1YUUPy9qA/6bOnr2LohbwbNZv9tDlNj8cdhxUz
+cKiS+c7Qsz+YCcrp19QRiJoQae/gUqz7kmUZQgyPmDd+ArE0V+kDZEE=
+-----END X509 CRL-----

apps/emqx/test/emqx_crl_cache_SUITE_data/intermediate-revoked-no-dp.crl.pem

@@ -0,0 +1,19 @@
+-----BEGIN X509 CRL-----
+MIIC/TCB5gIBATANBgkqhkiG9w0BAQsFADBrMQswCQYDVQQGEwJTRTESMBAGA1UE
+CAwJU3RvY2tob2xtMRIwEAYDVQQKDAlNeU9yZ05hbWUxGTAXBgNVBAsMEE15SW50
+ZXJtZWRpYXRlQ0ExGTAXBgNVBAMMEE15SW50ZXJtZWRpYXRlQ0EXDTIzMDExODEz
+Mjc1M1oXDTMzMDExNTEzMjc1M1owFTATAgIQAhcNMjMwMTEyMTMwODE2WqAwMC4w
+HwYDVR0jBBgwFoAUTHCGOxVSibq1D5KqrrExQRO+c/MwCwYDVR0UBAQCAhACMA0G
+CSqGSIb3DQEBCwUAA4ICAQCxoRYDc5MaBpDI+HQUX60+obFeZJdBkPO2wMb6HBQq
+e0lZM2ukS+4n5oGhRelsvmEz0qKvnYS6ISpuFzv4Qy6Vaun/KwIYAdXsEQVwDHsu
+Br4m1V01igjFnujowwR/7F9oPnZOmBaBdiyYbjgGV0YMF7sOfl4UO2MqI2GSGqVk
+63wELT1AXjx31JVoyATQOQkq1A5HKFYLEbFmdF/8lNfbxSCBY2tuJ+uWVQtzjM0y
+i+/owz5ez1BZ/Swx8akYhuvs8DVVTbjXydidVSrxt/QEf3+oJCzTA9qFqt4MH7gL
+6BAglCGtRiYTHqeMHrwddaHF2hzR61lHJlkMCL61yhVuL8WsEJ/AxVX0W3MfQ4Cw
+x/A6xIkgqtu+HtQnPyDcJxyaFHtFC+U67nSbEQySFvHfMw42DGdIGojKQCeUer9W
+ECFC8OATQwN2h//f8QkY7D0H3k/brrNYDfdFIcCti9iZiFrrPFxO7NbOTfkeKCt3
+7IwYduRc8DWKmS8c7j2re1KkdYnfE1sfwbn3trImkcET5tvDlVCZ1glnBQzk82PS
+HvKmSjD2pZI7upfLkoMgMhYyYJhYk7Mw2o4JXuddYGKmmw3bJyHkG/Ot5NAKjb7g
+k1QCeWzxO1xXm8PNDDFWMn351twUGDQ/cwrUw0ODeUZpfL0BtTn4YnfCLLTvZDxo
+Vg==
+-----END X509 CRL-----

apps/emqx/test/emqx_crl_cache_SUITE_data/intermediate-revoked.crl.pem

@@ -0,0 +1,20 @@
+-----BEGIN X509 CRL-----
+MIIDPDCCASQCAQEwDQYJKoZIhvcNAQELBQAwazELMAkGA1UEBhMCU0UxEjAQBgNV
+BAgMCVN0b2NraG9sbTESMBAGA1UECgwJTXlPcmdOYW1lMRkwFwYDVQQLDBBNeUlu
+dGVybWVkaWF0ZUNBMRkwFwYDVQQDDBBNeUludGVybWVkaWF0ZUNBFw0yMzAxMTIx
+MzA4MTZaFw0zMzAxMDkxMzA4MTZaMBUwEwICEAIXDTIzMDExMjEzMDgxNlqgbjBs
+MB8GA1UdIwQYMBaAFExwhjsVUom6tQ+Sqq6xMUETvnPzMDwGA1UdHAQ1MDOgLqAs
+hipodHRwOi8vbG9jYWxob3N0Ojk4NzgvaW50ZXJtZWRpYXRlLmNybC5wZW2EAf8w
+CwYDVR0UBAQCAhABMA0GCSqGSIb3DQEBCwUAA4ICAQCPadbaehEqLv4pwqF8em8T
+CW8TOQ4Vjz02uiVk9Bo0za1dQqQmwCBA6UE1BcOh+aWzQxBRz56NeUcfhgDxTntG
+xLs896N9MHIG6UxpqJH8cH+DXKHsQjvvCjXtiObmBQR1RiG5C1vEMkfzTt/WSrq5
+7blowLDs4NP6YbtqXEyyUkF7DQSUEUuIDWPQdx1f++nSpVaHWW4xpoO4umesaJco
+FuxaXQnZpTHHQfqUJVIL2Mmzvez9thgfKTV3vgkYrGiSLW2m2+Tfga30pUc0qaVI
+RrBVORVbcu9m1sV0aJyk96b2T/+i2FRR/np4TOcLgckBpHKeK2FH69lHFr0W/71w
+CErNTxahoh82Yi8POenu+S1m2sDnrF1FMf+ZG/i2wr0nW6/+zVGQsEOw77Spbmei
+dbEchu3iWF1XEO/n4zVBzl6a1o2RyVg+1pItYd5C5bPwcrfZnBrm4WECPxO+6rbW
+2/wz9Iku4XznTLqLEpXLAtenAdo73mLGC7riviX7mhcxfN2UjNfLuVGHmG8XwIsM
+Lgpr6DKaxHwpHgW3wA3SGJrY5dj0TvGWaoInrNt1cOMnIpoxRNy5+ko71Ubx3yrV
+RhbUMggd1GG1ct9uZn82v74RYF6J8Xcxn9vDFJu5LLT5kvfy414kdJeTXKqfKXA/
+atdUgFa0otoccn5FzyUuzg==
+-----END X509 CRL-----

apps/emqx/test/emqx_crl_cache_SUITE_data/intermediate.crl.pem

@@ -0,0 +1,20 @@
+-----BEGIN X509 CRL-----
+MIIDPDCCASQCAQEwDQYJKoZIhvcNAQELBQAwazELMAkGA1UEBhMCU0UxEjAQBgNV
+BAgMCVN0b2NraG9sbTESMBAGA1UECgwJTXlPcmdOYW1lMRkwFwYDVQQLDBBNeUlu
+dGVybWVkaWF0ZUNBMRkwFwYDVQQDDBBNeUludGVybWVkaWF0ZUNBFw0yMjA3MjAy
+MDIzNTNaFw0zMjEwMjUyMDIzNTNaMBUwEwICEAIXDTIyMDYxMzEyNDIwNVqgbjBs
+MB8GA1UdIwQYMBaAFCuv1TkzC1fSgTfzE1m1u5pRCJsVMDwGA1UdHAQ1MDOgLqAs
+hipodHRwOi8vbG9jYWxob3N0Ojk4NzgvaW50ZXJtZWRpYXRlLmNybC5wZW2EAf8w
+CwYDVR0UBAQCAhADMA0GCSqGSIb3DQEBCwUAA4ICAQBbWdqRFsIrG6coL6ln1RL+
+uhgW9l3XMmjNlyiYHHNzOgnlBok9xu9UdaVCOKC6GEthWSzSlBY1AZugje57DQQd
+RkIJol9am94lKMTjF/qhzFLiSjho8fwZGDGyES5YeZXkLqNMOf6m/drKaI3iofWf
+l63qU9jY8dnSrVDkwgCguUL2FTx60v5H9NPxSctQ3VDxDvDj0sTAcHFknQcZbfvY
+ZWpOYNS0FAJlQPVK9wUoDxI0LhrWDq5h/T1jcGO34fPT8RUA5HRtFVUevqSuOLWx
+WTfTx5oDeMZPJTvHWUcg4yMElHty4tEvtkFxLSYbZqj7qTU+mi/LAN3UKBH/gBEN
+y2OsJvFhVRgHf+zPYegf3WzBSoeaXNAJZ4UnRo34P9AL3Mrh+4OOUP++oYRKjWno
+pYtAmTrIwEYoLyisEhhZ6aD92f/Op3dIYsxwhHt0n0lKrbTmUfiJUAe7kUZ4PMn4
+Gg/OHlbEDaDxW1dCymjyRGl+3/8kjy7bkYUXCf7w6JBeL2Hw2dFp1Gh13NRjre93
+PYlSOvI6QNisYGscfuYPwefXogVrNjf/ttCksMa51tUk+ylw7ZMZqQjcPPSzmwKc
+5CqpnQHfolvRuN0xIVZiAn5V6/MdHm7ocrXxOkzWQyaoNODTq4js8h8eYXgAkt1w
+p1PTEFBucGud7uBDE6Ub6A==
+-----END X509 CRL-----

apps/emqx/test/emqx_crl_cache_SUITE_data/server.cert.pem

@@ -0,0 +1,35 @@
+-----BEGIN CERTIFICATE-----
+MIIGCTCCA/GgAwIBAgICEAAwDQYJKoZIhvcNAQELBQAwazELMAkGA1UEBhMCU0Ux
+EjAQBgNVBAgMCVN0b2NraG9sbTESMBAGA1UECgwJTXlPcmdOYW1lMRkwFwYDVQQL
+DBBNeUludGVybWVkaWF0ZUNBMRkwFwYDVQQDDBBNeUludGVybWVkaWF0ZUNBMB4X
+DTIzMDExMjEzMDgxNloXDTMzMDQxOTEzMDgxNloweDELMAkGA1UEBhMCU0UxEjAQ
+BgNVBAgMCVN0b2NraG9sbTESMBAGA1UEBwwJU3RvY2tob2xtMRIwEAYDVQQKDAlN
+eU9yZ05hbWUxGTAXBgNVBAsMEE15SW50ZXJtZWRpYXRlQ0ExEjAQBgNVBAMMCWxv
+Y2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKdU9FaA/n0Z
+TXkd10XA9l+UV9xKR65ZTy2ApCFlw2gGWLiUh96a6hX+GQZFUV7ECIDDf+7nC85o
+xo1Xyf0rHGABQ0uHlhqSemc12F9APIzRLlQkhtV4vMBBbGQFekje4F9bhY9JQtGd
+XJGmwsR+XWo6SUY7K5l9FuSSSRXC0kSYYQfSTPR/LrF6efdHf+ZN4huP7lM2qIFd
+afX+qBOI1/Y2LtITo2TaU/hXyKh9wEiuynoq0RZ2KkYQll5cKD9fSD+pW3Xm0XWX
+TQy4RZEe3WoYEQsklNw3NC92ocA/PQB9BGNO1fKhzDn6kW2HxDxruDKOuO/meGek
+ApCayu3e/I0CAwEAAaOCAagwggGkMAkGA1UdEwQCMAAwEQYJYIZIAYb4QgEBBAQD
+AgZAMDMGCWCGSAGG+EIBDQQmFiRPcGVuU1NMIEdlbmVyYXRlZCBTZXJ2ZXIgQ2Vy
+dGlmaWNhdGUwHQYDVR0OBBYEFGy5LQPzIelruJl7mL0mtUXM57XhMIGaBgNVHSME
+gZIwgY+AFExwhjsVUom6tQ+Sqq6xMUETvnPzoXOkcTBvMQswCQYDVQQGEwJTRTES
+MBAGA1UECAwJU3RvY2tob2xtMRIwEAYDVQQHDAlTdG9ja2hvbG0xEjAQBgNVBAoM
+CU15T3JnTmFtZTERMA8GA1UECwwITXlSb290Q0ExETAPBgNVBAMMCE15Um9vdENB
+ggIQADAOBgNVHQ8BAf8EBAMCBaAwEwYDVR0lBAwwCgYIKwYBBQUHAwEwOwYDVR0f
+BDQwMjAwoC6gLIYqaHR0cDovL2xvY2FsaG9zdDo5ODc4L2ludGVybWVkaWF0ZS5j
+cmwucGVtMDEGCCsGAQUFBwEBBCUwIzAhBggrBgEFBQcwAYYVaHR0cDovL2xvY2Fs
+aG9zdDo5ODc3MA0GCSqGSIb3DQEBCwUAA4ICAQCX3EQgiCVqLhnCNd0pmptxXPxo
+l1KyZkpdrFa/NgSqRhkuZSAkszwBDDS/gzkHFKEUhmqs6/UZwN4+Rr3LzrHonBiN
+aQ6GeNNXZ/3xAQfUCwjjGmz9Sgw6kaX19Gnk2CjI6xP7T+O5UmsMI9hHUepC9nWa
+XX2a0hsO/KOVu5ZZckI16Ek/jxs2/HEN0epYdvjKFAaVmzZZ5PATNjrPQXvPmq2r
+x++La+3bXZsrH8P2FhPpM5t/IxKKW/Tlpgz92c2jVSIHF5khSA/MFDC+dk80OFmm
+v4ZTPIMuZ//Q+wo0f9P48rsL9D27qS7CA+8pn9wu+cfnBDSt7JD5Yipa1gHz71fy
+YTa9qRxIAPpzW2v7TFZE8eSKFUY9ipCeM2BbdmCQGmq4+v36b5TZoyjH4k0UVWGo
+Gclos2cic5Vxi8E6hb7b7yZpjEfn/5lbCiGMfAnI6aoOyrWg6keaRA33kaLUEZiK
+OgFNbPkjiTV0ZQyLXf7uK9YFhpVzJ0dv0CFNse8rZb7A7PLn8VrV/ZFnJ9rPoawn
+t7ZGxC0d5BRSEyEeEgsQdxuY4m8OkE18zwhCkt2Qs3uosOWlIrYmqSEa0i/sPSQP
+jiwB4nEdBrf8ZygzuYjT5T9YRSwhVox4spS/Av8Ells5JnkuKAhCVv9gHxYwbj0c
+CzyLJgE1z9Tq63m+gQ==
+-----END CERTIFICATE-----

apps/emqx/test/emqx_crl_cache_SUITE_data/server.key.pem

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCnVPRWgP59GU15
+HddFwPZflFfcSkeuWU8tgKQhZcNoBli4lIfemuoV/hkGRVFexAiAw3/u5wvOaMaN
+V8n9KxxgAUNLh5YaknpnNdhfQDyM0S5UJIbVeLzAQWxkBXpI3uBfW4WPSULRnVyR
+psLEfl1qOklGOyuZfRbkkkkVwtJEmGEH0kz0fy6xenn3R3/mTeIbj+5TNqiBXWn1
+/qgTiNf2Ni7SE6Nk2lP4V8iofcBIrsp6KtEWdipGEJZeXCg/X0g/qVt15tF1l00M
+uEWRHt1qGBELJJTcNzQvdqHAPz0AfQRjTtXyocw5+pFth8Q8a7gyjrjv5nhnpAKQ
+msrt3vyNAgMBAAECggEABnWvIQ/Fw0qQxRYz00uJt1LguW5cqgxklBsdOvTUwFVO
+Y4HIZP2R/9tZV/ahF4l10pK5g52DxSoiUB6Ne6qIY+RolqfbUZdKBmX7vmGadM02
+fqUSV3dbwghEiO/1Mo74FnZQB6IKZFEw26aWakN+k7VAUufB3SEJGzXSgHaO63ru
+dFGSiYI8U+q+YnhUJjCnmI12fycNfy451TdUQtGZb6pNmm5HRUF6hpAV8Le9LojP
+Ql9eacPpsrzU15X5ElCQZ/f9iNh1bplcISuhrULgKUKOvAVrBlEK67uRVy6g98xA
+c/rgNLkbL/jZEsAc3/vHAyFgd3lABfwpBGLHej3QgQKBgQDFNYmfBNQr89HC5Zc+
+M6jXcAT/R+0GNczBTfC4iyNemwqsumSSRelNZ748UefKuS3F6Mvb2CBqE2LbB61G
+hrnCffG2pARjZ491SefRwghhWWVGLP1p8KliLgOGBehA1REgJb+XULncjuHZuh4O
+LVn3HVnWGxeBGg+yKa6Z4YQi3QKBgQDZN0O8ZcZY74lRJ0UjscD9mJ1yHlsssZag
+njkX/f0GR/iVpfaIxQNC3gvWUy2LsU0He9sidcB0cfej0j/qZObQyFsCB0+utOgy
++hX7gokV2pes27WICbNWE2lJL4QZRJgvf82OaEy57kfDrm+eK1XaSZTZ10P82C9u
+gAmMnontcQKBgGu29lhY9tqa7jOZ26Yp6Uri8JfO3XPK5u+edqEVvlfqL0Zw+IW8
+kdWpmIqx4f0kcA/tO4v03J+TvycLZmVjKQtGZ0PvCkaRRhY2K9yyMomZnmtaH4BB
+5wKtR1do2pauyg/ZDnDDswD5OfsGYWw08TK8YVlEqu3lIjWZ9rguKVIxAoGAZYUk
+zVqr10ks3pcCA2rCjkPT4lA5wKvHgI4ylPoKVfMxRY/pp4acvZXV5ne9o7pcDBFh
+G7v5FPNnEFPlt4EtN4tMragJH9hBZgHoYEJkG6islweg0lHmVWaBIMlqbfzXO+v5
+gINSyNuLAvP2CvCqEXmubhnkFrpbgMOqsuQuBqECgYB3ss2PDhBF+5qoWgqymFof
+1ovRPuQ9sPjWBn5IrCdoYITDnbBzBZERx7GLs6A/PUlWgST7jkb1PY/TxYSUfXzJ
+SNd47q0mCQ+IUdqUbHgpK9b1ncwLMsnexpYZdHJWRLgnUhOx7OMjJc/4iLCAFCoN
+3KJ7/V1keo7GBHOwnsFcCA==
+-----END PRIVATE KEY-----

apps/emqx/test/emqx_listeners_SUITE.erl

@@ -26,6 +26,8 @@
 
 -define(CERTS_PATH(CertName), filename:join(["../../lib/emqx/etc/certs/", CertName])).
 
+-define(SERVER_KEY_PASSWORD, "sErve7r8Key$!").
+
 all() -> emqx_common_test_helpers:all(?MODULE).
 
 init_per_suite(Config) ->
@@ -33,6 +35,7 @@ init_per_suite(Config) ->
     application:ensure_all_started(esockd),
     application:ensure_all_started(quicer),
     application:ensure_all_started(cowboy),
+    generate_tls_certs(Config),
     lists:foreach(fun set_app_env/1, NewConfig),
     Config.
 
@@ -45,11 +48,6 @@ init_per_testcase(Case, Config) when
 ->
     catch emqx_config_handler:stop(),
     {ok, _} = emqx_config_handler:start_link(),
-    case emqx_config:get([listeners], undefined) of
-        undefined -> ok;
-        Listeners -> emqx_config:put([listeners], maps:remove(quic, Listeners))
-    end,
-
     PrevListeners = emqx_config:get([listeners], #{}),
     PureListeners = remove_default_limiter(PrevListeners),
     PureListeners2 = PureListeners#{
@@ -185,6 +183,50 @@ t_wss_conn(_) ->
     {ok, Socket} = ssl:connect({127, 0, 0, 1}, 9998, [{verify, verify_none}], 1000),
     ok = ssl:close(Socket).
 
+t_quic_conn(Config) ->
+    Port = 24568,
+    DataDir = ?config(data_dir, Config),
+    SSLOpts = #{
+        password => ?SERVER_KEY_PASSWORD,
+        certfile => filename:join(DataDir, "server-password.pem"),
+        cacertfile => filename:join(DataDir, "ca.pem"),
+        keyfile => filename:join(DataDir, "server-password.key")
+    },
+    emqx_common_test_helpers:ensure_quic_listener(?FUNCTION_NAME, Port, #{ssl_options => SSLOpts}),
+    ct:pal("~p", [emqx_listeners:list()]),
+    {ok, Conn} = quicer:connect(
+        {127, 0, 0, 1},
+        Port,
+        [
+            {verify, verify_none},
+            {alpn, ["mqtt"]}
+        ],
+        1000
+    ),
+    ok = quicer:close_connection(Conn),
+    emqx_listeners:stop_listener(quic, ?FUNCTION_NAME, #{bind => Port}).
+
+t_ssl_password_cert(Config) ->
+    Port = 24568,
+    DataDir = ?config(data_dir, Config),
+    SSLOptsPWD = #{
+        password => ?SERVER_KEY_PASSWORD,
+        certfile => filename:join(DataDir, "server-password.pem"),
+        cacertfile => filename:join(DataDir, "ca.pem"),
+        keyfile => filename:join(DataDir, "server-password.key")
+    },
+    LConf = #{
+        enabled => true,
+        bind => {{127, 0, 0, 1}, Port},
+        mountpoint => <<>>,
+        zone => default,
+        ssl_options => SSLOptsPWD
+    },
+    ok = emqx_listeners:start_listener(ssl, ?FUNCTION_NAME, LConf),
+    {ok, SSLSocket} = ssl:connect("127.0.0.1", Port, [{verify, verify_none}]),
+    ssl:close(SSLSocket),
+    emqx_listeners:stop_listener(ssl, ?FUNCTION_NAME, LConf).
+
 t_format_bind(_) ->
     ?assertEqual(
         ":1883",
@@ -269,3 +311,10 @@ remove_default_limiter(Listeners) ->
         end,
         Listeners
     ).
+
+generate_tls_certs(Config) ->
+    DataDir = ?config(data_dir, Config),
+    emqx_common_test_helpers:gen_ca(DataDir, "ca"),
+    emqx_common_test_helpers:gen_host_cert("server-password", "ca", DataDir, #{
+        password => ?SERVER_KEY_PASSWORD
+    }).

apps/emqx/test/emqx_ocsp_cache_SUITE.erl

@@ -76,7 +76,7 @@ init_per_testcase(t_openssl_client, Config) ->
         [],
         Handler,
         #{
-            extra_mustache_vars => [{test_data_dir, DataDir}],
+            extra_mustache_vars => #{test_data_dir => DataDir},
             conf_file_path => ConfFilePath
         }
     ),

apps/emqx/test/emqx_quic_multistreams_SUITE.erl

@@ -1569,7 +1569,7 @@ t_multi_streams_remote_shutdown(Config) ->
 
     ok = stop_emqx(),
     %% Client should be closed
-    assert_client_die(C).
+    assert_client_die(C, 100, 50).
 
 t_multi_streams_remote_shutdown_with_reconnect(Config) ->
     erlang:process_flag(trap_exit, true),
@@ -2047,14 +2047,15 @@ via_stream({quic, _Conn, Stream}) ->
 assert_client_die(C) ->
     assert_client_die(C, 100, 10).
 assert_client_die(C, _, 0) ->
-    ct:fail("Client ~p did not die", [C]);
+    ct:fail("Client ~p did not die: stacktrace: ~p", [C, process_info(C, current_stacktrace)]);
 assert_client_die(C, Delay, Retries) ->
-    case catch emqtt:info(C) of
+    try emqtt:info(C) of
-        {'EXIT', {noproc, {gen_statem, call, [_, info, infinity]}}} ->
+        Info when is_list(Info) ->
-            ok;
-        _Other ->
             timer:sleep(Delay),
             assert_client_die(C, Delay, Retries - 1)
+    catch
+        exit:Error ->
+            ct:comment("client die with ~p", [Error])
     end.
 
 %% BUILD_WITHOUT_QUIC

apps/emqx/test/emqx_test_janitor.erl

@@ -65,7 +65,7 @@ terminate(_Reason, #{callbacks := Callbacks}) ->
 handle_call({push, Callback}, _From, State = #{callbacks := Callbacks}) ->
     {reply, ok, State#{callbacks := [Callback | Callbacks]}};
 handle_call(terminate, _From, State = #{callbacks := Callbacks}) ->
-    lists:foreach(fun(Fun) -> Fun() end, Callbacks),
+    lists:foreach(fun(Fun) -> catch Fun() end, Callbacks),
     {stop, normal, ok, State};
 handle_call(_Req, _From, State) ->
     {reply, error, State}.

apps/emqx_authn/src/emqx_authn.app.src

@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_authn, [
     {description, "EMQX Authentication"},
-    {vsn, "0.1.15"},
+    {vsn, "0.1.16"},
     {modules, []},
     {registered, [emqx_authn_sup, emqx_authn_registry]},
     {applications, [kernel, stdlib, emqx_resource, emqx_connector, ehttpc, epgsql, mysql, jose]},

apps/emqx_authn/src/emqx_authn_api.erl

@@ -1419,14 +1419,14 @@ request_user_create_examples() ->
             summary => <<"Regular user">>,
             value => #{
                 user_id => <<"user1">>,
-                password => <<"secret">>
+                password => <<"******">>
             }
         },
         super_user => #{
             summary => <<"Superuser">>,
             value => #{
                 user_id => <<"user2">>,
-                password => <<"secret">>,
+                password => <<"******">>,
                 is_superuser => true
             }
         }
@@ -1437,13 +1437,13 @@ request_user_update_examples() ->
         regular_user => #{
             summary => <<"Update regular user">>,
             value => #{
-                password => <<"newsecret">>
+                password => <<"******">>
             }
         },
         super_user => #{
             summary => <<"Update user and promote to superuser">>,
             value => #{
-                password => <<"newsecret">>,
+                password => <<"******">>,
                 is_superuser => true
             }
         }

apps/emqx_authz/etc/acl.conf

@@ -23,7 +23,7 @@
 %% -type(rule() :: {permission(), who(), access(), topics()} | {permission(), all}).
 %%--------------------------------------------------------------------
 
-{allow, {username, "^dashboard?"}, subscribe, ["$SYS/#"]}.
+{allow, {username, {re, "^dashboard$"}}, subscribe, ["$SYS/#"]}.
 
 {allow, {ipaddr, "127.0.0.1"}, all, ["$SYS/#", "#"]}.
 

apps/emqx_authz/src/emqx_authz.app.src

@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_authz, [
     {description, "An OTP application"},
-    {vsn, "0.1.15"},
+    {vsn, "0.1.16"},
     {registered, []},
     {mod, {emqx_authz_app, []}},
     {applications, [

apps/emqx_authz/src/emqx_authz_schema.erl

@@ -492,7 +492,9 @@ authz_fields() ->
                 ?ARRAY(?UNION(UnionMemberSelector)),
                 #{
                     default => [],
-                    desc => ?DESC(sources)
+                    desc => ?DESC(sources),
+                    %% doc_lift is force a root level reference instead of nesting sub-structs
+                    extra => #{doc_lift => true}
                 }
             )}
     ].

apps/emqx_authz/test/emqx_authz_SUITE.erl

@@ -26,6 +26,8 @@
 -include_lib("emqx/include/emqx_placeholder.hrl").
 -include_lib("snabbkaffe/include/snabbkaffe.hrl").
 
+-import(emqx_common_test_helpers, [on_exit/1]).
+
 all() ->
     emqx_common_test_helpers:all(?MODULE).
 
@@ -65,6 +67,7 @@ end_per_suite(_Config) ->
 
 init_per_testcase(TestCase, Config) when
     TestCase =:= t_subscribe_deny_disconnect_publishes_last_will_testament;
+    TestCase =:= t_publish_last_will_testament_banned_client_connecting;
     TestCase =:= t_publish_deny_disconnect_publishes_last_will_testament
 ->
     {ok, _} = emqx_authz:update(?CMD_REPLACE, []),
@@ -76,11 +79,15 @@ init_per_testcase(_, Config) ->
 
 end_per_testcase(TestCase, _Config) when
     TestCase =:= t_subscribe_deny_disconnect_publishes_last_will_testament;
+    TestCase =:= t_publish_last_will_testament_banned_client_connecting;
     TestCase =:= t_publish_deny_disconnect_publishes_last_will_testament
 ->
     {ok, _} = emqx:update_config([authorization, deny_action], ignore),
+    {ok, _} = emqx_authz:update(?CMD_REPLACE, []),
+    emqx_common_test_helpers:call_janitor(),
     ok;
 end_per_testcase(_TestCase, _Config) ->
+    emqx_common_test_helpers:call_janitor(),
     ok.
 
 set_special_configs(emqx_authz) ->
@@ -396,5 +403,63 @@ t_publish_last_will_testament_denied_topic(_Config) ->
 
     ok.
 
+%% client is allowed by ACL to publish to its LWT topic, is connected,
+%% and then gets banned and kicked out while connected.  Should not
+%% publish LWT.
+t_publish_last_will_testament_banned_client_connecting(_Config) ->
+    {ok, _} = emqx_authz:update(?CMD_REPLACE, [?SOURCE7]),
+    Username = <<"some_client">>,
+    ClientId = <<"some_clientid">>,
+    LWTPayload = <<"should not be published">>,
+    LWTTopic = <<"some_client/lwt">>,
+    ok = emqx:subscribe(<<"some_client/lwt">>),
+    {ok, C} = emqtt:start_link([
+        {clientid, ClientId},
+        {username, Username},
+        {will_topic, LWTTopic},
+        {will_payload, LWTPayload}
+    ]),
+    ?assertMatch({ok, _}, emqtt:connect(C)),
+
+    %% Now we ban the client while it is connected.
+    Now = erlang:system_time(second),
+    Who = {username, Username},
+    emqx_banned:create(#{
+        who => Who,
+        by => <<"test">>,
+        reason => <<"test">>,
+        at => Now,
+        until => Now + 120
+    }),
+    on_exit(fun() -> emqx_banned:delete(Who) end),
+    %% Now kick it as we do in the ban API.
+    process_flag(trap_exit, true),
+    ?check_trace(
+        begin
+            ok = emqx_cm:kick_session(ClientId),
+            receive
+                {deliver, LWTTopic, #message{payload = LWTPayload}} ->
+                    error(lwt_should_not_be_published_to_forbidden_topic)
+            after 2_000 -> ok
+            end,
+            ok
+        end,
+        fun(Trace) ->
+            ?assertMatch(
+                [
+                    #{
+                        client_banned := true,
+                        publishing_disallowed := false
+                    }
+                ],
+                ?of_kind(last_will_testament_publish_denied, Trace)
+            ),
+            ok
+        end
+    ),
+    ok = snabbkaffe:stop(),
+
+    ok.
+
 stop_apps(Apps) ->
     lists:foreach(fun application:stop/1, Apps).

apps/emqx_auto_subscribe/README.md

@@ -1,9 +1,54 @@
-emqx_auto_subscribe
+# Auto Subscribe
-=====
 
-An OTP application
+This application can help clients automatically subscribe to topics compiled from user definitions when they connect, and the clients no longer need to send the MQTT `SUBSCRIBE ` request.
 
-Build
+# How To Use
------
 
-    $ rebar3 compile
+Add the following configuration items to the `emqx.conf` file
+
+```yaml
+auto_subscribe {
+    topics = [
+        {
+            topic = "c/${clientid}"
+        },
+        {
+            topic = "client/${clientid}/username/${username}/host/${host}/port/${port}"
+            qos   = 1
+            rh    = 0
+            rap   = 0
+            nl    = 0
+        }
+    ]
+}
+```
+
+This example defines two templates, all of which will be compiled into the final topic by replacing placeholders like `${clientid}` `${port}` with the actual values when the client connects.
+
+# Configuration
+
+## Configuration Definition
+
+| Field          | Definition                    | Value Range                                                 | Default |
+| -------------- | ----------------------------- | ----------------------------------------------------------- | ------- |
+| auto_subscribe | Auto subscribe configuration  | topics                                                      | topics  |
+| topics         | Subscription Options          | Subscription configurations list. See `Subscription Option` | []      |
+
+## Subscription Option
+
+| Field | Definition                                                                                              | Value Range                                                     | Default          |
+|-------|---------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------|------------------|
+| topic | Required. Topic template.                                                                               | String, placeholders supported                                  | No default value |
+| qos   | Optional. Subscription QoS                                                                              | 0 or 1 or 2. Refer to the MQTT QoS definition                   | 0                |
+| rh    | Optional. MQTT version 5.0. Whether to send retain message when a subscription is created.              | 0: Not send the retain message </br>1: Send  the retain message | 0                |
+| rap   | Optional. MQTT version 5.0. When forwarding messages, Whether to send with retain flag                  | 0: Set retain 0</br>1: Keep retain flag                         | 0                |
+| nl    | Optional. MQTT version 5.0. Whether the message can be forwarded to the client when published by itself | 0: Forwarded to self</br>1: Not forwarded to self               | 0                |
+
+## Subscription Placeholders
+
+| Placeholder | Definition                             |
+| ----------- | -------------------------------------- |
+| ${clientid} | Client ID                              |
+| ${username} | Client Username                        |
+| ${ip}       | Client TCP connection local IP address |
+| ${port}     | Client TCP connection local Port       |

apps/emqx_bridge/src/emqx_bridge.app.src

@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_bridge, [
     {description, "EMQX bridges"},
-    {vsn, "0.1.14"},
+    {vsn, "0.1.15"},
     {registered, [emqx_bridge_sup]},
     {mod, {emqx_bridge_app, []}},
     {applications, [

apps/emqx_bridge/src/emqx_bridge.erl

@@ -67,7 +67,9 @@
     T == timescale;
     T == matrix;
     T == tdengine;
-    T == dynamo
+    T == dynamo;
+    T == rocketmq;
+    T == cassandra
 ).
 
 load() ->

apps/emqx_bridge/src/emqx_bridge_api.erl

@@ -51,10 +51,10 @@
     ?BAD_REQUEST(<<"Forbidden operation, bridge not enabled">>)
 ).
 
--define(BRIDGE_NOT_FOUND(BridgeType, BridgeName),
+-define(BRIDGE_NOT_FOUND(BRIDGE_TYPE, BRIDGE_NAME),
     ?NOT_FOUND(
-        <<"Bridge lookup failed: bridge named '", (BridgeName)/binary, "' of type ",
+        <<"Bridge lookup failed: bridge named '", (BRIDGE_NAME)/binary, "' of type ",
-            (bin(BridgeType))/binary, " does not exist.">>
+            (bin(BRIDGE_TYPE))/binary, " does not exist.">>
     )
 ).
 
@@ -218,7 +218,7 @@ info_example_basic(webhook) ->
             health_check_interval => 15000,
             auto_restart_interval => 15000,
             query_mode => async,
-            async_inflight_window => 100,
+            inflight_window => 100,
             max_queue_bytes => 100 * 1024 * 1024
         }
     };
@@ -235,7 +235,7 @@ mqtt_main_example() ->
         server => <<"127.0.0.1:1883">>,
         proto_ver => <<"v4">>,
         username => <<"foo">>,
-        password => <<"bar">>,
+        password => <<"******">>,
         clean_start => true,
         keepalive => <<"300s">>,
         retry_interval => <<"15s">>,
@@ -281,7 +281,7 @@ schema("/bridges") ->
         'operationId' => '/bridges',
         get => #{
             tags => [<<"bridges">>],
-            summary => <<"List Bridges">>,
+            summary => <<"List bridges">>,
             description => ?DESC("desc_api1"),
             responses => #{
                 200 => emqx_dashboard_swagger:schema_with_example(
@@ -292,7 +292,7 @@ schema("/bridges") ->
         },
         post => #{
             tags => [<<"bridges">>],
-            summary => <<"Create Bridge">>,
+            summary => <<"Create bridge">>,
             description => ?DESC("desc_api2"),
             'requestBody' => emqx_dashboard_swagger:schema_with_examples(
                 emqx_bridge_schema:post_request(),
@@ -309,7 +309,7 @@ schema("/bridges/:id") ->
         'operationId' => '/bridges/:id',
         get => #{
             tags => [<<"bridges">>],
-            summary => <<"Get Bridge">>,
+            summary => <<"Get bridge">>,
             description => ?DESC("desc_api3"),
             parameters => [param_path_id()],
             responses => #{
@@ -319,7 +319,7 @@ schema("/bridges/:id") ->
         },
         put => #{
             tags => [<<"bridges">>],
-            summary => <<"Update Bridge">>,
+            summary => <<"Update bridge">>,
             description => ?DESC("desc_api4"),
             parameters => [param_path_id()],
             'requestBody' => emqx_dashboard_swagger:schema_with_examples(
@@ -334,7 +334,7 @@ schema("/bridges/:id") ->
         },
         delete => #{
             tags => [<<"bridges">>],
-            summary => <<"Delete Bridge">>,
+            summary => <<"Delete bridge">>,
             description => ?DESC("desc_api5"),
             parameters => [param_path_id()],
             responses => #{
@@ -353,7 +353,7 @@ schema("/bridges/:id/metrics") ->
         'operationId' => '/bridges/:id/metrics',
         get => #{
             tags => [<<"bridges">>],
-            summary => <<"Get Bridge Metrics">>,
+            summary => <<"Get bridge metrics">>,
             description => ?DESC("desc_bridge_metrics"),
             parameters => [param_path_id()],
             responses => #{
@@ -367,7 +367,7 @@ schema("/bridges/:id/metrics/reset") ->
         'operationId' => '/bridges/:id/metrics/reset',
         put => #{
             tags => [<<"bridges">>],
-            summary => <<"Reset Bridge Metrics">>,
+            summary => <<"Reset bridge metrics">>,
             description => ?DESC("desc_api6"),
             parameters => [param_path_id()],
             responses => #{
@@ -382,7 +382,7 @@ schema("/bridges/:id/enable/:enable") ->
         put =>
             #{
                 tags => [<<"bridges">>],
-                summary => <<"Enable or Disable Bridge">>,
+                summary => <<"Enable or disable bridge">>,
                 desc => ?DESC("desc_enable_bridge"),
                 parameters => [param_path_id(), param_path_enable()],
                 responses =>
@@ -398,7 +398,7 @@ schema("/bridges/:id/:operation") ->
         'operationId' => '/bridges/:id/:operation',
         post => #{
             tags => [<<"bridges">>],
-            summary => <<"Stop or Restart Bridge">>,
+            summary => <<"Stop or restart bridge">>,
             description => ?DESC("desc_api7"),
             parameters => [
                 param_path_id(),
@@ -420,7 +420,7 @@ schema("/nodes/:node/bridges/:id/:operation") ->
         'operationId' => '/nodes/:node/bridges/:id/:operation',
         post => #{
             tags => [<<"bridges">>],
-            summary => <<"Stop/Restart Bridge">>,
+            summary => <<"Stop/restart bridge">>,
             description => ?DESC("desc_api8"),
             parameters => [
                 param_path_node(),
@@ -460,11 +460,10 @@ schema("/bridges_probe") ->
 '/bridges'(post, #{body := #{<<"type">> := BridgeType, <<"name">> := BridgeName} = Conf0}) ->
     case emqx_bridge:lookup(BridgeType, BridgeName) of
         {ok, _} ->
-            {400, error_msg('ALREADY_EXISTS', <<"bridge already exists">>)};
+            ?BAD_REQUEST('ALREADY_EXISTS', <<"bridge already exists">>);
         {error, not_found} ->
             Conf = filter_out_request_body(Conf0),
-            {ok, _} = emqx_bridge:create(BridgeType, BridgeName, Conf),
+            create_bridge(BridgeType, BridgeName, Conf)
-            lookup_from_all_nodes(BridgeType, BridgeName, 201)
     end;
 '/bridges'(get, _Params) ->
     Nodes = mria:running_nodes(),
@@ -475,9 +474,9 @@ schema("/bridges_probe") ->
                 [format_resource(Data, Node) || Data <- Bridges]
              || {Node, Bridges} <- lists:zip(Nodes, NodeBridges)
             ],
-            {200, zip_bridges(AllBridges)};
+            ?OK(zip_bridges(AllBridges));
         {error, Reason} ->
-            {500, error_msg('INTERNAL_ERROR', Reason)}
+            ?INTERNAL_ERROR(Reason)
     end.
 
 '/bridges/:id'(get, #{bindings := #{id := Id}}) ->
@@ -490,8 +489,7 @@ schema("/bridges_probe") ->
             {ok, _} ->
                 RawConf = emqx:get_raw_config([bridges, BridgeType, BridgeName], #{}),
                 Conf = deobfuscate(Conf1, RawConf),
-                {ok, _} = emqx_bridge:create(BridgeType, BridgeName, Conf),
+                update_bridge(BridgeType, BridgeName, Conf);
-                lookup_from_all_nodes(BridgeType, BridgeName, 200);
             {error, not_found} ->
                 ?BRIDGE_NOT_FOUND(BridgeType, BridgeName)
         end
@@ -509,16 +507,16 @@ schema("/bridges_probe") ->
                     end,
                 case emqx_bridge:check_deps_and_remove(BridgeType, BridgeName, AlsoDeleteActs) of
                     {ok, _} ->
-                        204;
+                        ?NO_CONTENT;
                     {error, {rules_deps_on_this_bridge, RuleIds}} ->
                         ?BAD_REQUEST(
                             {<<"Cannot delete bridge while active rules are defined for this bridge">>,
                                 RuleIds}
                         );
                     {error, timeout} ->
-                        {503, error_msg('SERVICE_UNAVAILABLE', <<"request timeout">>)};
+                        ?SERVICE_UNAVAILABLE(<<"request timeout">>);
                     {error, Reason} ->
-                        {500, error_msg('INTERNAL_ERROR', Reason)}
+                        ?INTERNAL_ERROR(Reason)
                 end;
             {error, not_found} ->
                 ?BRIDGE_NOT_FOUND(BridgeType, BridgeName)
@@ -535,7 +533,7 @@ schema("/bridges_probe") ->
             ok = emqx_bridge_resource:reset_metrics(
                 emqx_bridge_resource:resource_id(BridgeType, BridgeName)
             ),
-            {204}
+            ?NO_CONTENT
         end
     ).
 
@@ -546,9 +544,9 @@ schema("/bridges_probe") ->
             Params1 = maybe_deobfuscate_bridge_probe(Params),
             case emqx_bridge_resource:create_dry_run(ConnType, maps:remove(<<"type">>, Params1)) of
                 ok ->
-                    204;
+                    ?NO_CONTENT;
                 {error, Reason} when not is_tuple(Reason); element(1, Reason) =/= 'exit' ->
-                    {400, error_msg('TEST_FAILED', to_hr_reason(Reason))}
+                    ?BAD_REQUEST('TEST_FAILED', Reason)
             end;
         BadRequest ->
             BadRequest
@@ -582,7 +580,7 @@ do_lookup_from_all_nodes(BridgeType, BridgeName, SuccCode, FormatFun) ->
         {ok, [{error, not_found} | _]} ->
             ?BRIDGE_NOT_FOUND(BridgeType, BridgeName);
         {error, Reason} ->
-            {500, error_msg('INTERNAL_ERROR', Reason)}
+            ?INTERNAL_ERROR(Reason)
     end.
 
 lookup_from_local_node(BridgeType, BridgeName) ->
@@ -591,6 +589,20 @@ lookup_from_local_node(BridgeType, BridgeName) ->
         Error -> Error
     end.
 
+create_bridge(BridgeType, BridgeName, Conf) ->
+    create_or_update_bridge(BridgeType, BridgeName, Conf, 201).
+
+update_bridge(BridgeType, BridgeName, Conf) ->
+    create_or_update_bridge(BridgeType, BridgeName, Conf, 200).
+
+create_or_update_bridge(BridgeType, BridgeName, Conf, HttpStatusCode) ->
+    case emqx_bridge:create(BridgeType, BridgeName, Conf) of
+        {ok, _} ->
+            lookup_from_all_nodes(BridgeType, BridgeName, HttpStatusCode);
+        {error, #{kind := validation_error} = Reason} ->
+            ?BAD_REQUEST(map_to_json(Reason))
+    end.
+
 '/bridges/:id/enable/:enable'(put, #{bindings := #{id := Id, enable := Enable}}) ->
     ?TRY_PARSE_ID(
         Id,
@@ -600,15 +612,15 @@ lookup_from_local_node(BridgeType, BridgeName) ->
             OperFunc ->
                 case emqx_bridge:disable_enable(OperFunc, BridgeType, BridgeName) of
                     {ok, _} ->
-                        204;
+                        ?NO_CONTENT;
                     {error, {pre_config_update, _, bridge_not_found}} ->
                         ?BRIDGE_NOT_FOUND(BridgeType, BridgeName);
                     {error, {_, _, timeout}} ->
-                        {503, error_msg('SERVICE_UNAVAILABLE', <<"request timeout">>)};
+                        ?SERVICE_UNAVAILABLE(<<"request timeout">>);
                     {error, timeout} ->
-                        {503, error_msg('SERVICE_UNAVAILABLE', <<"request timeout">>)};
+                        ?SERVICE_UNAVAILABLE(<<"request timeout">>);
                     {error, Reason} ->
-                        {500, error_msg('INTERNAL_ERROR', Reason)}
+                        ?INTERNAL_ERROR(Reason)
                 end
         end
     ).
@@ -728,7 +740,7 @@ pick_bridges_by_id(Type, Name, BridgesAllNodes) ->
 
 format_bridge_info([FirstBridge | _] = Bridges) ->
     Res = maps:without([node, metrics], FirstBridge),
-    NodeStatus = collect_status(Bridges),
+    NodeStatus = node_status(Bridges),
     redact(Res#{
         status => aggregate_status(NodeStatus),
         node_status => NodeStatus
@@ -741,8 +753,8 @@ format_bridge_metrics(Bridges) ->
         node_metrics => NodeMetrics
     }.
 
-collect_status(Bridges) ->
+node_status(Bridges) ->
-    [maps:with([node, status], B) || B <- Bridges].
+    [maps:with([node, status, status_reason], B) || B <- Bridges].
 
 aggregate_status(AllStatus) ->
     Head = fun([A | _]) -> A end,
@@ -813,12 +825,16 @@ format_resource(
         )
     ).
 
-format_resource_data(#{status := Status, metrics := Metrics}) ->
+format_resource_data(ResData) ->
-    #{status => Status, metrics => format_metrics(Metrics)};
+    maps:fold(fun format_resource_data/3, #{}, maps:with([status, metrics, error], ResData)).
-format_resource_data(#{status := Status}) ->
-    #{status => Status}.
 
-format_metrics(#{
+format_resource_data(error, undefined, Result) ->
+    Result;
+format_resource_data(error, Error, Result) ->
+    Result#{status_reason => emqx_misc:readable_error_msg(Error)};
+format_resource_data(
+    metrics,
+    #{
         counters := #{
             'dropped' := Dropped,
             'dropped.other' := DroppedOther,
@@ -837,9 +853,13 @@ format_metrics(#{
         rate := #{
             matched := #{current := Rate, last5m := Rate5m, max := RateMax}
         }
-}) ->
+    },
+    Result
+) ->
     Queued = maps:get('queuing', Gauges, 0),
     SentInflight = maps:get('inflight', Gauges, 0),
+    Result#{
+        metrics =>
             ?METRICS(
                 Dropped,
                 DroppedOther,
@@ -858,7 +878,10 @@ format_metrics(#{
                 Rate5m,
                 RateMax,
                 Rcvd
-    ).
+            )
+    };
+format_resource_data(K, V, Result) ->
+    Result#{K => V}.
 
 fill_defaults(Type, RawConf) ->
     PackedConf = pack_bridge_conf(Type, RawConf),
@@ -900,6 +923,7 @@ filter_out_request_body(Conf) ->
         <<"type">>,
         <<"name">>,
         <<"status">>,
+        <<"status_reason">>,
         <<"node_status">>,
         <<"node_metrics">>,
         <<"metrics">>,
@@ -907,9 +931,6 @@ filter_out_request_body(Conf) ->
     ],
     maps:without(ExtraConfs, Conf).
 
-error_msg(Code, Msg) ->
-    #{code => Code, message => emqx_misc:readable_error_msg(Msg)}.
-
 bin(S) when is_list(S) ->
     list_to_binary(S);
 bin(S) when is_atom(S) ->
@@ -920,30 +941,31 @@ bin(S) when is_binary(S) ->
 call_operation(NodeOrAll, OperFunc, Args = [_Nodes, BridgeType, BridgeName]) ->
     case is_ok(do_bpapi_call(NodeOrAll, OperFunc, Args)) of
         Ok when Ok =:= ok; is_tuple(Ok), element(1, Ok) =:= ok ->
-            204;
+            ?NO_CONTENT;
         {error, not_implemented} ->
             %% Should only happen if we call `start` on a node that is
             %% still on an older bpapi version that doesn't support it.
             maybe_try_restart(NodeOrAll, OperFunc, Args);
         {error, timeout} ->
-            {503, error_msg('SERVICE_UNAVAILABLE', <<"request timeout">>)};
+            ?SERVICE_UNAVAILABLE(<<"Request timeout">>);
         {error, {start_pool_failed, Name, Reason}} ->
-            {503,
+            ?SERVICE_UNAVAILABLE(
-                error_msg(
+                bin(io_lib:format("Failed to start ~p pool for reason ~p", [Name, Reason]))
-                    'SERVICE_UNAVAILABLE',
+            );
-                    bin(
-                        io_lib:format(
-                            "failed to start ~p pool for reason ~p",
-                            [Name, Reason]
-                        )
-                    )
-                )};
         {error, not_found} ->
-            ?BRIDGE_NOT_FOUND(BridgeType, BridgeName);
+            BridgeId = emqx_bridge_resource:bridge_id(BridgeType, BridgeName),
+            ?SLOG(warning, #{
+                msg => "bridge_inconsistent_in_cluster_for_call_operation",
+                reason => not_found,
+                type => BridgeType,
+                name => BridgeName,
+                bridge => BridgeId
+            }),
+            ?SERVICE_UNAVAILABLE(<<"Bridge not found on remote node: ", BridgeId/binary>>);
         {error, {node_not_found, Node}} ->
             ?NOT_FOUND(<<"Node not found: ", (atom_to_binary(Node))/binary>>);
         {error, Reason} when not is_tuple(Reason); element(1, Reason) =/= 'exit' ->
-            ?BAD_REQUEST(to_hr_reason(Reason))
+            ?BAD_REQUEST(Reason)
     end.
 
 maybe_try_restart(all, start_bridges_to_all_nodes, Args) ->
@@ -951,7 +973,7 @@ maybe_try_restart(all, start_bridges_to_all_nodes, Args) ->
 maybe_try_restart(Node, start_bridge_to_node, Args) ->
     call_operation(Node, restart_bridge_to_node, Args);
 maybe_try_restart(_, _, _) ->
-    501.
+    ?NOT_IMPLEMENTED.
 
 do_bpapi_call(all, Call, Args) ->
     maybe_unwrap(
@@ -982,19 +1004,6 @@ supported_versions(start_bridge_to_node) -> [2, 3];
 supported_versions(start_bridges_to_all_nodes) -> [2, 3];
 supported_versions(_Call) -> [1, 2, 3].
 
-to_hr_reason(nxdomain) ->
-    <<"Host not found">>;
-to_hr_reason(econnrefused) ->
-    <<"Connection refused">>;
-to_hr_reason({unauthorized_client, _}) ->
-    <<"Unauthorized client">>;
-to_hr_reason({not_authorized, _}) ->
-    <<"Not authorized">>;
-to_hr_reason({malformed_username_or_password, _}) ->
-    <<"Malformed username or password">>;
-to_hr_reason(Reason) ->
-    Reason.
-
 redact(Term) ->
     emqx_misc:redact(Term).
 
@@ -1018,3 +1027,8 @@ deobfuscate(NewConf, OldConf) ->
         #{},
         NewConf
     ).
+
+map_to_json(M) ->
+    emqx_json:encode(
+        emqx_map_lib:jsonable_map(M, fun(K, V) -> {K, emqx_map_lib:binary_string(V)} end)
+    ).

apps/emqx_bridge/src/schema/emqx_bridge_compatible_config.erl

@@ -86,7 +86,7 @@ default_ssl() ->
 
 default_resource_opts() ->
     #{
-        <<"async_inflight_window">> => 100,
+        <<"inflight_window">> => 100,
         <<"auto_restart_interval">> => <<"60s">>,
         <<"health_check_interval">> => <<"15s">>,
         <<"max_queue_bytes">> => <<"1GB">>,

apps/emqx_bridge/src/schema/emqx_bridge_schema.erl

@@ -106,6 +106,12 @@ common_bridge_fields() ->
 status_fields() ->
     [
         {"status", mk(status(), #{desc => ?DESC("desc_status")})},
+        {"status_reason",
+            mk(binary(), #{
+                required => false,
+                desc => ?DESC("desc_status_reason"),
+                example => <<"Connection refused">>
+            })},
         {"node_status",
             mk(
                 hoconsc:array(ref(?MODULE, "node_status")),
@@ -190,7 +196,13 @@ fields("node_metrics") ->
 fields("node_status") ->
     [
         node_name(),
-        {"status", mk(status(), #{})}
+        {"status", mk(status(), #{})},
+        {"status_reason",
+            mk(binary(), #{
+                required => false,
+                desc => ?DESC("desc_status_reason"),
+                example => <<"Connection refused">>
+            })}
     ].
 
 desc(bridges) ->

apps/emqx_bridge/test/emqx_bridge_webhook_SUITE.erl

@@ -172,7 +172,7 @@ bridge_async_config(#{port := Port} = Config) ->
         "  request_timeout = \"~ps\"\n"
         "  body = \"${id}\""
         "  resource_opts {\n"
-        "    async_inflight_window = 100\n"
+        "    inflight_window = 100\n"
         "    auto_restart_interval = \"60s\"\n"
         "    health_check_interval = \"15s\"\n"
         "    max_queue_bytes = \"1GB\"\n"

apps/emqx_coap/.gitignore

@@ -0,0 +1,19 @@
+.rebar3
+_*
+.eunit
+*.o
+*.beam
+*.plt
+*.swp
+*.swo
+.erlang.cookie
+ebin
+log
+erl_crash.dump
+.rebar
+logs
+_build
+.idea
+*.iml
+rebar3.crashdump
+*~

apps/emqx_coap/README.md

@@ -0,0 +1,31 @@
+# emqx_coap
+
+The CoAP gateway implements publish, subscribe, and receive messages as standard
+with [Publish-Subscribe Broker for the CoAP](https://datatracker.ietf.org/doc/html/draft-ietf-core-coap-pubsub-09).
+
+## Quick Start
+
+In EMQX 5.0, CoAP gateways can be configured and enabled through the Dashboard.
+
+It can also be enabled via the HTTP API or emqx.conf, e.g. In emqx.conf:
+
+```properties
+gateway.coap {
+
+  mountpoint = "coap/"
+
+  connection_required = false
+
+  listeners.udp.default {
+    bind = "5683"
+    max_connections = 1024000
+    max_conn_rate = 1000
+  }
+}
+```
+
+> Note:
+> Configuring the gateway via emqx.conf requires changes on a per-node basis,
+> but configuring it via Dashboard or the HTTP API will take effect across the cluster.
+
+More documentations: [CoAP Gateway](https://www.emqx.io/docs/en/v5.0/gateway/coap.html)