Merge pull request #10553 from qzhuyan/dev/william/tls-root-fun-verify-partial-chain

v4.4: feat(listener): TLS partial_chain validation

changes/v4.4.18-en.md

@@ -7,4 +7,8 @@
   The parameters of certain actions support using placeholder syntax to dynamically fill in the content of strings. The format of the placeholder syntax is `${key}`.
   Prior to the improvement, the `key` in `${key}` could only contain letters, numbers, and underscores. Now the `key` supports any UTF8 character after the improvement.
 
+- Adds a new feature to enable partial certificate chain validation for TLS listeners[#10553](https://github.com/emqx/emqx/pull/10553).
+  If partial_chain is set to 'true', the last certificate in cacertfile is treated as the terminal of the certificate trust-chain. That is, the TLS handshake does not require full trust-chain, and EMQX will not try to validate the chain all the way up to the root CA.
+  
 ## Bug fixes
+

changes/v4.4.18-zh.md

@@ -7,5 +7,9 @@
   某些动作的参数支持使用占位符语法，来动态的填充字符串的内容，占位符语法的格式为 `${key}`。
   改进前，`${key}` 中的 `key` 只能包含字母、数字和下划线。改进后 `key` 支持任意的 UTF8 字符了。
 
+- 增加了一个新的功能，为TLS监听器启用部分证书链验证[#10553](https://github.com/emqx/emqx/pull/10553)。
+  如果 partial_chain 设置为“true”，cacertfile 中的最后一个证书将被视为证书信任链的顶端证书。 也就是说，TLS 握手不需要完整的链，并且 EMQX 不会尝试一直验证链直到根 CA。
+
+  
 ## 修复
 

priv/emqx.schema

@@ -1646,6 +1646,10 @@ end}.
   {datatype, atom}
 ]}.
 
+{mapping, "listener.ssl.$name.partial_chain", "emqx.listeners", [
+  {datatype, atom}
+]}.
+
 {mapping, "listener.ssl.$name.fail_if_no_peer_cert", "emqx.listeners", [
   {datatype, {enum, [true, false]}}
 ]}.
@@ -2377,6 +2381,7 @@ end}.
                           {certfile,   cuttlefish:conf_get(Prefix ++ ".certfile", Conf, undefined)},
                           {cacertfile, cuttlefish:conf_get(Prefix ++ ".cacertfile", Conf, undefined)},
                           {verify,     cuttlefish:conf_get(Prefix ++ ".verify", Conf, undefined)},
+                          {partial_chain, cuttlefish:conf_get(Prefix ++ ".partial_chain", Conf, undefined)},
                           {fail_if_no_peer_cert, cuttlefish:conf_get(Prefix ++ ".fail_if_no_peer_cert", Conf, undefined)},
                           {secure_renegotiate, cuttlefish:conf_get(Prefix ++ ".secure_renegotiate", Conf, undefined)},
                           {reuse_sessions, cuttlefish:conf_get(Prefix ++ ".reuse_sessions", Conf, undefined)},

src/emqx.appup.src

@@ -2,21 +2,33 @@
 %% Unless you know what you are doing, DO NOT edit manually!!
 {VSN,
   [{"4.4.18",
-    [{load_module,emqx_relup,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_listeners,brutal_purge,soft_purge,[]},
+     {load_module,emqx_tls_lib,brutal_purge,soft_purge,[]},
+     {add_module,emqx_const_v2},
+     {load_module,emqx_relup,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]}]},
    {"4.4.17",
-    [{load_module,emqx_relup,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_listeners,brutal_purge,soft_purge,[]},
+     {load_module,emqx_tls_lib,brutal_purge,soft_purge,[]},
+     {add_module,emqx_const_v2},
+     {load_module,emqx_relup,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]},
      {load_module,emqx_plugins,brutal_purge,soft_purge,[]}]},
    {"4.4.16",
-    [{load_module,emqx_channel,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_listeners,brutal_purge,soft_purge,[]},
+     {load_module,emqx_tls_lib,brutal_purge,soft_purge,[]},
+     {add_module,emqx_const_v2},
+     {load_module,emqx_channel,brutal_purge,soft_purge,[]},
      {load_module,emqx_pool,brutal_purge,soft_purge,[]},
      {load_module,emqx_misc,brutal_purge,soft_purge,[]},
      {load_module,emqx_relup,brutal_purge,soft_purge,[]},
      {load_module,emqx_plugins,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]}]},
    {"4.4.15",
-    [{load_module,emqx_channel,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_listeners,brutal_purge,soft_purge,[]},
+     {load_module,emqx_tls_lib,brutal_purge,soft_purge,[]},
+     {add_module,emqx_const_v2},
+     {load_module,emqx_channel,brutal_purge,soft_purge,[]},
      {load_module,emqx_pool,brutal_purge,soft_purge,[]},
      {load_module,emqx,brutal_purge,soft_purge,[]},
      {load_module,emqx_misc,brutal_purge,soft_purge,[]},
@@ -25,7 +37,10 @@
      {load_module,emqx_relup,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]}]},
    {"4.4.14",
-    [{load_module,emqx_channel,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_listeners,brutal_purge,soft_purge,[]},
+     {load_module,emqx_tls_lib,brutal_purge,soft_purge,[]},
+     {add_module,emqx_const_v2},
+     {load_module,emqx_channel,brutal_purge,soft_purge,[]},
      {load_module,emqx_pool,brutal_purge,soft_purge,[]},
      {load_module,emqx_vm,brutal_purge,soft_purge,[]},
      {load_module,emqx_keepalive,brutal_purge,soft_purge,[]},
@@ -41,7 +56,10 @@
      {load_module,emqx_app,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_actions_trans,brutal_purge,soft_purge,[]}]},
    {"4.4.13",
-    [{load_module,emqx_channel,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_listeners,brutal_purge,soft_purge,[]},
+     {load_module,emqx_tls_lib,brutal_purge,soft_purge,[]},
+     {add_module,emqx_const_v2},
+     {load_module,emqx_channel,brutal_purge,soft_purge,[]},
      {load_module,emqx_pool,brutal_purge,soft_purge,[]},
      {load_module,emqx_vm,brutal_purge,soft_purge,[]},
      {load_module,emqx_keepalive,brutal_purge,soft_purge,[]},
@@ -58,7 +76,10 @@
      {load_module,emqx_relup,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]}]},
    {"4.4.12",
-    [{load_module,emqx_channel,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_listeners,brutal_purge,soft_purge,[]},
+     {load_module,emqx_tls_lib,brutal_purge,soft_purge,[]},
+     {add_module,emqx_const_v2},
+     {load_module,emqx_channel,brutal_purge,soft_purge,[]},
      {load_module,emqx_pool,brutal_purge,soft_purge,[]},
      {load_module,emqx_vm,brutal_purge,soft_purge,[]},
      {load_module,emqx_keepalive,brutal_purge,soft_purge,[]},
@@ -75,7 +96,10 @@
      {load_module,emqx_relup,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]}]},
    {"4.4.11",
-    [{load_module,emqx_pool,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_listeners,brutal_purge,soft_purge,[]},
+     {load_module,emqx_tls_lib,brutal_purge,soft_purge,[]},
+     {add_module,emqx_const_v2},
+     {load_module,emqx_pool,brutal_purge,soft_purge,[]},
      {load_module,emqx_vm,brutal_purge,soft_purge,[]},
      {load_module,emqx_keepalive,brutal_purge,soft_purge,[]},
      {load_module,emqx_plugins,brutal_purge,soft_purge,[]},
@@ -94,7 +118,9 @@
      {load_module,emqx_session,brutal_purge,soft_purge,[]},
      {load_module,emqx_channel,brutal_purge,soft_purge,[]}]},
    {"4.4.10",
-    [{load_module,emqx_pool,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_tls_lib,brutal_purge,soft_purge,[]},
+     {add_module,emqx_const_v2},
+     {load_module,emqx_pool,brutal_purge,soft_purge,[]},
      {load_module,emqx_vm,brutal_purge,soft_purge,[]},
      {load_module,emqx_keepalive,brutal_purge,soft_purge,[]},
      {load_module,emqx_plugins,brutal_purge,soft_purge,[]},
@@ -122,7 +148,9 @@
      {apply,{application,set_env,
                          [gen_rpc,insecure_auth_fallback_allowed,true]}}]},
    {"4.4.9",
-    [{load_module,emqx_pool,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_tls_lib,brutal_purge,soft_purge,[]},
+     {add_module,emqx_const_v2},
+     {load_module,emqx_pool,brutal_purge,soft_purge,[]},
      {load_module,emqx_vm,brutal_purge,soft_purge,[]},
      {load_module,emqx_keepalive,brutal_purge,soft_purge,[]},
      {load_module,emqx_broker,brutal_purge,soft_purge,[]},
@@ -155,7 +183,9 @@
      {apply,{application,set_env,
                          [gen_rpc,insecure_auth_fallback_allowed,true]}}]},
    {"4.4.8",
-    [{load_module,emqx_pool,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_tls_lib,brutal_purge,soft_purge,[]},
+     {add_module,emqx_const_v2},
+     {load_module,emqx_pool,brutal_purge,soft_purge,[]},
      {load_module,emqx_vm,brutal_purge,soft_purge,[]},
      {load_module,emqx_keepalive,brutal_purge,soft_purge,[]},
      {load_module,emqx_broker,brutal_purge,soft_purge,[]},
@@ -189,7 +219,9 @@
      {apply,{application,set_env,
                          [gen_rpc,insecure_auth_fallback_allowed,true]}}]},
    {"4.4.7",
-    [{load_module,emqx_pool,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_tls_lib,brutal_purge,soft_purge,[]},
+     {add_module,emqx_const_v2},
+     {load_module,emqx_pool,brutal_purge,soft_purge,[]},
      {load_module,emqx_vm,brutal_purge,soft_purge,[]},
      {load_module,emqx_keepalive,brutal_purge,soft_purge,[]},
      {load_module,emqx_broker,brutal_purge,soft_purge,[]},
@@ -223,7 +255,9 @@
      {apply,{application,set_env,
                          [gen_rpc,insecure_auth_fallback_allowed,true]}}]},
    {"4.4.6",
-    [{load_module,emqx_pool,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_tls_lib,brutal_purge,soft_purge,[]},
+     {add_module,emqx_const_v2},
+     {load_module,emqx_pool,brutal_purge,soft_purge,[]},
      {load_module,emqx_vm,brutal_purge,soft_purge,[]},
      {load_module,emqx_keepalive,brutal_purge,soft_purge,[]},
      {load_module,emqx_broker,brutal_purge,soft_purge,[]},
@@ -257,7 +291,9 @@
      {apply,{application,set_env,
                          [gen_rpc,insecure_auth_fallback_allowed,true]}}]},
    {"4.4.5",
-    [{load_module,emqx_pool,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_tls_lib,brutal_purge,soft_purge,[]},
+     {add_module,emqx_const_v2},
+     {load_module,emqx_pool,brutal_purge,soft_purge,[]},
      {load_module,emqx_vm,brutal_purge,soft_purge,[]},
      {load_module,emqx_keepalive,brutal_purge,soft_purge,[]},
      {load_module,emqx_broker,brutal_purge,soft_purge,[]},
@@ -293,7 +329,9 @@
      {apply,{application,set_env,
                          [gen_rpc,insecure_auth_fallback_allowed,true]}}]},
    {"4.4.4",
-    [{load_module,emqx_pool,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_tls_lib,brutal_purge,soft_purge,[]},
+     {add_module,emqx_const_v2},
+     {load_module,emqx_pool,brutal_purge,soft_purge,[]},
      {load_module,emqx_vm,brutal_purge,soft_purge,[]},
      {load_module,emqx_keepalive,brutal_purge,soft_purge,[]},
      {load_module,emqx_banned,brutal_purge,soft_purge,[]},
@@ -335,7 +373,9 @@
      {apply,{application,set_env,
                          [gen_rpc,insecure_auth_fallback_allowed,true]}}]},
    {"4.4.3",
-    [{load_module,emqx_pool,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_tls_lib,brutal_purge,soft_purge,[]},
+     {add_module,emqx_const_v2},
+     {load_module,emqx_pool,brutal_purge,soft_purge,[]},
      {load_module,emqx_vm,brutal_purge,soft_purge,[]},
      {load_module,emqx_keepalive,brutal_purge,soft_purge,[]},
      {load_module,emqx_banned,brutal_purge,soft_purge,[]},
@@ -383,7 +423,9 @@
      {apply,{application,set_env,
                          [gen_rpc,insecure_auth_fallback_allowed,true]}}]},
    {"4.4.2",
-    [{load_module,emqx_pool,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_tls_lib,brutal_purge,soft_purge,[]},
+     {add_module,emqx_const_v2},
+     {load_module,emqx_pool,brutal_purge,soft_purge,[]},
      {load_module,emqx_vm,brutal_purge,soft_purge,[]},
      {load_module,emqx_keepalive,brutal_purge,soft_purge,[]},
      {load_module,emqx_banned,brutal_purge,soft_purge,[]},
@@ -432,7 +474,9 @@
      {apply,{application,set_env,
                          [gen_rpc,insecure_auth_fallback_allowed,true]}}]},
    {"4.4.1",
-    [{load_module,emqx_pool,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_tls_lib,brutal_purge,soft_purge,[]},
+     {add_module,emqx_const_v2},
+     {load_module,emqx_pool,brutal_purge,soft_purge,[]},
      {load_module,emqx_vm,brutal_purge,soft_purge,[]},
      {load_module,emqx_keepalive,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_actions_trans,brutal_purge,soft_purge,[]},
@@ -485,7 +529,9 @@
      {apply,{application,set_env,
                          [gen_rpc,insecure_auth_fallback_allowed,true]}}]},
    {"4.4.0",
-    [{load_module,emqx_pool,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_tls_lib,brutal_purge,soft_purge,[]},
+     {add_module,emqx_const_v2},
+     {load_module,emqx_pool,brutal_purge,soft_purge,[]},
      {load_module,emqx_vm,brutal_purge,soft_purge,[]},
      {load_module,emqx_keepalive,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_actions_trans,brutal_purge,soft_purge,[]},
@@ -541,21 +587,29 @@
                          [gen_rpc,insecure_auth_fallback_allowed,true]}}]},
    {<<".*">>,[]}],
   [{"4.4.18",
-    [{load_module,emqx_relup,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_listeners,brutal_purge,soft_purge,[]},
+     {load_module,emqx_tls_lib,brutal_purge,soft_purge,[]},
+     {load_module,emqx_relup,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]}]},
    {"4.4.17",
-    [{load_module,emqx_relup,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_listeners,brutal_purge,soft_purge,[]},
+     {load_module,emqx_tls_lib,brutal_purge,soft_purge,[]},
+     {load_module,emqx_relup,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]},
      {load_module,emqx_plugins,brutal_purge,soft_purge,[]}]},
    {"4.4.16",
-    [{load_module,emqx_channel,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_listeners,brutal_purge,soft_purge,[]},
+     {load_module,emqx_tls_lib,brutal_purge,soft_purge,[]},
+     {load_module,emqx_channel,brutal_purge,soft_purge,[]},
      {load_module,emqx_pool,brutal_purge,soft_purge,[]},
      {load_module,emqx_misc,brutal_purge,soft_purge,[]},
      {load_module,emqx_plugins,brutal_purge,soft_purge,[]},
      {load_module,emqx_relup,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]}]},
    {"4.4.15",
-    [{load_module,emqx_channel,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_listeners,brutal_purge,soft_purge,[]},
+     {load_module,emqx_tls_lib,brutal_purge,soft_purge,[]},
+     {load_module,emqx_channel,brutal_purge,soft_purge,[]},
      {load_module,emqx_pool,brutal_purge,soft_purge,[]},
      {load_module,emqx,brutal_purge,soft_purge,[]},
      {load_module,emqx_plugins,brutal_purge,soft_purge,[]},
@@ -564,7 +618,9 @@
      {load_module,emqx_relup,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]}]},
    {"4.4.14",
-    [{load_module,emqx_channel,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_listeners,brutal_purge,soft_purge,[]},
+     {load_module,emqx_tls_lib,brutal_purge,soft_purge,[]},
+     {load_module,emqx_channel,brutal_purge,soft_purge,[]},
      {load_module,emqx_pool,brutal_purge,soft_purge,[]},
      {load_module,emqx_vm,brutal_purge,soft_purge,[]},
      {load_module,emqx_plugins,brutal_purge,soft_purge,[]},
@@ -580,7 +636,9 @@
      {load_module,emqx_app,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_actions_trans,brutal_purge,soft_purge,[]}]},
    {"4.4.13",
-    [{load_module,emqx_channel,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_listeners,brutal_purge,soft_purge,[]},
+     {load_module,emqx_tls_lib,brutal_purge,soft_purge,[]},
+     {load_module,emqx_channel,brutal_purge,soft_purge,[]},
      {load_module,emqx_pool,brutal_purge,soft_purge,[]},
      {load_module,emqx_vm,brutal_purge,soft_purge,[]},
      {load_module,emqx_plugins,brutal_purge,soft_purge,[]},
@@ -597,7 +655,9 @@
      {load_module,emqx_relup,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]}]},
    {"4.4.12",
-    [{load_module,emqx_channel,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_listeners,brutal_purge,soft_purge,[]},
+     {load_module,emqx_tls_lib,brutal_purge,soft_purge,[]},
+     {load_module,emqx_channel,brutal_purge,soft_purge,[]},
      {load_module,emqx_pool,brutal_purge,soft_purge,[]},
      {load_module,emqx_vm,brutal_purge,soft_purge,[]},
      {load_module,emqx_keepalive,brutal_purge,soft_purge,[]},
@@ -614,7 +674,9 @@
      {load_module,emqx_relup,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]}]},
    {"4.4.11",
-    [{load_module,emqx_pool,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_listeners,brutal_purge,soft_purge,[]},
+     {load_module,emqx_tls_lib,brutal_purge,soft_purge,[]},
+     {load_module,emqx_pool,brutal_purge,soft_purge,[]},
      {load_module,emqx_vm,brutal_purge,soft_purge,[]},
      {load_module,emqx_keepalive,brutal_purge,soft_purge,[]},
      {load_module,emqx_broker,brutal_purge,soft_purge,[]},
@@ -633,7 +695,8 @@
      {load_module,emqx_session,brutal_purge,soft_purge,[]},
      {delete_module,emqx_cover}]},
    {"4.4.10",
-    [{load_module,emqx_pool,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_tls_lib,brutal_purge,soft_purge,[]},
+     {load_module,emqx_pool,brutal_purge,soft_purge,[]},
      {load_module,emqx_vm,brutal_purge,soft_purge,[]},
      {load_module,emqx_keepalive,brutal_purge,soft_purge,[]},
      {load_module,emqx_broker,brutal_purge,soft_purge,[]},
@@ -658,7 +721,8 @@
      {delete_module,emqx_crl_cache},
      {delete_module,emqx_ocsp_cache}]},
    {"4.4.9",
-    [{load_module,emqx_pool,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_tls_lib,brutal_purge,soft_purge,[]},
+     {load_module,emqx_pool,brutal_purge,soft_purge,[]},
      {load_module,emqx_vm,brutal_purge,soft_purge,[]},
      {load_module,emqx_keepalive,brutal_purge,soft_purge,[]},
      {load_module,emqx_broker,brutal_purge,soft_purge,[]},
@@ -687,7 +751,8 @@
      {delete_module,emqx_crl_cache},
      {delete_module,emqx_ocsp_cache}]},
    {"4.4.8",
-    [{load_module,emqx_pool,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_tls_lib,brutal_purge,soft_purge,[]},
+     {load_module,emqx_pool,brutal_purge,soft_purge,[]},
      {load_module,emqx_vm,brutal_purge,soft_purge,[]},
      {load_module,emqx_keepalive,brutal_purge,soft_purge,[]},
      {load_module,emqx_broker,brutal_purge,soft_purge,[]},
@@ -717,7 +782,8 @@
      {delete_module,emqx_crl_cache},
      {delete_module,emqx_ocsp_cache}]},
    {"4.4.7",
-    [{load_module,emqx_pool,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_tls_lib,brutal_purge,soft_purge,[]},
+     {load_module,emqx_pool,brutal_purge,soft_purge,[]},
      {load_module,emqx_vm,brutal_purge,soft_purge,[]},
      {load_module,emqx_keepalive,brutal_purge,soft_purge,[]},
      {load_module,emqx_broker,brutal_purge,soft_purge,[]},
@@ -747,7 +813,8 @@
      {delete_module,emqx_crl_cache},
      {delete_module,emqx_ocsp_cache}]},
    {"4.4.6",
-    [{load_module,emqx_pool,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_tls_lib,brutal_purge,soft_purge,[]},
+     {load_module,emqx_pool,brutal_purge,soft_purge,[]},
      {load_module,emqx_vm,brutal_purge,soft_purge,[]},
      {load_module,emqx_keepalive,brutal_purge,soft_purge,[]},
      {load_module,emqx_broker,brutal_purge,soft_purge,[]},
@@ -777,7 +844,8 @@
      {delete_module,emqx_crl_cache},
      {delete_module,emqx_ocsp_cache}]},
    {"4.4.5",
-    [{load_module,emqx_pool,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_tls_lib,brutal_purge,soft_purge,[]},
+     {load_module,emqx_pool,brutal_purge,soft_purge,[]},
      {load_module,emqx_vm,brutal_purge,soft_purge,[]},
      {load_module,emqx_keepalive,brutal_purge,soft_purge,[]},
      {load_module,emqx_broker,brutal_purge,soft_purge,[]},
@@ -809,7 +877,8 @@
      {delete_module,emqx_crl_cache},
      {delete_module,emqx_ocsp_cache}]},
    {"4.4.4",
-    [{load_module,emqx_pool,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_tls_lib,brutal_purge,soft_purge,[]},
+     {load_module,emqx_pool,brutal_purge,soft_purge,[]},
      {load_module,emqx_vm,brutal_purge,soft_purge,[]},
      {load_module,emqx_keepalive,brutal_purge,soft_purge,[]},
      {load_module,emqx_banned,brutal_purge,soft_purge,[]},
@@ -847,7 +916,8 @@
      {delete_module,emqx_crl_cache},
      {delete_module,emqx_ocsp_cache}]},
    {"4.4.3",
-    [{load_module,emqx_pool,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_tls_lib,brutal_purge,soft_purge,[]},
+     {load_module,emqx_pool,brutal_purge,soft_purge,[]},
      {load_module,emqx_vm,brutal_purge,soft_purge,[]},
      {load_module,emqx_keepalive,brutal_purge,soft_purge,[]},
      {load_module,emqx_banned,brutal_purge,soft_purge,[]},
@@ -890,7 +960,8 @@
      {delete_module,emqx_crl_cache},
      {delete_module,emqx_ocsp_cache}]},
    {"4.4.2",
-    [{load_module,emqx_pool,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_tls_lib,brutal_purge,soft_purge,[]},
+     {load_module,emqx_pool,brutal_purge,soft_purge,[]},
      {load_module,emqx_vm,brutal_purge,soft_purge,[]},
      {load_module,emqx_keepalive,brutal_purge,soft_purge,[]},
      {load_module,emqx_banned,brutal_purge,soft_purge,[]},
@@ -934,7 +1005,8 @@
      {delete_module,emqx_crl_cache},
      {delete_module,emqx_ocsp_cache}]},
    {"4.4.1",
-    [{load_module,emqx_pool,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_tls_lib,brutal_purge,soft_purge,[]},
+     {load_module,emqx_pool,brutal_purge,soft_purge,[]},
      {load_module,emqx_vm,brutal_purge,soft_purge,[]},
      {load_module,emqx_keepalive,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_actions_trans,brutal_purge,soft_purge,[]},
@@ -982,7 +1054,8 @@
      {delete_module,emqx_crl_cache},
      {delete_module,emqx_ocsp_cache}]},
    {"4.4.0",
-    [{load_module,emqx_pool,brutal_purge,soft_purge,[]},
+    [{load_module,emqx_tls_lib,brutal_purge,soft_purge,[]},
+     {load_module,emqx_pool,brutal_purge,soft_purge,[]},
      {load_module,emqx_vm,brutal_purge,soft_purge,[]},
      {load_module,emqx_keepalive,brutal_purge,soft_purge,[]},
      {load_module,emqx_rule_actions_trans,brutal_purge,soft_purge,[]},

src/emqx_const_v2.erl

@@ -0,0 +1,45 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%
+%% @doc Never update this module, create a v3 instead.
+%%--------------------------------------------------------------------
+
+-module(emqx_const_v2).
+
+-export([ make_tls_root_fun/2
+        ]).
+
+%% @doc Build a root fun for verify TLS partial_chain.
+%% The `InputChain' is composed by OTP SSL with local cert store
+%% AND the cert (chain if any) from the client.
+%% @end
+make_tls_root_fun(cacert_from_cacertfile, [Trusted]) ->
+  %% Allow only one trusted ca cert, and just return the defined trusted CA cert,
+  fun(_InputChain) ->
+      %% Note, returing `trusted_ca` doesn't really mean it accepts the connection
+      %% OTP SSL app will do the path validation, signature validation subsequently.
+      {trusted_ca, Trusted}
+  end;
+make_tls_root_fun(cacert_from_cacertfile, [TrustedOne, TrustedTwo]) ->
+  %% Allow two trusted CA certs in case of CA cert renewal
+  %% This is a little expensive call as it compares the binaries.
+  fun(InputChain) ->
+      case lists:member(TrustedOne, InputChain) of
+        true ->
+          {trusted_ca, TrustedOne};
+        false ->
+          {trusted_ca, TrustedTwo}
+      end
+  end.

src/emqx_listeners.erl

@@ -138,7 +138,8 @@ start_listener(tcp, ListenOn, Options) ->
 start_listener(Proto, ListenOn, Options0) when Proto == ssl; Proto == tls ->
     ListenerID = proplists:get_value(listener_id, Options0),
     Options1 = proplists:delete(listener_id, Options0),
-    Options = emqx_ocsp_cache:inject_sni_fun(ListenerID, Options1),
+    Options2 = emqx_ocsp_cache:inject_sni_fun(ListenerID, Options1),
+    Options = emqx_tls_lib:inject_root_fun(Options2),
     ok = maybe_register_crl_urls(Options),
     start_mqtt_listener('mqtt:ssl', ListenOn, Options);
 

src/emqx_tls_lib.erl

@@ -22,8 +22,12 @@
         , default_ciphers/1
         , integral_ciphers/2
         , drop_tls13_for_old_otp/1
+        , inject_root_fun/1
+        , opt_partial_chain/1
         ]).
 
+-include("logger.hrl").
+
 %% non-empty string
 -define(IS_STRING(L), (is_list(L) andalso L =/= [] andalso is_integer(hd(L)))).
 %% non-empty list of strings
@@ -170,8 +174,49 @@ drop_tls13(SslOpts0) ->
         Ciphers -> replace(SslOpts1, ciphers, Ciphers -- ?TLSV13_EXCLUSIVE_CIPHERS)
     end.
 
+inject_root_fun(Options) ->
+    case proplists:get_value(ssl_options, Options) of
+        undefined ->
+            Options;
+        SslOpts ->
+            replace(Options, ssl_options, opt_partial_chain(SslOpts))
+    end.
+
+%% @doc enable TLS partial_chain validation if set.
+-spec opt_partial_chain(SslOpts :: proplists:proplist()) -> NewSslOpts :: proplists:proplist().
+opt_partial_chain(SslOpts) ->
+    case proplists:get_value(partial_chain, SslOpts, undefined) of
+        undefined ->
+            SslOpts;
+        false ->
+            SslOpts;
+        V when V =:= cacert_from_cacertfile orelse V == true ->
+            replace(SslOpts, partial_chain, rootfun_trusted_ca_from_cacertfile(1, SslOpts));
+        V when V =:= two_cacerts_from_cacertfile -> %% for certificate rotations
+            replace(SslOpts, partial_chain, rootfun_trusted_ca_from_cacertfile(2, SslOpts))
+    end.
+
 replace(Opts, Key, Value) -> [{Key, Value} | proplists:delete(Key, Opts)].
 
+%% @doc Helper, make TLS root_fun
+rootfun_trusted_ca_from_cacertfile(NumOfCerts, SslOpts) ->
+    Cacertfile = proplists:get_value(cacertfile, SslOpts, undefined),
+    try do_rootfun_trusted_ca_from_cacertfile(NumOfCerts, Cacertfile)
+    catch _Error:_Info:ST ->
+            %% The cacertfile will be checked by OTP SSL as well and OTP choice to be silent on this.
+            %% We are touching security sutffs, don't leak extra info..
+            ?LOG(error, "Failed to look for trusted cacert from cacertfile. Stacktrace: ~p", [ST]),
+            throw({error, ?FUNCTION_NAME})
+    end.
+do_rootfun_trusted_ca_from_cacertfile(NumOfCerts, Cacertfile) ->
+    {ok, PemBin} = file:read_file(Cacertfile),
+    %% The last one or two should be the top parent in the chain if it is a chain
+    Certs = public_key:pem_decode(PemBin),
+    Pos = length(Certs) - NumOfCerts + 1,
+    Trusted = [ CADer || {'Certificate', CADer, _} <-
+                             lists:sublist(public_key:pem_decode(PemBin), Pos, NumOfCerts)],
+    emqx_const_v2:make_tls_root_fun(cacert_from_cacertfile, Trusted).
+
 -if(?OTP_RELEASE > 22).
 -ifdef(TEST).
 -include_lib("eunit/include/eunit.hrl").
@@ -194,5 +239,5 @@ drop_tls13_no_versions_cipers_test() ->
 has_tlsv13_cipher(Ciphers) ->
     lists:any(fun(C) -> lists:member(C, Ciphers) end, ?TLSV13_EXCLUSIVE_CIPHERS).
 
--endif.
+-endif. %% TEST
--endif.
+-endif. %% OTP_RELEASE > 22

test/emqx_listener_tls_verify_chain_SUITE.erl

@@ -0,0 +1,173 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+-module(emqx_listener_tls_verify_chain_SUITE).
+
+-compile(export_all).
+-compile(nowarn_export_all).
+
+-include_lib("emqx/include/emqx.hrl").
+-include_lib("emqx/include/emqx_mqtt.hrl").
+-include_lib("eunit/include/eunit.hrl").
+-include_lib("common_test/include/ct.hrl").
+
+-import(emqx_test_tls_certs_helper, [ fail_when_ssl_error/1
+                                    , fail_when_no_ssl_alert/2
+                                    , generate_tls_certs/1
+                                    ]).
+
+
+all() -> emqx_ct:all(?MODULE).
+
+init_per_suite(Config) ->
+    generate_tls_certs(Config),
+    application:ensure_all_started(esockd),
+    [{ssl_config, ssl_config_verify_peer()} | Config].
+
+end_per_suite(_Config) ->
+    application:stop(esockd).
+
+t_conn_fail_with_intermediate_ca_cert(Config) ->
+  Port = emqx_test_tls_certs_helper:select_free_port(ssl),
+  DataDir = ?config(data_dir, Config),
+  Options = [{ssl_options, [ {cacertfile, filename:join(DataDir, "intermediate1.pem")}
+                           , {certfile, filename:join(DataDir, "server1.pem")}
+                           , {keyfile, filename:join(DataDir, "server1.key")}
+                           | ?config(ssl_config, Config)
+                           ]}],
+  emqx_listeners:start_listener(ssl, Port, Options),
+  {ok, Socket} = ssl:connect({127, 0, 0, 1}, Port, [{keyfile,  filename:join(DataDir, "client1.key")},
+                                                    {certfile,  filename:join(DataDir, "client1.pem")}
+                                                   ], 1000),
+
+  fail_when_no_ssl_alert(Socket, unknown_ca),
+  ok = ssl:close(Socket).
+
+
+t_conn_fail_with_other_intermediate_ca_cert(Config) ->
+  Port = emqx_test_tls_certs_helper:select_free_port(ssl),
+  DataDir = ?config(data_dir, Config),
+  Options = [{ssl_options, [ {cacertfile, filename:join(DataDir, "intermediate1.pem")}
+                           , {certfile, filename:join(DataDir, "server1.pem")}
+                           , {keyfile, filename:join(DataDir, "server1.key")}
+                           | ?config(ssl_config, Config)
+                           ]}],
+  emqx_listeners:start_listener(ssl, Port, Options),
+  {ok, Socket} = ssl:connect({127, 0, 0, 1}, Port, [{keyfile,  filename:join(DataDir, "client2.key")},
+                                                    {certfile,  filename:join(DataDir, "client2.pem")}
+                                                   ], 1000),
+
+  fail_when_no_ssl_alert(Socket, unknown_ca),
+  ok = ssl:close(Socket).
+
+t_conn_success_with_server_client_composed_complete_chain(Config) ->
+  Port = emqx_test_tls_certs_helper:select_free_port(ssl),
+  DataDir = ?config(data_dir, Config),
+  %% Server has root ca cert
+  Options = [{ssl_options, [ {cacertfile, filename:join(DataDir, "root.pem")}
+                           , {certfile, filename:join(DataDir, "server2.pem")}
+                           , {keyfile, filename:join(DataDir, "server2.key")}
+                           | ?config(ssl_config, Config)
+                           ]}],
+  %% Client has complete chain
+  emqx_listeners:start_listener(ssl, Port, Options),
+  {ok, Socket} = ssl:connect({127, 0, 0, 1}, Port, [{keyfile, filename:join(DataDir, "client2.key")},
+                                                    {certfile, filename:join(DataDir, "client2-intermediate2-bundle.pem")}
+                                                   ], 1000),
+  fail_when_ssl_error(Socket),
+  ok = ssl:close(Socket).
+
+t_conn_success_with_other_signed_client_composed_complete_chain(Config) ->
+  Port = emqx_test_tls_certs_helper:select_free_port(ssl),
+  DataDir = ?config(data_dir, Config),
+  %% Server has root ca cert
+  Options = [{ssl_options, [ {cacertfile, filename:join(DataDir, "root.pem")}
+                           , {certfile, filename:join(DataDir, "server1.pem")}
+                           , {keyfile, filename:join(DataDir, "server1.key")}
+                           | ?config(ssl_config, Config)
+                           ]}],
+  %% Client has partial_chain
+  emqx_listeners:start_listener(ssl, Port, Options),
+  {ok, Socket} = ssl:connect({127, 0, 0, 1}, Port, [{keyfile, filename:join(DataDir, "client2.key")},
+                                                    {certfile, filename:join(DataDir, "client2-intermediate2-bundle.pem")}
+                                                   ], 1000),
+  fail_when_ssl_error(Socket),
+  ok = ssl:close(Socket).
+
+t_conn_success_with_renewed_intermediate_root_bundle(Config) ->
+  Port = emqx_test_tls_certs_helper:select_free_port(ssl),
+  DataDir = ?config(data_dir, Config),
+  %% Server has root ca cert
+  Options = [{ssl_options, [ {cacertfile, filename:join(DataDir, "intermediate1_renewed-root-bundle.pem")}
+                           , {certfile, filename:join(DataDir, "server1.pem")}
+                           , {keyfile, filename:join(DataDir, "server1.key")}
+                           | ?config(ssl_config, Config)
+                           ]}],
+  emqx_listeners:start_listener(ssl, Port, Options),
+  {ok, Socket} = ssl:connect({127, 0, 0, 1}, Port, [{keyfile, filename:join(DataDir, "client1.key")},
+                                                    {certfile, filename:join(DataDir, "client1.pem")}
+                                                   ], 1000),
+  fail_when_ssl_error(Socket),
+  ok = ssl:close(Socket).
+
+t_conn_success_with_client_complete_cert_chain(Config) ->
+  Port = emqx_test_tls_certs_helper:select_free_port(ssl),
+  DataDir = ?config(data_dir, Config),
+  Options = [{ssl_options, [ {cacertfile, filename:join(DataDir, "root.pem")}
+                           , {certfile, filename:join(DataDir, "server2.pem")}
+                           , {keyfile, filename:join(DataDir, "server2.key")}
+                           | ?config(ssl_config, Config)
+                           ]}],
+  emqx_listeners:start_listener(ssl, Port, Options),
+  {ok, Socket} = ssl:connect({127, 0, 0, 1}, Port, [{keyfile,  filename:join(DataDir, "client2.key")},
+                                                    {certfile,  filename:join(DataDir, "client2-complete-bundle.pem")}
+                                                   ], 1000),
+  fail_when_ssl_error(Socket),
+  ok = ssl:close(Socket).
+
+t_conn_fail_with_server_partial_chain(Config) ->
+  Port = emqx_test_tls_certs_helper:select_free_port(ssl),
+  DataDir = ?config(data_dir, Config),
+  Options = [{ssl_options, [{cacertfile, filename:join(DataDir, "intermediate2.pem")} %% imcomplete at server side
+                           , {certfile, filename:join(DataDir, "server2.pem")}
+                           , {keyfile, filename:join(DataDir, "server2.key")}
+                           | ?config(ssl_config, Config)
+                           ]}],
+  emqx_listeners:start_listener(ssl, Port, Options),
+  {ok, Socket} = ssl:connect({127, 0, 0, 1}, Port, [{keyfile,  filename:join(DataDir, "client2.key")},
+                                                    {certfile,  filename:join(DataDir, "client2-complete-bundle.pem")}
+                                                   ], 1000),
+  fail_when_no_ssl_alert(Socket, unknown_ca),
+  ok = ssl:close(Socket).
+
+t_conn_fail_without_root_cacert(Config) ->
+  Port = emqx_test_tls_certs_helper:select_free_port(ssl),
+  DataDir = ?config(data_dir, Config),
+  Options = [{ssl_options, [ {cacertfile, filename:join(DataDir, "intermediate2.pem")}
+                           , {certfile, filename:join(DataDir, "server2.pem")}
+                           , {keyfile, filename:join(DataDir, "server2.key")}
+                           | ?config(ssl_config, Config)
+                           ]}],
+  emqx_listeners:start_listener(ssl, Port, Options),
+  {ok, Socket} = ssl:connect({127, 0, 0, 1}, Port, [{keyfile,  filename:join(DataDir, "client2.key")},
+                                                    {certfile,  filename:join(DataDir, "client2-intermediate2-bundle.pem")}
+                                                   ], 1000),
+  fail_when_no_ssl_alert(Socket, unknown_ca),
+  ok = ssl:close(Socket).
+
+ssl_config_verify_peer() ->
+  [ {verify, verify_peer}
+  , {fail_if_no_peer_cert, true}
+  ].

test/emqx_listener_tls_verify_partial_chain_SUITE.erl

@@ -0,0 +1,418 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+-module(emqx_listener_tls_verify_partial_chain_SUITE).
+
+-compile(export_all).
+-compile(nowarn_export_all).
+
+-include_lib("emqx/include/emqx.hrl").
+-include_lib("emqx/include/emqx_mqtt.hrl").
+-include_lib("eunit/include/eunit.hrl").
+-include_lib("common_test/include/ct.hrl").
+
+-import(emqx_test_tls_certs_helper, [ fail_when_ssl_error/1
+                                    , fail_when_no_ssl_alert/2
+                                    , generate_tls_certs/1
+                                    ]).
+
+
+all() -> emqx_ct:all(?MODULE).
+
+init_per_suite(Config) ->
+    generate_tls_certs(Config),
+    application:ensure_all_started(esockd),
+    [{ssl_config, ssl_config_verify_partial_chain()} | Config].
+
+end_per_suite(_Config) ->
+    application:stop(esockd).
+
+
+t_conn_success_with_server_intermediate_cacert_and_client_cert(Config) ->
+  Port = emqx_test_tls_certs_helper:select_free_port(ssl),
+  DataDir = ?config(data_dir, Config),
+  Options = [{ssl_options, [ {cacertfile, filename:join(DataDir, "intermediate1.pem")}
+                           , {certfile, filename:join(DataDir, "server1.pem")}
+                           , {keyfile, filename:join(DataDir, "server1.key")}
+                           | ?config(ssl_config, Config)
+                           ]}],
+  emqx_listeners:start_listener(ssl, Port, Options),
+  {ok, Socket} = ssl:connect({127, 0, 0, 1}, Port, [{keyfile,  filename:join(DataDir, "client1.key")},
+                                                    {certfile,  filename:join(DataDir, "client1.pem")}
+                                                   ], 1000),
+  fail_when_ssl_error(Socket),
+  ssl:close(Socket).
+
+t_conn_success_with_intermediate_cacert_bundle(Config) ->
+  Port = emqx_test_tls_certs_helper:select_free_port(ssl),
+  DataDir = ?config(data_dir, Config),
+  Options = [{ssl_options, [ {cacertfile, filename:join(DataDir, "server1-intermediate1-bundle.pem")}
+                           , {certfile, filename:join(DataDir, "server1.pem")}
+                           , {keyfile, filename:join(DataDir, "server1.key")}
+                           | ?config(ssl_config, Config)
+                           ]}],
+  emqx_listeners:start_listener(ssl, Port, Options),
+  {ok, Socket} = ssl:connect({127, 0, 0, 1}, Port, [{keyfile,  filename:join(DataDir, "client1.key")},
+                                                    {certfile,  filename:join(DataDir, "client1.pem")}
+                                                   ], 1000),
+  fail_when_ssl_error(Socket),
+  ssl:close(Socket).
+
+t_conn_success_with_renewed_intermediate_cacert(Config) ->
+  Port = emqx_test_tls_certs_helper:select_free_port(ssl),
+  DataDir = ?config(data_dir, Config),
+  Options = [{ssl_options, [ {cacertfile, filename:join(DataDir, "intermediate1_renewed.pem")}
+                           , {certfile, filename:join(DataDir, "server1.pem")}
+                           , {keyfile, filename:join(DataDir, "server1.key")}
+                           | ?config(ssl_config, Config)
+                           ]}],
+  emqx_listeners:start_listener(ssl, Port, Options),
+  {ok, Socket} = ssl:connect({127, 0, 0, 1}, Port, [{keyfile,  filename:join(DataDir, "client1.key")},
+                                                    {certfile,  filename:join(DataDir, "client1.pem")}
+                                                   ], 1000),
+  fail_when_ssl_error(Socket),
+  ssl:close(Socket).
+
+t_conn_fail_with_renewed_intermediate_cacert_and_client_using_old_complete_bundle(Config) ->
+  Port = emqx_test_tls_certs_helper:select_free_port(ssl),
+  DataDir = ?config(data_dir, Config),
+  Options = [{ssl_options, [ {cacertfile, filename:join(DataDir, "intermediate2_renewed.pem")}
+                           , {certfile, filename:join(DataDir, "server2.pem")}
+                           , {keyfile, filename:join(DataDir, "server2.key")}
+                           | ?config(ssl_config, Config)
+                           ]}],
+  emqx_listeners:start_listener(ssl, Port, Options),
+  {ok, Socket} = ssl:connect({127, 0, 0, 1}, Port, [{keyfile,  filename:join(DataDir, "client2.key")},
+                                                    {certfile,  filename:join(DataDir, "client2-complete-bundle.pem")}
+                                                   ], 1000),
+  fail_when_no_ssl_alert(Socket, unknown_ca),
+  ssl:close(Socket).
+
+t_conn_fail_with_renewed_intermediate_cacert_and_client_using_old_bundle(Config) ->
+  Port = emqx_test_tls_certs_helper:select_free_port(ssl),
+  DataDir = ?config(data_dir, Config),
+  Options = [{ssl_options, [ {cacertfile, filename:join(DataDir, "intermediate2_renewed.pem")}
+                           , {certfile, filename:join(DataDir, "server2.pem")}
+                           , {keyfile, filename:join(DataDir, "server2.key")}
+                           | ?config(ssl_config, Config)
+                           ]}],
+  emqx_listeners:start_listener(ssl, Port, Options),
+  {ok, Socket} = ssl:connect({127, 0, 0, 1}, Port, [{keyfile,  filename:join(DataDir, "client2.key")},
+                                                    {certfile, filename:join(DataDir, "client2-intermediate2-bundle.pem")}
+                                                   ], 1000),
+  fail_when_no_ssl_alert(Socket, unknown_ca),
+  ssl:close(Socket).
+
+t_conn_success_with_old_and_renewed_intermediate_cacert_and_client_provides_renewed_client_cert(Config) ->
+  Port = emqx_test_tls_certs_helper:select_free_port(ssl),
+  DataDir = ?config(data_dir, Config),
+  Options = [{ssl_options, [ {cacertfile, filename:join(DataDir, "intermediate2_renewed_old-bundle.pem")}
+                           , {certfile, filename:join(DataDir, "server2.pem")}
+                           , {keyfile, filename:join(DataDir, "server2.key")}
+                           , {partial_chain, two_cacerts_from_cacertfile}
+                           | ?config(ssl_config, Config)
+                           ]}],
+  emqx_listeners:start_listener(ssl, Port, Options),
+  {ok, Socket} = ssl:connect({127, 0, 0, 1}, Port, [{keyfile,  filename:join(DataDir, "client2.key")},
+                                                    {certfile, filename:join(DataDir, "client2_renewed.pem")}
+                                                   ], 1000),
+  fail_when_ssl_error(Socket),
+  ssl:close(Socket).
+
+%% Note, this is good to have for usecase coverage
+t_conn_success_with_new_intermediate_cacert_and_client_provides_renewed_client_cert_signed_by_old_intermediate(Config) ->
+  Port = emqx_test_tls_certs_helper:select_free_port(ssl),
+  DataDir = ?config(data_dir, Config),
+  Options = [{ssl_options, [ {cacertfile, filename:join(DataDir, "intermediate2_renewed.pem")}
+                           , {certfile, filename:join(DataDir, "server2.pem")}
+                           , {keyfile, filename:join(DataDir, "server2.key")}
+                           | ?config(ssl_config, Config)
+                           ]}],
+  emqx_listeners:start_listener(ssl, Port, Options),
+  {ok, Socket} = ssl:connect({127, 0, 0, 1}, Port, [{keyfile,  filename:join(DataDir, "client2.key")},
+                                                    {certfile, filename:join(DataDir, "client2_renewed.pem")}
+                                                   ], 1000),
+  fail_when_ssl_error(Socket),
+  ssl:close(Socket).
+
+%% @doc server should build a partial_chain with old version of ca cert.
+t_conn_success_with_old_and_renewed_intermediate_cacert_and_client_provides_client_cert(Config) ->
+  Port = emqx_test_tls_certs_helper:select_free_port(ssl),
+  DataDir = ?config(data_dir, Config),
+  Options = [{ssl_options, [ {cacertfile, filename:join(DataDir, "intermediate2_renewed_old-bundle.pem")}
+                           , {certfile, filename:join(DataDir, "server2.pem")}
+                           , {keyfile, filename:join(DataDir, "server2.key")}
+                           , {partial_chain, two_cacerts_from_cacertfile}
+                           | ?config(ssl_config, Config)
+                           ]}],
+  emqx_listeners:start_listener(ssl, Port, Options),
+  {ok, Socket} = ssl:connect({127, 0, 0, 1}, Port, [{keyfile,  filename:join(DataDir, "client2.key")},
+                                                    {certfile, filename:join(DataDir, "client2.pem")}
+                                                   ], 1000),
+  fail_when_ssl_error(Socket),
+  ssl:close(Socket).
+
+%% @doc verify when config does not allow two versions of certs from same trusted CA.
+t_conn_fail_with_renewed_and_old_intermediate_cacert_and_client_using_old_bundle(Config) ->
+  Port = emqx_test_tls_certs_helper:select_free_port(ssl),
+  DataDir = ?config(data_dir, Config),
+  Options = [{ssl_options, [ {cacertfile, filename:join(DataDir, "intermediate2_renewed_old-bundle.pem")}
+                           , {certfile, filename:join(DataDir, "server2.pem")}
+                           , {keyfile, filename:join(DataDir, "server2.key")}
+                           | ?config(ssl_config, Config)
+                           ]}],
+  emqx_listeners:start_listener(ssl, Port, Options),
+  {ok, Socket} = ssl:connect({127, 0, 0, 1}, Port, [{keyfile,  filename:join(DataDir, "client2.key")},
+                                                    {certfile, filename:join(DataDir, "client2-intermediate2-bundle.pem")}
+                                                   ], 1000),
+  fail_when_no_ssl_alert(Socket, unknown_ca),
+  ssl:close(Socket).
+
+%% @doc verify when config (two_cacerts_from_cacertfile) allows two versions of certs from same trusted CA.
+t_conn_success_with_old_and_renewed_intermediate_cacert_bundle_and_client_using_old_bundle(Config) ->
+  Port = emqx_test_tls_certs_helper:select_free_port(ssl),
+  DataDir = ?config(data_dir, Config),
+  Options = [{ssl_options, [ {cacertfile, filename:join(DataDir, "intermediate2_renewed_old-bundle.pem")}
+                           , {certfile, filename:join(DataDir, "server2.pem")}
+                           , {keyfile, filename:join(DataDir, "server2.key")}
+                           , {partial_chain, two_cacerts_from_cacertfile}
+                           | ?config(ssl_config, Config)
+                           ]}],
+  emqx_listeners:start_listener(ssl, Port, Options),
+  {ok, Socket} = ssl:connect({127, 0, 0, 1}, Port, [{keyfile,  filename:join(DataDir, "client2.key")},
+                                                    {certfile, filename:join(DataDir, "client2-intermediate2-bundle.pem")}
+                                                   ], 1000),
+  fail_when_ssl_error(Socket),
+  ssl:close(Socket).
+
+%% @doc: verify even if listener has old/new intermediate2 certs,
+%%       client1 should not able to connect with old intermediate2 cert.
+%%  In this case, listener verify_fun returns {trusted_ca, Oldintermediate2Cert} but OTP should still fail the validation
+%%  since the client1 cert is not signed by Oldintermediate2Cert (trusted CA cert).
+%% @end
+t_fail_success_with_old_and_renewed_intermediate_cacert_bundle_and_client_using_all_CAcerts(Config) ->
+  Port = emqx_test_tls_certs_helper:select_free_port(ssl),
+  DataDir = ?config(data_dir, Config),
+  Options = [{ssl_options, [ {cacertfile, filename:join(DataDir, "intermediate2_renewed_old-bundle.pem")}
+                           , {certfile, filename:join(DataDir, "server2.pem")}
+                           , {keyfile, filename:join(DataDir, "server2.key")}
+                           , {partial_chain, two_cacerts_from_cacertfile}
+                           | ?config(ssl_config, Config)
+                           ]}],
+  emqx_listeners:start_listener(ssl, Port, Options),
+  {ok, Socket} = ssl:connect({127, 0, 0, 1}, Port, [{keyfile,  filename:join(DataDir, "client1.key")},
+                                                    {certfile, filename:join(DataDir, "all-CAcerts-bundle.pem")}
+                                                   ], 1000),
+  fail_when_no_ssl_alert(Socket, unknown_ca),
+  ssl:close(Socket).
+
+t_conn_fail_with_renewed_intermediate_cacert_other_client(Config) ->
+  Port = emqx_test_tls_certs_helper:select_free_port(ssl),
+  DataDir = ?config(data_dir, Config),
+  Options = [{ssl_options, [ {cacertfile, filename:join(DataDir, "intermediate1_renewed.pem")}
+                           , {certfile, filename:join(DataDir, "server1.pem")}
+                           , {keyfile, filename:join(DataDir, "server1.key")}
+                           | ?config(ssl_config, Config)
+                           ]}],
+  emqx_listeners:start_listener(ssl, Port, Options),
+  {ok, Socket} = ssl:connect({127, 0, 0, 1}, Port, [{keyfile,  filename:join(DataDir, "client2.key")},
+                                                    {certfile,  filename:join(DataDir, "client2.pem")}
+                                                   ], 1000),
+  fail_when_no_ssl_alert(Socket, unknown_ca),
+  ssl:close(Socket).
+
+t_conn_fail_with_intermediate_cacert_bundle_but_incorrect_order(Config) ->
+  Port = emqx_test_tls_certs_helper:select_free_port(ssl),
+  DataDir = ?config(data_dir, Config),
+  Options = [{ssl_options, [ {cacertfile, filename:join(DataDir, "intermediate1-server1-bundle.pem")}
+                           , {certfile, filename:join(DataDir, "server1.pem")}
+                           , {keyfile, filename:join(DataDir, "server1.key")}
+                           | ?config(ssl_config, Config)
+                           ]}],
+  emqx_listeners:start_listener(ssl, Port, Options),
+  {ok, Socket} = ssl:connect({127, 0, 0, 1}, Port, [{keyfile,  filename:join(DataDir, "client1.key")},
+                                                    {certfile,  filename:join(DataDir, "client1.pem")}
+                                                   ], 1000),
+  fail_when_no_ssl_alert(Socket, unknown_ca),
+  ssl:close(Socket).
+
+t_conn_fail_when_singed_by_other_intermediate_ca(Config) ->
+  Port = emqx_test_tls_certs_helper:select_free_port(ssl),
+  DataDir = ?config(data_dir, Config),
+  Options = [{ssl_options, [ {cacertfile, filename:join(DataDir, "intermediate1.pem")}
+                           , {certfile, filename:join(DataDir, "server1.pem")}
+                           , {keyfile, filename:join(DataDir, "server1.key")}
+                           | ?config(ssl_config, Config)
+                           ]}],
+  emqx_listeners:start_listener(ssl, Port, Options),
+  {ok, Socket} = ssl:connect({127, 0, 0, 1}, Port, [{keyfile,  filename:join(DataDir, "client2.key")},
+                                                    {certfile,  filename:join(DataDir, "client2.pem")}
+                                                   ], 1000),
+  fail_when_no_ssl_alert(Socket, unknown_ca),
+  ok = ssl:close(Socket).
+
+t_conn_success_with_complete_chain_that_server_root_cacert_and_client_complete_cert_chain(Config) ->
+  Port = emqx_test_tls_certs_helper:select_free_port(ssl),
+  DataDir = ?config(data_dir, Config),
+  Options = [{ssl_options, [ {cacertfile, filename:join(DataDir, "root.pem")}
+                           , {certfile, filename:join(DataDir, "server2.pem")}
+                           , {keyfile, filename:join(DataDir, "server2.key")}
+                           | ?config(ssl_config, Config)
+                           ]}],
+  emqx_listeners:start_listener(ssl, Port, Options),
+  {ok, Socket} = ssl:connect({127, 0, 0, 1}, Port, [{keyfile,  filename:join(DataDir, "client2.key")},
+                                                    {certfile,  filename:join(DataDir, "client2-complete-bundle.pem")}
+                                                   ], 1000),
+  fail_when_ssl_error(Socket),
+  ok = ssl:close(Socket).
+
+t_conn_fail_with_other_client_complete_cert_chain(Config) ->
+  Port = emqx_test_tls_certs_helper:select_free_port(ssl),
+  DataDir = ?config(data_dir, Config),
+  Options = [{ssl_options, [ {cacertfile, filename:join(DataDir, "intermediate1.pem")}
+                           , {certfile, filename:join(DataDir, "server1.pem")}
+                           , {keyfile, filename:join(DataDir, "server1.key")}
+                           | ?config(ssl_config, Config)
+                           ]}],
+  emqx_listeners:start_listener(ssl, Port, Options),
+  {ok, Socket} = ssl:connect({127, 0, 0, 1}, Port, [{keyfile,  filename:join(DataDir, "client2.key")},
+                                                    {certfile,  filename:join(DataDir, "client2-complete-bundle.pem")}
+                                                   ], 1000),
+  fail_when_no_ssl_alert(Socket, unknown_ca),
+  ok = ssl:close(Socket).
+
+t_conn_fail_with_server_intermediate_and_other_client_complete_cert_chain(Config) ->
+  Port = emqx_test_tls_certs_helper:select_free_port(ssl),
+  DataDir = ?config(data_dir, Config),
+  Options = [{ssl_options, [ {cacertfile, filename:join(DataDir, "intermediate1-root-bundle.pem")}
+                           , {certfile, filename:join(DataDir, "server1.pem")}
+                           , {keyfile, filename:join(DataDir, "server1.key")}
+                           | ?config(ssl_config, Config)
+                           ]}],
+  emqx_listeners:start_listener(ssl, Port, Options),
+  {ok, Socket} = ssl:connect({127, 0, 0, 1}, Port, [{keyfile,  filename:join(DataDir, "client2.key")},
+                                                    {certfile,  filename:join(DataDir, "client2-complete-bundle.pem")}
+                                                   ], 1000),
+  fail_when_ssl_error(Socket),
+  ok = ssl:close(Socket).
+
+t_conn_success_with_server_intermediate_cacert_and_client_complete_chain(Config) ->
+  Port = emqx_test_tls_certs_helper:select_free_port(ssl),
+  DataDir = ?config(data_dir, Config),
+  Options = [{ssl_options, [ {cacertfile, filename:join(DataDir, "intermediate2.pem")}
+                           , {certfile, filename:join(DataDir, "server2.pem")}
+                           , {keyfile, filename:join(DataDir, "server2.key")}
+                           | ?config(ssl_config, Config)
+                           ]}],
+  emqx_listeners:start_listener(ssl, Port, Options),
+  {ok, Socket} = ssl:connect({127, 0, 0, 1}, Port, [{keyfile,  filename:join(DataDir, "client2.key")},
+                                                    {certfile,  filename:join(DataDir, "client2-complete-bundle.pem")}
+                                                   ], 1000),
+  fail_when_ssl_error(Socket),
+  ok = ssl:close(Socket).
+
+t_conn_fail_with_server_intermediate_chain_and_client_other_incomplete_cert_chain(Config) ->
+  Port = emqx_test_tls_certs_helper:select_free_port(ssl),
+  DataDir = ?config(data_dir, Config),
+  Options = [{ssl_options, [ {cacertfile, filename:join(DataDir, "intermediate1.pem")}
+                           , {certfile, filename:join(DataDir, "server1.pem")}
+                           , {keyfile, filename:join(DataDir, "server1.key")}
+                           | ?config(ssl_config, Config)
+                           ]}],
+  emqx_listeners:start_listener(ssl, Port, Options),
+  {ok, Socket} = ssl:connect({127, 0, 0, 1}, Port, [{keyfile,  filename:join(DataDir, "client2.key")},
+                                                    {certfile,  filename:join(DataDir, "client2-intermediate2-bundle.pem")}
+                                                   ], 1000),
+  fail_when_no_ssl_alert(Socket, unknown_ca),
+  ok = ssl:close(Socket).
+
+t_conn_fail_with_server_intermediate_and_other_client_root_chain(Config) ->
+  Port = emqx_test_tls_certs_helper:select_free_port(ssl),
+  DataDir = ?config(data_dir, Config),
+  Options = [{ssl_options, [ {cacertfile, filename:join(DataDir, "intermediate1.pem")}
+                           , {certfile, filename:join(DataDir, "server1.pem")}
+                           , {keyfile, filename:join(DataDir, "server1.key")}
+                           | ?config(ssl_config, Config)
+                           ]}],
+  emqx_listeners:start_listener(ssl, Port, Options),
+  {ok, Socket} = ssl:connect({127, 0, 0, 1}, Port, [{keyfile,  filename:join(DataDir, "client2.key")},
+                                                    {certfile,  filename:join(DataDir, "client2-root-bundle.pem")}
+                                                   ], 1000),
+  fail_when_no_ssl_alert(Socket, unknown_ca),
+  ok = ssl:close(Socket).
+
+t_conn_success_with_server_intermediate_and_client_root_chain(Config) ->
+  Port = emqx_test_tls_certs_helper:select_free_port(ssl),
+  DataDir = ?config(data_dir, Config),
+  Options = [{ssl_options, [ {cacertfile, filename:join(DataDir, "intermediate2.pem")}
+                           , {certfile, filename:join(DataDir, "server2.pem")}
+                           , {keyfile, filename:join(DataDir, "server2.key")}
+                           | ?config(ssl_config, Config)
+                           ]}],
+  emqx_listeners:start_listener(ssl, Port, Options),
+  {ok, Socket} = ssl:connect({127, 0, 0, 1}, Port, [{keyfile,  filename:join(DataDir, "client2.key")},
+                                                    {certfile,  filename:join(DataDir, "client2-root-bundle.pem")}
+                                                   ], 1000),
+  fail_when_ssl_error(Socket),
+  ok = ssl:close(Socket).
+
+
+%% @doc once rootCA cert present in cacertfile, sibling CA signed Client cert could connect.
+t_conn_success_with_server_all_CA_bundle_and_client_root_chain(Config) ->
+  Port = emqx_test_tls_certs_helper:select_free_port(ssl),
+  DataDir = ?config(data_dir, Config),
+  Options = [{ssl_options, [ {cacertfile, filename:join(DataDir, "all-CAcerts-bundle.pem")}
+                           , {certfile, filename:join(DataDir, "server1.pem")}
+                           , {keyfile, filename:join(DataDir, "server1.key")}
+                           | ?config(ssl_config, Config)
+                           ]}],
+  emqx_listeners:start_listener(ssl, Port, Options),
+  {ok, Socket} = ssl:connect({127, 0, 0, 1}, Port, [{keyfile,  filename:join(DataDir, "client2.key")},
+                                                    {certfile,  filename:join(DataDir, "client2-root-bundle.pem")}
+                                                   ], 1000),
+  fail_when_ssl_error(Socket),
+  ok = ssl:close(Socket).
+
+t_conn_fail_with_server_two_IA_bundle_and_client_root_chain(Config) ->
+  Port = emqx_test_tls_certs_helper:select_free_port(ssl),
+  DataDir = ?config(data_dir, Config),
+  Options = [{ssl_options, [ {cacertfile, filename:join(DataDir, "two-intermediates-bundle.pem")}
+                           , {certfile, filename:join(DataDir, "server1.pem")}
+                           , {keyfile, filename:join(DataDir, "server1.key")}
+                           | ?config(ssl_config, Config)
+                           ]}],
+  emqx_listeners:start_listener(ssl, Port, Options),
+  {ok, Socket} = ssl:connect({127, 0, 0, 1}, Port, [{keyfile,  filename:join(DataDir, "client2.key")},
+                                                    {certfile,  filename:join(DataDir, "client2-root-bundle.pem")}
+                                                   ], 1000),
+  fail_when_no_ssl_alert(Socket, unknown_ca),
+  ok = ssl:close(Socket).
+
+t_error_handling_invalid_cacertfile(Config) ->
+  Port = emqx_test_tls_certs_helper:select_free_port(ssl),
+  DataDir = ?config(data_dir, Config),
+  Options = [{ssl_options, [ {cacertfile, filename:join(DataDir, "server2.key")} %% trigger error
+                           , {certfile, filename:join(DataDir, "server2.pem")}
+                           , {keyfile, filename:join(DataDir, "server2.key")}
+                           | ?config(ssl_config, Config)
+                           ]}],
+  ?assertException(throw, {error, rootfun_trusted_ca_from_cacertfile}, emqx_listeners:start_listener(ssl, Port, Options)).
+
+
+ssl_config_verify_partial_chain() ->
+  [ {verify, verify_peer}
+  , {fail_if_no_peer_cert, true}
+  , {partial_chain, true}
+  ].

test/emqx_test_tls_certs_helper.erl

@@ -0,0 +1,260 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_test_tls_certs_helper).
+-export([ gen_ca/2
+        , gen_host_cert/3
+        , gen_host_cert/4
+
+        , select_free_port/1
+        , generate_tls_certs/1
+
+        , fail_when_ssl_error/1
+        , fail_when_ssl_error/2
+        , fail_when_no_ssl_alert/2
+        , fail_when_no_ssl_alert/3
+
+
+        ]).
+
+-include_lib("common_test/include/ct.hrl").
+
+%%-------------------------------------------------------------------------------
+%% TLS certs
+%%-------------------------------------------------------------------------------
+gen_ca(Path, Name) ->
+    %% Generate ca.pem and ca.key which will be used to generate certs
+    %% for hosts server and clients
+    ECKeyFile = eckey_name(Path),
+    filelib:ensure_dir(ECKeyFile),
+    os:cmd("openssl ecparam -name secp256r1 > " ++ ECKeyFile),
+    Cmd = lists:flatten(
+        io_lib:format(
+            "openssl req -new -x509 -nodes "
+            "-newkey ec:~s "
+            "-keyout ~s -out ~s -days 3650 "
+            "-addext basicConstraints=CA:TRUE "
+            "-subj \"/C=SE/O=TEST CA\"",
+            [
+                ECKeyFile,
+                ca_key_name(Path, Name),
+                ca_cert_name(Path, Name)
+            ]
+        )
+    ),
+    os:cmd(Cmd).
+
+ca_cert_name(Path, Name) ->
+    filename(Path, "~s.pem", [Name]).
+ca_key_name(Path, Name) ->
+    filename(Path, "~s.key", [Name]).
+
+eckey_name(Path) ->
+    filename(Path, "ec.key", []).
+
+gen_host_cert(H, CaName, Path) ->
+    gen_host_cert(H, CaName, Path, #{}).
+
+gen_host_cert(H, CaName, Path, Opts) ->
+    ECKeyFile = eckey_name(Path),
+    CN = str(H),
+    HKey = filename(Path, "~s.key", [H]),
+    HCSR = filename(Path, "~s.csr", [H]),
+    HCSR2 = filename(Path, "~s.csr", [H]),
+    HPEM = filename(Path, "~s.pem", [H]),
+    HPEM2 = filename(Path, "~s_renewed.pem", [H]),
+    HEXT = filename(Path, "~s.extfile", [H]),
+    PasswordArg =
+        case maps:get(password, Opts, undefined) of
+            undefined ->
+                " -nodes ";
+            Password ->
+                io_lib:format(" -passout pass:'~s' ", [Password])
+        end,
+
+    create_file(
+      HEXT,
+      "keyUsage=digitalSignature,keyAgreement,keyCertSign\n"
+      "basicConstraints=CA:TRUE \n"
+      "subjectAltName=DNS:~s\n",
+      [CN]
+    ),
+
+    CSR_Cmd = csr_cmd(PasswordArg, ECKeyFile, HKey, HCSR, CN),
+    CSR_Cmd2 = csr_cmd(PasswordArg, ECKeyFile, HKey, HCSR2, CN),
+
+    CERT_Cmd = cert_sign_cmd(HEXT, HCSR, ca_cert_name(Path, CaName), ca_key_name(Path, CaName), HPEM),
+    %% 2nd cert for testing renewed cert.
+    CERT_Cmd2 = cert_sign_cmd(HEXT, HCSR2, ca_cert_name(Path, CaName), ca_key_name(Path, CaName), HPEM2),
+    ct:pal(os:cmd(CSR_Cmd)),
+    ct:pal(os:cmd(CSR_Cmd2)),
+    ct:pal(os:cmd(CERT_Cmd)),
+    ct:pal(os:cmd(CERT_Cmd2)),
+    file:delete(HEXT).
+
+cert_sign_cmd(ExtFile, CSRFile, CACert, CAKey, OutputCert)->
+    lists:flatten(
+            io_lib:format(
+                "openssl x509 -req "
+                "-extfile ~s "
+                "-in ~s -CA ~s -CAkey ~s -CAcreateserial "
+                "-out ~s -days 500",
+                [
+                    ExtFile,
+                    CSRFile,
+                    CACert,
+                    CAKey,
+                    OutputCert
+                ]
+            )
+        ).
+
+csr_cmd(PasswordArg, ECKeyFile, HKey, HCSR, CN) ->
+    lists:flatten(
+      io_lib:format(
+        "openssl req -new ~s -newkey ec:~s "
+        "-keyout ~s -out ~s "
+        "-addext \"subjectAltName=DNS:~s\" "
+        "-addext basicConstraints=CA:TRUE "
+        "-addext keyUsage=digitalSignature,keyAgreement,keyCertSign "
+        "-subj \"/C=SE/O=TEST/CN=~s\"",
+        [PasswordArg, ECKeyFile, HKey, HCSR, CN, CN]
+       )
+     ).
+
+filename(Path, F, A) ->
+    filename:join(Path, str(io_lib:format(F, A))).
+
+str(Arg) ->
+    binary_to_list(iolist_to_binary(Arg)).
+
+create_file(Filename, Fmt, Args) ->
+    filelib:ensure_dir(Filename),
+    {ok, F} = file:open(Filename, [write]),
+    try
+        io:format(F, Fmt, Args)
+    after
+        file:close(F)
+    end,
+    ok.
+
+%% @doc get unused port from OS
+-spec select_free_port(tcp | udp | ssl | quic) -> inets:port_number().
+select_free_port(tcp) ->
+    select_free_port(gen_tcp, listen);
+select_free_port(udp) ->
+    select_free_port(gen_udp, open);
+select_free_port(ssl) ->
+    select_free_port(tcp);
+select_free_port(quic) ->
+    select_free_port(udp).
+
+select_free_port(GenModule, Fun) when
+    GenModule == gen_tcp orelse
+        GenModule == gen_udp
+->
+    {ok, S} = GenModule:Fun(0, [{reuseaddr, true}]),
+    {ok, Port} = inet:port(S),
+    ok = GenModule:close(S),
+    case os:type() of
+        {unix, darwin} ->
+            %% in MacOS, still get address_in_use after close port
+            timer:sleep(500);
+        _ ->
+            skip
+    end,
+    ct:pal("Select free OS port: ~p", [Port]),
+    Port.
+
+%% @doc fail the test if ssl_error recvd
+%%      post check for success conn establishment
+fail_when_ssl_error(Socket) ->
+    fail_when_ssl_error(Socket, 1000).
+fail_when_ssl_error(Socket, Timeout) ->
+  receive
+    {ssl_error, Socket, _} ->
+      ct:fail("Handshake failed!")
+  after Timeout ->
+      ok
+  end.
+
+%% @doc fail the test if no ssl_error recvd
+fail_when_no_ssl_alert(Socket, Alert) ->
+    fail_when_no_ssl_alert(Socket, Alert, 1000).
+fail_when_no_ssl_alert(Socket, Alert, Timeout) ->
+  receive
+    {ssl_error, Socket, {tls_alert, {Alert, AlertInfo}}} ->
+        ct:pal("alert info: ~p~n", [AlertInfo]);
+    {ssl_error, Socket, Other} ->
+        ct:fail("recv unexpected ssl_error: ~p~n", [Other])
+  after Timeout ->
+      ct:fail("No expected alert: ~p from Socket: ~p ", [Alert, Socket])
+  end.
+
+%% @doc Generate TLS cert chain for tests
+generate_tls_certs(Config) ->
+  DataDir = ?config(data_dir, Config),
+  gen_ca(DataDir, "root"),
+  gen_host_cert("intermediate1", "root", DataDir),
+  gen_host_cert("intermediate2", "root", DataDir),
+  gen_host_cert("server1", "intermediate1", DataDir),
+  gen_host_cert("client1", "intermediate1", DataDir),
+  gen_host_cert("server2", "intermediate2", DataDir),
+  gen_host_cert("client2", "intermediate2", DataDir),
+
+  %% Build bundles below
+  os:cmd(io_lib:format("cat ~p ~p ~p > ~p", [filename:join(DataDir, "client2.pem"),
+                                             filename:join(DataDir, "intermediate2.pem"),
+                                             filename:join(DataDir, "root.pem"),
+                                             filename:join(DataDir, "client2-complete-bundle.pem")
+                                            ])),
+  os:cmd(io_lib:format("cat ~p ~p > ~p", [filename:join(DataDir, "client2.pem"),
+                                          filename:join(DataDir, "intermediate2.pem"),
+                                          filename:join(DataDir, "client2-intermediate2-bundle.pem")
+                                         ])),
+  os:cmd(io_lib:format("cat ~p ~p > ~p", [filename:join(DataDir, "client2.pem"),
+                                          filename:join(DataDir, "root.pem"),
+                                          filename:join(DataDir, "client2-root-bundle.pem")
+                                         ])),
+  os:cmd(io_lib:format("cat ~p ~p > ~p", [filename:join(DataDir, "server1.pem"),
+                                          filename:join(DataDir, "intermediate1.pem"),
+                                          filename:join(DataDir, "server1-intermediate1-bundle.pem")
+                                         ])),
+  os:cmd(io_lib:format("cat ~p ~p > ~p", [filename:join(DataDir, "intermediate1.pem"),
+                                          filename:join(DataDir, "server1.pem"),
+                                          filename:join(DataDir, "intermediate1-server1-bundle.pem")
+                                         ])),
+  os:cmd(io_lib:format("cat ~p ~p > ~p", [filename:join(DataDir, "intermediate1_renewed.pem"),
+                                          filename:join(DataDir, "root.pem"),
+                                          filename:join(DataDir, "intermediate1_renewed-root-bundle.pem")
+                                         ])),
+  os:cmd(io_lib:format("cat ~p ~p > ~p", [filename:join(DataDir, "intermediate2.pem"),
+                                          filename:join(DataDir, "intermediate2_renewed.pem"),
+                                          filename:join(DataDir, "intermediate2_renewed_old-bundle.pem")
+                                         ])),
+  os:cmd(io_lib:format("cat ~p ~p > ~p", [filename:join(DataDir, "intermediate1.pem"),
+                                          filename:join(DataDir, "root.pem"),
+                                          filename:join(DataDir, "intermediate1-root-bundle.pem")
+                                         ])),
+  os:cmd(io_lib:format("cat ~p ~p ~p > ~p", [filename:join(DataDir, "root.pem"),
+                                             filename:join(DataDir, "intermediate2.pem"),
+                                             filename:join(DataDir, "intermediate1.pem"),
+                                             filename:join(DataDir, "all-CAcerts-bundle.pem")
+                                            ])),
+  os:cmd(io_lib:format("cat ~p ~p > ~p", [filename:join(DataDir, "intermediate2.pem"),
+                                          filename:join(DataDir, "intermediate1.pem"),
+                                          filename:join(DataDir, "two-intermediates-bundle.pem")
+                                         ])).