chore: fix Erlang distribution over TLS

bin/emqx

@@ -267,13 +267,6 @@ if [ -d "$ERTS_DIR/lib" ]; then
     export LD_LIBRARY_PATH="$ERTS_DIR/lib:$LD_LIBRARY_PATH"
 fi
 
-if [ -z "$WITH_EPMD" ]; then
-    EPMD_ARG="-start_epmd false -epmd_module ekka_epmd -proto_dist ekka"
-else
-    PROTO_DIST=$(grep -E '^[ \t]*cluster.proto_dist[ \t]*=[ \t]*' "$RUNNER_ETC_DIR/emqx.conf" 2> /dev/null | tail -1 | awk -F"= " '{print $NF}')
-    EPMD_ARG="-start_epmd true -proto_dist $PROTO_DIST"
-fi
-
 # Warn the user if ulimit -n is less than 1024
 ULIMIT_F=$(ulimit -n)
 if [ "$ULIMIT_F" -lt 1024 ]; then
@@ -463,6 +456,7 @@ wait_for() {
             return 1
         fi
         WAIT_TIME=$((WAIT_TIME - 1))
+
         sleep 1
     done
 }
@@ -523,6 +517,29 @@ case "$1" in
         ;;
 esac
 
+EPMD_ARG="-start_epmd false -epmd_module ekka_epmd -proto_dist ekka"
+PROTO_DIST=$(grep -E '^[ \t]*cluster.proto_dist[ \t]*=[ \t]*' "$RUNNER_ETC_DIR/emqx.conf" 2> /dev/null | tail -1 | awk -F"= " '{print $NF}')
+# this environment variable is required by ekka_dist module
+# because proto_dist is overriden to ekka, and there is a lack of ekka_tls module
+export EKKA_PROTO_DIST_MOD="${PROTO_DIST:-inet_tcp}"
+if [ "$EKKA_PROTO_DIST_MOD" = 'inet_tls' ]; then
+    SSL_DIST_OPTFILE=${EMQX_SSL_DIST_OPTFILE:-"$RUNNER_ETC_DIR/ssl_dist.conf"}
+    case "$SSL_DIST_OPTFILE" in
+        *\ *)
+            set +x
+            echoerr "Got space in: $SSL_DIST_OPTFILE"
+            echoerr "No space is allowed for Erlang distribution over SSL option file path."
+            echoerr "Configure it from environment varialbe EMQX_SSL_DIST_OPTFILE."
+            echoerr "Or make sure emqx root path '$RUNNER_ROOT_DIR' has no space"
+            exit 1
+            ;;
+        *)
+            true
+            ;;
+    esac
+    EPMD_ARG="${EPMD_ARG} -ssl_dist_optfile $SSL_DIST_OPTFILE"
+fi
+
 if [ "$IS_BOOT_COMMAND" = 'no' ]; then
     # for non-boot commands, inspect vm.<time>.args for node name
     # shellcheck disable=SC2012

bin/nodetool

@@ -55,6 +55,11 @@ main(Args) ->
             halt(1)
     end,
 
+
+    %% Mute logger from now on.
+    %% Otherwise Erlang distribution over TLS (inet_tls_dist) warning logs
+    %% and supervisor reports may contaminate io:format outputs
+    logger:set_primary_config(level, none),
     case RestArgs of
         ["getpid"] ->
             io:format("~p\n", [list_to_integer(rpc:call(TargetNode, os, getpid, []))]);

etc/emqx_cloud/vm.args

@@ -32,14 +32,6 @@
 ## (Disabled by default..use with caution!)
 #-heart
 
-## Specify the erlang distributed protocol.
-## Can be one of: inet_tcp, inet6_tcp, inet_tls
-#-proto_dist inet_tcp
-
-## Specify SSL Options in the file if using SSL for Erlang Distribution.
-## Used only when -proto_dist set to inet_tls
-#-ssl_dist_optfile {{ platform_etc_dir }}/ssl_dist.conf
-
 ## Specifies the net_kernel tick time in seconds.
 ## This is the approximate time a connected node may be unresponsive until
 ## it is considered down and thereby disconnected.

etc/emqx_edge/vm.args

@@ -31,14 +31,6 @@
 ## (Disabled by default..use with caution!)
 -heart
 
-## Specify the erlang distributed protocol.
-## Can be one of: inet_tcp, inet6_tcp, inet_tls
-#-proto_dist inet_tcp
-
-## Specify SSL Options in the file if using SSL for Erlang Distribution.
-## Used only when -proto_dist set to inet_tls
-#-ssl_dist_optfile {{ platform_etc_dir }}/ssl_dist.conf
-
 ## Specifies the net_kernel tick time in seconds.
 ## This is the approximate time a connected node may be unresponsive until
 ## it is considered down and thereby disconnected.

etc/ssl_dist.conf

@@ -5,7 +5,10 @@
 [{server,
   [{certfile, "{{ platform_etc_dir }}/certs/cert.pem"},
    {keyfile, "{{ platform_etc_dir }}/certs/key.pem"},
-   {secure_renegotiate, true},
+   {cacertfile, "{{ platform_etc_dir }}/certs/cacert.pem"},
-   {depth, 0}]},
+   {verify, verify_none}
+  ]},
  {client,
-  [{secure_renegotiate, true}]}].
+  [
+   {verify, verify_none}
+  ]}].

scripts/start-two-nodes-in-docker.sh

@@ -44,6 +44,7 @@ docker run -d -it --restart=always --name "$NODE1" \
   -e EMQX_NODE_NAME="emqx@$NODE1" \
   -e EMQX_NODE_COOKIE="$COOKIE" \
   -e WAIT_FOR_ERLANG=60 \
+  -e EMQX_CLUSTER__PROTO_DIST='inet_tls' \
   -p 18083:18083 \
   -v "$PROJ_DIR"/_build/"${REL_DIR}"/rel/emqx:/built \
   "$IMAGE" sh -c 'cp -r /built /emqx && /emqx/bin/emqx console'
@@ -53,6 +54,7 @@ docker run -d -it --restart=always --name "$NODE2" \
   -e EMQX_NODE_NAME="emqx@$NODE2" \
   -e EMQX_NODE_COOKIE="$COOKIE" \
   -e WAIT_FOR_ERLANG=60 \
+  -e EMQX_CLUSTER__PROTO_DIST='inet_tls' \
   -p 18084:18083 \
   -v "$PROJ_DIR"/_build/"${REL_DIR}"/rel/emqx:/built \
   "$IMAGE" sh -c 'cp -r /built /emqx && /emqx/bin/emqx console'