From 0fe3083cd762c475cabab8d40077db2da6daa5f1 Mon Sep 17 00:00:00 2001
From: x1001100011 <x1001100011@gmail.com>
Date: Wed, 9 Jun 2021 21:46:21 -0700
Subject: [PATCH 1/4] build: add erlang 24.0.1 for ci

---
 .ci/docker-compose-file/docker-compose.yaml | 24 +++++++++++++++--
 .github/workflows/build_packages.yaml       | 11 +++++++-
 .github/workflows/build_slim_packages.yaml  |  4 ++-
 .github/workflows/check_deps_integrity.yaml |  8 +++++-
 .github/workflows/run_emqx_app_tests.yaml   |  8 +++++-
 .github/workflows/run_fvt_tests.yaml        | 10 +++++--
 .github/workflows/run_test_cases.yaml       | 30 ++++++++++++++++-----
 apps/emqx_retainer/rebar.config             |  2 +-
 8 files changed, 82 insertions(+), 15 deletions(-)

diff --git a/.ci/docker-compose-file/docker-compose.yaml b/.ci/docker-compose-file/docker-compose.yaml
index a502aeb6e..5ba6498f3 100644
--- a/.ci/docker-compose-file/docker-compose.yaml
+++ b/.ci/docker-compose-file/docker-compose.yaml
@@ -1,8 +1,8 @@
 version: '3.9'
 
 services:
-  erlang:
-    container_name: erlang
+  erlang23:
+    container_name: erlang23
     image: emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04
     env_file:
       - conf.env
@@ -21,6 +21,26 @@ services:
     working_dir: /emqx
     tty: true
 
+  erlang24:
+    container_name: erlang24
+    image: emqx/build-env:erl24.0.1-emqx-1-ubuntu20.04
+    env_file:
+      - conf.env
+    environment:
+      GITHUB_ACTIONS: ${GITHUB_ACTIONS}
+      GITHUB_TOKEN: ${GITHUB_TOKEN}
+      GITHUB_RUN_ID: ${GITHUB_RUN_ID}
+      GITHUB_SHA: ${GITHUB_SHA}
+      GITHUB_RUN_NUMBER: ${GITHUB_RUN_NUMBER}
+      GITHUB_EVENT_NAME: ${GITHUB_EVENT_NAME}
+      GITHUB_REF: ${GITHUB_REF}
+    networks:
+      - emqx_bridge
+    volumes:
+      - ../..:/emqx
+    working_dir: /emqx
+    tty: true
+
 networks:
   emqx_bridge:
     driver: bridge
diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml
index 0b8316b61..a5c156cb5 100644
--- a/.github/workflows/build_packages.yaml
+++ b/.github/workflows/build_packages.yaml
@@ -10,8 +10,14 @@ on:
 
 jobs:
   prepare:
+    strategy:
+      matrix:
+        container:
+          - "emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04"
+          - "emqx/build-env:erl24.0.1-emqx-1-ubuntu20.04"
+
     runs-on: ubuntu-20.04
-    container: emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04
+    container: ${{ matrix.container }}
 
     outputs:
       profiles: ${{ steps.set_profile.outputs.profiles}}
@@ -213,6 +219,9 @@ jobs:
       fail-fast: false
       matrix:
         profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
+        erl_otp:
+          - 23.2.7.2-emqx-2
+          - 24.0.1-emqx-1
         arch:
           - amd64
           - arm64
diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml
index 162959040..afee6c2fe 100644
--- a/.github/workflows/build_slim_packages.yaml
+++ b/.github/workflows/build_slim_packages.yaml
@@ -16,6 +16,8 @@ jobs:
       matrix:
         erl_otp:
         - erl23.2.7.2-emqx-2
+        - erl24.0.1-emqx-1
+
         os:
         - ubuntu20.04
         - centos7
@@ -43,7 +45,7 @@ jobs:
       with:
         name: rebar3.crashdump
         path: ./rebar3.crashdump
-    - name: pakcages test
+    - name: packages test
       run: |
         export CODE_PATH=$GITHUB_WORKSPACE
         .ci/build_packages/tests.sh
diff --git a/.github/workflows/check_deps_integrity.yaml b/.github/workflows/check_deps_integrity.yaml
index 0aa8f7903..64564469c 100644
--- a/.github/workflows/check_deps_integrity.yaml
+++ b/.github/workflows/check_deps_integrity.yaml
@@ -4,8 +4,14 @@ on: [pull_request]
 
 jobs:
   check_deps_integrity:
+    strategy:
+      matrix:
+        container:
+          - "emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04"
+          - "emqx/build-env:erl24.0.1-emqx-1-ubuntu20.04"
+
     runs-on: ubuntu-20.04
-    container: emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04
+    container: ${{ matrix.container }}
 
     steps:
       - uses: actions/checkout@v2
diff --git a/.github/workflows/run_emqx_app_tests.yaml b/.github/workflows/run_emqx_app_tests.yaml
index 56089ffcf..6e64d7747 100644
--- a/.github/workflows/run_emqx_app_tests.yaml
+++ b/.github/workflows/run_emqx_app_tests.yaml
@@ -9,8 +9,14 @@ on:
 
 jobs:
     check_all:
+        strategy:
+          matrix:
+            container:
+              - "emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04"
+              - "emqx/build-env:erl24.0.1-emqx-1-ubuntu20.04"
+
         runs-on: ubuntu-20.04
-        container: emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04
+        container: ${{ matrix.container }}
 
         steps:
         - uses: actions/checkout@v2
diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml
index 108434cb5..6fb8f4ea6 100644
--- a/.github/workflows/run_fvt_tests.yaml
+++ b/.github/workflows/run_fvt_tests.yaml
@@ -183,8 +183,15 @@ jobs:
             exit $RESULT
 
     relup_test:
+        strategy:
+          matrix:
+            container:
+              - "emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04"
+              - "emqx/build-env:erl24.0.1-emqx-1-ubuntu20.04"
+
         runs-on: ubuntu-20.04
-        container: emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04
+        container: ${{ matrix.container }}
+
         defaults:
           run:
             shell: bash
@@ -295,4 +302,3 @@ jobs:
           with:
             name: lux_logs
             path: lux_logs
-
diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml
index 812058111..3e5a1e346 100644
--- a/.github/workflows/run_test_cases.yaml
+++ b/.github/workflows/run_test_cases.yaml
@@ -9,8 +9,14 @@ on:
 
 jobs:
     run_static_analysis:
+        strategy:
+          matrix:
+            container:
+              - "emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04"
+              - "emqx/build-env:erl24.0.1-emqx-1-ubuntu20.04"
+
         runs-on: ubuntu-20.04
-        container: emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04
+        container: ${{ matrix.container }}
 
         steps:
         - uses: actions/checkout@v2
@@ -26,8 +32,14 @@ jobs:
           run: make dialyzer
 
     run_proper_test:
+        strategy:
+          matrix:
+            container:
+              - "emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04"
+              - "emqx/build-env:erl24.0.1-emqx-1-ubuntu20.04"
+
         runs-on: ubuntu-20.04
-        container: emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04
+        container: ${{ matrix.container }}
 
         steps:
         - uses: actions/checkout@v2
@@ -41,6 +53,12 @@ jobs:
           run: make proper
 
     run_common_test:
+        strategy:
+          matrix:
+            otp_release:
+              - "erlang23"
+              - "erlang24"
+
         runs-on: ubuntu-20.04
 
         steps:
@@ -73,15 +91,15 @@ jobs:
                 up -d --build
         - name: run eunit
           run: |
-            docker exec -i erlang bash -c "make eunit"
+            docker exec -i ${{ matrix.otp_release }} bash -c "make eunit"
         - name: run common test
           run: |
-            docker exec -i erlang bash -c "make ct"
+            docker exec -i ${{ matrix.otp_release }} bash -c "make ct"
         - name: run cover
           run: |
             printenv > .env
-            docker exec -i erlang bash -c "make cover"
-            docker exec --env-file .env -i erlang bash -c "make coveralls"
+            docker exec -i ${{ matrix.otp_release }} bash -c "make cover"
+            docker exec --env-file .env -i ${{ matrix.otp_release }} bash -c "make coveralls"
         - name: cat rebar.crashdump
           if: failure()
           run: if [ -f 'rebar3.crashdump' ];then cat 'rebar3.crashdump'; fi
diff --git a/apps/emqx_retainer/rebar.config b/apps/emqx_retainer/rebar.config
index 7e762cb72..b49f979ac 100644
--- a/apps/emqx_retainer/rebar.config
+++ b/apps/emqx_retainer/rebar.config
@@ -19,6 +19,6 @@
  [{test,
    [{deps,
      [
-      {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.2.3"}}}]}
+      {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.4.0"}}}]}
    ]}
  ]}.

From cd92f73ca15b218cbc219ada9c0e9e63be9dd1ef Mon Sep 17 00:00:00 2001
From: Zaiming Shi <zmstone@gmail.com>
Date: Mon, 9 Aug 2021 17:39:26 +0200
Subject: [PATCH 2/4] test: add debug print

---
 .ci/build_packages/tests.sh | 8 ++++++--
 .tool-versions              | 2 +-
 bin/emqx                    | 2 +-
 3 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/.ci/build_packages/tests.sh b/.ci/build_packages/tests.sh
index 3b08f838a..b01d6ca27 100755
--- a/.ci/build_packages/tests.sh
+++ b/.ci/build_packages/tests.sh
@@ -1,11 +1,10 @@
 #!/bin/bash
 set -x -e -u
+export DEBUG=1
 export CODE_PATH=${CODE_PATH:-"/emqx"}
 export EMQX_NAME=${EMQX_NAME:-"emqx"}
 export PACKAGE_PATH="${CODE_PATH}/_packages/${EMQX_NAME}"
 export RELUP_PACKAGE_PATH="${CODE_PATH}/_upgrade_base"
-# export EMQX_NODE_NAME="emqx-on-$(uname -m)@127.0.0.1"
-# export EMQX_NODE_COOKIE=$(date +%s%N)
 
 case "$(uname -m)" in
     x86_64)
@@ -122,6 +121,9 @@ run_test(){
         tee -a "$emqx_env_vars" <<EOF
 export EMQX_ZONE__EXTERNAL__SERVER_KEEPALIVE=60
 export EMQX_MQTT__MAX_TOPIC_ALIAS=10
+export EMQX_LOG__CONSOLE_HANDLER__LEVEL=debug
+export EMQX_LOG__FILE_HANDLERS__EMQX_LOG__LEVEL=debug
+export EMQX_LOG__PRIMARY_LEVEL=debug
 EOF
         ## for ARM, due to CI env issue, skip start of quic listener for the moment
         [[ $(arch) == *arm* || $(arch) == aarch64 ]] && tee -a "$emqx_env_vars" <<EOF
@@ -151,6 +153,8 @@ EOF
     # shellcheck disable=SC2009 # pgrep does not support Extended Regular Expressions
     ps -ef | grep -E '\-progname\s.+emqx\s'
     if ! emqx 'stop'; then
+        # shellcheck disable=SC2009 # pgrep does not support Extended Regular Expressions
+        ps -ef | grep -E '\-progname\s.+emqx\s'
         echo "ERROR: failed_to_stop_emqx_with_the_stop_command"
         cat /var/log/emqx/erlang.log.1 || true
         cat /var/log/emqx/emqx.log.1 || true
diff --git a/.tool-versions b/.tool-versions
index b87853803..3eb01f497 100644
--- a/.tool-versions
+++ b/.tool-versions
@@ -1 +1 @@
-erlang 24.0.1-emqx-1
+erlang 24.0.5-emqx-1
diff --git a/bin/emqx b/bin/emqx
index 6b6a6504c..d21a234f2 100755
--- a/bin/emqx
+++ b/bin/emqx
@@ -489,7 +489,7 @@ case "$1" in
             # also log to syslog
             logger -t "${REL_NAME}[${PID}]" "STOP: $msg"
             # log to user console
-            echoerr "STOP: $msg"
+            echoerr "stop failed, $msg"
             exit 1
         fi
         logger -t "${REL_NAME}[${PID}]" "STOP: OK"

From 1a80b9dc4764191bd11853436fbf87887354b6f1 Mon Sep 17 00:00:00 2001
From: Zaiming Shi <zmstone@gmail.com>
Date: Mon, 9 Aug 2021 22:28:25 +0200
Subject: [PATCH 3/4] fix: bin/emqx change use ps -p to check if pid is running

---
 bin/emqx | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/bin/emqx b/bin/emqx
index d21a234f2..37ad53f48 100755
--- a/bin/emqx
+++ b/bin/emqx
@@ -302,9 +302,11 @@ bootstrapd() {
 # check if a PID is down
 is_down() {
     PID="$1"
-    if kill -s 0 "$PID" 2>/dev/null; then
+    if ps -p "$PID" >/dev/null; then
+        # still alive
         return 1
     fi
+    # it's gone
     return 0
 }
 
@@ -484,7 +486,8 @@ case "$1" in
             exit 1
         fi
         WAIT_TIME="${WAIT_FOR_ERLANG_STOP:-60}"
-        if ! wait_for "$WAIT_TIME" is_down "$PID"; then
+        if ! wait_for "$WAIT_TIME" 'is_down' "$PID"; then
+            ps -p "$PID"
             msg="dangling after ${WAIT_TIME} seconds"
             # also log to syslog
             logger -t "${REL_NAME}[${PID}]" "STOP: $msg"

From 31b1646611d4fb67c656c8f483de7f5217c69091 Mon Sep 17 00:00:00 2001
From: Zaiming Shi <zmstone@gmail.com>
Date: Mon, 9 Aug 2021 23:32:56 +0200
Subject: [PATCH 4/4] feat: inspect parent pid if emqx pid enters defunct state

---
 bin/emqx | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/bin/emqx b/bin/emqx
index 37ad53f48..a286d2801 100755
--- a/bin/emqx
+++ b/bin/emqx
@@ -303,7 +303,15 @@ bootstrapd() {
 is_down() {
     PID="$1"
     if ps -p "$PID" >/dev/null; then
-        # still alive
+        # still around
+        # shellcheck disable=SC2009 # this grep pattern is not a part of the progra names
+        if ps -p "$PID" | grep -q 'defunct'; then
+            # zombie state, print parent pid
+            parent="$(ps -o ppid= -p "$PID" | tr -d ' ')"
+            echo "WARN: $PID is marked <defunct>, parent:"
+            ps -p "$parent"
+            return 0
+        fi
         return 1
     fi
     # it's gone
@@ -487,7 +495,6 @@ case "$1" in
         fi
         WAIT_TIME="${WAIT_FOR_ERLANG_STOP:-60}"
         if ! wait_for "$WAIT_TIME" 'is_down' "$PID"; then
-            ps -p "$PID"
             msg="dangling after ${WAIT_TIME} seconds"
             # also log to syslog
             logger -t "${REL_NAME}[${PID}]" "STOP: $msg"